diff options
Diffstat (limited to 'erts/emulator/beam')
115 files changed, 23356 insertions, 12488 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index e2a79d6e4f..b97705ed96 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -41,8 +41,7 @@ static erts_smp_rwmtx_t atom_table_lock; #define atom_read_unlock() erts_smp_rwmtx_runlock(&atom_table_lock) #define atom_write_lock() erts_smp_rwmtx_rwlock(&atom_table_lock) #define atom_write_unlock() erts_smp_rwmtx_rwunlock(&atom_table_lock) -#define atom_init_lock() erts_smp_rwmtx_init(&atom_table_lock, \ - "atom_tab") + #if 0 #define ERTS_ATOM_PUT_OPS_STAT #endif @@ -304,12 +303,17 @@ init_atom_table(void) HashFunctions f; int i; Atom a; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT erts_smp_atomic_init(&atom_put_ops, 0); #endif - atom_init_lock(); + erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); + f.hash = (H_FUN) atom_hash; f.cmp = (HCMP_FUN) atom_cmp; f.alloc = (HALLOC_FUN) atom_alloc; diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 57c8b08223..68d64fb7b0 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2010. All Rights Reserved. +# Copyright Ericsson AB 1996-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -65,6 +65,7 @@ atom EXIT='EXIT' atom aborted atom abs_path atom absoluteURI +atom ac atom active atom all atom all_but_first @@ -75,6 +76,7 @@ atom allocator_sizes atom alloc_util_allocators atom allow_passive_connect atom already_loaded +atom amd64 atom anchored atom and atom andalso @@ -100,8 +102,15 @@ atom band atom big atom bif_return_trap atom binary +atom binary_bin_to_list_trap +atom binary_copy_trap +atom binary_longest_prefix_trap +atom binary_longest_suffix_trap +atom binary_match_trap +atom binary_matches_trap atom block atom blocked +atom bm atom bnot atom bor atom bxor @@ -111,10 +120,12 @@ atom bsl atom bsr atom bsr_anycrlf atom bsr_unicode +atom build_type atom busy_dist_port atom busy_port atom call atom call_count +atom call_time atom caller atom capture atom case_clause @@ -256,6 +267,7 @@ atom info atom info_msg atom initial_call atom input +atom internal atom internal_error atom internal_status atom instruction_counts @@ -368,6 +380,7 @@ atom old_heap_size atom on_load atom open atom open_error +atom opt atom or atom ordered_set atom orelse @@ -422,6 +435,7 @@ atom raw atom re atom re_pattern atom re_run_trap +atom read_concurrency atom ready_input atom ready_output atom ready_async @@ -453,6 +467,7 @@ atom scheduler atom scheduler_id atom schedulers_online atom scheme +atom scope atom sensitive atom sequential_tracer atom sequential_trace_token diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index b1feec7074..d76a7d8e9f 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -39,10 +39,10 @@ static Eterm check_process_code(Process* rp, Module* modp); static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp); static void delete_export_references(Eterm module); static int purge_module(int module); -static int is_native(Eterm* code); +static int is_native(BeamInstr* code); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); -static void remove_from_address_table(Eterm* code); +static void remove_from_address_table(BeamInstr* code); Eterm load_module_2(BIF_ALIST_2) @@ -142,7 +142,7 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) if ((modp = erts_get_module(BIF_ARG_1)) == NULL) { return am_undefined; } - return (is_native(modp->code) || + return ((modp->code && is_native(modp->code)) || (modp->old_code != 0 && is_native(modp->old_code))) ? am_true : am_false; } @@ -175,8 +175,12 @@ check_process_code_2(BIF_ALIST_2) Eterm res; if (internal_pid_index(BIF_ARG_1) >= erts_max_processes) goto error; - rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_MAIN); +#ifdef ERTS_SMP + rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, ERTS_PROC_LOCK_MAIN); +#else + rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); +#endif if (!rp) { BIF_RET(am_false); } @@ -187,8 +191,10 @@ check_process_code_2(BIF_ALIST_2) modp = erts_get_module(BIF_ARG_2); res = check_process_code(rp, modp); #ifdef ERTS_SMP - if (BIF_P != rp) + if (BIF_P != rp) { + erts_resume(rp, ERTS_PROC_LOCK_MAIN); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + } #endif BIF_RET(res); } @@ -337,15 +343,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) ep->code[0] == BIF_ARG_1 && ep->code[4] != 0) { ep->address = (void *) ep->code[4]; - ep->code[3] = 0; ep->code[4] = 0; } } modp->code[MI_ON_LOAD_FUNCTION_PTR] = 0; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { - Eterm* code; - Eterm* end; + BeamInstr* code; + BeamInstr* end; /* * The on_load function failed. Remove the loaded code. @@ -354,7 +359,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) */ erts_total_code_size -= modp->code_length; code = modp->code; - end = (Eterm *)((char *)code + modp->code_length); + end = (BeamInstr *)((char *)code + modp->code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->catches, code, modp->code_length); erts_free(ERTS_ALC_T_CODE, (void *) code); @@ -368,7 +373,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) BIF_RET(am_true); } - static void set_default_trace_pattern(Eterm module) { @@ -397,13 +401,13 @@ set_default_trace_pattern(Eterm module) static Eterm check_process_code(Process* rp, Module* modp) { - Eterm* start; + BeamInstr* start; char* mod_start; Uint mod_size; - Eterm* end; + BeamInstr* end; Eterm* sp; #ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp; + struct erl_off_heap_header* oh; int done_gc = 0; #endif @@ -418,7 +422,7 @@ check_process_code(Process* rp, Module* modp) * Pick up limits for the module. */ start = modp->old_code; - end = (Eterm *)((char *)start + modp->old_code_length); + end = (BeamInstr *)((char *)start + modp->old_code_length); mod_start = (char *) start; mod_size = modp->old_code_length; @@ -471,27 +475,27 @@ check_process_code(Process* rp, Module* modp) #ifndef HYBRID /* FIND ME! */ rescan: - for (funp = MSO(rp).funs; funp; funp = funp->next) { - Eterm* fun_code; - - fun_code = funp->fe->address; - - if (INSIDE((Eterm *) funp->fe->address)) { - if (done_gc) { - return am_true; - } else { - /* - * Try to get rid of this fun by garbage collecting. - * Clear both fvalue and ftrace to make sure they - * don't hold any funs. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; - FLAGS(rp) |= F_NEED_FULLSWEEP; - (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - goto rescan; + for (oh = MSO(rp).first; oh; oh = oh->next) { + if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { + ErlFunThing* funp = (ErlFunThing*) oh; + + if (INSIDE((BeamInstr *) funp->fe->address)) { + if (done_gc) { + return am_true; + } else { + /* + * Try to get rid of this fun by garbage collecting. + * Clear both fvalue and ftrace to make sure they + * don't hold any funs. + */ + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + done_gc = 1; + FLAGS(rp) |= F_NEED_FULLSWEEP; + (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + goto rescan; + } } } } @@ -576,7 +580,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(val, mod_start, mod_size)) { + if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { return 1; } break; @@ -596,7 +600,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(val, mod_start, mod_size)) { + if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { return 1; } break; @@ -617,8 +621,8 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) static int purge_module(int module) { - Eterm* code; - Eterm* end; + BeamInstr* code; + BeamInstr* end; Module* modp; /* @@ -653,7 +657,7 @@ purge_module(int module) ASSERT(erts_total_code_size >= modp->old_code_length); erts_total_code_size -= modp->old_code_length; code = modp->old_code; - end = (Eterm *)((char *)code + modp->old_code_length); + end = (BeamInstr *)((char *)code + modp->old_code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old_catches, code, modp->old_code_length); erts_free(ERTS_ALC_T_CODE, (void *) code); @@ -665,7 +669,7 @@ purge_module(int module) } static void -remove_from_address_table(Eterm* code) +remove_from_address_table(BeamInstr* code) { int i; @@ -738,11 +742,11 @@ delete_export_references(Eterm module) Export *ep = export_list(i); if (ep != NULL && (ep->code[0] == module)) { if (ep->address == ep->code+3 && - (ep->code[3] == (Eterm) em_apply_bif)) { + (ep->code[3] == (BeamInstr) em_apply_bif)) { continue; } ep->address = ep->code+3; - ep->code[3] = (Uint) em_call_error_handler; + ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; MatchSetUnref(ep->match_prog_set); ep->match_prog_set = NULL; @@ -774,7 +778,7 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) } static int -is_native(Eterm* code) +is_native(BeamInstr* code) { return ((Eterm *)code[MI_FUNCTIONS])[1] != 0; } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 1abf1dc10c..31910888d1 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -30,6 +30,7 @@ #include "error.h" #include "erl_binary.h" #include "beam_bp.h" +#include "erl_term.h" /* ************************************************************************* ** Macros @@ -100,6 +101,11 @@ do { \ (b)->prev = (a); \ } while (0) + +#define BREAK_IS_BIF (1) +#define BREAK_IS_ERL (0) + + /* ************************************************************************* ** Local prototypes */ @@ -109,24 +115,42 @@ do { \ */ static int set_break(Eterm mfa[3], int specified, - Binary *match_spec, Uint break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); static int set_module_break(Module *modp, Eterm mfa[3], int specified, - Binary *match_spec, Uint break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); -static int set_function_break(Module *modp, Uint *pc, - Binary *match_spec, Uint break_op, +static int set_function_break(Module *modp, BeamInstr *pc, int bif, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); static int clear_break(Eterm mfa[3], int specified, - Uint break_op); + BeamInstr break_op); static int clear_module_break(Module *modp, Eterm mfa[3], int specified, - Uint break_op); -static int clear_function_break(Module *modp, Uint *pc, - Uint break_op); - -static BpData *is_break(Uint *pc, Uint break_op); - + BeamInstr break_op); +static int clear_function_break(Module *modp, BeamInstr *pc, int bif, + BeamInstr break_op); + +static BpData *is_break(BeamInstr *pc, BeamInstr break_op); +static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op); + +/* bp_hash */ +#define BP_TIME_ADD(pi0, pi1) \ + do { \ + Uint r; \ + (pi0)->count += (pi1)->count; \ + (pi0)->s_time += (pi1)->s_time; \ + (pi0)->us_time += (pi1)->us_time; \ + r = (pi0)->us_time / 1000000; \ + (pi0)->s_time += r; \ + (pi0)->us_time = (pi0)->us_time % 1000000; \ + } while(0) + +static void bp_hash_init(bp_time_hash_t *hash, Uint n); +static void bp_hash_rehash(bp_time_hash_t *hash, Uint n); +static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem); +static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t *sitem); +static void bp_hash_delete(bp_time_hash_t *hash); /* ************************************************************************* @@ -145,7 +169,7 @@ erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, match_spec, - (Uint) BeamOp(op_i_trace_breakpoint), 0, tracer_pid); + (BeamInstr) BeamOp(op_i_trace_breakpoint), 0, tracer_pid); } int @@ -153,87 +177,84 @@ erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, match_spec, - (Uint) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); + (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); } +/* set breakpoint data for on exported bif entry */ + void -erts_set_mtrace_bif(Uint *pc, Binary *match_spec, Eterm tracer_pid) { - BpDataTrace *bdt; +erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); +} - bdt = (BpDataTrace *) pc[-4]; - if (bdt) { - MatchSetUnref(bdt->match_spec); - MatchSetRef(match_spec); - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - } else { - bdt = Alloc(sizeof(BpDataTrace)); - BpInit((BpData *) bdt, 0); - MatchSetRef(match_spec); - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - pc[-4] = (Uint) bdt; - } +void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) { + set_function_break(NULL, pc, BREAK_IS_BIF, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +} + +void erts_clear_time_trace_bif(BeamInstr *pc) { + clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_time_breakpoint)); } int erts_set_debug_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, NULL, - (Uint) BeamOp(op_i_debug_breakpoint), 0, NIL); + (BeamInstr) BeamOp(op_i_debug_breakpoint), 0, NIL); } int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return set_break(mfa, specified, NULL, - (Uint) BeamOp(op_i_count_breakpoint), count_op, NIL); + (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL); } - +int +erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + return set_break(mfa, specified, NULL, + (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +} int erts_clear_trace_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_trace_breakpoint)); + (BeamInstr) BeamOp(op_i_trace_breakpoint)); } int erts_clear_mtrace_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_mtrace_breakpoint)); + (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); } void -erts_clear_mtrace_bif(Uint *pc) { - BpDataTrace *bdt; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); - - bdt = (BpDataTrace *) pc[-4]; - if (bdt) { - if (bdt->match_spec) { - MatchSetUnref(bdt->match_spec); - } - Free(bdt); - } - pc[-4] = (Uint) NULL; +erts_clear_mtrace_bif(BeamInstr *pc) { + clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); } int erts_clear_debug_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_debug_breakpoint)); + (BeamInstr) BeamOp(op_i_debug_breakpoint)); } int erts_clear_count_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, - (Uint) BeamOp(op_i_count_breakpoint)); + (BeamInstr) BeamOp(op_i_count_breakpoint)); +} + +int +erts_clear_time_break(Eterm mfa[3], int specified) { + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + return clear_break(mfa, specified, + (BeamInstr) BeamOp(op_i_time_breakpoint)); } int @@ -250,10 +271,10 @@ erts_clear_module_break(Module *modp) { } int -erts_clear_function_break(Module *modp, Uint *pc) { +erts_clear_function_break(Module *modp, BeamInstr *pc) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); ASSERT(modp); - return clear_function_break(modp, pc, 0); + return clear_function_break(modp, pc, BREAK_IS_ERL, 0); } @@ -261,13 +282,16 @@ erts_clear_function_break(Module *modp, Uint *pc) { /* * SMP NOTE: Process p may have become exiting on return! */ -Uint -erts_trace_break(Process *p, Uint *pc, Eterm *args, +BeamInstr +erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, Uint32 *ret_flags, Eterm *tracer_pid) { Eterm tpid1, tpid2; - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; - - ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); + BpData **bds = (BpData **) (pc)[-4]; + BpDataTrace *bdt = NULL; + + ASSERT(bds); + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + bdt = (BpDataTrace *) bds[bp_sched2ix_proc(p)]; ASSERT(bdt); bdt = (BpDataTrace *) bdt->next; ASSERT(bdt); @@ -286,7 +310,7 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args, bdt->tracer_pid = tpid2; ErtsSmpBPUnlock(bdt); } - pc[-4] = (Uint) bdt; + bds[bp_sched2ix_proc(p)] = (BpData *) bdt; return bdt->orig_instr; } @@ -296,14 +320,17 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args, * SMP NOTE: Process p may have become exiting on return! */ Uint32 -erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local, +erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local, Eterm *tracer_pid) { - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; - + BpData **bds = (BpData **) (pc)[-4]; + BpDataTrace *bdt = NULL; + + ASSERT(tracer_pid); - if (bdt) { + if (bds) { Eterm tpid1, tpid2; Uint32 flags; + bdt = (BpDataTrace *)bds[bp_sched2ix_proc(p)]; ErtsSmpBPLock(bdt); tpid1 = tpid2 = bdt->tracer_pid; @@ -326,9 +353,9 @@ erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local, int -erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { +erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { BpDataTrace *bdt = - (BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_trace_breakpoint)); + (BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_trace_breakpoint)); if (bdt) { if (match_spec_ret) { @@ -345,9 +372,9 @@ erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { } int -erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { +erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { BpDataTrace *bdt = - (BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_mtrace_breakpoint)); + (BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); if (bdt) { if (match_spec_ret) { @@ -364,74 +391,455 @@ erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { } int -erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; - - if (bdt) { - if (match_spec_ret) { - *match_spec_ret = bdt->match_spec; - } - if (tracer_pid_ret) { - ErtsSmpBPLock(bdt); - *tracer_pid_ret = bdt->tracer_pid; - ErtsSmpBPUnlock(bdt); - } - return !0; - } - return 0; -} - -int -erts_is_native_break(Uint *pc) { +erts_is_native_break(BeamInstr *pc) { #ifdef HIPE - ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); - return pc[0] == (Uint) BeamOp(op_hipe_trap_call) - || pc[0] == (Uint) BeamOp(op_hipe_trap_call_closure); + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + return pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call) + || pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); #else return 0; #endif } int -erts_is_count_break(Uint *pc, Sint *count_ret) { +erts_is_count_break(BeamInstr *pc, Sint *count_ret) { BpDataCount *bdc = - (BpDataCount *) is_break(pc, (Uint) BeamOp(op_i_count_breakpoint)); + (BpDataCount *) is_break(pc, (BeamInstr) BeamOp(op_i_count_breakpoint)); if (bdc) { if (count_ret) { - ErtsSmpBPLock(bdc); - *count_ret = bdc->count; - ErtsSmpBPUnlock(bdc); + *count_ret = (Sint) erts_smp_atomic_read(&bdc->acount); } return !0; } return 0; } -Uint * +int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { + Uint i, ix; + bp_time_hash_t hash; + Uint size; + Eterm *hp, t; + bp_data_time_item_t *item = NULL; + BpDataTime *bdt = (BpDataTime *) is_break(pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + if (bdt) { + if (retval) { + /* collect all hashes to one hash */ + bp_hash_init(&hash, 64); + /* foreach threadspecific hash */ + for (i = 0; i < bdt->n; i++) { + bp_data_time_item_t *sitem; + + /* foreach hash bucket not NIL*/ + for(ix = 0; ix < bdt->hash[i].n; ix++) { + item = &(bdt->hash[i].item[ix]); + if (item->pid != NIL) { + sitem = bp_hash_get(&hash, item); + if (sitem) { + BP_TIME_ADD(sitem, item); + } else { + bp_hash_put(&hash, item); + } + } + } + } + /* *retval should be NIL or term from previous bif in export entry */ + + if (hash.used > 0) { + size = (5 + 2)*hash.used; + hp = HAlloc(p, size); + + for(ix = 0; ix < hash.n; ix++) { + item = &(hash.item[ix]); + if (item->pid != NIL) { + t = TUPLE4(hp, item->pid, + make_small(item->count), + make_small(item->s_time), + make_small(item->us_time)); + hp += 5; + *retval = CONS(hp, t, *retval); hp += 2; + } + } + } + bp_hash_delete(&hash); + } + return !0; + } + + return 0; +} + + +BeamInstr * erts_find_local_func(Eterm mfa[3]) { Module *modp; - Uint** code_base; - Uint* code_ptr; + BeamInstr** code_base; + BeamInstr* code_ptr; Uint i,n; if ((modp = erts_get_module(mfa[0])) == NULL) return NULL; - if ((code_base = (Uint **) modp->code) == NULL) + if ((code_base = (BeamInstr **) modp->code) == NULL) return NULL; - n = (Uint) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; - ASSERT(((Uint) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); + ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); ASSERT(mfa[0] == ((Eterm) code_ptr[2])); if (mfa[1] == ((Eterm) code_ptr[3]) && - ((Uint) mfa[2]) == code_ptr[4]) { + ((BeamInstr) mfa[2]) == code_ptr[4]) { return code_ptr + 5; } } return NULL; } +/* bp_hash */ +ERTS_INLINE Uint bp_sched2ix() { +#ifdef ERTS_SMP + ErtsSchedulerData *esdp; + esdp = erts_get_scheduler_data(); + return esdp->no - 1; +#else + return 0; +#endif +} +static void bp_hash_init(bp_time_hash_t *hash, Uint n) { + Uint size = sizeof(bp_data_time_item_t)*n; + Uint i; + + hash->n = n; + hash->used = 0; + + hash->item = (bp_data_time_item_t *)Alloc(size); + sys_memzero(hash->item, size); + + for(i = 0; i < n; ++i) { + hash->item[i].pid = NIL; + } +} + +static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) { + bp_data_time_item_t *item = NULL; + Uint size = sizeof(bp_data_time_item_t)*n; + Uint ix; + Uint hval; + + item = (bp_data_time_item_t *)Alloc(size); + sys_memzero(item, size); + + for( ix = 0; ix < n; ++ix) { + item[ix].pid = NIL; + } + + /* rehash, old hash -> new hash */ + + for( ix = 0; ix < hash->n; ix++) { + if (hash->item[ix].pid != NIL) { + + hval = ((hash->item[ix].pid) >> 4) % n; /* new n */ + + while (item[hval].pid != NIL) { + hval = (hval + 1) % n; + } + item[hval].pid = hash->item[ix].pid; + item[hval].count = hash->item[ix].count; + item[hval].s_time = hash->item[ix].s_time; + item[hval].us_time = hash->item[ix].us_time; + } + } + + Free(hash->item); + hash->n = n; + hash->item = item; +} +static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem) { + Eterm pid = sitem->pid; + Uint hval = (pid >> 4) % hash->n; + bp_data_time_item_t *item = NULL; + + item = hash->item; + + while (item[hval].pid != pid) { + if (item[hval].pid == NIL) return NULL; + hval = (hval + 1) % hash->n; + } + + return &(item[hval]); +} + +static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t* sitem) { + Uint hval; + float r = 0.0; + bp_data_time_item_t *item; + + /* make sure that the hash is not saturated */ + /* if saturated, rehash it */ + + r = hash->used / (float) hash->n; + + if (r > 0.7f) { + bp_hash_rehash(hash, hash->n * 2); + } + /* Do hval after rehash */ + hval = (sitem->pid >> 4) % hash->n; + + /* find free slot */ + item = hash->item; + + while (item[hval].pid != NIL) { + hval = (hval + 1) % hash->n; + } + item = &(hash->item[hval]); + + item->pid = sitem->pid; + item->s_time = sitem->s_time; + item->us_time = sitem->us_time; + item->count = sitem->count; + hash->used++; + + return item; +} + +static void bp_hash_delete(bp_time_hash_t *hash) { + hash->n = 0; + hash->used = 0; + Free(hash->item); + hash->item = NULL; +} + +static void bp_time_diff(bp_data_time_item_t *item, /* out */ + process_breakpoint_time_t *pbt, /* in */ + Uint ms, Uint s, Uint us) { + int dms,ds,dus; + + dms = ms - pbt->ms; + ds = s - pbt->s; + dus = us - pbt->us; + + /* get_sys_now may return zero difftime, + * this is ok. + */ + + ASSERT(dms >= 0 || ds >= 0 || dus >= 0); + + if (dus < 0) { + dus += 1000000; + ds -= 1; + } + if (ds < 0) { + ds += 1000000; + } + + item->s_time = ds; + item->us_time = dus; +} + +void erts_schedule_time_break(Process *p, Uint schedule) { + Uint ms, s, us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; + + ASSERT(p); + + pbt = ERTS_PROC_GET_CALL_TIME(p); + + if (pbt) { + + switch(schedule) { + case ERTS_BP_CALL_TIME_SCHEDULE_EXITING : + break; + case ERTS_BP_CALL_TIME_SCHEDULE_OUT : + /* When a process is scheduled _out_, + * timestamp it and add its delta to + * the previous breakpoint. + */ + + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + if (pbdt) { + get_sys_now(&ms,&s,&us); + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + break; + case ERTS_BP_CALL_TIME_SCHEDULE_IN : + /* When a process is scheduled _in_, + * timestamp it and remove the previous + * timestamp in the psd. + */ + get_sys_now(&ms,&s,&us); + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + break; + default : + ASSERT(0); + /* will never happen */ + break; + } + } /* pbt */ +} + +/* call_time breakpoint + * Accumulated times are added to the previous bp, + * not the current one. The current one is saved + * for future reference. + * The previous breakpoint is stored in the process it self, the psd. + * We do not need to store in a stack frame. + * There is no need for locking, each thread has its own + * area in each bp to save data. + * Since we need to diffrentiate between processes for each bp, + * every bp has a hash (per thread) to process-bp statistics. + * - egil + */ + +void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type) { + Uint ms,s,us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; + + ASSERT(p); + ASSERT(p->status == P_RUNNING); + + /* get previous timestamp and breakpoint + * from the process psd */ + + pbt = ERTS_PROC_GET_CALL_TIME(p); + get_sys_now(&ms,&s,&us); + + switch(type) { + /* get pbt + * timestamp = t0 + * lookup bdt from code + * set ts0 to pbt + * add call count here? + */ + case ERTS_BP_CALL_TIME_CALL: + case ERTS_BP_CALL_TIME_TAIL_CALL: + + if (pbt) { + ASSERT(pbt->pc); + /* add time to previous code */ + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + /* previous breakpoint */ + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + /* if null then the breakpoint was removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + + } else { + /* first call of process to instrumented function */ + pbt = Alloc(sizeof(process_breakpoint_time_t)); + (void *) ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCK_MAIN, pbt); + } + /* add count to this code */ + sitem.pid = p->id; + sitem.count = 1; + sitem.s_time = 0; + sitem.us_time = 0; + + /* this breakpoint */ + ASSERT(bdt); + h = &(bdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + + pbt->pc = pc; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + break; + + case ERTS_BP_CALL_TIME_RETURN: + /* get pbt + * lookup bdt from code + * timestamp = t1 + * get ts0 from pbt + * get item from bdt->hash[bp_hash(p->id)] + * ack diff (t1, t0) to item + */ + + if(pbt) { + /* might have been removed due to + * trace_pattern(false) + */ + ASSERT(pbt->pc); + + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + /* previous breakpoint */ + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + /* beware, the trace_pattern might have been removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + + pbt->pc = pc; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + } + break; + default : + ASSERT(0); + /* will never happen */ + break; + } +} /* ************************************************************************* @@ -440,7 +848,7 @@ erts_find_local_func(Eterm mfa[3]) { static int set_break(Eterm mfa[3], int specified, - Binary *match_spec, Eterm break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid) { Module *modp; @@ -470,45 +878,54 @@ static int set_break(Eterm mfa[3], int specified, } static int set_module_break(Module *modp, Eterm mfa[3], int specified, - Binary *match_spec, Uint break_op, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid) { - Uint** code_base; - Uint* code_ptr; + BeamInstr** code_base; + BeamInstr* code_ptr; int num_processed = 0; Uint i,n; ASSERT(break_op); ASSERT(modp); - code_base = (Uint **) modp->code; + code_base = (BeamInstr **) modp->code; if (code_base == NULL) { return 0; } - n = (Uint) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; - ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI)); + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) && (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) { - Uint *pc = code_ptr+5; + BeamInstr *pc = code_ptr+5; num_processed += - set_function_break(modp, pc, match_spec, + set_function_break(modp, pc, BREAK_IS_ERL, match_spec, break_op, count_op, tracer_pid); } } return num_processed; } -static int set_function_break(Module *modp, Uint *pc, - Binary *match_spec, Uint break_op, +static int set_function_break(Module *modp, BeamInstr *pc, int bif, + Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid) { - BpData *bd, **r; + + BeamInstr **code_base = NULL; + BpData *bd, **r, ***rs; size_t size; - Uint **code_base = (Uint **)modp->code; + Uint ix = 0; - ASSERT(code_base); - ASSERT(code_base <= (Uint **)pc); - ASSERT((Uint **)pc < code_base + (modp->code_length/sizeof(Uint *))); + if (bif == BREAK_IS_ERL) { + code_base = (BeamInstr **)modp->code; + ASSERT(code_base); + ASSERT(code_base <= (BeamInstr **)pc); + ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *))); + } else { + ASSERT(*pc == (BeamInstr) em_apply_bif); + ASSERT(modp == NULL); + } + /* * Currently no trace support for native code. */ @@ -517,8 +934,9 @@ static int set_function_break(Module *modp, Uint *pc, } /* Do not allow two breakpoints of the same kind */ if ( (bd = is_break(pc, break_op))) { - if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) - || break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) + || break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { + BpDataTrace *bdt = (BpDataTrace *) bd; Binary *old_match_spec; @@ -531,71 +949,115 @@ static int set_function_break(Module *modp, Uint *pc, ErtsSmpBPUnlock(bdt); MatchSetUnref(old_match_spec); } else { + BpDataCount *bdc = (BpDataCount *) bd; + erts_aint_t count = 0; + erts_aint_t res = 0; + ASSERT(! match_spec); ASSERT(is_nil(tracer_pid)); - if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) { - BpDataCount *bdc = (BpDataCount *) bd; - ErtsSmpBPLock(bdc); + if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { if (count_op == erts_break_stop) { - if (bdc->count >= 0) { - bdc->count = -bdc->count-1; /* Stop call counter */ + count = erts_smp_atomic_read(&bdc->acount); + if (count >= 0) { + while(1) { + res = erts_smp_atomic_cmpxchg(&bdc->acount, -count - 1, count); + if ((res == count) || count < 0) break; + count = res; + } } } else { - bdc->count = 0; /* Reset call counter */ + /* Reset call counter */ + erts_smp_atomic_set(&bdc->acount, 0); } - ErtsSmpBPUnlock(bdc); + + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + + if (count_op == erts_break_stop) { + bdt->pause = 1; + } else { + bdt->pause = 0; + for (i = 0; i < bdt->n; i++) { + bp_hash_delete(&(bdt->hash[i])); + bp_hash_init(&(bdt->hash[i]), 32); + } + } + } else { ASSERT (! count_op); } } return 1; } - if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) || - break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { size = sizeof(BpDataTrace); } else { ASSERT(! match_spec); ASSERT(is_nil(tracer_pid)); - if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) { - if (count_op == erts_break_reset - || count_op == erts_break_stop) { + if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { + if (count_op == erts_break_reset || count_op == erts_break_stop) { /* Do not insert a new breakpoint */ return 1; } size = sizeof(BpDataCount); + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + if (count_op == erts_break_reset || count_op == erts_break_stop) { + /* Do not insert a new breakpoint */ + return 1; + } + size = sizeof(BpDataTime); } else { ASSERT(! count_op); - ASSERT(break_op == (Uint) BeamOp(op_i_debug_breakpoint)); + ASSERT(break_op == (BeamInstr) BeamOp(op_i_debug_breakpoint)); size = sizeof(BpDataDebug); } } - r = (BpData **) (pc-4); + rs = (BpData ***) (pc-4); + if (! *rs) { + size_t ssize = sizeof(BeamInstr) * erts_no_schedulers; + *rs = (BpData **) Alloc(ssize); + sys_memzero(*rs, ssize); + } + + r = &((*rs)[0]); + if (! *r) { - ASSERT(*pc != (Uint) BeamOp(op_i_trace_breakpoint)); - ASSERT(*pc != (Uint) BeamOp(op_i_mtrace_breakpoint)); - ASSERT(*pc != (Uint) BeamOp(op_i_debug_breakpoint)); - ASSERT(*pc != (Uint) BeamOp(op_i_count_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_trace_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_debug_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_count_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_time_breakpoint)); /* First breakpoint; create singleton ring */ bd = Alloc(size); BpInit(bd, *pc); - *pc = break_op; *r = bd; + if (bif == BREAK_IS_ERL) { + *pc = break_op; + } } else { - ASSERT(*pc == (Uint) BeamOp(op_i_trace_breakpoint) || - *pc == (Uint) BeamOp(op_i_mtrace_breakpoint) || - *pc == (Uint) BeamOp(op_i_debug_breakpoint) || - *pc == (Uint) BeamOp(op_i_count_breakpoint)); - if (*pc == (Uint) BeamOp(op_i_debug_breakpoint)) { + ASSERT(*pc == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_mtrace_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_debug_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_time_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_count_breakpoint) || + *pc == (BeamInstr) em_apply_bif); + if (*pc == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { /* Debug bp must be last, so if it is also first; * it must be singleton. */ - ASSERT(BpSingleton(*r)); + ASSERT(BpSingleton(*r)); /* Insert new bp first in the ring, i.e second to last. */ bd = Alloc(size); BpInitAndSpliceNext(bd, *pc, *r); - *pc = break_op; - } else if ((*r)->prev->orig_instr - == (Uint) BeamOp(op_i_debug_breakpoint)) { + if (bif == BREAK_IS_ERL) { + *pc = break_op; + } + } else if ((*r)->prev->orig_instr + == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { /* Debug bp last in the ring; insert new second to last. */ bd = Alloc(size); BpInitAndSplicePrev(bd, (*r)->prev->orig_instr, *r); @@ -608,25 +1070,43 @@ static int set_function_break(Module *modp, Uint *pc, *r = bd; } } + for (ix = 1; ix < erts_no_schedulers; ++ix) { + (*rs)[ix] = (*rs)[0]; + } + + bd->this_instr = break_op; /* Init the bp type specific data */ - if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) || - break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { BpDataTrace *bdt = (BpDataTrace *) bd; MatchSetRef(match_spec); bdt->match_spec = match_spec; bdt->tracer_pid = tracer_pid; - } else if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) { + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + + bdt->pause = 0; + bdt->n = erts_no_schedulers; + bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); + + for (i = 0; i < bdt->n; i++) { + bp_hash_init(&(bdt->hash[i]), 32); + } + } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { BpDataCount *bdc = (BpDataCount *) bd; + erts_smp_atomic_init(&bdc->acount, 0); + } - bdc->count = 0; + if (bif == BREAK_IS_ERL) { + ++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); } - ++(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]); return 1; } -static int clear_break(Eterm mfa[3], int specified, Uint break_op) +static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op) { int num_processed = 0; Module *modp; @@ -652,52 +1132,71 @@ static int clear_break(Eterm mfa[3], int specified, Uint break_op) } static int clear_module_break(Module *m, Eterm mfa[3], int specified, - Uint break_op) { - Uint** code_base; - Uint* code_ptr; + BeamInstr break_op) { + BeamInstr** code_base; + BeamInstr* code_ptr; int num_processed = 0; - Uint i,n; + Uint i; + BeamInstr n; ASSERT(m); - code_base = (Uint **) m->code; + code_base = (BeamInstr **) m->code; if (code_base == NULL) { return 0; } - n = (Uint) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) && (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) { - Uint *pc = code_ptr + 5; + BeamInstr *pc = code_ptr + 5; num_processed += - clear_function_break(m, pc, break_op); + clear_function_break(m, pc, BREAK_IS_ERL, break_op); } } return num_processed; } -static int clear_function_break(Module *m, Uint *pc, Uint break_op) { +static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr break_op) { BpData *bd; - Uint **code_base = (Uint **)m->code; - - ASSERT(code_base); - ASSERT(code_base <= (Uint **)pc); - ASSERT((Uint **)pc < code_base + (m->code_length/sizeof(Uint *))); + Uint ix = 0; + BeamInstr **code_base = NULL; + + if (bif == BREAK_IS_ERL) { + code_base = (BeamInstr **)m->code; + ASSERT(code_base); + ASSERT(code_base <= (BeamInstr **)pc); + ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *))); + } else { + ASSERT(*pc == (BeamInstr) em_apply_bif); + ASSERT(m == NULL); + } + /* * Currently no trace support for native code. */ if (erts_is_native_break(pc)) { return 0; } + while ( (bd = is_break(pc, break_op))) { /* Remove all breakpoints of this type. * There should be only one of each type, * but break_op may be 0 which matches any type. */ - Uint op; - BpData **r = (BpData **) (pc-4); + BeamInstr op; + BpData ***rs = (BpData ***) (pc - 4); + BpData **r = NULL; + +#ifdef DEBUG + for (ix = 1; ix < erts_no_schedulers; ++ix) { + ASSERT((*rs)[ix] == (*rs)[0]); + } +#endif + r = &((*rs)[0]); + ASSERT(*r); /* Find opcode for this breakpoint */ if (break_op) { @@ -713,8 +1212,11 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) { if (BpSingleton(bd)) { ASSERT(*r == bd); /* Only one breakpoint to remove */ - *r = NULL; - *pc = bd->orig_instr; + if (bif == BREAK_IS_ERL) { + *pc = bd->orig_instr; + } + Free(*rs); + *rs = NULL; } else { BpData *bd_prev = bd->prev; @@ -726,22 +1228,64 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) { bd_prev->orig_instr = bd->orig_instr; } else if (bd_prev == *r) { /* We removed the first breakpoint in the ring */ - *pc = bd->orig_instr; + if (bif == BREAK_IS_ERL) { + *pc = bd->orig_instr; + } } else { bd_prev->orig_instr = bd->orig_instr; } } - if (op == (Uint) BeamOp(op_i_trace_breakpoint) || - op == (Uint) BeamOp(op_i_mtrace_breakpoint)) { + if (op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || + op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { BpDataTrace *bdt = (BpDataTrace *) bd; - MatchSetUnref(bdt->match_spec); } + if (op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + Uint j = 0; + Process *h_p = NULL; + bp_data_time_item_t *item = NULL; + process_breakpoint_time_t *pbt = NULL; + + /* remove all psd associated with the hash + * and then delete the hash. + * ... sigh ... + */ + + for( i = 0; i < bdt->n; ++i) { + if (bdt->hash[i].used) { + for (j = 0; j < bdt->hash[i].n; ++j) { + item = &(bdt->hash[i].item[j]); + if (item->pid != NIL) { + h_p = process_tab[internal_pid_index(item->pid)]; + if (h_p) { + pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL); + if (pbt) { + Free(pbt); + } + } + } + } + } + bp_hash_delete(&(bdt->hash[i])); + } + Free(bdt->hash); + bdt->hash = NULL; + bdt->n = 0; + } Free(bd); - ASSERT(((Uint) code_base[MI_NUM_BREAKPOINTS]) > 0); - --(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]); - } + if (bif == BREAK_IS_ERL) { + ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0); + --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); + } + if (*rs) { + for (ix = 1; ix < erts_no_schedulers; ++ix) { + (*rs)[ix] = (*rs)[0]; + } + } + } /* while bd != NULL */ return 1; } @@ -754,32 +1298,63 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) { ** returned. The program counter must point to the first executable ** (breakpoint) instruction of the function. */ -static BpData *is_break(Uint *pc, Uint break_op) { - ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); + +BpData *erts_get_time_break(Process *p, BeamInstr *pc) { + return get_break(p, pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); +} + +static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) { + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (! erts_is_native_break(pc)) { - BpData *bd = (BpData *) pc[-4]; - - if (break_op == 0) { - return bd; - } - if (*pc == break_op) { - ASSERT(bd); - return bd->next; - } - if (! bd){ + BpData **rs = (BpData **) pc[-4]; + BpData *bd = NULL, *ebd = NULL; + + if (! rs) { return NULL; } + + bd = ebd = rs[bp_sched2ix_proc(p)]; + ASSERT(bd); + if (bd->this_instr == break_op) { + return bd; + } + bd = bd->next; - while (bd != (BpData *) pc[-4]) { + while (bd != ebd) { ASSERT(bd); - if (bd->orig_instr == break_op) { - bd = bd->next; + if (bd->this_instr == break_op) { ASSERT(bd); return bd; - } else { - bd = bd->next; } + bd = bd->next; } } return NULL; } + +static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { + BpData **rs = (BpData **) pc[-4]; + BpData *bd = NULL, *ebd = NULL; + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + + if (! rs) { + return NULL; + } + + bd = ebd = rs[bp_sched2ix()]; + ASSERT(bd); + if ( (break_op == 0) || (bd->this_instr == break_op)) { + return bd; + } + + bd = bd->next; + while (bd != ebd) { + ASSERT(bd); + if (bd->this_instr == break_op) { + ASSERT(bd); + return bd; + } + bd = bd->next; + } + return NULL; +} diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 44e6b294d8..bd8a7249a7 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -27,28 +27,46 @@ -/* -** Common struct to all bp_data_* -** -** Two gotchas: -** -** 1) The type of bp_data structure in the ring is deduced from the -** orig_instr field of the structure _before_ in the ring, except for -** the first structure in the ring that has its instruction in -** pc[0] of the code to execute. -** -** 2) pc[-4] points to the _last_ structure in the ring before the -** breakpoints are being executed. -** -** So, as an example, when a breakpointed function starts to execute, -** the first instruction that is a breakpoint instruction at pc[0] finds -** its data at ((BpData *) pc[-4])->next and has to cast that pointer -** to the correct bp_data type. +/* A couple of gotchas: + * + * The breakpoint structure from BeamInstr, + * In beam_emu where the instruction counter pointer, I (or pc), + * points to the *current* instruction. At that time, if the instruction + * is a breakpoint instruction the pc looks like the following, + * + * I[-5] | op_i_func_info_IaaI | scheduler specific entries + * I[-4] | BpData** bpa | --> | BpData * bdas1 | ... | BpData * bdasN | + * I[-3] | Tagged Module | | | + * I[-2] | Tagged Function | V V + * I[-1] | Arity | BpData -> BpData -> BpData -> BpData + * I[0] | The bp instruction | ^ * the bp wheel * | + * |------------------------------ + * + * Common struct to all bp_data_* + * + * 1) The type of bp_data structure in the ring is deduced from the + * orig_instr field of the structure _before_ in the ring, except for + * the first structure in the ring that has its instruction in + * pc[0] of the code to execute. + * This is valid as long as you don't search for the function while it is + * being executed by something else. Or is in the middle of its rotation for + * any other reason. + * A key, the bp beam instruction, is included for this reason. + * + * 2) pc[-4][sched_id - 1] points to the _last_ structure in the ring before the + * breakpoints are being executed. + * + * So, as an example, when a breakpointed function starts to execute, + * the first instruction that is a breakpoint instruction at pc[0] finds + * its data at ((BpData **) pc[-4][sched_id - 1])->next and has to cast that pointer + * to the correct bp_data type. */ + typedef struct bp_data { struct bp_data *next; /* Doubly linked ring pointers */ struct bp_data *prev; /* -"- */ - Uint orig_instr; /* The original instruction to execute */ + BeamInstr orig_instr; /* The original instruction to execute */ + BeamInstr this_instr; /* key */ } BpData; /* ** All the following bp_data_.. structs must begin the same way @@ -57,7 +75,8 @@ typedef struct bp_data { typedef struct bp_data_trace { struct bp_data *next; struct bp_data *prev; - Uint orig_instr; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ Binary *match_spec; Eterm tracer_pid; } BpDataTrace; @@ -65,18 +84,58 @@ typedef struct bp_data_trace { typedef struct bp_data_debug { struct bp_data *next; struct bp_data *prev; - Uint orig_instr; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ } BpDataDebug; -typedef struct bp_data_count { /* Call count */ +typedef struct bp_data_count { /* Call count */ struct bp_data *next; struct bp_data *prev; - Uint orig_instr; - Sint count; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ + erts_smp_atomic_t acount; } BpDataCount; +typedef struct { + Eterm pid; + Sint count; + Uint s_time; + Uint us_time; +} bp_data_time_item_t; + +typedef struct { + Uint n; + Uint used; + bp_data_time_item_t *item; +} bp_time_hash_t; + +typedef struct bp_data_time { /* Call time */ + struct bp_data *next; + struct bp_data *prev; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ + Uint pause; + Uint n; + bp_time_hash_t *hash; +} BpDataTime; + +typedef struct { + Uint ms; + Uint s; + Uint us; + BeamInstr *pc; +} process_breakpoint_time_t; /* used within psd */ + extern erts_smp_spinlock_t erts_bp_lock; +#define ERTS_BP_CALL_TIME_SCHEDULE_IN (0) +#define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) +#define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) + +#define ERTS_BP_CALL_TIME_CALL (0) +#define ERTS_BP_CALL_TIME_RETURN (1) +#define ERTS_BP_CALL_TIME_TAIL_CALL (2) + #ifdef ERTS_SMP #define ErtsSmpBPLock(BDC) erts_smp_spin_lock(&erts_bp_lock) #define ErtsSmpBPUnlock(BDC) erts_smp_spin_unlock(&erts_bp_lock) @@ -85,31 +144,46 @@ extern erts_smp_spinlock_t erts_bp_lock; #define ErtsSmpBPUnlock(BDC) #endif -#define ErtsCountBreak(pc,instr_result) \ -do { \ - BpDataCount *bdc = (BpDataCount *) (pc)[-4]; \ - \ - ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bdc); \ - bdc = (BpDataCount *) bdc->next; \ - ASSERT(bdc); \ - (pc)[-4] = (Uint) bdc; \ - ErtsSmpBPLock(bdc); \ - if (bdc->count >= 0) bdc->count++; \ - ErtsSmpBPUnlock(bdc); \ - *(instr_result) = bdc->orig_instr; \ +ERTS_INLINE Uint bp_sched2ix(void); + +#ifdef ERTS_SMP +#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1) +#else +#define bp_sched2ix_proc(p) (0) +#endif + +#define ErtsCountBreak(p, pc,instr_result) \ +do { \ + BpData **bds = (BpData **) (pc)[-4]; \ + BpDataCount *bdc = NULL; \ + Uint ix = bp_sched2ix_proc( (p) ); \ + erts_aint_t count = 0; \ + \ + ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ + ASSERT(bds); \ + bdc = (BpDataCount *) bds[ix]; \ + bdc = (BpDataCount *) bdc->next; \ + ASSERT(bdc); \ + bds[ix] = (BpData *) bdc; \ + count = erts_smp_atomic_read(&bdc->acount); \ + if (count >= 0) erts_smp_atomic_inc(&bdc->acount); \ + *(instr_result) = bdc->orig_instr; \ } while (0) -#define ErtsBreakSkip(pc,instr_result) \ -do { \ - BpData *bd = (BpData *) (pc)[-4]; \ - \ - ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bd); \ - bd = bd->next; \ - ASSERT(bd); \ - (pc)[-4] = (Uint) bd; \ - *(instr_result) = bd->orig_instr; \ +#define ErtsBreakSkip(p, pc,instr_result) \ +do { \ + BpData **bds = (BpData **) (pc)[-4]; \ + BpData *bd = NULL; \ + Uint ix = bp_sched2ix_proc( (p) ); \ + \ + ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ + ASSERT(bds); \ + bd = bds[ix]; \ + ASSERT(bd); \ + bd = bd->next; \ + ASSERT(bd); \ + bds[ix] = bd; \ + *(instr_result) = bd->orig_instr; \ } while (0) enum erts_break_op{ @@ -133,9 +207,9 @@ int erts_clear_trace_break(Eterm mfa[3], int specified); int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid); int erts_clear_mtrace_break(Eterm mfa[3], int specified); -void erts_set_mtrace_bif(Uint *pc, Binary *match_spec, +void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid); -void erts_clear_mtrace_bif(Uint *pc); +void erts_clear_mtrace_bif(BeamInstr *pc); int erts_set_debug_break(Eterm mfa[3], int specified); int erts_clear_debug_break(Eterm mfa[3], int specified); int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op); @@ -144,22 +218,33 @@ int erts_clear_count_break(Eterm mfa[3], int specified); int erts_clear_break(Eterm mfa[3], int specified); int erts_clear_module_break(Module *modp); -int erts_clear_function_break(Module *modp, Uint *pc); +int erts_clear_function_break(Module *modp, BeamInstr *pc); -Uint erts_trace_break(Process *p, Uint *pc, Eterm *args, +BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, Uint32 *ret_flags, Eterm *tracer_pid); -Uint32 erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, +Uint32 erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local, Eterm *tracer_pid); -int erts_is_trace_break(Uint *pc, Binary **match_spec_ret, +int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret); -int erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, +int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_rte); -int erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, +int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret); -int erts_is_native_break(Uint *pc); -int erts_is_count_break(Uint *pc, Sint *count_ret); +int erts_is_native_break(BeamInstr *pc); +int erts_is_count_break(BeamInstr *pc, Sint *count_ret); +int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); + +void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type); +void erts_schedule_time_break(Process *p, Uint out); +int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op); +int erts_clear_time_break(Eterm mfa[3], int specified); + +int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); +void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); +void erts_clear_time_trace_bif(BeamInstr *pc); +BpData *erts_get_time_break(Process *p, BeamInstr *pc); -Uint *erts_find_local_func(Eterm mfa[3]); +BeamInstr *erts_find_local_func(Eterm mfa[3]); #endif /* _BEAM_BP_H */ diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index d5cef1cad2..e795b4efbd 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -26,7 +26,7 @@ /* XXX: should use dynamic reallocation */ #define TABSIZ (16*1024) static struct { - Eterm *cp; + BeamInstr *cp; unsigned cdr; } beam_catches[TABSIZ]; @@ -39,7 +39,7 @@ void beam_catches_init(void) high_mark = 0; } -unsigned beam_catches_cons(Eterm *cp, unsigned cdr) +unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr) { int i; @@ -65,7 +65,7 @@ unsigned beam_catches_cons(Eterm *cp, unsigned cdr) return i; } -Eterm *beam_catches_car(unsigned i) +BeamInstr *beam_catches_car(unsigned i) { if( i >= TABSIZ ) { fprintf(stderr, @@ -75,7 +75,7 @@ Eterm *beam_catches_car(unsigned i) return beam_catches[i].cp; } -void beam_catches_delmod(unsigned head, Eterm *code, unsigned code_bytes) +void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes) { unsigned i, cdr; diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h index ccf33d5e86..6223427f0d 100644 --- a/erts/emulator/beam/beam_catches.h +++ b/erts/emulator/beam/beam_catches.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -23,9 +23,9 @@ #define BEAM_CATCHES_NIL (-1) void beam_catches_init(void); -unsigned beam_catches_cons(Eterm* cp, unsigned cdr); -Eterm *beam_catches_car(unsigned i); -void beam_catches_delmod(unsigned head, Eterm* code, unsigned code_bytes); +unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr); +BeamInstr *beam_catches_car(unsigned i); +void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes); #define catch_pc(x) beam_catches_car(catch_val((x))) diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 4242a4161e..fffb172c68 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -43,12 +43,12 @@ #else # define HEXF "%08bpX" #endif +#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) void dbg_bt(Process* p, Eterm* sp); -void dbg_where(Eterm* addr, Eterm x0, Eterm* reg); +void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); -static void print_big(int to, void *to_arg, Eterm* addr); -static int print_op(int to, void *to_arg, int op, int size, Eterm* addr); +static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); Eterm erts_debug_same_2(Process* p, Eterm term1, Eterm term2) { @@ -124,6 +124,57 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) BIF_ERROR(p, BADARG); } +#if 0 /* Kept for conveninence when hard debugging. */ +void debug_dump_code(BeamInstr *I, int num) +{ + BeamInstr *code_ptr = I; + BeamInstr *end = code_ptr + num; + erts_dsprintf_buf_t *dsbufp; + BeamInstr instr; + int i; + + dsbufp = erts_create_tmp_dsbuf(0); + while (code_ptr < end) { + erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); + instr = (BeamInstr) code_ptr[0]; + for (i = 0; i < NUM_SPECIFIC_OPS; i++) { + if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { + code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, + i, opc[i].sz-1, code_ptr+1) + 1; + break; + } + } + if (i >= NUM_SPECIFIC_OPS) { + erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, + "unknown " HEXF "\n", instr); + code_ptr++; + } + } + dsbufp->str[dsbufp->str_len] = 0; + erts_fprintf(stderr,"%s", dsbufp->str); + erts_destroy_tmp_dsbuf(dsbufp); +} +#endif + +BIF_RETTYPE +erts_debug_instructions_0(BIF_ALIST_0) +{ + int i = 0; + Uint needed = num_instructions * 2; + Eterm* hp; + Eterm res = NIL; + + for (i = 0; i < num_instructions; i++) { + needed += 2*strlen(opc[i].name); + } + hp = HAlloc(BIF_P, needed); + for (i = num_instructions-1; i >= 0; i--) { + Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name)); + res = erts_bld_cons(&hp, 0, s, res); + } + return res; +} + Eterm erts_debug_disassemble_1(Process* p, Eterm addr) { @@ -132,16 +183,16 @@ erts_debug_disassemble_1(Process* p, Eterm addr) Eterm* tp; Eterm bin; Eterm mfa; - Eterm* funcinfo = NULL; /* Initialized to eliminate warning. */ - Uint* code_base; - Uint* code_ptr = NULL; /* Initialized to eliminate warning. */ - Uint instr; - Uint uaddr; + BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ + BeamInstr* code_base; + BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ + BeamInstr instr; + BeamInstr uaddr; Uint hsz; int i; - if (term_to_Uint(addr, &uaddr)) { - code_ptr = (Uint *) uaddr; + if (term_to_UWord(addr, &uaddr)) { + code_ptr = (BeamInstr *) uaddr; if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) { BIF_RET(am_false); } @@ -180,14 +231,14 @@ erts_debug_disassemble_1(Process* p, Eterm addr) * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((Eterm *) ep->address) - 5; + code_ptr = ((BeamInstr *) ep->address) - 5; funcinfo = code_ptr+2; } else if (modp == NULL || (code_base = modp->code) == NULL) { BIF_RET(am_undef); } else { n = code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; i++) { - code_ptr = (Uint *) code_base[MI_FUNCTIONS+i]; + code_ptr = (BeamInstr *) code_base[MI_FUNCTIONS+i]; if (code_ptr[3] == name && code_ptr[4] == arity) { funcinfo = code_ptr+2; break; @@ -203,9 +254,9 @@ erts_debug_disassemble_1(Process* p, Eterm addr) dsbufp = erts_create_tmp_dsbuf(0); erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); - instr = (Uint) code_ptr[0]; + instr = (BeamInstr) code_ptr[0]; for (i = 0; i < NUM_SPECIFIC_OPS; i++) { - if (instr == (Uint) BeamOp(i) && opc[i].name[0] != '\0') { + if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, i, opc[i].sz-1, code_ptr+1) + 1; break; @@ -216,15 +267,15 @@ erts_debug_disassemble_1(Process* p, Eterm addr) "unknown " HEXF "\n", instr); code_ptr++; } - bin = new_binary(p, (byte *) dsbufp->str, (int) dsbufp->str_len); + bin = new_binary(p, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hsz = 4+4; - (void) erts_bld_uint(NULL, &hsz, (Uint) code_ptr); + (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); - addr = erts_bld_uint(&hp, NULL, (Uint) code_ptr); + addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); ASSERT(is_atom(funcinfo[0])); ASSERT(is_atom(funcinfo[1])); - mfa = TUPLE3(hp, funcinfo[0], funcinfo[1], make_small(funcinfo[2])); + mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); hp += 4; return TUPLE3(hp, addr, bin, mfa); } @@ -236,20 +287,20 @@ dbg_bt(Process* p, Eterm* sp) while (sp < stack) { if (is_CP(*sp)) { - Eterm* addr = find_function_from_pc(cp_val(*sp)); + BeamInstr* addr = find_function_from_pc(cp_val(*sp)); if (addr) erts_fprintf(stderr, HEXF ": %T:%T/%bpu\n", - addr, addr[0], addr[1], addr[2]); + addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]); } sp++; } } void -dbg_where(Eterm* addr, Eterm x0, Eterm* reg) +dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) { - Eterm* f = find_function_from_pc(addr); + BeamInstr* f = find_function_from_pc(addr); if (f == NULL) { erts_fprintf(stderr, "???\n"); @@ -259,7 +310,7 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg) addr = f; arity = addr[2]; - erts_fprintf(stderr, HEXF ": %T:%T(", addr, addr[0], addr[1]); + erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]); for (i = 0; i < arity; i++) erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0); erts_fprintf(stderr, ")\n"); @@ -267,18 +318,19 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg) } static int -print_op(int to, void *to_arg, int op, int size, Eterm* addr) +print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) { int i; - Uint tag; + BeamInstr tag; char* sign; char* start_prog; /* Start of program for packer. */ char* prog; /* Current position in packer program. */ - Uint stack[8]; /* Stack for packer. */ - Uint* sp = stack; /* Points to next free position. */ - Uint packed = 0; /* Accumulator for packed operations. */ - Uint args[8]; /* Arguments for this instruction. */ - Uint* ap; /* Pointer to arguments. */ + BeamInstr stack[8]; /* Stack for packer. */ + BeamInstr* sp = stack; /* Points to next free position. */ + BeamInstr packed = 0; /* Accumulator for packed operations. */ + BeamInstr args[8]; /* Arguments for this instruction. */ + BeamInstr* ap; /* Pointer to arguments. */ + BeamInstr* unpacked; /* Unpacked arguments */ start_prog = opc[op].pack; @@ -288,7 +340,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) * Avoid copying because instructions containing bignum operands * are bigger than actually declared. */ - ap = (Uint *) addr; + ap = (BeamInstr *) addr; } else { /* * Copy all arguments to a local buffer for the unpacking. @@ -324,9 +376,15 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) packed >>= BEAM_TIGHT_SHIFT; break; case '6': /* Shift 16 steps */ - *ap++ = packed & 0xffff; - packed >>= 16; + *ap++ = packed & BEAM_LOOSE_MASK; + packed >>= BEAM_LOOSE_SHIFT; break; +#ifdef ARCH_64 + case 'w': /* Shift 32 steps */ + *ap++ = packed & BEAM_WIDE_MASK; + packed >>= BEAM_WIDE_SHIFT; + break; +#endif case 'p': *sp++ = *--ap; break; @@ -353,7 +411,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) break; case 'x': /* x(N) */ if (reg_index(ap[0]) == 0) { - erts_print(to, to_arg, "X[0]"); + erts_print(to, to_arg, "x[0]"); } else { erts_print(to, to_arg, "x(%d)", reg_index(ap[0])); } @@ -390,11 +448,11 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) case 'i': /* Tagged integer */ case 'c': /* Tagged constant */ case 'q': /* Tagged literal */ - erts_print(to, to_arg, "%T", *ap); + erts_print(to, to_arg, "%T", (Eterm) *ap); ap++; break; case 'A': - erts_print(to, to_arg, "%d", arityval(ap[0])); + erts_print(to, to_arg, "%d", arityval( (Eterm) ap[0])); ap++; break; case 'd': /* Destination (x(0), x(N), y(N)) */ @@ -421,30 +479,36 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) ap++; break; case 'f': /* Destination label */ - erts_print(to, to_arg, "f(%X)", *ap); - ap++; + { + BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); + if (f+3 != (BeamInstr *) *ap) { + erts_print(to, to_arg, "f(" HEXF ")", *ap); + } else { + erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + } + ap++; + } break; case 'p': /* Pointer (to label) */ { - Eterm* f = find_function_from_pc((Eterm *)*ap); - - if (f+3 != (Eterm *) *ap) { - erts_print(to, to_arg, "p(%X)", *ap); + BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); + if (f+3 != (BeamInstr *) *ap) { + erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", f[0], f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); } ap++; } break; case 'j': /* Pointer (to label) */ - erts_print(to, to_arg, "j(%X)", *ap); + erts_print(to, to_arg, "j(" HEXF ")", *ap); ap++; break; case 'e': /* Export entry */ { Export* ex = (Export *) *ap; erts_print(to, to_arg, - "%T:%T/%bpu", ex->code[0], ex->code[1], ex->code[2]); + "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]); ap++; } break; @@ -467,7 +531,8 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) ap++; break; case 'P': /* Byte offset into tuple (see beam_load.c) */ - erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm*)) - 1); + case 'Q': /* Like 'P', but packable */ + erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm)) - 1); ap++; break; case 'l': /* fr(N) */ @@ -487,62 +552,90 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr) * Print more information about certain instructions. */ + unpacked = ap; ap = addr + size; switch (op) { - case op_i_select_val_sfI: + case op_i_select_val_rfI: + case op_i_select_val_xfI: + case op_i_select_val_yfI: { int n = ap[-1]; while (n > 0) { - erts_print(to, to_arg, "%T f(%X) ", ap[0], ap[1]); + erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); ap += 2; size += 2; n--; } } break; - case op_i_jump_on_val_sfII: + case op_i_select_tuple_arity_rfI: + case op_i_select_tuple_arity_xfI: + case op_i_select_tuple_arity_yfI: + { + int n = ap[-1]; + + while (n > 0) { + Uint arity = arityval(ap[0]); + erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]); + ap += 2; + size += 2; + n--; + } + } + break; + case op_i_jump_on_val_rfII: + case op_i_jump_on_val_xfII: + case op_i_jump_on_val_yfII: { int n; for (n = ap[-2]; n > 0; n--) { - erts_print(to, to_arg, "f(%X) ", ap[0]); + erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); ap++; size++; } } break; - case op_i_select_big_sf: - while (ap[0]) { - int arity = thing_arityval(ap[0]); - print_big(to, to_arg, ap); - size += arity+1; - ap += arity+1; - erts_print(to, to_arg, " f(%X) ", ap[0]); - ap++; - size++; + case op_i_jump_on_val_zero_rfI: + case op_i_jump_on_val_zero_xfI: + case op_i_jump_on_val_zero_yfI: + { + int n; + for (n = ap[-1]; n > 0; n--) { + erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); + ap++; + size++; + } + } + break; + case op_i_put_tuple_rI: + case op_i_put_tuple_xI: + case op_i_put_tuple_yI: + { + int n = unpacked[-1]; + + while (n > 0) { + if (!is_header(ap[0])) { + erts_print(to, to_arg, " %T", (Eterm) ap[0]); + } else { + switch ((ap[0] >> 2) & 0x03) { + case R_REG_DEF: + erts_print(to, to_arg, " x(0)"); + break; + case X_REG_DEF: + erts_print(to, to_arg, " x(%d)", ap[0] >> 4); + break; + case Y_REG_DEF: + erts_print(to, to_arg, " y(%d)", ap[0] >> 4); + break; + } + } + ap++, size++, n--; + } } - ap++; - size++; break; } erts_print(to, to_arg, "\n"); return size; } - -static void -print_big(int to, void *to_arg, Eterm* addr) -{ - int i; - int k; - - i = BIG_SIZE(addr); - if (BIG_SIGN(addr)) - erts_print(to, to_arg, "-#integer(%d) = {", i); - else - erts_print(to, to_arg, "#integer(%d) = {", i); - erts_print(to, to_arg, "%d", BIG_DIGIT(addr, 0)); - for (k = 1; k < i; k++) - erts_print(to, to_arg, ",%d", BIG_DIGIT(addr, k)); - erts_print(to, to_arg, "}"); -} diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 2f7f48193d..fb90a7d4f7 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -117,6 +117,7 @@ do { \ #endif #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) +#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) /* @@ -138,8 +139,8 @@ do { \ #define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10))) #else #define VALID_INSTR(IP) \ - ((Sint)LabelAddr(emulator_loop) <= (Sint)(IP) && \ - (Sint)(IP) < (Sint)LabelAddr(end_emulator_loop)) + ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \ + (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop)) #endif /* NO_JUMP_TABLE */ #define SET_CP(p, ip) \ @@ -181,11 +182,11 @@ do { \ #define StoreBifResult(Dst, Result) \ do { \ - Eterm* stb_next; \ + BeamInstr* stb_next; \ Eterm stb_reg; \ stb_reg = Arg(Dst); \ I += (Dst) + 2; \ - stb_next = (Eterm *) *I; \ + stb_next = (BeamInstr *) *I; \ CHECK_TERM(Result); \ switch (beam_reg_tag(stb_reg)) { \ case R_REG_DEF: \ @@ -205,7 +206,7 @@ do { \ c_p->cp = 0; \ } while(0) -#define RESTORE_CP(X) SET_CP(c_p, cp_val(*(X))) +#define RESTORE_CP(X) SET_CP(c_p, (BeamInstr *) cp_val(*(X))) #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y)) @@ -213,13 +214,13 @@ do { \ * Special Beam instructions. */ -Eterm beam_apply[2]; -Eterm beam_exit[1]; -Eterm beam_continue_exit[1]; +BeamInstr beam_apply[2]; +BeamInstr beam_exit[1]; +BeamInstr beam_continue_exit[1]; -Eterm* em_call_error_handler; -Eterm* em_apply_bif; -Eterm* em_call_traced_function; +BeamInstr* em_call_error_handler; +BeamInstr* em_apply_bif; +BeamInstr* em_call_traced_function; /* NOTE These should be the only variables containing trace instructions. @@ -227,9 +228,10 @@ Eterm* em_call_traced_function; ** for the refering variable (one of these), and rouge references ** will most likely cause chaos. */ -Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ -Eterm beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */ +BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ +BeamInstr beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */ +BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* * All Beam instructions in numerical order. @@ -319,6 +321,7 @@ extern int count_instructions; # define POST_BIF_GC_SWAPIN_0(_p, _res) \ ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ PROCESS_MAIN_CHK_LOCKS((_p)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ _res = erts_gc_after_bif_call((_p), (_res), NULL, 0); \ E = (_p)->stop; \ @@ -326,6 +329,7 @@ extern int count_instructions; HTOP = HEAP_TOP((_p)) # define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity) \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ PROCESS_MAIN_CHK_LOCKS((_p)); \ if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ @@ -342,6 +346,8 @@ extern int count_instructions; #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N))) #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N))) #define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N))) +#define Qb(N) (N) +#define Ib(N) (N) #define x(N) reg[N] #define y(N) E[N] #define r(N) x##N @@ -363,6 +369,7 @@ extern int count_instructions; reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ r(0) = reg[0]; \ SWAPIN; \ @@ -416,6 +423,7 @@ extern int count_instructions; reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ r(0) = reg[0]; \ SWAPIN; \ @@ -438,6 +446,7 @@ extern int count_instructions; reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ r(0) = reg[0]; \ SWAPIN; \ @@ -460,6 +469,7 @@ extern int count_instructions; reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ if (Live > 0) { \ r(0) = reg[0]; \ @@ -470,6 +480,13 @@ extern int count_instructions; HEAP_SPACE_VERIFIED(need); \ } while (0) +#define TestHeapPutList(Need, Reg) \ + do { \ + TestHeap((Need), 1); \ + PutList(Reg, r(0), r(0), StoreSimpleDest); \ + CHECK_TERM(r(0)); \ + } while (0) + #ifdef HYBRID #ifdef INCREMENTAL #define TestGlobalHeap(Nh, Live, hp) \ @@ -514,6 +531,11 @@ extern int count_instructions; SWAPIN; \ } while (0) +#define PutTuple(Dst, Arity) \ + do { \ + Dst = make_tuple(HTOP); \ + pt_arity = (Arity); \ + } while (0) /* * Check that we haven't used the reductions and jump to function pointed to by @@ -522,8 +544,8 @@ extern int count_instructions; #define DispatchMacro() \ do { \ - Eterm* dis_next; \ - dis_next = (Eterm *) *I; \ + BeamInstr* dis_next; \ + dis_next = (BeamInstr *) *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -535,8 +557,8 @@ extern int count_instructions; #define DispatchMacroFun() \ do { \ - Eterm* dis_next; \ - dis_next = (Eterm *) *I; \ + BeamInstr* dis_next; \ + dis_next = (BeamInstr *) *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -590,7 +612,7 @@ extern int count_instructions; ASSERT(VALID_INSTR(*I)); \ Goto(*I) -#define PreFetch(N, Dst) do { Dst = (Eterm *) *(I + N + 1); } while (0) +#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0) #define NextPF(N, Dst) \ I += N + 1; \ ASSERT(VALID_INSTR(Dst)); \ @@ -644,7 +666,7 @@ extern int count_instructions; #define DeallocateReturn(Deallocate) \ do { \ int words_to_pop = (Deallocate); \ - SET_I(cp_val(*E)); \ + SET_I((BeamInstr *) cp_val(*E)); \ E = ADD_BYTE_OFFSET(E, words_to_pop); \ CHECK_TERM(r(0)); \ Goto(*I); \ @@ -657,79 +679,77 @@ extern int count_instructions; #define MoveCall(Src, Dest, CallDest, Size) \ (Dest) = (Src); \ SET_CP(c_p, I+Size+1); \ - SET_I((Eterm *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveCallLast(Src, Dest, CallDest, Deallocate) \ (Dest) = (Src); \ RESTORE_CP(E); \ E = ADD_BYTE_OFFSET(E, (Deallocate)); \ - SET_I((Eterm *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveCallOnly(Src, Dest, CallDest) \ (Dest) = (Src); \ - SET_I((Eterm *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); +#define MoveJump(Src) \ + r(0) = (Src); \ + SET_I((BeamInstr *) Arg(0)); \ + Goto(*I); + #define GetList(Src, H, T) do { \ Eterm* tmp_ptr = list_val(Src); \ H = CAR(tmp_ptr); \ T = CDR(tmp_ptr); } while (0) -#define GetTupleElement(Src, Element, Dest) \ - do { \ - tmp_arg1 = (Eterm) (((unsigned char *) tuple_val(Src)) + (Element)); \ - (Dest) = (*(Eterm *)tmp_arg1); \ +#define GetTupleElement(Src, Element, Dest) \ + do { \ + tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + \ + (Element)); \ + (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1)); \ } while (0) -#define ExtractNextElement(Dest) \ - tmp_arg1 += sizeof(Eterm); \ - (Dest) = (* (Eterm *) (((unsigned char *) tmp_arg1))) +#define ExtractNextElement(Dest) \ + tmp_arg1 += sizeof(Eterm); \ + (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1)))) -#define ExtractNextElement2(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) tmp_arg1)[1]; \ - ene_dstp[1] = ((Eterm *) tmp_arg1)[2]; \ - tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ +#define ExtractNextElement2(Dest) \ + do { \ + Eterm* ene_dstp = &(Dest); \ + ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ + ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ + tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ } while (0) #define ExtractNextElement3(Dest) \ do { \ Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) tmp_arg1)[1]; \ - ene_dstp[1] = ((Eterm *) tmp_arg1)[2]; \ - ene_dstp[2] = ((Eterm *) tmp_arg1)[3]; \ + ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ + ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ + ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ tmp_arg1 += 3*sizeof(Eterm); \ } while (0) #define ExtractNextElement4(Dest) \ do { \ Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) tmp_arg1)[1]; \ - ene_dstp[1] = ((Eterm *) tmp_arg1)[2]; \ - ene_dstp[2] = ((Eterm *) tmp_arg1)[3]; \ - ene_dstp[3] = ((Eterm *) tmp_arg1)[4]; \ + ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ + ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ + ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ + ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4]; \ tmp_arg1 += 4*sizeof(Eterm); \ } while (0) #define ExtractElement(Element, Dest) \ do { \ tmp_arg1 += (Element); \ - (Dest) = (* (Eterm *) tmp_arg1); \ + (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ } while (0) -#define PutTuple(Arity, Src, Dest) \ - ASSERT(is_arity_value(Arity)); \ - Dest = make_tuple(HTOP); \ - HTOP[0] = (Arity); \ - HTOP[1] = (Src); \ - HTOP += 2 - -#define Put(Word) *HTOP++ = (Word) - #define EqualImmed(X, Y, Action) if (X != Y) { Action; } +#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -759,8 +779,13 @@ extern int count_instructions; #define IsTuple(X, Action) if (is_not_tuple(X)) Action -#define IsArity(Pointer, Arity, Fail) \ - if (*(Eterm *)(tmp_arg1 = (Eterm)tuple_val(Pointer)) != (Arity)) { Fail; } +#define IsArity(Pointer, Arity, Fail) \ + if (*(Eterm *) \ + EXPAND_POINTER(tmp_arg1 = (Eterm) \ + COMPRESS_POINTER(tuple_val(Pointer))) != (Arity)) \ + { \ + Fail; \ + } #define IsFunction(X, Action) \ do { \ @@ -776,11 +801,14 @@ extern int count_instructions; } \ } while (0) -#define IsTupleOfArity(Src, Arity, Fail) \ - do { \ - if (is_not_tuple(Src) || *(Eterm *)(tmp_arg1 = (Eterm) tuple_val(Src)) != Arity) { \ - Fail; \ - } \ +#define IsTupleOfArity(Src, Arity, Fail) \ + do { \ + if (is_not_tuple(Src) || \ + *(Eterm *) \ + EXPAND_POINTER(tmp_arg1 = \ + (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \ + Fail; \ + } \ } while (0) #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } @@ -791,7 +819,7 @@ extern int count_instructions; #define IsBitstring(Src, Fail) \ if (is_not_binary(Src)) { Fail; } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define BsSafeMul(A, B, Fail, Target) \ do { Uint64 _res = (A) * (B); \ if (_res / B != A) { Fail; } \ @@ -973,34 +1001,55 @@ extern int count_instructions; #define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; } #define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; } -static BifFunction translate_gc_bif(void* gcf); -static Eterm* handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf); -static Eterm* next_catch(Process* c_p, Eterm *reg); +/* + * process_main() is already huge, so we want to avoid inlining + * into it. Especially functions that are seldom used. + */ +#ifdef __GNUC__ +# define NOINLINE __attribute__((__noinline__)) +#else +# define NOINLINE +#endif + +/* + * The following functions are called directly by process_main(). + * Don't inline them. + */ +static BifFunction translate_gc_bif(void* gcf) NOINLINE; +static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, + Eterm* reg, BifFunction bf) NOINLINE; +static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, + Eterm* reg, Eterm func) NOINLINE; +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; +static BeamInstr* apply(Process* p, Eterm module, Eterm function, + Eterm args, Eterm* reg) NOINLINE; +static BeamInstr* call_fun(Process* p, int arity, + Eterm* reg, Eterm args) NOINLINE; +static BeamInstr* apply_fun(Process* p, Eterm fun, + Eterm args, Eterm* reg) NOINLINE; +static Eterm new_fun(Process* p, Eterm* reg, + ErlFunEntry* fe, int num_free) NOINLINE; + + +/* + * Functions not directly called by process_main(). OK to inline. + */ +static BeamInstr* next_catch(Process* c_p, Eterm *reg); static void terminate_proc(Process* c_p, Eterm Value); static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc); -static void save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, +static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args); static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); -static Eterm call_error_handler(Process* p, Eterm* ip, Eterm* reg); -static Eterm call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg); -static Uint* fixed_apply(Process* p, Eterm* reg, Uint arity); -static Eterm* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg); -static int hibernate(Process* c_p, Eterm module, Eterm function, - Eterm args, Eterm* reg); -static Eterm* call_fun(Process* p, int arity, Eterm* reg, Eterm args); -static Eterm* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg); -static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free); - -#if defined(_OSE_) || defined(VXWORKS) + +#if defined(VXWORKS) static int init_done; #endif void init_emulator(void) { -#if defined(_OSE_) || defined(VXWORKS) +#if defined(VXWORKS) init_done = 0; #endif process_main(); @@ -1039,7 +1088,7 @@ init_emulator(void) */ void process_main(void) { -#if !defined(_OSE_) && !defined(VXWORKS) +#if !defined(VXWORKS) static int init_done = 0; #endif Process* c_p = NULL; @@ -1078,7 +1127,7 @@ void process_main(void) /* * Pointer to next threaded instruction. */ - register Eterm *I REG_I = NULL; + register BeamInstr *I REG_I = NULL; /* Number of reductions left. This function * returns to the scheduler when FCALLS reaches zero. @@ -1090,9 +1139,14 @@ void process_main(void) */ register Eterm tmp_arg1 REG_tmp_arg1 = NIL; register Eterm tmp_arg2 REG_tmp_arg2 = NIL; - Eterm tmp_big[2]; /* Temporary buffer for small bignums. */ +#if HEAP_ON_C_STACK + Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */ +#else + Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ +#endif #ifndef ERTS_SMP +#if !HALFWORD_HEAP static Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers -- not used directly, but * through 'reg', because using it directly @@ -1100,7 +1154,7 @@ void process_main(void) * while using it through reg needs only * one. */ - +#endif /* * Floating point registers. */ @@ -1130,6 +1184,8 @@ void process_main(void) Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ + Eterm pt_arity; /* Used by do_put_tuple */ + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1141,13 +1197,17 @@ void process_main(void) * Note: c_p->arity must be set to reflect the number of useful terms in * c_p->arg_reg before calling the scheduler. */ - if (!init_done) { init_done = 1; goto init_emulator; } #ifndef ERTS_SMP +#if !HALFWORD_HEAP reg = save_reg; /* XXX: probably wastes a register on x86 */ +#else + /* Registers need to be heap allocated (correct memory range) for tracing to work */ + reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); +#endif #endif c_p = NULL; reds_used = 0; @@ -1158,7 +1218,12 @@ void process_main(void) do_schedule1: PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); +#if HALFWORD_HEAP + ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0); +#endif + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); #ifdef DEBUG pid = c_p->id; #endif @@ -1168,11 +1233,14 @@ void process_main(void) reg = c_p->scheduler_data->save_reg; freg = c_p->scheduler_data->freg; #endif +#if !HEAP_ON_C_STACK + tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; +#endif ERL_BITS_RELOAD_STATEP(c_p); { int reds; Eterm* argp; - Eterm* next; + BeamInstr *next; int i; argp = c_p->arg_reg; @@ -1199,7 +1267,7 @@ void process_main(void) FCALLS = REDS_IN(c_p) = reds; } - next = (Eterm *) *I; + next = (BeamInstr *) *I; r(0) = c_p->arg_reg[0]; #ifdef HARDDEBUG if (c_p->arity > 0) { @@ -1223,6 +1291,52 @@ void process_main(void) #define STORE_ARITH_RESULT(res) StoreBifResult(2, (res)); #define ARITH_FUNC(name) erts_gc_##name + { + Eterm increment_reg_val; + Eterm increment_val; + Uint live; + Eterm result; + + OpCase(i_increment_yIId): + increment_reg_val = yb(Arg(0)); + goto do_increment; + + OpCase(i_increment_xIId): + increment_reg_val = xb(Arg(0)); + goto do_increment; + + OpCase(i_increment_rIId): + increment_reg_val = r(0); + I--; + + do_increment: + increment_val = Arg(1); + if (is_small(increment_reg_val)) { + Sint i = signed_val(increment_reg_val) + increment_val; + ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); + if (MY_IS_SSMALL(i)) { + result = make_small(i); + store_result: + StoreBifResult(3, result); + } + } + + live = Arg(2); + SWAPOUT; + reg[0] = r(0); + reg[live] = increment_reg_val; + reg[live+1] = make_small(increment_val); + result = erts_gc_mixed_plus(c_p, reg, live); + r(0) = reg[0]; + SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (is_value(result)) { + goto store_result; + } + ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); + goto find_func_info; + } + OpCase(i_plus_jId): { Eterm result; @@ -1286,12 +1400,58 @@ void process_main(void) } Next(1); + { + Eterm is_eq_exact_lit_val; + + OpCase(i_is_eq_exact_literal_xfc): + is_eq_exact_lit_val = xb(Arg(0)); + I++; + goto do_is_eq_exact_literal; + + OpCase(i_is_eq_exact_literal_yfc): + is_eq_exact_lit_val = yb(Arg(0)); + I++; + goto do_is_eq_exact_literal; + + OpCase(i_is_eq_exact_literal_rfc): + is_eq_exact_lit_val = r(0); + + do_is_eq_exact_literal: + if (!eq(Arg(1), is_eq_exact_lit_val)) { + ClauseFail(); + } + Next(2); + } + + { + Eterm is_ne_exact_lit_val; + + OpCase(i_is_ne_exact_literal_xfc): + is_ne_exact_lit_val = xb(Arg(0)); + I++; + goto do_is_ne_exact_literal; + + OpCase(i_is_ne_exact_literal_yfc): + is_ne_exact_lit_val = yb(Arg(0)); + I++; + goto do_is_ne_exact_literal; + + OpCase(i_is_ne_exact_literal_rfc): + is_ne_exact_lit_val = r(0); + + do_is_ne_exact_literal: + if (eq(Arg(1), is_ne_exact_lit_val)) { + ClauseFail(); + } + Next(2); + } + OpCase(i_move_call_only_fcr): { r(0) = Arg(1); } /* FALL THROUGH */ OpCase(i_call_only_f): { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Dispatch(); } @@ -1302,7 +1462,7 @@ void process_main(void) OpCase(i_call_last_fP): { RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Dispatch(); } @@ -1313,7 +1473,7 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_f): { SET_CP(c_p, I+2); - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Dispatch(); } @@ -1349,7 +1509,7 @@ void process_main(void) Dispatchx(); OpCase(init_y): { - Eterm* next; + BeamInstr *next; PreFetch(1, next); make_blank(yb(Arg(0))); @@ -1357,7 +1517,7 @@ void process_main(void) } OpCase(i_trim_I): { - Eterm* next; + BeamInstr *next; Uint words; Uint cp; @@ -1369,6 +1529,17 @@ void process_main(void) NextPF(1, next); } + OpCase(move_x1_c): { + x(1) = Arg(0); + Next(1); + } + + OpCase(move_x2_c): { + x(2) = Arg(0); + Next(1); + } + + OpCase(return): { SET_I(c_p->cp); /* @@ -1382,30 +1553,6 @@ void process_main(void) Goto(*I); } - OpCase(test_heap_1_put_list_Iy): { - Eterm* next; - - PreFetch(2, next); - TestHeap(Arg(0), 1); - PutList(yb(Arg(1)), r(0), r(0), StoreSimpleDest); - CHECK_TERM(r(0)); - NextPF(2, next); - } - - OpCase(put_string_IId): - { - unsigned char* s; - int len; - Eterm result; - - len = Arg(0); /* Length. */ - result = NIL; - for (s = (unsigned char *) Arg(1); len > 0; s--, len--) { - PutList(make_small(*s), result, result, StoreSimpleDest); - } - StoreBifResult(2, result); - } - /* * Send is almost a standard call-BIF with two arguments, except for: * 1) It cannot be traced. @@ -1414,7 +1561,7 @@ void process_main(void) */ OpCase(send): { - Eterm* next; + BeamInstr *next; Eterm result; PRE_BIF_SWAPOUT(c_p); @@ -1429,7 +1576,7 @@ void process_main(void) NextPF(0, next); } else if (c_p->freason == TRAP) { SET_CP(c_p, I+1); - SET_I((Eterm *) c_p->def_arg_reg[3]); + SET_I(*((BeamInstr **) (BeamInstr) ((c_p)->def_arg_reg + 3))); SWAPIN; r(0) = c_p->def_arg_reg[0]; x(1) = c_p->def_arg_reg[1]; @@ -1438,24 +1585,36 @@ void process_main(void) goto find_func_info; } - OpCase(i_element_jssd): { - Eterm index; - Eterm tuple; - - /* - * Inlined version of element/2 for speed. - */ - GetArg2(1, index, tuple); - if (is_small(index) && is_tuple(tuple)) { - Eterm* tp = tuple_val(tuple); - - if ((signed_val(index) >= 1) && - (signed_val(index) <= arityval(*tp))) { - Eterm result = tp[signed_val(index)]; - StoreBifResult(3, result); - } - } - } + { + Eterm element_index; + Eterm element_tuple; + + OpCase(i_element_xjsd): + element_tuple = xb(Arg(0)); + I++; + goto do_element; + + OpCase(i_element_yjsd): + element_tuple = yb(Arg(0)); + I++; + goto do_element; + + OpCase(i_element_rjsd): + element_tuple = r(0); + /* Fall through */ + + do_element: + GetArg1(1, element_index); + if (is_small(element_index) && is_tuple(element_tuple)) { + Eterm* tp = tuple_val(element_tuple); + + if ((signed_val(element_index) >= 1) && + (signed_val(element_index) <= arityval(*tp))) { + Eterm result = tp[signed_val(element_index)]; + StoreBifResult(2, result); + } + } + } /* Fall through */ OpCase(badarg_j): @@ -1463,24 +1622,32 @@ void process_main(void) c_p->freason = BADARG; goto lb_Cl_error; - OpCase(i_fast_element_jIsd): { - Eterm tuple; - - /* - * Inlined version of element/2 for even more speed. - * The first argument is an untagged integer >= 1. - * The second argument is guaranteed to be a register operand. - */ - GetArg1(2, tuple); - if (is_tuple(tuple)) { - Eterm* tp = tuple_val(tuple); - tmp_arg2 = Arg(1); - if (tmp_arg2 <= arityval(*tp)) { - Eterm result = tp[tmp_arg2]; - StoreBifResult(3, result); - } - } + { + Eterm fast_element_tuple; + + OpCase(i_fast_element_rjId): + fast_element_tuple = r(0); + + do_fast_element: + if (is_tuple(fast_element_tuple)) { + Eterm* tp = tuple_val(fast_element_tuple); + Eterm pos = Arg(1); /* Untagged integer >= 1 */ + if (pos <= arityval(*tp)) { + Eterm result = tp[pos]; + StoreBifResult(2, result); + } + } goto badarg; + + OpCase(i_fast_element_xjId): + fast_element_tuple = xb(Arg(0)); + I++; + goto do_fast_element; + + OpCase(i_fast_element_yjId): + fast_element_tuple = yb(Arg(0)); + I++; + goto do_fast_element; } OpCase(catch_yf): @@ -1506,6 +1673,7 @@ void process_main(void) SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; } @@ -1531,6 +1699,10 @@ void process_main(void) /* * Skeleton for receive statement: * + * recv_mark L1 Optional + * call make_ref/monitor Optional + * ... + * recv_set L1 Optional * L1: <-------------------+ * <-----------+ | * | | @@ -1549,13 +1721,41 @@ void process_main(void) * */ + OpCase(recv_mark_f): { + /* + * Save the current position in message buffer and the + * the label for the loop_rec/2 instruction for the + * the receive statement. + */ + c_p->msg.mark = (BeamInstr *) Arg(0); + c_p->msg.saved_last = c_p->msg.last; + Next(1); + } + + OpCase(i_recv_set): { + /* + * If the mark is valid (points to the loop_rec/2 + * instruction that follows), we know that the saved + * position points to the first message that could + * possibly be matched out. + * + * If the mark is invalid, we do nothing, meaning that + * we will look through all messages in the message queue. + */ + if (c_p->msg.mark == (BeamInstr *) (I+1)) { + c_p->msg.save = c_p->msg.saved_last; + } + I++; + /* Fall through to the loop_rec/2 instruction */ + } + /* * Pick up the next message and place it in x(0). * If no message, jump to a wait or wait_timeout instruction. */ OpCase(i_loop_rec_fr): { - Eterm* next; + BeamInstr *next; ErlMessage* msgp; loop_rec__: @@ -1579,7 +1779,7 @@ void process_main(void) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); else { #endif - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); /* Jump to a wait or wait_timeout instruction */ #ifdef ERTS_SMP } @@ -1592,6 +1792,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); }, { + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); r(0) = reg[0]; SWAPIN; @@ -1615,7 +1816,7 @@ void process_main(void) * Remove a (matched) message from the message queue. */ OpCase(remove_message): { - Eterm* next; + BeamInstr *next; ErlMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -1650,6 +1851,7 @@ void process_main(void) CANCEL_TIMER(c_p); free_message(msgp); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); NextPF(0, next); @@ -1660,7 +1862,7 @@ void process_main(void) * message didn't match), then jump to the loop_rec instruction. */ OpCase(loop_rec_end_f): { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); SAVE_MESSAGE(c_p); goto loop_rec__; } @@ -1690,12 +1892,12 @@ void process_main(void) } GetArg1(1, timeout_value); if (timeout_value != make_small(0)) { -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP Uint time_val; #endif if (is_small(timeout_value) && signed_val(timeout_value) > 0 && -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP ((unsigned_val(timeout_value) >> 32) == 0) #else 1 @@ -1706,14 +1908,18 @@ void process_main(void) * c_p->def_arg_reg[0]. Note that it is safe to use this * location because there are no living x registers in * a receive statement. + * Note that for the halfword emulator, the two first elements + * of the array are used. */ - c_p->def_arg_reg[0] = (Eterm) (I+3); + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, unsigned_val(timeout_value)); } else if (timeout_value == am_infinity) { c_p->flags |= F_TIMO; -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP } else if (term_to_Uint(timeout_value, &time_val)) { - c_p->def_arg_reg[0] = (Eterm) (I+3); + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; set_timer(c_p, time_val); #endif } else { /* Wrong time */ @@ -1742,7 +1948,7 @@ void process_main(void) wait2: { ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - c_p->i = (Eterm *) Arg(0); /* L1 */ + c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; c_p->status = P_WAITING; @@ -1770,7 +1976,8 @@ void process_main(void) * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. */ if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - c_p->def_arg_reg[0] = (Eterm) (I+3); + BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; + *p = I+3; set_timer(c_p, Arg(1)); } goto wait2; @@ -1785,7 +1992,7 @@ void process_main(void) } OpCase(timeout): { - Eterm* next; + BeamInstr *next; PreFetch(0, next); if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { @@ -1799,14 +2006,93 @@ void process_main(void) NextPF(0, next); } - OpCase(i_select_val_sfI): - GetArg1(0, tmp_arg1); + + { + Eterm select_val2; + + OpCase(i_select_tuple_arity2_yfAfAf): + select_val2 = yb(Arg(0)); + goto do_select_tuple_arity2; + + OpCase(i_select_tuple_arity2_xfAfAf): + select_val2 = xb(Arg(0)); + goto do_select_tuple_arity2; + + OpCase(i_select_tuple_arity2_rfAfAf): + select_val2 = r(0); + I--; + + do_select_tuple_arity2: + if (is_not_tuple(select_val2)) { + goto select_val2_fail; + } + select_val2 = *tuple_val(select_val2); + goto do_select_val2; + + OpCase(i_select_val2_yfcfcf): + select_val2 = yb(Arg(0)); + goto do_select_val2; + + OpCase(i_select_val2_xfcfcf): + select_val2 = xb(Arg(0)); + goto do_select_val2; + + OpCase(i_select_val2_rfcfcf): + select_val2 = r(0); + I--; + + do_select_val2: + if (select_val2 == Arg(2)) { + I += 2; + } else if (select_val2 == Arg(4)) { + I += 4; + } + + select_val2_fail: + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + } + + { + Eterm select_val; + + OpCase(i_select_tuple_arity_xfI): + select_val = xb(Arg(0)); + goto do_select_tuple_arity; + + OpCase(i_select_tuple_arity_yfI): + select_val = yb(Arg(0)); + goto do_select_tuple_arity; + + OpCase(i_select_tuple_arity_rfI): + select_val = r(0); + I--; + + do_select_tuple_arity: + if (is_tuple(select_val)) { + select_val = *tuple_val(select_val); + goto do_binary_search; + } + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + + OpCase(i_select_val_xfI): + select_val = xb(Arg(0)); + goto do_binary_search; + + OpCase(i_select_val_yfI): + select_val = yb(Arg(0)); + goto do_binary_search; + + OpCase(i_select_val_rfI): + select_val = r(0); + I--; do_binary_search: { struct Pairs { - Eterm val; - Eterm* addr; + BeamInstr val; + BeamInstr* addr; }; struct Pairs* low; struct Pairs* high; @@ -1837,48 +2123,98 @@ void process_main(void) unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1); mid = (struct Pairs*)((char*)low + boffset); - if (tmp_arg1 < mid->val) { + if (select_val < mid->val) { high = mid; - } else if (tmp_arg1 > mid->val) { + } else if (select_val > mid->val) { low = mid + 1; } else { SET_I(mid->addr); Goto(*I); } } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } + } - OpCase(i_jump_on_val_zero_sfI): { - Eterm index; - - GetArg1(0, index); - if (is_small(index)) { - index = signed_val(index); - if (index < Arg(2)) { - SET_I((Eterm *) (&Arg(3))[index]); + Eterm jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_yfI): + jump_on_val_zero_index = yb(Arg(0)); + goto do_jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_xfI): + jump_on_val_zero_index = xb(Arg(0)); + goto do_jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_rfI): + jump_on_val_zero_index = r(0); + I--; + + do_jump_on_val_zero_index: + if (is_small(jump_on_val_zero_index)) { + jump_on_val_zero_index = signed_val(jump_on_val_zero_index); + if (jump_on_val_zero_index < Arg(2)) { + SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]); Goto(*I); } } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); Goto(*I); } - OpCase(i_jump_on_val_sfII): { - Eterm index; + Eterm jump_on_val_index; - GetArg1(0, index); - if (is_small(index)) { - index = (Uint) (signed_val(index) - Arg(3)); - if (index < Arg(2)) { - SET_I((Eterm *) (&Arg(4))[index]); + + OpCase(i_jump_on_val_yfII): + jump_on_val_index = yb(Arg(0)); + goto do_jump_on_val_index; + + OpCase(i_jump_on_val_xfII): + jump_on_val_index = xb(Arg(0)); + goto do_jump_on_val_index; + + OpCase(i_jump_on_val_rfII): + jump_on_val_index = r(0); + I--; + + do_jump_on_val_index: + if (is_small(jump_on_val_index)) { + jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); + if (jump_on_val_index < Arg(2)) { + SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]); Goto(*I); } } - SET_I((Eterm *) Arg(1)); + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + } + + do_put_tuple: { + Eterm* hp = HTOP; + + *hp++ = make_arityval(pt_arity); + + do { + Eterm term = *I++; + switch (term & _TAG_IMMED1_MASK) { + case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = r(0); + break; + case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = x(term >> _TAG_IMMED1_SIZE); + break; + case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = y(term >> _TAG_IMMED1_SIZE); + break; + default: + *hp++ = term; + break; + } + } while (--pt_arity != 0); + HTOP = hp; Goto(*I); } @@ -1909,13 +2245,14 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, arg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { StoreBifResult(3, result); } - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } @@ -1937,6 +2274,7 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, arg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; @@ -1955,7 +2293,7 @@ void process_main(void) GcBifFunction bf; Eterm arg; Eterm result; - Uint live = Arg(3); + Uint live = (Uint) Arg(3); GetArg1(2, arg); reg[0] = r(0); @@ -1966,6 +2304,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); result = (*bf)(c_p, reg, live); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -1976,7 +2315,7 @@ void process_main(void) StoreBifResult(4, result); } if (Arg(0) != 0) { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } reg[0] = arg; @@ -1984,6 +2323,83 @@ void process_main(void) goto post_error_handling; } + OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1 + and i_gc_bif3 */ + { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm result; + Uint live = (Uint) Arg(2); + + reg[0] = r(0); + reg[live++] = tmp_arg1; + reg[live] = tmp_arg2; + bf = (GcBifFunction) Arg(1); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + result = (*bf)(c_p, reg, live); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + r(0) = reg[0]; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + if (is_value(result)) { + StoreBifResult(3, result); + } + if (Arg(0) != 0) { + SET_I((BeamInstr *) Arg(0)); + Goto(*I); + } + reg[0] = tmp_arg1; + reg[1] = tmp_arg2; + I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + goto post_error_handling; + } + + OpCase(i_gc_bif3_jIsId): + { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm arg; + Eterm result; + Uint live = (Uint) Arg(3); + + GetArg1(2, arg); + reg[0] = r(0); + reg[live++] = arg; + reg[live++] = tmp_arg1; + reg[live] = tmp_arg2; + bf = (GcBifFunction) Arg(1); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + result = (*bf)(c_p, reg, live); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + r(0) = reg[0]; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + if (is_value(result)) { + StoreBifResult(4, result); + } + if (Arg(0) != 0) { + SET_I((BeamInstr *) Arg(0)); + Goto(*I); + } + reg[0] = arg; + reg[1] = tmp_arg1; + reg[2] = tmp_arg2; + I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + goto post_error_handling; + } + /* * Guards bifs and, or, xor in guards. */ @@ -1998,13 +2414,14 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, tmp_arg1, tmp_arg2); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { StoreBifResult(2, result); } - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } @@ -2021,6 +2438,7 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, tmp_arg1, tmp_arg2); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); if (is_value(result)) { @@ -2040,7 +2458,7 @@ void process_main(void) */ OpCase(call_bif0_e): { - Eterm (*bf)(Process*, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2073,9 +2491,9 @@ void process_main(void) OpCase(call_bif1_e): { - Eterm (*bf)(Process*, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; - Eterm* next; + BeamInstr *next; c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { @@ -2108,9 +2526,9 @@ void process_main(void) OpCase(call_bif2_e): { - Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; - Eterm* next; + BeamInstr *next; PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2145,9 +2563,9 @@ void process_main(void) OpCase(call_bif3_e): { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; - Eterm* next; + BeamInstr *next; PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2168,7 +2586,7 @@ void process_main(void) } else if (c_p->freason == TRAP) { call_bif_trap3: SET_CP(c_p, I+2); - SET_I((Eterm *)c_p->def_arg_reg[3]); + SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); SWAPIN; r(0) = c_p->def_arg_reg[0]; x(1) = c_p->def_arg_reg[1]; @@ -2276,7 +2694,7 @@ void process_main(void) lb_Cl_error: { if (Arg(0) != 0) { OpCase(jump_f): { - SET_I((Eterm *) Arg(0)); + SET_I((BeamInstr *) Arg(0)); Goto(*I); } } @@ -2444,23 +2862,25 @@ void process_main(void) OpCase(i_int_bnot_jsId): { - GetArg1(1, tmp_arg1); - if (is_small(tmp_arg1)) { - tmp_arg1 = make_small(~signed_val(tmp_arg1)); + Eterm bnot_val; + + GetArg1(1, bnot_val); + if (is_small(bnot_val)) { + bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = Arg(2); SWAPOUT; reg[0] = r(0); - reg[live] = tmp_arg1; - tmp_arg1 = erts_gc_bnot(c_p, reg, live); + reg[live] = bnot_val; + bnot_val = erts_gc_bnot(c_p, reg, live); r(0) = reg[0]; SWAPIN; ERTS_HOLE_CHECK(c_p); - if (is_nil(tmp_arg1)) { + if (is_nil(bnot_val)) { goto lb_Cl_error; } } - StoreBifResult(3, tmp_arg1); + StoreBifResult(3, bnot_val); } badarith: @@ -2468,7 +2888,7 @@ void process_main(void) goto lb_Cl_error; OpCase(i_apply): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; @@ -2483,13 +2903,13 @@ void process_main(void) } OpCase(i_apply_last_P): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatch(); @@ -2499,7 +2919,7 @@ void process_main(void) } OpCase(i_apply_only): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); SWAPIN; @@ -2513,7 +2933,7 @@ void process_main(void) } OpCase(apply_I): { - Eterm* next; + BeamInstr *next; reg[0] = r(0); SWAPOUT; @@ -2530,7 +2950,7 @@ void process_main(void) } OpCase(apply_last_IP): { - Eterm* next; + BeamInstr *next; reg[0] = r(0); SWAPOUT; @@ -2538,7 +2958,7 @@ void process_main(void) SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatch(); @@ -2548,7 +2968,7 @@ void process_main(void) } OpCase(i_apply_fun): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); @@ -2563,14 +2983,14 @@ void process_main(void) } OpCase(i_apply_fun_last_P): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatchfun(); @@ -2579,7 +2999,7 @@ void process_main(void) } OpCase(i_apply_fun_only): { - Eterm* next; + BeamInstr *next; SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); @@ -2593,10 +3013,11 @@ void process_main(void) } OpCase(i_call_fun_I): { - Eterm* next; + BeamInstr *next; SWAPOUT; reg[0] = r(0); + next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); SWAPIN; if (next != NULL) { @@ -2609,7 +3030,7 @@ void process_main(void) } OpCase(i_call_fun_last_IP): { - Eterm* next; + BeamInstr *next; SWAPOUT; reg[0] = r(0); @@ -2617,7 +3038,7 @@ void process_main(void) SWAPIN; if (next != NULL) { r(0) = reg[0]; - SET_CP(c_p, (Eterm *) E[0]); + SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatchfun(); @@ -2714,123 +3135,10 @@ void process_main(void) goto do_schedule1; } - OpCase(i_select_tuple_arity_sfI): - { - GetArg1(0, tmp_arg1); - - if (is_tuple(tmp_arg1)) { - tmp_arg1 = *tuple_val(tmp_arg1); - goto do_binary_search; - } - SET_I((Eterm *) Arg(1)); - Goto(*I); - } - - OpCase(i_select_big_sf): - { - Eterm* bigp; - Uint arity; - Eterm* given; - Uint given_arity; - Uint given_size; - - GetArg1(0, tmp_arg1); - if (is_big(tmp_arg1)) { - - /* - * The loader has sorted the bignumbers in descending order - * on the arity word. Therefore, we know that the search - * has failed as soon as we encounter an arity word less than - * the arity word of the given number. There is a zero word - * (less than any valid arity word) stored after the last bignumber. - */ - - given = big_val(tmp_arg1); - given_arity = given[0]; - given_size = thing_arityval(given_arity); - bigp = &Arg(2); - while ((arity = bigp[0]) > given_arity) { - bigp += thing_arityval(arity) + 2; - } - while (bigp[0] == given_arity) { - if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) { - SET_I((Eterm *) bigp[given_size+1]); - Goto(*I); - } - bigp += thing_arityval(arity) + 2; - } - } - - /* - * Failed. - */ - - SET_I((Eterm *) Arg(1)); - Goto(*I); - } - -#ifdef ARCH_64 - OpCase(i_select_float_sfI): - { - Uint f; - int n; - struct ValLabel { - Uint f; - Eterm* addr; - }; - struct ValLabel* ptr; - - GetArg1(0, tmp_arg1); - ASSERT(is_float(tmp_arg1)); - f = float_val(tmp_arg1)[1]; - n = Arg(2); - ptr = (struct ValLabel *) &Arg(3); - while (n-- > 0) { - if (ptr->f == f) { - SET_I(ptr->addr); - Goto(*I); - } - ptr++; - } - SET_I((Eterm *) Arg(1)); - Goto(*I); - } -#else - OpCase(i_select_float_sfI): - { - Uint fpart1; - Uint fpart2; - int n; - struct ValLabel { - Uint fpart1; - Uint fpart2; - Eterm* addr; - }; - struct ValLabel* ptr; - - GetArg1(0, tmp_arg1); - ASSERT(is_float(tmp_arg1)); - fpart1 = float_val(tmp_arg1)[1]; - fpart2 = float_val(tmp_arg1)[2]; - - n = Arg(2); - ptr = (struct ValLabel *) &Arg(3); - while (n-- > 0) { - if (ptr->fpart1 == fpart1 && ptr->fpart2 == fpart2) { - SET_I(ptr->addr); - Goto(*I); - } - ptr++; - } - SET_I((Eterm *) Arg(1)); - Goto(*I); - } -#endif - OpCase(set_tuple_element_sdP): { Eterm element; Eterm tuple; - Eterm* next; + BeamInstr *next; Eterm* p; PreFetch(3, next); @@ -2872,15 +3180,17 @@ void process_main(void) the first argument. We also handle atom tags in the first argument for backwards compatibility. */ - GetArg2(0, tmp_arg1, tmp_arg2); - c_p->fvalue = tmp_arg2; + Eterm raise_val1; + Eterm raise_val2; + GetArg2(0, raise_val1, raise_val2); + c_p->fvalue = raise_val2; if (c_p->freason == EXC_NULL) { /* a safety check for the R10-0 case; should not happen */ c_p->ftrace = NIL; c_p->freason = EXC_ERROR; } /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */ - switch (tmp_arg1) { + switch (raise_val1) { case am_throw: c_p->freason = EXC_THROWN & ~EXF_SAVETRACE; break; @@ -2896,8 +3206,8 @@ void process_main(void) passed from a user! Currently only expecting generated calls. */ struct StackTrace *s; - c_p->ftrace = tmp_arg1; - s = get_trace_from_exc(tmp_arg1); + c_p->ftrace = raise_val1; + s = get_trace_from_exc(raise_val1); if (s == NULL) { c_p->freason = EXC_ERROR; } else { @@ -2908,11 +3218,24 @@ void process_main(void) goto find_func_info; } - OpCase(badmatch_s): { - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = BADMATCH; - } + { + Eterm badmatch_val; + + OpCase(badmatch_y): + badmatch_val = yb(Arg(0)); + goto do_badmatch; + + OpCase(badmatch_x): + badmatch_val = xb(Arg(0)); + goto do_badmatch; + + OpCase(badmatch_r): + badmatch_val = r(0); + + do_badmatch: + c_p->fvalue = badmatch_val; + c_p->freason = BADMATCH; + } /* Fall through here */ find_func_info: { @@ -2935,12 +3258,11 @@ void process_main(void) */ SWAPOUT; reg[0] = r(0); - tmp_arg1 = call_error_handler(c_p, I-3, reg); + I = call_error_handler(c_p, I-3, reg, am_undefined_function); r(0) = reg[0]; SWAPIN; - if (tmp_arg1) { - SET_I(c_p->i); - Dispatch(); + if (I) { + Goto(*I); } /* Fall through */ @@ -2963,128 +3285,154 @@ void process_main(void) } } - OpCase(call_nif): - { - /* - * call_nif is always first instruction in function: - * - * I[-3]: Module - * I[-2]: Function - * I[-1]: Arity - * I[0]: &&call_nif - * I[1]: Function pointer to NIF function - * I[2]: Pointer to erl_module_nif - */ - BifFunction vbf; - - c_p->current = I-3; /* current and vbf set to please handle_error */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; - PROCESS_MAIN_CHK_LOCKS(c_p); - tmp_arg2 = I[-1]; - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + { + Eterm nif_bif_result; + Eterm bif_nif_arity; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); - reg[0] = r(0); - tmp_arg1 = (*fp)(&env, tmp_arg2, reg); - erts_post_nif(&env); - } - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); - goto apply_bif_or_nif_epilogue; - - OpCase(apply_bif): - /* - * At this point, I points to the code[3] in the export entry for - * the BIF: - * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function - */ + OpCase(call_nif): + { + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + */ + BifFunction vbf; - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ - c_p->i = I; /* In case we apply check_process_code/2. */ - c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; - vbf = (BifFunction) Arg(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - tmp_arg2 = I[-1]; - ASSERT(tmp_arg2 <= 3); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - switch (tmp_arg2) { - case 3: + c_p->current = I-3; /* current and vbf set to please handle_error */ + SWAPOUT; + c_p->fcalls = FCALLS - 1; + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = I[-1]; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); + reg[0] = r(0); + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + erts_post_nif(&env); } - break; - case 2: - { - Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), x(1), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + goto apply_bif_or_nif_epilogue; + + OpCase(apply_bif): + /* + * At this point, I points to the code[3] in the export entry for + * the BIF: + * + * code[0]: Module + * code[1]: Function + * code[2]: Arity + * code[3]: &&apply_bif + * code[4]: Function pointer to BIF function + */ + + c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->i = I; /* In case we apply check_process_code/2. */ + c_p->arity = 0; /* To allow garbage collection on ourselves + * (check_process_code/2). + */ + SWAPOUT; + c_p->fcalls = FCALLS - 1; + vbf = (BifFunction) Arg(0); + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = I[-1]; + ASSERT(bif_nif_arity <= 3); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + switch (bif_nif_arity) { + case 3: + { + Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 2: + { + Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), x(1), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 1: + { + Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 0: + { + Eterm (*bf)(Process*, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + break; + } + default: + erl_exit(1, "apply_bif: invalid arity: %u\n", + bif_nif_arity); } - break; - case 1: - { - Eterm (*bf)(Process*, Eterm, Uint*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + + apply_bif_or_nif_epilogue: + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_HOLE_CHECK(c_p); + if (c_p->mbuf) { + reg[0] = r(0); + nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, + reg, bif_nif_arity); + r(0) = reg[0]; } - break; - case 0: - { - Eterm (*bf)(Process*, Uint*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); - break; + SWAPIN; /* There might have been a garbage collection. */ + FCALLS = c_p->fcalls; + if (is_value(nif_bif_result)) { + r(0) = nif_bif_result; + CHECK_TERM(r(0)); + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); + } else if (c_p->freason == TRAP) { + SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); + r(0) = c_p->def_arg_reg[0]; + x(1) = c_p->def_arg_reg[1]; + x(2) = c_p->def_arg_reg[2]; + if (c_p->flags & F_HIBERNATE_SCHED) { + c_p->flags &= ~F_HIBERNATE_SCHED; + goto do_schedule; + } + Dispatch(); } - } -apply_bif_or_nif_epilogue: - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_HOLE_CHECK(c_p); - if (c_p->mbuf) { reg[0] = r(0); - tmp_arg1 = erts_gc_after_bif_call(c_p, tmp_arg1, reg, tmp_arg2); - r(0) = reg[0]; + I = handle_error(c_p, c_p->cp, reg, vbf); + goto post_error_handling; } - SWAPIN; /* There might have been a garbage collection. */ - FCALLS = c_p->fcalls; - if (is_value(tmp_arg1)) { - r(0) = tmp_arg1; - CHECK_TERM(r(0)); - SET_I(c_p->cp); - Goto(*I); - } else if (c_p->freason == TRAP) { - SET_I((Eterm *)c_p->def_arg_reg[3]); - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; - x(2) = c_p->def_arg_reg[2]; - Dispatch(); - } - reg[0] = r(0); - I = handle_error(c_p, c_p->cp, reg, vbf); - goto post_error_handling; } OpCase(i_get_sd): @@ -3097,17 +3445,25 @@ apply_bif_or_nif_epilogue: StoreBifResult(1, result); } - OpCase(i_put_tuple_only_Ad): { - tmp_arg1 = make_tuple(HTOP); - *HTOP++ = Arg(0); - StoreBifResult(1, tmp_arg1); - } + { + Eterm case_end_val; - OpCase(case_end_s): - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = EXC_CASE_CLAUSE; - goto find_func_info; + OpCase(case_end_x): + case_end_val = xb(Arg(0)); + goto do_case_end; + + OpCase(case_end_y): + case_end_val = yb(Arg(0)); + goto do_case_end; + + OpCase(case_end_r): + case_end_val = r(0); + + do_case_end: + c_p->fvalue = case_end_val; + c_p->freason = EXC_CASE_CLAUSE; + goto find_func_info; + } OpCase(if_end): c_p->freason = EXC_IF_CLAUSE; @@ -3120,10 +3476,13 @@ apply_bif_or_nif_epilogue: } OpCase(try_case_end_s): - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = EXC_TRY_CLAUSE; - goto find_func_info; + { + Eterm try_case_end_val; + GetArg1(0, try_case_end_val); + c_p->fvalue = try_case_end_val; + c_p->freason = EXC_TRY_CLAUSE; + goto find_func_info; + } /* * Construction of binaries using new instructions. @@ -3270,12 +3629,12 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = num_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(c_p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); new_binary = make_binary(pb); goto do_bits_sub_bin; } @@ -3371,13 +3730,13 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = tmp_arg1; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm)); StoreBifResult(2, make_binary(pb)); } @@ -3413,42 +3772,6 @@ apply_bif_or_nif_epilogue: } } - OpCase(i_bs_bits_to_bytes_rjd): { - tmp_arg1 = r(0); - goto do_bits_to_bytes; - } - - OpCase(i_bs_bits_to_bytes_yjd): { - tmp_arg1 = yb(Arg(0)); - I++; - goto do_bits_to_bytes; - - OpCase(i_bs_bits_to_bytes_xjd): { - tmp_arg1 = xb(Arg(0)); - I++; - } - - do_bits_to_bytes: - { - if (is_valid_bit_size(tmp_arg1)) { - tmp_arg1 = make_small(unsigned_val(tmp_arg1) >> 3); - } else { - Uint bytes; - if (!term_to_Uint(tmp_arg1, &bytes)) { - goto badarg; - } - tmp_arg1 = bytes; - if ((tmp_arg1 & 0x07) != 0) { - goto badarg; - } - SWAPOUT; - tmp_arg1 = erts_make_integer(tmp_arg1 >> 3, c_p); - HTOP = HEAP_TOP(c_p); - } - StoreBifResult(1, tmp_arg1); - } - } - OpCase(i_bs_add_jId): { Uint Unit = Arg(1); if (is_both_small(tmp_arg1, tmp_arg2)) { @@ -3486,7 +3809,7 @@ apply_bif_or_nif_epilogue: /* * Now we know that one of the arguments is - * not at small. We must convert both arguments + * not a small. We must convert both arguments * to Uints and check for errors at the same time. * * Error checking is tricky. @@ -3534,7 +3857,7 @@ apply_bif_or_nif_epilogue: OpCase(bs_put_string_II): { - Eterm* next; + BeamInstr *next; PreFetch(2, next); erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0))); NextPF(2, next); @@ -3705,21 +4028,22 @@ apply_bif_or_nif_epilogue: { Eterm header; - Eterm* next; + BeamInstr *next; Uint slots; + Eterm context; OpCase(i_bs_start_match2_rfIId): { - tmp_arg1 = r(0); + context = r(0); do_start_match: slots = Arg(2); - if (!is_boxed(tmp_arg1)) { + if (!is_boxed(context)) { ClauseFail(); } PreFetch(4, next); - header = *boxed_val(tmp_arg1); + header = *boxed_val(context); if (header_is_bin_matchstate(header)) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); Uint actual_slots = HEADER_NUM_SLOTS(header); ms->save_offset[0] = ms->mb.offset; if (actual_slots < slots) { @@ -3727,8 +4051,8 @@ apply_bif_or_nif_epilogue: Uint live = Arg(1); Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, tmp_arg1); - ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + TestHeapPreserve(wordsneeded, live, context); + ms = (ErlBinMatchState *) boxed_val(context); dst = (ErlBinMatchState *) HTOP; *dst = *ms; *HTOP = HEADER_BIN_MATCHSTATE(slots); @@ -3740,12 +4064,12 @@ apply_bif_or_nif_epilogue: Eterm result; Uint live = Arg(1); Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, tmp_arg1); + TestHeapPreserve(wordsneeded, live, context); HEAP_TOP(c_p) = HTOP; #ifdef DEBUG c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ #endif - result = erts_bs_start_match_2(c_p, tmp_arg1, slots); + result = erts_bs_start_match_2(c_p, context, slots); HTOP = HEAP_TOP(c_p); HEAP_SPACE_VERIFIED(0); if (is_non_value(result)) { @@ -3759,19 +4083,19 @@ apply_bif_or_nif_epilogue: NextPF(4, next); } OpCase(i_bs_start_match2_xfIId): { - tmp_arg1 = xb(Arg(0)); + context = xb(Arg(0)); I++; goto do_start_match; } OpCase(i_bs_start_match2_yfIId): { - tmp_arg1 = yb(Arg(0)); + context = yb(Arg(0)); I++; goto do_start_match; } } OpCase(bs_test_zero_tail2_fr): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(1, next); @@ -3783,7 +4107,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_zero_tail2_fx): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); @@ -3795,7 +4119,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_tail_imm2_frI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); _mb = ms_matchbuffer(r(0)); @@ -3805,7 +4129,7 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } OpCase(bs_test_tail_imm2_fxI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(3, next); _mb = ms_matchbuffer(xb(Arg(1))); @@ -3816,7 +4140,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_unit_frI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); _mb = ms_matchbuffer(r(0)); @@ -3826,7 +4150,7 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } OpCase(bs_test_unit_fxI): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(3, next); _mb = ms_matchbuffer(xb(Arg(1))); @@ -3837,7 +4161,7 @@ apply_bif_or_nif_epilogue: } OpCase(bs_test_unit8_fr): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(1, next); _mb = ms_matchbuffer(r(0)); @@ -3847,7 +4171,7 @@ apply_bif_or_nif_epilogue: NextPF(1, next); } OpCase(bs_test_unit8_fx): { - Eterm* next; + BeamInstr *next; ErlBinMatchBuffer *_mb; PreFetch(2, next); _mb = ms_matchbuffer(xb(Arg(1))); @@ -3857,93 +4181,105 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } + { + Eterm bs_get_integer8_context; + OpCase(i_bs_get_integer_8_rfd): { - tmp_arg1 = r(0); - goto do_bs_get_integer_8; - } + bs_get_integer8_context = r(0); + goto do_bs_get_integer_8; + } OpCase(i_bs_get_integer_8_xfd): { - tmp_arg1 = xb(Arg(0)); - I++; - } + bs_get_integer8_context = xb(Arg(0)); + I++; + } do_bs_get_integer_8: { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 8) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); - } else { - _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); - _mb->offset += 8; + ErlBinMatchBuffer *_mb; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer8_context); + if (_mb->size - _mb->offset < 8) { + ClauseFail(); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); + } else { + _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); + _mb->offset += 8; + } + StoreBifResult(1, _result); } - StoreBifResult(1, _result); } - OpCase(i_bs_get_integer_16_rfd): { - tmp_arg1 = r(0); + { + Eterm bs_get_integer_16_context; + + OpCase(i_bs_get_integer_16_rfd): + bs_get_integer_16_context = r(0); goto do_bs_get_integer_16; - } - OpCase(i_bs_get_integer_16_xfd): { - tmp_arg1 = xb(Arg(0)); + OpCase(i_bs_get_integer_16_xfd): + bs_get_integer_16_context = xb(Arg(0)); I++; - } - do_bs_get_integer_16: { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 16) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); - } else { - _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); - _mb->offset += 16; + do_bs_get_integer_16: + { + ErlBinMatchBuffer *_mb; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer_16_context); + if (_mb->size - _mb->offset < 16) { + ClauseFail(); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); + } else { + _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); + _mb->offset += 16; + } + StoreBifResult(1, _result); } - StoreBifResult(1, _result); } - OpCase(i_bs_get_integer_32_rfId): { - tmp_arg1 = r(0); + { + Eterm bs_get_integer_32_context; + + OpCase(i_bs_get_integer_32_rfId): + bs_get_integer_32_context = r(0); goto do_bs_get_integer_32; - } + - OpCase(i_bs_get_integer_32_xfId): { - tmp_arg1 = xb(Arg(0)); + OpCase(i_bs_get_integer_32_xfId): + bs_get_integer_32_context = xb(Arg(0)); I++; - } - do_bs_get_integer_32: { - ErlBinMatchBuffer *_mb; - Uint32 _integer; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 32) { ClauseFail(); } - if (BIT_OFFSET(_mb->offset) != 0) { - _integer = erts_bs_get_unaligned_uint32(_mb); - } else { - _integer = get_int32(_mb->base + _mb->offset/8); - } - _mb->offset += 32; -#ifndef ARCH_64 - if (IS_USMALL(0, _integer)) { + + do_bs_get_integer_32: + { + ErlBinMatchBuffer *_mb; + Uint32 _integer; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer_32_context); + if (_mb->size - _mb->offset < 32) { ClauseFail(); } + if (BIT_OFFSET(_mb->offset) != 0) { + _integer = erts_bs_get_unaligned_uint32(_mb); + } else { + _integer = get_int32(_mb->base + _mb->offset/8); + } + _mb->offset += 32; +#if !defined(ARCH_64) || HALFWORD_HEAP + if (IS_USMALL(0, _integer)) { #endif - _result = make_small(_integer); -#ifndef ARCH_64 - } else { - TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); - _result = uint_to_big((Uint) _integer, HTOP); - HTOP += BIG_UINT_HEAP_SIZE; - HEAP_SPACE_VERIFIED(0); - } + _result = make_small(_integer); +#if !defined(ARCH_64) || HALFWORD_HEAP + } else { + TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); + _result = uint_to_big((Uint) _integer, HTOP); + HTOP += BIG_UINT_HEAP_SIZE; + HEAP_SPACE_VERIFIED(0); + } #endif - StoreBifResult(2, _result); + StoreBifResult(2, _result); + } } /* Operands: Size Live Fail Flags Dst */ @@ -4041,54 +4377,64 @@ apply_bif_or_nif_epilogue: StoreBifResult(3, result); } - /* Operands: MatchContext Fail Dst */ + { + Eterm get_utf8_context; + + /* Operands: MatchContext Fail Dst */ OpCase(i_bs_get_utf8_rfd): { - tmp_arg1 = r(0); - goto do_bs_get_utf8; - } + get_utf8_context = r(0); + goto do_bs_get_utf8; + } OpCase(i_bs_get_utf8_xfd): { - tmp_arg1 = xb(Arg(0)); - I++; - } + get_utf8_context = xb(Arg(0)); + I++; + } - /* - * tmp_arg1 = match_context - * Operands: Fail Dst - */ + /* + * get_utf8_context = match_context + * Operands: Fail Dst + */ - do_bs_get_utf8: { - Eterm result = erts_bs_get_utf8(ms_matchbuffer(tmp_arg1)); - if (is_non_value(result)) { - ClauseFail(); + do_bs_get_utf8: { + Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(1, result); } - StoreBifResult(1, result); } - /* Operands: MatchContext Fail Flags Dst */ + { + Eterm get_utf16_context; + + /* Operands: MatchContext Fail Flags Dst */ OpCase(i_bs_get_utf16_rfId): { - tmp_arg1 = r(0); - goto do_bs_get_utf16; - } + get_utf16_context = r(0); + goto do_bs_get_utf16; + } OpCase(i_bs_get_utf16_xfId): { - tmp_arg1 = xb(Arg(0)); - I++; - } + get_utf16_context = xb(Arg(0)); + I++; + } - /* - * tmp_arg1 = match_context - * Operands: Fail Flags Dst - */ - do_bs_get_utf16: { - Eterm result = erts_bs_get_utf16(ms_matchbuffer(tmp_arg1), Arg(1)); - if (is_non_value(result)) { - ClauseFail(); + /* + * get_utf16_context = match_context + * Operands: Fail Flags Dst + */ + do_bs_get_utf16: { + Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), + Arg(1)); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(2, result); } - StoreBifResult(2, result); } { + Eterm context_to_binary_context; ErlBinMatchBuffer* mb; ErlSubBin* sb; Uint size; @@ -4097,27 +4443,29 @@ apply_bif_or_nif_epilogue: Uint hole_size; OpCase(bs_context_to_binary_r): { - tmp_arg1 = x0; + context_to_binary_context = x0; I -= 2; goto do_context_to_binary; } /* Unfortunately, inlining can generate this instruction. */ OpCase(bs_context_to_binary_y): { - tmp_arg1 = yb(Arg(0)); + context_to_binary_context = yb(Arg(0)); goto do_context_to_binary0; } OpCase(bs_context_to_binary_x): { - tmp_arg1 = xb(Arg(0)); + context_to_binary_context = xb(Arg(0)); do_context_to_binary0: I--; } do_context_to_binary: - if (is_boxed(tmp_arg1) && header_is_bin_matchstate(*boxed_val(tmp_arg1))) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + if (is_boxed(context_to_binary_context) && + header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { + ErlBinMatchState* ms; + ms = (ErlBinMatchState *) boxed_val(context_to_binary_context); mb = &ms->mb; offs = ms->save_offset[0]; size = mb->size - offs; @@ -4126,17 +4474,17 @@ apply_bif_or_nif_epilogue: Next(2); OpCase(i_bs_get_binary_all_reuse_rfI): { - tmp_arg1 = x0; + context_to_binary_context = x0; goto do_bs_get_binary_all_reuse; } OpCase(i_bs_get_binary_all_reuse_xfI): { - tmp_arg1 = xb(Arg(0)); + context_to_binary_context = xb(Arg(0)); I++; } do_bs_get_binary_all_reuse: - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(context_to_binary_context); size = mb->size - mb->offset; if (size % Arg(1) != 0) { ClauseFail(); @@ -4145,7 +4493,7 @@ apply_bif_or_nif_epilogue: do_bs_get_binary_all_reuse_common: orig = mb->orig; - sb = (ErlSubBin *) boxed_val(tmp_arg1); + sb = (ErlSubBin *) boxed_val(context_to_binary_context); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); @@ -4161,18 +4509,20 @@ apply_bif_or_nif_epilogue: } { + Eterm match_string_context; + OpCase(i_bs_match_string_rfII): { - tmp_arg1 = r(0); + match_string_context = r(0); goto do_bs_match_string; } OpCase(i_bs_match_string_xfII): { - tmp_arg1 = xb(Arg(0)); + match_string_context = xb(Arg(0)); I++; } do_bs_match_string: { - Eterm* next; + BeamInstr *next; byte* bytes; Uint bits; ErlBinMatchBuffer* mb; @@ -4181,7 +4531,7 @@ apply_bif_or_nif_epilogue: PreFetch(3, next); bits = Arg(1); bytes = (byte *) Arg(2); - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(match_string_context); if (mb->size - mb->offset < bits) { ClauseFail(); } @@ -4199,7 +4549,7 @@ apply_bif_or_nif_epilogue: } OpCase(i_bs_save2_rI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(1, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); @@ -4207,7 +4557,7 @@ apply_bif_or_nif_epilogue: NextPF(1, next); } OpCase(i_bs_save2_xI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(2, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); @@ -4216,7 +4566,7 @@ apply_bif_or_nif_epilogue: } OpCase(i_bs_restore2_rI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(1, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); @@ -4224,7 +4574,7 @@ apply_bif_or_nif_epilogue: NextPF(1, next); } OpCase(i_bs_restore2_xI): { - Eterm* next; + BeamInstr *next; ErlBinMatchState *_ms; PreFetch(2, next); _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); @@ -4241,7 +4591,7 @@ apply_bif_or_nif_epilogue: * deallocate not followed by a return, and that should work. */ OpCase(deallocate_I): { - Eterm* next; + BeamInstr *next; PreFetch(1, next); D(Arg(0)); @@ -4264,7 +4614,7 @@ apply_bif_or_nif_epilogue: */ OpCase(call_traced_function): { if (IS_TRACED_FL(c_p, F_TRACE_CALLS)) { - unsigned offset = offsetof(Export, code) + 3*sizeof(Eterm); + unsigned offset = offsetof(Export, code) + 3*sizeof(BeamInstr); Export* ep = (Export *) (((char *)I)-offset); Uint32 flags; @@ -4274,6 +4624,7 @@ apply_bif_or_nif_epilogue: ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); flags = erts_call_trace(c_p, ep->code, ep->match_prog_set, reg, 0, &c_p->tracer_proc); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -4285,32 +4636,32 @@ apply_bif_or_nif_epilogue: /* SWAPOUT, SWAPIN was done and r(0) was saved above */ PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect(c_p, 3, reg, ep->code[2]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); r(0) = reg[0]; SWAPIN; } E -= 3; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm)(ep->code))); + ASSERT(is_CP((BeamInstr)(ep->code))); ASSERT(is_internal_pid(c_p->tracer_proc) || is_internal_port(c_p->tracer_proc)); - E[2] = make_cp(c_p->cp); + E[2] = make_cp(c_p->cp); /* Code in lower range on halfword */ E[1] = am_true; /* Process tracer */ E[0] = make_cp(ep->code); - c_p->cp = (Eterm*) - make_cp(flags & MATCH_SET_EXCEPTION_TRACE - ? beam_exception_trace : beam_return_trace); + c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) + ? beam_exception_trace : beam_return_trace; erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); c_p->trace_flags |= F_EXCEPTION_TRACE; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } } - SET_I((Uint *) Arg(0)); + SET_I((BeamInstr *)Arg(0)); Dispatch(); } OpCase(return_trace): { - Uint* code = (Uint *) E[0]; + BeamInstr* code = (BeamInstr *) (UWord) E[0]; SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); @@ -4318,44 +4669,118 @@ apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; - SET_I((Eterm *) E[2]); + SET_I((BeamInstr *) cp_val(E[2])); E += 3; Goto(*I); } OpCase(i_count_breakpoint): { - Uint real_I; + BeamInstr real_I; - ErtsCountBreak((Uint *) I, &real_I); + ErtsCountBreak(c_p, (BeamInstr *) I, &real_I); ASSERT(VALID_INSTR(real_I)); Goto(real_I); } + /* need to send mfa instead of bdt pointer + * the pointer might be deallocated. + */ + + OpCase(i_time_breakpoint): { + BeamInstr real_I; + BpData **bds = (BpData **) (I)[-4]; + BpDataTime *bdt = NULL; + Uint ix = 0; +#ifdef ERTS_SMP + ix = c_p->scheduler_data->no - 1; +#else + ix = 0; +#endif + bdt = (BpDataTime *)bds[ix]; + + ASSERT((I)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(bdt); + bdt = (BpDataTime *) bdt->next; + ASSERT(bdt); + bds[ix] = (BpData *) bdt; + real_I = bdt->orig_instr; + ASSERT(VALID_INSTR(real_I)); + + if (IS_TRACED_FL(c_p, F_TRACE_CALLS) && !(bdt->pause)) { + if ( (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) || + (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) || + (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace))) { + /* This _IS_ a tail recursive call */ + SWAPOUT; + erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_TAIL_CALL); + SWAPIN; + } else { + SWAPOUT; + erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_CALL); + + /* r register needs to be copied to the array + * for the garbage collector + */ + ASSERT(c_p->htop <= E && E <= c_p->hend); + if (E - 2 < HTOP) { + reg[0] = r(0); + PROCESS_MAIN_CHK_LOCKS(c_p); + FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + r(0) = reg[0]; + } + SWAPIN; + + ASSERT(c_p->htop <= E && E <= c_p->hend); + + E -= 2; + E[0] = make_cp(I); + E[1] = make_cp(c_p->cp); /* original return address */ + c_p->cp = beam_return_time_trace; + } + } + + Goto(real_I); + } + + OpCase(i_return_time_trace): { + BeamInstr *pc = (BeamInstr *) (UWord) E[0]; + SWAPOUT; + erts_trace_time_break(c_p, pc, NULL, ERTS_BP_CALL_TIME_RETURN); + SWAPIN; + c_p->cp = NULL; + SET_I((BeamInstr *) cp_val(E[1])); + E += 2; + Goto(*I); + } + OpCase(i_trace_breakpoint): if (! IS_TRACED_FL(c_p, F_TRACE_CALLS)) { - Uint real_I; + BeamInstr real_I; - ErtsBreakSkip((Uint *) I, &real_I); + ErtsBreakSkip(c_p, (BeamInstr *) I, &real_I); Goto(real_I); } /* Fall through to next case */ OpCase(i_mtrace_breakpoint): { - Uint real_I; + BeamInstr real_I; Uint32 flags; Eterm tracer_pid; - Uint *cpp; + Uint* cpp; int return_to_trace = 0, need = 0; flags = 0; SWAPOUT; reg[0] = r(0); - if (*cp_val((Eterm)c_p->cp) - == (Uint) OpCode(return_trace)) { - cpp = (Uint*)&E[2]; - } else if (*cp_val((Eterm)c_p->cp) - == (Uint) OpCode(i_return_to_trace)) { + if (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) { + cpp = &E[2]; + } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace)) { + return_to_trace = !0; + cpp = &E[0]; + } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) { return_to_trace = !0; - cpp = (Uint*)&E[0]; + cpp = &E[0]; } else { cpp = NULL; } @@ -4364,19 +4789,21 @@ apply_bif_or_nif_epilogue: * return_trace and/or i_return_to_trace stackframes * on the stack, they are not intermixed with y registers */ - Eterm *cp_save = c_p->cp; + BeamInstr *cp_save = c_p->cp; for (;;) { ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) { + if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { cpp += 3; - } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) { + } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { return_to_trace = !0; cpp += 1; + } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_time_trace)) { + cpp += 2; } else break; } - c_p->cp = (Eterm *) *cpp; - ASSERT(is_CP((Eterm)c_p->cp)); + c_p->cp = (BeamInstr *) cp_val(*cpp); + ASSERT(is_CP(*cpp)); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -4403,6 +4830,7 @@ apply_bif_or_nif_epilogue: /* SWAPOUT was done and r(0) was saved above */ PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect(c_p, need, reg, I[-1]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); r(0) = reg[0]; SWAPIN; @@ -4412,12 +4840,12 @@ apply_bif_or_nif_epilogue: E -= 1; ASSERT(c_p->htop <= E && E <= c_p->hend); E[0] = make_cp(c_p->cp); - c_p->cp = (Eterm *) make_cp(beam_return_to_trace); + c_p->cp = (BeamInstr *) beam_return_to_trace; } if (flags & MATCH_SET_RX_TRACE) { E -= 3; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm) (I - 3))); + ASSERT(is_CP((Eterm) (UWord) (I - 3))); ASSERT(am_true == tracer_pid || is_internal_pid(tracer_pid) || is_internal_port(tracer_pid)); E[2] = make_cp(c_p->cp); @@ -4425,9 +4853,9 @@ apply_bif_or_nif_epilogue: E[0] = make_cp(I - 3); /* We ARE at the beginning of an instruction, the funcinfo is above i. */ - c_p->cp = (Eterm*) - make_cp(flags & MATCH_SET_EXCEPTION_TRACE - ? beam_exception_trace : beam_return_trace); + c_p->cp = + (flags & MATCH_SET_EXCEPTION_TRACE) + ? beam_exception_trace : beam_return_trace; erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); c_p->trace_flags |= F_EXCEPTION_TRACE; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); @@ -4440,10 +4868,10 @@ apply_bif_or_nif_epilogue: Uint *cpp = (Uint*) E; for(;;) { ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) { + if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { do ++cpp; while(is_not_CP(*cpp)); cpp += 2; - } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) { + } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { do ++cpp; while(is_not_CP(*cpp)); } else break; } @@ -4454,7 +4882,7 @@ apply_bif_or_nif_epilogue: SWAPIN; } c_p->cp = NULL; - SET_I((Eterm *) E[0]); + SET_I((BeamInstr *) cp_val(E[0])); E += 1; Goto(*I); } @@ -4465,7 +4893,7 @@ apply_bif_or_nif_epilogue: OpCase(i_global_cons): { - Eterm *next; + BeamInstr *next; #ifdef HYBRID Eterm *hp; @@ -4487,7 +4915,7 @@ apply_bif_or_nif_epilogue: OpCase(i_global_tuple): { - Eterm *next; + BeamInstr *next; int len; #ifdef HYBRID Eterm list; @@ -4522,7 +4950,7 @@ apply_bif_or_nif_epilogue: OpCase(i_global_copy): { - Eterm *next; + BeamInstr *next; PreFetch(0,next); #ifdef HYBRID if (!IS_CONST(r(0))) @@ -4551,7 +4979,7 @@ apply_bif_or_nif_epilogue: OpCase(fmove_ql): { Eterm fr = Arg(1); - Eterm* next; + BeamInstr *next; PreFetch(2, next); GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); @@ -4561,7 +4989,7 @@ apply_bif_or_nif_epilogue: OpCase(fmove_dl): { Eterm targ1; Eterm fr = Arg(1); - Eterm* next; + BeamInstr *next; PreFetch(2, next); GetR(0, targ1); @@ -4570,7 +4998,7 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } - OpCase(fmove_new_ld): { + OpCase(fmove_ld): { Eterm fr = Arg(0); Eterm dest = make_float(HTOP); @@ -4582,7 +5010,7 @@ apply_bif_or_nif_epilogue: OpCase(fconv_dl): { Eterm targ1; Eterm fr = Arg(1); - Eterm* next; + BeamInstr *next; GetR(0, targ1); PreFetch(2, next); @@ -4600,18 +5028,13 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } - /* - * Old allocating fmove. - */ - - #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); #else OpCase(fclearerror): { - Eterm* next; + BeamInstr *next; PreFetch(0, next); ERTS_FP_CHECK_INIT(c_p); @@ -4619,7 +5042,7 @@ apply_bif_or_nif_epilogue: } OpCase(i_fcheckerror): { - Eterm* next; + BeamInstr *next; PreFetch(0, next); ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith); @@ -4633,7 +5056,7 @@ apply_bif_or_nif_epilogue: OpCase(i_fadd_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4642,7 +5065,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fsub_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4651,7 +5074,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fmul_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4660,7 +5083,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fdiv_lll): { - Eterm* next; + BeamInstr *next; PreFetch(3, next); ERTS_FP_CHECK_INIT(c_p); @@ -4669,7 +5092,7 @@ apply_bif_or_nif_epilogue: NextPF(3, next); } OpCase(i_fnegate_ll): { - Eterm* next; + BeamInstr *next; PreFetch(2, next); ERTS_FP_CHECK_INIT(c_p); @@ -4736,7 +5159,7 @@ apply_bif_or_nif_epilogue: neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; - switch( c_p->def_arg_reg[3] ) { + switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */ case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); MoveReturn(reg[0], r(0)); @@ -4748,7 +5171,7 @@ apply_bif_or_nif_epilogue: /* This can be used to call any function value, but currently it's only used to call closures referring to unloaded modules. */ { - Eterm *next; + BeamInstr *next; next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); SWAPIN; @@ -4805,7 +5228,8 @@ apply_bif_or_nif_epilogue: OpCase(i_hibernate): { SWAPOUT; - if (hibernate(c_p, r(0), x(1), x(2), reg)) { + if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { + c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } else { I = handle_error(c_p, I, reg, hibernate_3); @@ -4816,12 +5240,11 @@ apply_bif_or_nif_epilogue: OpCase(i_debug_breakpoint): { SWAPOUT; reg[0] = r(0); - tmp_arg1 = call_breakpoint_handler(c_p, I-3, reg); + I = call_error_handler(c_p, I-3, reg, am_breakpoint); r(0) = reg[0]; SWAPIN; - if (tmp_arg1) { - SET_I(c_p->i); - Dispatch(); + if (I) { + Goto(*I); } goto no_error_handler; } @@ -4881,13 +5304,15 @@ apply_bif_or_nif_epilogue: em_call_error_handler = OpCode(call_error_handler); em_call_traced_function = OpCode(call_traced_function); em_apply_bif = OpCode(apply_bif); - beam_apply[0] = (Eterm) OpCode(i_apply); - beam_apply[1] = (Eterm) OpCode(normal_exit); - beam_exit[0] = (Eterm) OpCode(error_action_code); - beam_continue_exit[0] = (Eterm) OpCode(continue_exit); - beam_return_to_trace[0] = (Eterm) OpCode(i_return_to_trace); - beam_return_trace[0] = (Eterm) OpCode(return_trace); - beam_exception_trace[0] = (Eterm) OpCode(return_trace); /* UGLY */ + + beam_apply[0] = (BeamInstr) OpCode(i_apply); + beam_apply[1] = (BeamInstr) OpCode(normal_exit); + beam_exit[0] = (BeamInstr) OpCode(error_action_code); + beam_continue_exit[0] = (BeamInstr) OpCode(continue_exit); + beam_return_to_trace[0] = (BeamInstr) OpCode(i_return_to_trace); + beam_return_trace[0] = (BeamInstr) OpCode(return_trace); + beam_exception_trace[0] = (BeamInstr) OpCode(return_trace); /* UGLY */ + beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace); /* * Enter all BIFs into the export table. @@ -4897,8 +5322,10 @@ apply_bif_or_nif_epilogue: bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[3] = (Eterm) OpCode(apply_bif); - ep->code[4] = (Eterm) bif_table[i].f; + ep->code[3] = (BeamInstr) OpCode(apply_bif); + ep->code[4] = (BeamInstr) bif_table[i].f; + /* XXX: set func info for bifs */ + ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; @@ -4943,6 +5370,10 @@ translate_gc_bif(void* gcf) return round_1; } else if (gcf == erts_gc_trunc_1) { return round_1; + } else if (gcf == erts_gc_binary_part_2) { + return binary_part_2; + } else if (gcf == erts_gc_binary_part_3) { + return binary_part_3; } else { erl_exit(1, "bad gc bif"); } @@ -5001,8 +5432,8 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { * at the point of the original exception. */ -static Eterm* -handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf) +static BeamInstr* +handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) { Eterm* hp; Eterm Value = c_p->fvalue; @@ -5056,7 +5487,7 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf) /* Find a handler or die */ if ((c_p->catches > 0 || IS_TRACED_FL(c_p, F_EXCEPTION_TRACE)) && !(c_p->freason & EXF_PANIC)) { - Eterm *new_pc; + BeamInstr *new_pc; /* The Beam handler code (catch_end or try_end) checks reg[0] for THE_NON_VALUE to see if the previous code finished abnormally. If so, reg[1], reg[2] and reg[3] should hold the @@ -5082,30 +5513,37 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf) /* * Find the nearest catch handler */ -static Eterm* +static BeamInstr* next_catch(Process* c_p, Eterm *reg) { int active_catches = c_p->catches > 0; int have_return_to_trace = 0; Eterm *ptr, *prev, *return_to_trace_ptr = NULL; - Uint i_return_trace = beam_return_trace[0]; - Uint i_return_to_trace = beam_return_to_trace[0]; + + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_time_trace = beam_return_time_trace[0]; + ptr = prev = c_p->stop; ASSERT(is_CP(*ptr)); ASSERT(ptr <= STACK_START(c_p)); if (ptr == STACK_START(c_p)) return NULL; if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace)) + *cp_val(*ptr) != i_return_to_trace && + *cp_val(*ptr) != i_return_time_trace )) && c_p->cp) { /* Can not follow cp here - code may be unloaded */ - Uint *cpp = cp_val((Eterm) c_p->cp); + BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { - erts_trace_exception(c_p, (Eterm*) ptr[0], + erts_trace_exception(c_p, cp_val(ptr[0]), reg[1], reg[2], ptr+1); /* Skip return_trace parameters */ ptr += 2; } else if (cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; + } else if (cpp == beam_return_time_trace) { + /* Skip return_trace parameters */ + ptr += 1; } else if (cpp == beam_return_to_trace) { have_return_to_trace = !0; /* Record next cp */ } @@ -5123,7 +5561,7 @@ next_catch(Process* c_p, Eterm *reg) { if (is_catch(*ptr) && active_catches) goto found_catch; } if (cp_val(*prev) == beam_exception_trace) { - erts_trace_exception(c_p, (Eterm*) ptr[0], + erts_trace_exception(c_p, cp_val(ptr[0]), reg[1], reg[2], ptr+1); } /* Skip return_trace parameters */ @@ -5135,6 +5573,13 @@ next_catch(Process* c_p, Eterm *reg) { } have_return_to_trace = !0; /* Record next cp */ return_to_trace_ptr = NULL; + } else if (*cp_val(*prev) == i_return_time_trace) { + /* Skip stack frame variables */ + while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) { + if (is_catch(*ptr) && active_catches) goto found_catch; + } + /* Skip return_trace parameters */ + ptr += 1; } else { if (have_return_to_trace) { /* Record this cp as possible return_to trace cp */ @@ -5252,7 +5697,7 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { */ static void -save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, +save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args) { struct StackTrace* s; int sz; @@ -5263,7 +5708,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, } /* Create a container for the exception data */ - sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm)*depth + sz = (offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth + sizeof(Eterm) - 1) / sizeof(Eterm); s = (struct StackTrace *) HAlloc(c_p, 1 + sz); /* The following fields are inside the bignum */ @@ -5298,7 +5743,6 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, ASSERT(c_p->current); s->current = c_p->current; a = s->current[2]; - ASSERT(s->current[2] <= 3); } /* Save first stack entry */ ASSERT(pc); @@ -5350,9 +5794,10 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, /* Save the actual stack trace */ if (depth > 0) { - Eterm *ptr, *prev = s->depth ? s->trace[s->depth-1] : NULL; - Uint i_return_trace = beam_return_trace[0]; - Uint i_return_to_trace = beam_return_to_trace[0]; + Eterm *ptr; + BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL; + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; /* * Traverse the stack backwards and add all unique continuation * pointers to the buffer, up to the maximum stack trace size. @@ -5365,7 +5810,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, *cp_val(*ptr) != i_return_to_trace)) && c_p->cp) { /* Can not follow cp here - code may be unloaded */ - Uint *cpp = cp_val((Eterm) c_p->cp); + BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace || cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; @@ -5385,7 +5830,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf, /* Skip stack frame variables */ do ++ptr; while (is_not_CP(*ptr)); } else { - Eterm *cp = (Eterm *)(*ptr); + BeamInstr *cp = cp_val(*ptr); if (cp != prev) { /* Record non-duplicates only */ prev = cp; @@ -5457,9 +5902,9 @@ build_stacktrace(Process* c_p, Eterm exc) { struct StackTrace* s; Eterm args; int depth; - Eterm* current; + BeamInstr* current; Eterm Where = NIL; - Eterm* next_p = &Where; + Eterm *next_p = &Where; if (! (s = get_trace_from_exc(exc))) { return NIL; @@ -5523,7 +5968,7 @@ build_stacktrace(Process* c_p, Eterm exc) { * Finally, we go through the saved continuation pointers. */ for (i = 0; i < depth; i++) { - Eterm *fi = find_function_from_pc((Eterm *) s->trace[i]); + BeamInstr *fi = find_function_from_pc((BeamInstr *) s->trace[i]); if (fi == NULL) continue; mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2])); hp += 4; @@ -5539,8 +5984,8 @@ build_stacktrace(Process* c_p, Eterm exc) { } -static Eterm -call_error_handler(Process* p, Eterm* fi, Eterm* reg) +static BeamInstr* +call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -5552,62 +5997,12 @@ call_error_handler(Process* p, Eterm* fi, Eterm* reg) /* * Search for the error_handler module. */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_function, 3); - if (ep == NULL) { /* No error handler */ - p->current = fi; - p->freason = EXC_UNDEF; - return 0; - } - p->i = ep->address; - - /* - * Create a list with all arguments in the x registers. - */ - - arity = fi[2]; - sz = 2 * arity; - if (HeapWordsLeft(p) < sz) { - erts_garbage_collect(p, sz, reg, arity); - } - hp = HEAP_TOP(p); - HEAP_TOP(p) += sz; - args = NIL; - for (i = arity-1; i >= 0; i--) { - args = CONS(hp, reg[i], args); - hp += 2; - } - - /* - * Set up registers for call to error_handler:undefined_function/3. - */ - reg[0] = fi[0]; - reg[1] = fi[1]; - reg[2] = args; - return 1; -} - -static Eterm -call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg) -{ - Eterm* hp; - Export* ep; - int arity; - Eterm args; - Uint sz; - int i; - - /* - * Search for error handler module. - */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_breakpoint, 3); + ep = erts_find_function(erts_proc_get_error_handler(p), func, 3); if (ep == NULL) { /* No error handler */ p->current = fi; p->freason = EXC_UNDEF; return 0; } - p->i = ep->address; /* * Create a list with all arguments in the x registers. @@ -5627,15 +6022,14 @@ call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg) } /* - * Set up registers for call to error_handler:breakpoint/3. + * Set up registers for call to error_handler:<func>/3. */ reg[0] = fi[0]; reg[1] = fi[1]; reg[2] = args; - return 1; + return ep->address; } - static Export* apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg) @@ -5681,7 +6075,7 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, return ep; } -static Uint* +static BeamInstr* apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) { int arity; @@ -5763,7 +6157,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) return ep->address; } -static Uint* +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) { Export* ep; @@ -5812,8 +6206,8 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) return ep->address; } -static int -hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) +int +erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) { int arity; Eterm tmp; @@ -5869,7 +6263,7 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) c_p->stop = STACK_START(c_p); c_p->catches = 0; c_p->i = beam_apply; - c_p->cp = (Eterm *) beam_apply+1; + c_p->cp = (BeamInstr *) beam_apply+1; /* * If there are no waiting messages, garbage collect and @@ -5884,22 +6278,25 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) c_p->fvalue = NIL; PROCESS_MAIN_CHK_LOCKS(c_p); erts_garbage_collect_hibernate(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - c_p->status = P_WAITING; #ifdef ERTS_SMP ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); if (c_p->msg.len > 0) erts_add_to_runq(c_p); + else #endif + c_p->status = P_WAITING; } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->current = bif_export[BIF_hibernate_3]->code; + c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } -static Uint* +static BeamInstr* call_fun(Process* p, /* Current process. */ int arity, /* Number of arguments for Fun. */ Eterm* reg, /* Contents of registers. */ @@ -5919,7 +6316,7 @@ call_fun(Process* p, /* Current process. */ if (is_fun_header(hdr)) { ErlFunThing* funp = (ErlFunThing *) fun_val(fun); ErlFunEntry* fe; - Eterm* code_ptr; + BeamInstr* code_ptr; Eterm* var_ptr; int actual_arity; unsigned num_free; @@ -6010,12 +6407,17 @@ call_fun(Process* p, /* Current process. */ reg[0] = module; reg[1] = fun; reg[2] = args; + reg[3] = NIL; return ep->address; } } } else if (is_export_header(hdr)) { - Export* ep = (Export *) (export_val(fun))[1]; - int actual_arity = (int) ep->code[2]; + Export *ep; + int actual_arity; + + ep = *((Export **) (export_val(fun) + 1)); + actual_arity = (int) ep->code[2]; + if (arity == actual_arity) { return ep->address; } else { @@ -6082,7 +6484,7 @@ call_fun(Process* p, /* Current process. */ } } -static Eterm* +static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) { int arity; @@ -6125,6 +6527,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) if (HEAP_LIMIT(p) - HEAP_TOP(p) <= needed) { PROCESS_MAIN_CHK_LOCKS(p); erts_garbage_collect(p, needed, reg, num_free); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); PROCESS_MAIN_CHK_LOCKS(p); } hp = p->htop; @@ -6134,8 +6537,8 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) erts_refc_inc(&fe->refc, 2); funp->thing_word = HEADER_FUN; #ifndef HYBRID /* FIND ME! */ - funp->next = MSO(p).funs; - MSO(p).funs = funp; + funp->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) funp; #endif funp->fe = fe; funp->num_free = num_free; @@ -6176,7 +6579,7 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->address == ep->code+3 && (ep->code[3] == (Uint) em_apply_bif); + return ep->address == ep->code+3 && (ep->code[3] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 99fab28dce..57fe25453d 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -51,9 +51,9 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); #define EXPORTED 2 #ifdef NO_JUMP_TABLE -# define BeamOpCode(Op) ((Uint)(Op)) +# define BeamOpCode(Op) ((BeamInstr)(Op)) #else -# define BeamOpCode(Op) ((Eterm)beam_ops[Op]) +# define BeamOpCode(Op) ((BeamInstr)beam_ops[Op]) #endif #if defined(WORDS_BIGENDIAN) @@ -89,13 +89,12 @@ typedef struct { } Label; /* - * Type for a operand for a generic instruction. + * Type for an operand for a generic instruction. */ typedef struct { unsigned type; /* Type of operand. */ - Uint val; /* Value of operand. */ - Uint bigarity; /* Arity for bignumbers (only). */ + BeamInstr val; /* Value of operand. */ } GenOpArg; /* @@ -142,7 +141,7 @@ typedef struct { typedef struct { Eterm function; /* Tagged atom for function. */ int arity; /* Arity. */ - Eterm* address; /* Address to function in code. */ + BeamInstr* address; /* Address to function in code. */ } ExportEntry; #define MakeIffId(a, b, c, d) \ @@ -274,13 +273,12 @@ typedef struct { int num_functions; /* Number of functions in module. */ int num_labels; /* Number of labels. */ int code_buffer_size; /* Size of code buffer in words. */ - Eterm* code; /* Loaded code. */ + BeamInstr* code; /* Loaded code. */ int ci; /* Current index into loaded code. */ Label* labels; - Uint put_strings; /* Linked list of put_string instructions. */ - Uint new_bs_put_strings; /* Linked list of i_new_bs_put_string instructions. */ + BeamInstr new_bs_put_strings; /* Linked list of i_new_bs_put_string instructions. */ StringPatch* string_patches; /* Linked list of position into string table to patch. */ - Uint catches; /* Linked list of catch_yf instructions. */ + BeamInstr catches; /* Linked list of catch_yf instructions. */ unsigned loaded_size; /* Final size of code when loaded. */ byte mod_md5[16]; /* MD5 for module code. */ int may_load_nif; /* true if NIFs may later be loaded for this module */ @@ -327,11 +325,6 @@ typedef struct { Literal* literals; /* Array of literals. */ LiteralPatch* literal_patches; /* Operands that need to be patched. */ Uint total_literal_size; /* Total heap size for all literals. */ - - /* - * Floating point. - */ - int new_float_instructions; /* New allocation scheme for floating point. */ } LoaderState; typedef struct { @@ -341,7 +334,7 @@ typedef struct { #define GetTagAndValue(Stp, Tag, Val) \ do { \ - Uint __w; \ + BeamInstr __w; \ GetByte(Stp, __w); \ Tag = __w & 0x07; \ if ((__w & 0x08) == 0) { \ @@ -388,7 +381,7 @@ typedef struct { goto load_error; \ } else { \ int __n = (N); \ - Uint __result = 0; \ + BeamInstr __result = 0; \ Stp->file_left -= (unsigned) __n; \ while (__n-- > 0) { \ __result = __result << 8 | *Stp->file_p++; \ @@ -465,7 +458,7 @@ static int bin_load(Process *c_p, ErtsProcLocks c_p_locks, static void init_state(LoaderState* stp); static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - Eterm* code, Uint size, Uint catches); + BeamInstr* code, Uint size, BeamInstr catches); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); static int load_atom_table(LoaderState* stp); @@ -477,19 +470,16 @@ static int read_code_header(LoaderState* stp); static int load_code(LoaderState* stp); static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOpArg Tuple, GenOpArg Dst); -static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, +static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, + GenOpArg TypeFail, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); static GenOp* gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); -static GenOp* gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail, - GenOpArg Size, GenOpArg* Rest); +static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S, + GenOpArg Fail, GenOpArg Size, + GenOpArg* Rest); static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); -static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func, - GenOpArg arity, GenOpArg label); -static GenOp* -gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, - GenOpArg Src, GenOpArg Dst); static int freeze_code(LoaderState* stp); @@ -499,8 +489,8 @@ static void load_printf(int line, LoaderState* context, char *fmt, ...); static int transform_engine(LoaderState* st); static void id_to_string(Uint id, char* s); static void new_genop(LoaderState* stp); -static int get_int_val(LoaderState* stp, Uint len_code, Uint* result); -static int get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result); +static int get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result); +static int get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result); static int new_label(LoaderState* stp); static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); @@ -513,7 +503,7 @@ static Eterm compilation_info_for_module(Process* p, Eterm mod); static Eterm native_addresses(Process* p, Eterm mod); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); -static int safe_mul(Uint a, Uint b, Uint* resp); +static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -591,7 +581,18 @@ erts_load_module(Process *c_p, } return result; } - +/* #define LOAD_MEMORY_HARD_DEBUG 1*/ + +#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) +/* Requires allocators ERTS_ALLOC_UTIL_HARD_DEBUG also set in erl_alloc_util.h */ +extern void check_allocators(void); +extern void check_allocated_block(Uint type, void *blk); +#define CHKALLOC() check_allocators() +#define CHKBLK(TYPE,BLK) if ((BLK) != NULL) check_allocated_block((TYPE),(BLK)) +#else +#define CHKALLOC() /* nothing */ +#define CHKBLK(TYPE,BLK) /* nothing */ +#endif static int bin_load(Process *c_p, ErtsProcLocks c_p_locks, @@ -608,6 +609,12 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Scan the IFF file. */ +#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) + erts_fprintf(stderr,"Loading a module\n"); +#endif + + CHKALLOC(); + CHKBLK(ERTS_ALC_T_CODE,state.code); state.file_name = "IFF header for Beam file"; state.file_p = bytes; state.file_left = unloaded_size; @@ -619,6 +626,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the header for the code chunk. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "code chunk header", CODE_CHUNK); if (!read_code_header(&state)) { goto load_error; @@ -628,6 +636,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the atom table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "atom table", ATOM_CHUNK); if (!load_atom_table(&state)) { goto load_error; @@ -637,6 +646,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the import table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "import table", IMP_CHUNK); if (!load_import_table(&state)) { goto load_error; @@ -646,6 +656,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the lambda (fun) table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); if (state.chunks[LAMBDA_CHUNK].size > 0) { define_file(&state, "lambda (fun) table", LAMBDA_CHUNK); if (!read_lambda_table(&state)) { @@ -657,6 +668,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the literal table. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); if (state.chunks[LITERAL_CHUNK].size > 0) { define_file(&state, "literals table (constant pool)", LITERAL_CHUNK); if (!read_literal_table(&state)) { @@ -668,18 +680,25 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Load the code chunk. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); state.file_name = "code chunk"; state.file_p = state.code_start; state.file_left = state.code_size; - if (!load_code(&state) || !freeze_code(&state)) { + if (!load_code(&state)) { goto load_error; } + CHKBLK(ERTS_ALC_T_CODE,state.code); + if (!freeze_code(&state)) { + goto load_error; + } + /* * Read and validate the export table. (This must be done after * loading the code, because it contains labels.) */ + CHKBLK(ERTS_ALC_T_CODE,state.code); define_file(&state, "export table", EXP_CHUNK); if (!read_export_table(&state)) { goto load_error; @@ -690,16 +709,25 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * exported and imported functions. This can't fail. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); rval = insert_new_code(c_p, c_p_locks, state.group_leader, state.module, state.code, state.loaded_size, state.catches); if (rval < 0) { goto load_error; } + CHKBLK(ERTS_ALC_T_CODE,state.code); final_touch(&state); /* * Loading succeded. */ + CHKBLK(ERTS_ALC_T_CODE,state.code); +#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) + erts_fprintf(stderr,"Loaded %T\n",*modp); +#if 0 + debug_dump_code(state.code,state.ci); +#endif +#endif rval = 0; state.code = NULL; /* Prevent code from being freed. */ *modp = state.module; @@ -784,14 +812,13 @@ init_state(LoaderState* stp) stp->total_literal_size = 0; stp->literal_patches = 0; stp->string_patches = 0; - stp->new_float_instructions = 0; stp->may_load_nif = 0; stp->on_load = 0; } static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, Eterm* code, Uint size, Uint catches) + Eterm group_leader, Eterm module, BeamInstr* code, Uint size, BeamInstr catches) { Module* modp; int rval; @@ -833,7 +860,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modules[i] = modules[i-1]; } modules[i].start = code; - modules[i].end = (Eterm *) (((byte *)code) + size); + modules[i].end = (BeamInstr *) (((byte *)code) + size); num_loaded_modules++; mid_module = &modules[num_loaded_modules/2]; return 0; @@ -1083,7 +1110,7 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_find_export_entry(mod, func, arity)) != NULL) { - if (e->code[3] == (Uint) em_apply_bif) { + if (e->code[3] == (BeamInstr) em_apply_bif) { stp->import[i].bf = (BifFunction) e->code[4]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; @@ -1151,7 +1178,7 @@ read_export_table(LoaderState* stp) * redefine). */ if ((e = erts_find_export_entry(stp->module, func, arity)) != NULL) { - if (e->code[3] == (Uint) em_apply_bif) { + if (e->code[3] == (BeamInstr) em_apply_bif) { int j; for (j = 0; j < sizeof(allow_redef)/sizeof(allow_redef[0]); j++) { @@ -1220,7 +1247,7 @@ static int read_literal_table(LoaderState* stp) { int i; - Uint uncompressed_sz; + BeamInstr uncompressed_sz; byte* uncompressed = 0; GetInt(stp, 4, uncompressed_sz); @@ -1338,19 +1365,20 @@ read_code_header(LoaderState* stp) * Initialize code area. */ stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0); - stp->code = (Eterm*) erts_alloc(ERTS_ALC_T_CODE, - sizeof(Eterm) * stp->code_buffer_size); + stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, + sizeof(BeamInstr) * stp->code_buffer_size); stp->code[MI_NUM_FUNCTIONS] = stp->num_functions; stp->ci = MI_FUNCTIONS + stp->num_functions + 1; stp->code[MI_ATTR_PTR] = 0; + stp->code[MI_ATTR_SIZE] = 0; stp->code[MI_ATTR_SIZE_ON_HEAP] = 0; stp->code[MI_COMPILE_PTR] = 0; + stp->code[MI_COMPILE_SIZE] = 0; stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; stp->code[MI_NUM_BREAKPOINTS] = 0; - stp->put_strings = 0; stp->new_bs_put_strings = 0; stp->catches = 0; return 1; @@ -1365,29 +1393,30 @@ read_code_header(LoaderState* stp) LoadError2(Stp, "bad tag %d; expected %d", Actual, Expected); \ } else {} -#define Need(w) \ - ASSERT(ci <= code_buffer_size); \ - if (code_buffer_size < ci+(w)) { \ - code_buffer_size = erts_next_heap_size(ci+(w), 0); \ - stp->code = code \ - = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, \ - (void *) code, \ - code_buffer_size * sizeof(Eterm)); \ - } +#define CodeNeed(w) do { \ + ASSERT(ci <= code_buffer_size); \ + if (code_buffer_size < ci+(w)) { \ + code_buffer_size = erts_next_heap_size(ci+(w), 0); \ + stp->code = code \ + = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ + (void *) code, \ + code_buffer_size * sizeof(BeamInstr)); \ + } \ +} while (0) +#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) static int load_code(LoaderState* stp) { int i; - int tmp; int ci; int last_func_start = 0; char* sign; int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ - Eterm* code; + BeamInstr* code; int code_buffer_size; int specific; Uint last_label = 0; /* Number of last label. */ @@ -1446,7 +1475,7 @@ load_code(LoaderState* stp) if (((First) & 0x08) == 0) { \ Val = (First) >> 4; \ } else if (((First) & 0x10) == 0) { \ - Uint __w; \ + BeamInstr __w; \ GetByte(Stp, __w); \ Val = (((First) >> 5) << 8) | __w; \ } else { \ @@ -1455,7 +1484,7 @@ load_code(LoaderState* stp) } while (0) for (arg = 0; arg < arity; arg++) { - Uint first; + BeamInstr first; GetByte(stp, first); last_op->a[arg].type = first & 0x07; @@ -1464,7 +1493,7 @@ load_code(LoaderState* stp) if ((first & 0x08) == 0) { last_op->a[arg].val = first >> 4; } else if ((first & 0x10) == 0) { - Uint w; + BeamInstr w; GetByte(stp, w); ASSERT(first < 0x800); last_op->a[arg].val = ((first >> 5) << 8) | w; @@ -1523,7 +1552,7 @@ load_code(LoaderState* stp) break; case TAG_z: { - Uint ext_tag; + BeamInstr ext_tag; unsigned tag; GetValue(stp, first, ext_tag); @@ -1531,14 +1560,15 @@ load_code(LoaderState* stp) case 0: /* Floating point number */ { Eterm* hp; -# ifndef ARCH_64 +/* XXX:PaN - Halfword should use ARCH_64 variant instead */ +#if !defined(ARCH_64) || HALFWORD_HEAP Uint high, low; # endif last_op->a[arg].val = new_literal(stp, &hp, FLOAT_SIZE_OBJECT); hp[0] = HEADER_FLONUM; last_op->a[arg].type = TAG_q; -# ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP GetInt(stp, 8, hp[1]); # else GetInt(stp, 4, high); @@ -1575,12 +1605,11 @@ load_code(LoaderState* stp) break; case 3: /* Allocation list. */ { - Uint n; - Uint type; - Uint val; - Uint words = 0; + BeamInstr n; + BeamInstr type; + BeamInstr val; + BeamInstr words = 0; - stp->new_float_instructions = 1; GetTagAndValue(stp, tag, n); VerifyTag(stp, tag, TAG_u); while (n-- > 0) { @@ -1607,7 +1636,7 @@ load_code(LoaderState* stp) } case 4: /* Literal. */ { - Uint val; + BeamInstr val; GetTagAndValue(stp, tag, val); VerifyTag(stp, tag, TAG_u); @@ -1734,7 +1763,7 @@ load_code(LoaderState* stp) } stp->specific_op = specific; - Need(opc[stp->specific_op].sz+2); /* Extra margin for packing */ + CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */ code[ci++] = BeamOpCode(stp->specific_op); } @@ -1772,7 +1801,7 @@ load_code(LoaderState* stp) case 'c': /* Tagged constant */ switch (tag) { case TAG_i: - code[ci++] = make_small(tmp_op->a[arg].val); + code[ci++] = (BeamInstr) make_small((Uint) tmp_op->a[arg].val); break; case TAG_a: code[ci++] = tmp_op->a[arg].val; @@ -1802,7 +1831,7 @@ load_code(LoaderState* stp) code[ci++] = make_yreg(tmp_op->a[arg].val); break; case TAG_i: - code[ci++] = make_small(tmp_op->a[arg].val); + code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val); break; case TAG_a: code[ci++] = tmp_op->a[arg].val; @@ -1896,12 +1925,12 @@ load_code(LoaderState* stp) if (stp->import[i].bf == NULL) { LoadError1(stp, "not a BIF: import table index %d", i); } - code[ci++] = (Eterm) stp->import[i].bf; + code[ci++] = (BeamInstr) stp->import[i].bf; break; - case 'P': /* Byte offset into tuple */ + case 'P': /* Byte offset into tuple or stack */ + case 'Q': /* Like 'P', but packable */ VerifyTag(stp, tag, TAG_u); - tmp = tmp_op->a[arg].val; - code[ci++] = (Eterm) ((tmp_op->a[arg].val+1) * sizeof(Eterm *)); + code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm)); break; case 'l': /* Floating point register. */ VerifyTag(stp, tag_to_letter[tag], *sign); @@ -1919,75 +1948,14 @@ load_code(LoaderState* stp) } /* - * Load any list arguments using the primitive tags. - */ - - for ( ; arg < tmp_op->arity; arg++) { - switch (tmp_op->a[arg].type) { - case TAG_i: - Need(1); - code[ci++] = make_small(tmp_op->a[arg].val); - break; - case TAG_u: - case TAG_a: - case TAG_v: - Need(1); - code[ci++] = tmp_op->a[arg].val; - break; - case TAG_f: - Need(1); - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; - ci++; - break; - case TAG_q: - { - Eterm lit; - - lit = stp->literals[tmp_op->a[arg].val].term; - if (is_big(lit)) { - Eterm* bigp; - Uint size; - - bigp = big_val(lit); - size = bignum_header_arity(*bigp); - Need(size+1); - code[ci++] = *bigp++; - while (size-- > 0) { - code[ci++] = *bigp++; - } - } else if (is_float(lit)) { -#ifdef ARCH_64 - Need(1); - code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1]; -#else - Eterm* fptr; - - fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1; - Need(2); - code[ci++] = *fptr++; - code[ci++] = *fptr; -#endif - } else { - LoadError0(stp, "literal is neither float nor big"); - } - } - break; - default: - LoadError1(stp, "unsupported primitive type '%c'", - tag_to_letter[tmp_op->a[arg].type]); - } - } - - /* * The packing engine. */ if (opc[stp->specific_op].pack[0]) { char* prog; /* Program for packing engine. */ - Uint stack[8]; /* Stack. */ - Uint* sp = stack; /* Points to next free position. */ - Uint packed = 0; /* Accumulator for packed operations. */ - + BeamInstr stack[8]; /* Stack. */ + BeamInstr* sp = stack; /* Points to next free position. */ + BeamInstr packed = 0; /* Accumulator for packed operations. */ + for (prog = opc[stp->specific_op].pack; *prog; prog++) { switch (*prog) { case 'g': /* Get instruction; push on stack. */ @@ -2000,8 +1968,13 @@ load_code(LoaderState* stp) packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci]; break; case '6': /* Shift 16 steps */ - packed = (packed << 16) | code[--ci]; + packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci]; + break; +#ifdef ARCH_64 + case 'w': /* Shift 32 steps */ + packed = (packed << BEAM_WIDE_SHIFT) | code[--ci]; break; +#endif case 'p': /* Put instruction (from stack). */ code[ci++] = *--sp; break; @@ -2017,6 +1990,58 @@ load_code(LoaderState* stp) } /* + * Load any list arguments using the primitive tags. + */ + + for ( ; arg < tmp_op->arity; arg++) { + switch (tmp_op->a[arg].type) { + case TAG_i: + CodeNeed(1); + code[ci++] = make_small(tmp_op->a[arg].val); + break; + case TAG_u: + case TAG_a: + case TAG_v: + CodeNeed(1); + code[ci++] = tmp_op->a[arg].val; + break; + case TAG_f: + CodeNeed(1); + code[ci] = stp->labels[tmp_op->a[arg].val].patches; + stp->labels[tmp_op->a[arg].val].patches = ci; + ci++; + break; + case TAG_r: + CodeNeed(1); + code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) | + TAG_PRIMARY_HEADER; + break; + case TAG_x: + CodeNeed(1); + code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | + (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + break; + case TAG_y: + CodeNeed(1); + code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | + (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + break; + case TAG_n: + CodeNeed(1); + code[ci++] = NIL; + break; + case TAG_q: + CodeNeed(1); + new_literal_patch(stp, ci); + code[ci++] = tmp_op->a[arg].val; + break; + default: + LoadError1(stp, "unsupported primitive type '%c'", + tag_to_letter[tmp_op->a[arg].type]); + } + } + + /* * Handle a few special cases. */ switch (stp->specific_op) { @@ -2037,9 +2062,9 @@ load_code(LoaderState* stp) /* Must make room for call_nif op */ int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); ASSERT(pad > 0 && pad < MIN_FUNC_SZ); - Need(pad); - sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(Eterm)); - sys_memset(&code[finfo_ix], 0, pad*sizeof(Eterm)); + CodeNeed(pad); + sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(BeamInstr)); + sys_memset(&code[finfo_ix], 0, pad*sizeof(BeamInstr)); ci += pad; stp->labels[last_label].value += pad; } @@ -2050,6 +2075,7 @@ load_code(LoaderState* stp) */ stp->function = code[ci-2]; stp->arity = code[ci-1]; + ASSERT(stp->labels[last_label].value == ci - FINFO_SZ); offset = MI_FUNCTIONS + function_number; code[offset] = stp->labels[last_label].patches; @@ -2072,34 +2098,6 @@ load_code(LoaderState* stp) /* Remember offset for the on_load function. */ stp->on_load = ci; break; - case op_put_string_IId: - { - /* - * At entry: - * - * code[ci-4] &&lb_put_string_IId - * code[ci-3] length of string - * code[ci-2] offset into string table - * code[ci-1] destination register - * - * Since we don't know the address of the string table yet, - * just check the offset and length for validity, and use - * the instruction field as a link field to link all put_string - * instructions into a single linked list. At exit: - * - * code[ci-4] pointer to next put_string instruction (or 0 - * if this is the last) - */ - Uint offset = code[ci-2]; - Uint len = code[ci-3]; - unsigned strtab_size = stp->chunks[STR_CHUNK].size; - if (offset > strtab_size || offset + len > strtab_size) { - LoadError2(stp, "invalid string reference %d, size %d", offset, len); - } - code[ci-4] = stp->put_strings; - stp->put_strings = ci - 4; - } - break; case op_bs_put_string_II: { /* @@ -2161,7 +2159,6 @@ load_code(LoaderState* stp) } } -#undef Need load_error: return 0; @@ -2212,11 +2209,12 @@ use_jump_tab(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) } /* - * Predicate to test whether all values in a table are big numbers. + * Predicate to test whether all values in a table are either + * floats or bignums. */ static int -all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) +floats_or_bignums(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) { int i; @@ -2228,9 +2226,6 @@ all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) if (Rest[i].type != TAG_q) { return 0; } - if (is_not_big(stp->literals[Rest[i].val].term)) { - return 0; - } if (Rest[i+1].type != TAG_f) { return 0; } @@ -2290,6 +2285,14 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) return 0; } +static int +same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label) +{ + return Target.type = TAG_f && Label.type == TAG_u && + Target.val == Label.val; +} + + /* * Generate an instruction for element/2. */ @@ -2301,23 +2304,23 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_element_4; op->arity = 4; - op->a[0] = Fail; - op->a[1] = Index; - op->a[2] = Tuple; - op->a[3] = Dst; op->next = NULL; - /* - * If safe, generate a faster instruction. - */ - if (Index.type == TAG_i && Index.val > 0 && (Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; - op->a[1].type = TAG_u; - op->a[1].val = Index.val; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = Index.val; + op->a[3] = Dst; + } else { + op->op = genop_i_element_4; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2] = Index; + op->a[3] = Dst; } return op; @@ -2373,7 +2376,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, GenOpArg Flags, GenOpArg Dst) { GenOp* op; - Uint bits; + UWord bits; NEW_GENOP(stp, op); @@ -2568,8 +2571,6 @@ binary_too_big_bits(LoaderState* stp, GenOpArg Size) return Size.type == TAG_u && (((Size.val+7)/8) >> (8*sizeof(Uint)-3) != 0); } -#define new_float_allocation(Stp) ((Stp)->new_float_instructions) - static GenOp* gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, GenOpArg Unit, GenOpArg Flags, GenOpArg Src) @@ -2782,6 +2783,52 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, return op; } +static GenOp* +gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, + GenOpArg Live, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + op->op = genop_i_increment_4; + op->arity = 4; + op->next = NULL; + op->a[0] = Reg; + op->a[1].type = TAG_u; + op->a[1].val = Integer.val; + op->a[2] = Live; + op->a[3] = Dst; + return op; +} + +static GenOp* +gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, + GenOpArg Live, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + op->op = genop_i_increment_4; + op->arity = 4; + op->next = NULL; + op->a[0] = Reg; + op->a[1].type = TAG_u; + op->a[1].val = -Integer.val; + op->a[2] = Live; + op->a[3] = Dst; + return op; +} + +/* + * Test whether the negation of the given number is small. + */ +static int +negation_is_small(LoaderState* stp, GenOpArg Int) +{ + return Int.type == TAG_i && IS_SSMALL(-Int.val); +} + + static int smp(LoaderState* stp) { @@ -2838,14 +2885,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP } else if (Time.type == TAG_q) { Eterm big; @@ -2856,11 +2903,13 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) if (big_arity(big) > 1 || big_sign(big)) { goto error; } else { - (void) term_to_Uint(big, &op->a[1].val); + Uint u; + (void) term_to_Uint(big, &u); + op->a[1].val = (BeamInstr) u; } #endif } else { -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP error: #endif op->op = genop_i_wait_error_0; @@ -2883,14 +2932,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#ifndef ARCH_64 +#if !defined(ARCH_64) || HALFWORD_HEAP } else if (Time.type == TAG_q) { Eterm big; @@ -2901,11 +2950,13 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) if (big_arity(big) > 1 || big_sign(big)) { goto error; } else { - (void) term_to_Uint(big, &op->a[1].val); + Uint u; + (void) term_to_Uint(big, &u); + op->a[1].val = (BeamInstr) u; } #endif } else { -#ifndef ARCH_64 +#if !defined(ARCH_64) || HALFWORD_HEAP error: #endif op->op = genop_i_wait_error_locked_0; @@ -2969,6 +3020,21 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, ASSERT(op->a[i].val < op->a[i+2].val); } #endif + + /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + op->op = genop_i_select_tuple_arity2_6; + op->arity--; + op->a[2].type = TAG_u; + op->a[2].val = arityval(op->a[3].val); + op->a[3] = op->a[4]; + op->a[4].type = TAG_u; + op->a[4].val = arityval(op->a[5].val); + op->a[5] = op->a[6]; + } + return op; } @@ -2978,18 +3044,24 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, */ static GenOp* -gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, - GenOpArg Size, GenOpArg* Rest) +gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail, + GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op1; GenOp* op2; GenOp* label; - Uint type; + GenOp* is_integer; int i; ASSERT(Size.val >= 2 && Size.val % 2 == 0); + NEW_GENOP(stp, is_integer); + is_integer->op = genop_is_integer_2; + is_integer->arity = 2; + is_integer->a[0] = TypeFail; + is_integer->a[1] = S; + NEW_GENOP(stp, label); label->op = genop_label_1; label->arity = 1; @@ -3015,15 +3087,13 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, op2->a[2].type = TAG_u; op2->a[2].val = 0; - op1->next = label; - label->next = op2; - op2->next = NULL; - - type = Rest[0].type; + /* + * Split the list. + */ ASSERT(Size.type == TAG_u); for (i = 0; i < Size.val; i += 2) { - GenOp* op = (Rest[i].type == type) ? op1 : op2; + GenOp* op = (Rest[i].type == TAG_q) ? op2 : op1; int dst = 3 + op->a[2].val; ASSERT(Rest[i+1].type == TAG_f); @@ -3032,13 +3102,36 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->arity += 2; op->a[2].val += 2; } + ASSERT(op1->a[2].val > 0); + ASSERT(op2->a[2].val > 0); /* - * None of the instructions should have zero elements in the list. + * Order the instruction sequence appropriately. */ - ASSERT(op1->a[2].val > 0); - ASSERT(op2->a[2].val > 0); + if (TypeFail.val == Fail.val) { + /* + * select_val L1 S ... (small numbers) + * label L1 + * is_integer Fail S + * select_val Fail S ... (bignums) + */ + op1->next = label; + label->next = is_integer; + is_integer->next = op2; + } else { + /* + * is_integer TypeFail S + * select_val L1 S ... (small numbers) + * label L1 + * select_val Fail S ... (bignums) + */ + is_integer->next = op1; + op1->next = label; + label->next = op2; + op1 = is_integer; + } + op2->next = NULL; return op1; } @@ -3060,6 +3153,29 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr ASSERT(Size.val >= 2 && Size.val % 2 == 0); /* + * If there is only one choice, don't generate a jump table. + */ + if (Size.val == 2) { + GenOp* jump; + + NEW_GENOP(stp, op); + op->arity = 3; + op->op = genop_is_ne_exact_3; + op->a[0] = Rest[1]; + op->a[1] = S; + op->a[2] = Rest[0]; + + NEW_GENOP(stp, jump); + jump->next = NULL; + jump->arity = 1; + jump->op = genop_jump_1; + jump->a[0] = Fail; + + op->next = jump; + return op; + } + + /* * Calculate the minimum and maximum values and size of jump table. */ @@ -3131,8 +3247,9 @@ genopargcompare(GenOpArg* a, GenOpArg* b) } /* - * Generate a select_val instruction. We know that a jump table is not suitable, - * and that all values are of the same type (integer, atoms, floats; never bignums). + * Generate a select_val instruction. We know that a jump table + * is not suitable, and that all values are of the same type + * (integer or atoms). */ static GenOp* @@ -3146,12 +3263,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, NEW_GENOP(stp, op); op->next = NULL; - if (Rest[0].type != TAG_q) { - op->op = genop_i_select_val_3; - } else { - ASSERT(is_float(stp->literals[Rest[0].val].term)); - op->op = genop_i_select_float_3; - } + op->op = genop_i_select_val_3; GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; @@ -3173,19 +3285,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, } #endif - return op; -} - -/* - * Compare function for qsort(). - */ + /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + op->op = genop_i_select_val2_6; + op->arity--; + op->a[2] = op->a[3]; + op->a[3] = op->a[4]; + op->a[4] = op->a[5]; + op->a[5] = op->a[6]; + } -static int -genbigcompare(GenOpArg* a, GenOpArg* b) -{ - int val = (int)(b->bigarity - a->bigarity); - - return val != 0 ? val : ((int) (a->val - b->val)); + return op; } /* @@ -3193,37 +3305,35 @@ genbigcompare(GenOpArg* a, GenOpArg* b) */ static GenOp* -gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail, +gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op; - int arity = Size.val + 2 + 1; - int size = Size.val / 2; + GenOp* jump; + GenOp** prev_next = &op; + int i; - NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_big_2; - GENOP_ARITY(op, arity); - op->a[0] = S; - op->a[1] = Fail; for (i = 0; i < Size.val; i += 2) { + GenOp* op; ASSERT(Rest[i].type == TAG_q); - op->a[i+2] = Rest[i]; - op->a[i+2].bigarity = *big_val(stp->literals[op->a[i+2].val].term); - op->a[i+3] = Rest[i+1]; - } - ASSERT(i+2 == arity-1); - op->a[arity-1].type = TAG_u; - op->a[arity-1].val = 0; - - /* - * Sort the values in descending arity order. - */ - - qsort(op->a+2, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genbigcompare); + NEW_GENOP(stp, op); + op->op = genop_is_ne_exact_3; + op->arity = 3; + op->a[0] = Rest[i+1]; + op->a[1] = S; + op->a[2] = Rest[i]; + *prev_next = op; + prev_next = &op->next; + } + + NEW_GENOP(stp, jump); + jump->next = NULL; + jump->op = genop_jump_1; + jump->arity = 1; + jump->a[0] = Fail; + *prev_next = jump; return op; } @@ -3241,7 +3351,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, int i; ASSERT(Size.type == TAG_u); - ASSERT(S.type == TAG_q); NEW_GENOP(stp, op); op->next = NULL; @@ -3252,18 +3361,32 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, * Search for a literal matching the controlling expression. */ - if (S.type == TAG_q) { - Eterm expr = stp->literals[S.val].term; - for (i = 0; i < Size.val; i += 2) { - if (Rest[i].type == TAG_q) { - Eterm term = stp->literals[Rest[i].val].term; - if (eq(term, expr)) { - ASSERT(Rest[i+1].type == TAG_f); - op->a[0] = Rest[i+1]; - return op; + switch (S.type) { + case TAG_q: + { + Eterm expr = stp->literals[S.val].term; + for (i = 0; i < Size.val; i += 2) { + if (Rest[i].type == TAG_q) { + Eterm term = stp->literals[Rest[i].val].term; + if (eq(term, expr)) { + ASSERT(Rest[i+1].type == TAG_f); + op->a[0] = Rest[i+1]; + return op; + } } } } + break; + case TAG_i: + case TAG_a: + for (i = 0; i < Size.val; i += 2) { + if (Rest[i].val == S.val && Rest[i].type == S.type) { + ASSERT(Rest[i+1].type == TAG_f); + op->a[0] = Rest[i+1]; + return op; + } + } + break; } /* @@ -3274,36 +3397,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, return op; } - -static GenOp* -gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg func, - GenOpArg arity, GenOpArg label) -{ - GenOp* fi; - GenOp* op; - - NEW_GENOP(stp, fi); - fi->op = genop_i_func_info_4; - fi->arity = 4; - fi->a[0].type = TAG_u; /* untagged Zero */ - fi->a[0].val = 0; - fi->a[1] = mod; - fi->a[2] = func; - fi->a[3] = arity; - - NEW_GENOP(stp, op); - op->op = genop_label_1; - op->arity = 1; - op->a[0] = label; - - fi->next = op; - op->next = NULL; - - return fi; -} - - - static GenOp* gen_make_fun2(LoaderState* stp, GenOpArg idx) { @@ -3321,15 +3414,21 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) op->op = genop_i_make_fun_2; op->arity = 2; op->a[0].type = TAG_u; - op->a[0].val = (Uint) fe; + op->a[0].val = (BeamInstr) fe; op->a[1].type = TAG_u; op->a[1].val = stp->lambdas[idx.val].num_free; op->next = NULL; return op; } - +/* + * Rewrite gc_bifs with one parameter (the common case). Utilized + * in ops.tab to rewrite instructions calling bif's in guards + * to use a garbage collecting implementation. The instructions + * are sometimes once again rewritten to handle literals (putting the + * parameter in the mostly unused r[0] before the instruction is executed). + */ static GenOp* -gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, +gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg Src, GenOpArg Dst) { GenOp* op; @@ -3341,22 +3440,24 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->a[0] = Fail; op->a[1].type = TAG_u; bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ if (bf == length_1) { - op->a[1].val = (Uint) (void *) erts_gc_length_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_length_1; } else if (bf == size_1) { - op->a[1].val = (Uint) (void *) erts_gc_size_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_size_1; } else if (bf == bit_size_1) { - op->a[1].val = (Uint) (void *) erts_gc_bit_size_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1; } else if (bf == byte_size_1) { - op->a[1].val = (Uint) (void *) erts_gc_byte_size_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1; } else if (bf == abs_1) { - op->a[1].val = (Uint) (void *) erts_gc_abs_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1; } else if (bf == float_1) { - op->a[1].val = (Uint) (void *) erts_gc_float_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_float_1; } else if (bf == round_1) { - op->a[1].val = (Uint) (void *) erts_gc_round_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_round_1; } else if (bf == trunc_1) { - op->a[1].val = (Uint) (void *) erts_gc_trunc_1; + op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1; } else { abort(); } @@ -3367,6 +3468,127 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, return op; } +/* + * This is used by the ops.tab rule that rewrites gc_bifs with two parameters + * The instruction returned is then again rewritten to an i_load instruction + * folowed by i_gc_bif2_jIId, to handle literals properly. + * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is + * always rewritten, regardless of if there actually are any literals. + */ +static GenOp* +gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, + GenOpArg S1, GenOpArg S2, GenOpArg Dst) +{ + GenOp* op; + BifFunction bf; + + NEW_GENOP(stp, op); + op->op = genop_ii_gc_bif2_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1].type = TAG_u; + bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ + if (bf == binary_part_2) { + op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; + } else { + abort(); + } + op->a[2] = S1; + op->a[3] = S2; + op->a[4] = Live; + op->a[5] = Dst; + op->next = NULL; + return op; +} + +/* + * This is used by the ops.tab rule that rewrites gc_bifs with three parameters + * The instruction returned is then again rewritten to a move instruction that + * uses r[0] for temp storage, followed by an i_load instruction, + * folowed by i_gc_bif3_jIsId, to handle literals properly. Rewriting + * always occur, as with the gc_bif2 counterpart. + */ +static GenOp* +gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, + GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst) +{ + GenOp* op; + BifFunction bf; + + NEW_GENOP(stp, op); + op->op = genop_ii_gc_bif3_7; + op->arity = 7; + op->a[0] = Fail; + op->a[1].type = TAG_u; + bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ + if (bf == binary_part_3) { + op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; + } else { + abort(); + } + op->a[2] = S1; + op->a[3] = S2; + op->a[4] = S3; + op->a[5] = Live; + op->a[6] = Dst; + op->next = NULL; + return op; +} + +static GenOp* +tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, + GenOpArg* Puts, GenOpArg S1, GenOpArg S2, GenOpArg S3, + GenOpArg S4, GenOpArg S5) +{ + GenOp* op; + int arity = Arity.val; /* Arity of tuple, not the instruction */ + int i; + + NEW_GENOP(stp, op); + op->next = NULL; + GENOP_ARITY(op, arity+2+5); + op->op = genop_i_put_tuple_2; + op->a[0] = Dst; + op->a[1].type = TAG_u; + op->a[1].val = arity + 5; + for (i = 0; i < arity; i++) { + op->a[i+2] = Puts[i]; + } + op->a[arity+2] = S1; + op->a[arity+3] = S2; + op->a[arity+4] = S3; + op->a[arity+5] = S4; + op->a[arity+6] = S5; + return op; +} + +static GenOp* +tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, + GenOpArg* Puts, GenOpArg S) +{ + GenOp* op; + int arity = Arity.val; /* Arity of tuple, not the instruction */ + int i; + + NEW_GENOP(stp, op); + op->next = NULL; + GENOP_ARITY(op, arity+2+1); + op->op = genop_i_put_tuple_2; + op->a[0] = Dst; + op->a[1].type = TAG_u; + op->a[1].val = arity + 1; + for (i = 0; i < arity; i++) { + op->a[i+2] = Puts[i]; + } + op->a[arity+2] = S; + return op; +} + + /* * Freeze the code in memory, move the string table into place, @@ -3376,7 +3598,8 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, static int freeze_code(LoaderState* stp) { - Eterm* code = stp->code; + BeamInstr* code = stp->code; + Uint *literal_end = NULL; Uint index; int i; byte* str_table; @@ -3401,46 +3624,49 @@ freeze_code(LoaderState* stp) * Calculate the final size of the code. */ - size = (stp->ci + stp->total_literal_size) * sizeof(Eterm) + + size = (stp->ci * sizeof(BeamInstr)) + (stp->total_literal_size * sizeof(Eterm)) + strtab_size + attr_size + compile_size; /* * Move the code to its final location. */ - code = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size); - + code = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size); + CHKBLK(ERTS_ALC_T_CODE,code); /* * Place a pointer to the op_int_code_end instruction in the * function table in the beginning of the file. */ - code[MI_FUNCTIONS+stp->num_functions] = (Eterm) (code + stp->ci - 1); + code[MI_FUNCTIONS+stp->num_functions] = (BeamInstr) (code + stp->ci - 1); + CHKBLK(ERTS_ALC_T_CODE,code); /* * Store the pointer to the on_load function. */ if (stp->on_load) { - code[MI_ON_LOAD_FUNCTION_PTR] = (Eterm) (code + stp->on_load); + code[MI_ON_LOAD_FUNCTION_PTR] = (BeamInstr) (code + stp->on_load); } else { code[MI_ON_LOAD_FUNCTION_PTR] = 0; } + CHKBLK(ERTS_ALC_T_CODE,code); + literal_end = (Uint *) (code+stp->ci); /* * Place the literal heap directly after the code and fix up all - * put_literal instructions that refer to it. + * instructions that refer to it. */ { - Eterm* ptr; - Eterm* low; - Eterm* high; + Uint* ptr; + Uint* low; + Uint* high; LiteralPatch* lp; - low = code+stp->ci; + low = (Uint *) (code+stp->ci); high = low + stp->total_literal_size; - code[MI_LITERALS_START] = (Eterm) low; - code[MI_LITERALS_END] = (Eterm) high; + code[MI_LITERALS_START] = (BeamInstr) low; + code[MI_LITERALS_END] = (BeamInstr) high; ptr = low; for (i = 0; i < stp->num_literals; i++) { Uint offset; @@ -3472,7 +3698,7 @@ freeze_code(LoaderState* stp) } lp = stp->literal_patches; while (lp != 0) { - Uint* op_ptr; + BeamInstr* op_ptr; Uint literal; Literal* lit; @@ -3485,53 +3711,55 @@ freeze_code(LoaderState* stp) op_ptr[0] = literal; lp = lp->next; } - stp->ci += stp->total_literal_size; + literal_end += stp->total_literal_size; } /* * Place the string table and, optionally, attributes, after the literal heap. */ + CHKBLK(ERTS_ALC_T_CODE,code); - sys_memcpy(code+stp->ci, stp->chunks[STR_CHUNK].start, strtab_size); - str_table = (byte *) (code+stp->ci); + sys_memcpy(literal_end, stp->chunks[STR_CHUNK].start, strtab_size); + CHKBLK(ERTS_ALC_T_CODE,code); + str_table = (byte *) literal_end; if (attr_size) { byte* attr = str_table + strtab_size; sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size); - code[MI_ATTR_PTR] = (Eterm) attr; - code[MI_ATTR_SIZE] = (Eterm) stp->chunks[ATTR_CHUNK].size; + code[MI_ATTR_PTR] = (BeamInstr) attr; + code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size; decoded_size = erts_decode_ext_size(attr, attr_size, 0); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of module attributes"); } code[MI_ATTR_SIZE_ON_HEAP] = decoded_size; } + CHKBLK(ERTS_ALC_T_CODE,code); if (compile_size) { byte* compile_info = str_table + strtab_size + attr_size; + CHKBLK(ERTS_ALC_T_CODE,code); sys_memcpy(compile_info, stp->chunks[COMPILE_CHUNK].start, stp->chunks[COMPILE_CHUNK].size); - code[MI_COMPILE_PTR] = (Eterm) compile_info; - code[MI_COMPILE_SIZE] = (Eterm) stp->chunks[COMPILE_CHUNK].size; + + CHKBLK(ERTS_ALC_T_CODE,code); + code[MI_COMPILE_PTR] = (BeamInstr) compile_info; + CHKBLK(ERTS_ALC_T_CODE,code); + code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; + CHKBLK(ERTS_ALC_T_CODE,code); decoded_size = erts_decode_ext_size(compile_info, compile_size, 0); + CHKBLK(ERTS_ALC_T_CODE,code); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of compilation information"); } + CHKBLK(ERTS_ALC_T_CODE,code); code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size; } - + CHKBLK(ERTS_ALC_T_CODE,code); /* - * Go through all put_strings instructions, restore the pointer to - * the instruction and convert string offsets to pointers (to the - * LAST character). + * Make sure that we have not overflowed the allocated code space. */ - - index = stp->put_strings; - while (index != 0) { - Uint next = code[index]; - code[index] = BeamOpCode(op_put_string_IId); - code[index+2] = (Uint) (str_table + code[index+2] + code[index+1] - 1); - index = next; - } + ASSERT(str_table + strtab_size + attr_size + compile_size == + ((byte *) code) + size); /* * Go through all i_new_bs_put_strings instructions, restore the pointer to @@ -3543,23 +3771,25 @@ freeze_code(LoaderState* stp) while (index != 0) { Uint next = code[index]; code[index] = BeamOpCode(op_bs_put_string_II); - code[index+2] = (Uint) (str_table + code[index+2]); + code[index+2] = (BeamInstr) (str_table + code[index+2]); index = next; } + CHKBLK(ERTS_ALC_T_CODE,code); { StringPatch* sp = stp->string_patches; while (sp != 0) { - Uint* op_ptr; + BeamInstr* op_ptr; byte* strp; op_ptr = code + sp->pos; strp = str_table + op_ptr[0]; - op_ptr[0] = (Eterm) strp; + op_ptr[0] = (BeamInstr) strp; sp = sp->next; } } + CHKBLK(ERTS_ALC_T_CODE,code); /* * Resolve all labels. @@ -3579,10 +3809,11 @@ freeze_code(LoaderState* stp) ASSERT(this_patch < stp->ci); next_patch = code[this_patch]; ASSERT(next_patch < stp->ci); - code[this_patch] = (Uint) (code + value); + code[this_patch] = (BeamInstr) (code + value); this_patch = next_patch; } } + CHKBLK(ERTS_ALC_T_CODE,code); /* * Fix all catch_yf instructions. @@ -3590,13 +3821,14 @@ freeze_code(LoaderState* stp) index = stp->catches; catches = BEAM_CATCHES_NIL; while (index != 0) { - Uint next = code[index]; + BeamInstr next = code[index]; code[index] = BeamOpCode(op_catch_yf); - catches = beam_catches_cons((Uint*)code[index+2], catches); + catches = beam_catches_cons((BeamInstr *)code[index+2], catches); code[index+2] = make_catch(catches); index = next; } stp->catches = catches; + CHKBLK(ERTS_ALC_T_CODE,code); /* * Save the updated code pointer and code size. @@ -3605,6 +3837,7 @@ freeze_code(LoaderState* stp) stp->code = code; stp->loaded_size = size; + CHKBLK(ERTS_ALC_T_CODE,code); return 1; load_error: @@ -3638,7 +3871,7 @@ final_touch(LoaderState* stp) * callable yet. */ ep->address = ep->code+3; - ep->code[4] = (Eterm) stp->export[i].address; + ep->code[4] = (BeamInstr) stp->export[i].address; } } @@ -3650,14 +3883,14 @@ final_touch(LoaderState* stp) Eterm mod; Eterm func; Uint arity; - Uint import; + BeamInstr import; Uint current; Uint next; mod = stp->import[i].module; func = stp->import[i].function; arity = stp->import[i].arity; - import = (Uint) erts_export_put(mod, func, arity); + import = (BeamInstr) erts_export_put(mod, func, arity); current = stp->import[i].patches; while (current != 0) { ASSERT(current < stp->ci); @@ -3675,7 +3908,7 @@ final_touch(LoaderState* stp) for (i = 0; i < stp->num_lambdas; i++) { unsigned entry_label = stp->lambdas[i].label; ErlFunEntry* fe = stp->lambdas[i].fe; - Eterm* code_ptr = (Eterm *) (stp->code + stp->labels[entry_label].value); + BeamInstr* code_ptr = (BeamInstr *) (stp->code + stp->labels[entry_label].value); if (fe->address[0] != 0) { /* @@ -3762,11 +3995,23 @@ transform_engine(LoaderState* st) if (i == 0) goto restart; break; +#if defined(TOP_is_eq) case TOP_is_eq: ASSERT(ap < instr->arity); if (*pc++ != instr->a[ap].val) goto restart; break; +#endif + case TOP_is_type_eq: + mask = *pc++; + + ASSERT(ap < instr->arity); + ASSERT(instr->a[ap].type < BEAM_NUM_TAGS); + if (((1 << instr->a[ap].type) & mask) == 0) + goto restart; + if (*pc++ != instr->a[ap].val) + goto restart; + break; case TOP_is_same_var: ASSERT(ap < instr->arity); i = *pc++; @@ -3807,7 +4052,7 @@ transform_engine(LoaderState* st) if (i >= st->num_imports || st->import[i].bf == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->code[4] != (Uint) st->import[i].bf) { + bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) { goto restart; } } @@ -3887,14 +4132,17 @@ transform_engine(LoaderState* st) case TOP_rest_args: { int n = *pc++; + int formal_arity = gen_opc[instr->op].arity; + int num_vars = n + (instr->arity - formal_arity); + int j = formal_arity; + var = erts_alloc(ERTS_ALC_T_LOADER_TMP, - instr->arity * sizeof(GenOpArg)); + num_vars * sizeof(GenOpArg)); for (i = 0; i < n; i++) { var[i] = def_vars[i]; } - while (i < instr->arity) { - var[i] = instr->a[i]; - i++; + while (i < num_vars) { + var[i++] = instr->a[j++]; } } break; @@ -4071,7 +4319,7 @@ load_printf(int line, LoaderState* context, char *fmt,...) static int -get_int_val(LoaderState* stp, Uint len_code, Uint* result) +get_int_val(LoaderState* stp, Uint len_code, BeamInstr* result) { Uint count; Uint val; @@ -4103,7 +4351,7 @@ get_int_val(LoaderState* stp, Uint len_code, Uint* result) static int -get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result) +get_erlang_integer(LoaderState* stp, Uint len_code, BeamInstr* result) { Uint count; Sint val; @@ -4124,11 +4372,12 @@ get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result) count = len_code + 2; } else { Uint tag; + UWord len_word; ASSERT(len_code == 7); - GetTagAndValue(stp, tag, len_code); + GetTagAndValue(stp, tag, len_word); VerifyTag(stp, TAG_u, tag); - count = len_code + 9; + count = len_word + 9; } /* @@ -4376,7 +4625,7 @@ functions_in_module(Process* p, /* Process whose heap to use. */ Eterm mod) /* Tagged atom for module. */ { Module* modp; - Eterm* code; + BeamInstr* code; int i; Uint num_functions; Eterm* hp; @@ -4394,9 +4643,9 @@ functions_in_module(Process* p, /* Process whose heap to use. */ num_functions = code[MI_NUM_FUNCTIONS]; hp = HAlloc(p, 5*num_functions); for (i = num_functions-1; i >= 0 ; i--) { - Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i]; - Eterm name = func_info[3]; - int arity = func_info[4]; + BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + Eterm name = (Eterm) func_info[3]; + int arity = (int) func_info[4]; Eterm tuple; ASSERT(is_atom(name)); @@ -4419,7 +4668,7 @@ static Eterm native_addresses(Process* p, Eterm mod) { Module* modp; - Eterm* code; + BeamInstr* code; int i; Eterm* hp; Uint num_functions; @@ -4442,9 +4691,9 @@ native_addresses(Process* p, Eterm mod) hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i]; - Eterm name = func_info[3]; - int arity = func_info[4]; + BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + Eterm name = (Eterm) func_info[3]; + int arity = (int) func_info[4]; Eterm tuple; ASSERT(is_atom(name)); @@ -4486,7 +4735,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */ Eterm tuple; if (ep->address == ep->code+3 && - ep->code[3] == (Eterm) em_call_error_handler) { + ep->code[3] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; } @@ -4519,7 +4768,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */ { Module* modp; - Eterm* code; + BeamInstr* code; Eterm* hp; byte* ext; Eterm result = NIL; @@ -4559,7 +4808,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ Eterm mod) /* Tagged atom for module. */ { Module* modp; - Eterm* code; + BeamInstr* code; Eterm* hp; byte* ext; Eterm result = NIL; @@ -4591,8 +4840,8 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ /* * Returns a pointer to {module, function, arity}, or NULL if not found. */ -Eterm* -find_function_from_pc(Eterm* pc) +BeamInstr * +find_function_from_pc(BeamInstr* pc) { Range* low = modules; Range* high = low + num_loaded_modules; @@ -4604,9 +4853,9 @@ find_function_from_pc(Eterm* pc) } else if (pc > mid->end) { low = mid + 1; } else { - Eterm** low1 = (Eterm **) (mid->start + MI_FUNCTIONS); - Eterm** high1 = low1 + mid->start[MI_NUM_FUNCTIONS]; - Eterm** mid1; + BeamInstr** low1 = (BeamInstr **) (mid->start + MI_FUNCTIONS); + BeamInstr** high1 = low1 + mid->start[MI_NUM_FUNCTIONS]; + BeamInstr** mid1; while (low1 < high1) { mid1 = low1 + (high1-low1) / 2; @@ -4719,10 +4968,10 @@ code_module_md5_1(Process* p, Eterm Bin) #define WORDS_PER_FUNCTION 6 -static Eterm* -make_stub(Eterm* fp, Eterm mod, Eterm func, Uint arity, Uint native, Eterm OpCode) +static BeamInstr* +make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { - fp[0] = (Eterm) BeamOp(op_i_func_info_IaaI); + fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); fp[1] = native; fp[2] = mod; fp[3] = func; @@ -4741,14 +4990,14 @@ static byte* stub_copy_info(LoaderState* stp, int chunk, /* Chunk: ATTR_CHUNK or COMPILE_CHUNK */ byte* info, /* Where to store info. */ - Eterm* ptr_word, /* Where to store pointer into info. */ - Eterm* size_word) /* Where to store size of info. */ + BeamInstr* ptr_word, /* Where to store pointer into info. */ + BeamInstr* size_word) /* Where to store size of info. */ { Sint decoded_size; Uint size = stp->chunks[chunk].size; if (size != 0) { memcpy(info, stp->chunks[chunk].start, size); - *ptr_word = (Eterm) info; + *ptr_word = (BeamInstr) info; decoded_size = erts_decode_ext_size(info, size, 0); if (decoded_size < 0) { return 0; @@ -4791,7 +5040,7 @@ stub_read_export_table(LoaderState* stp) } static void -stub_final_touch(LoaderState* stp, Eterm* fp) +stub_final_touch(LoaderState* stp, BeamInstr* fp) { int i; int n = stp->num_exps; @@ -4978,12 +5227,12 @@ Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) { LoaderState state; - Eterm Funcs; - Eterm Patchlist; + BeamInstr Funcs; + BeamInstr Patchlist; Eterm* tp; - Eterm* code = NULL; - Eterm* ptrs; - Eterm* fp; + BeamInstr* code = NULL; + BeamInstr* ptrs; + BeamInstr* fp; byte* info; Uint ci; int n; @@ -5072,7 +5321,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Allocate memory for the stub module. */ - code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(Eterm); + code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr); code_size += state.chunks[ATTR_CHUNK].size; code_size += state.chunks[COMPILE_CHUNK].size; code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); @@ -5086,8 +5335,10 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_NUM_FUNCTIONS] = n; code[MI_ATTR_PTR] = 0; + code[MI_ATTR_SIZE] = 0; code[MI_ATTR_SIZE_ON_HEAP] = 0; code[MI_COMPILE_PTR] = 0; + code[MI_COMPILE_SIZE] = 0; code[MI_COMPILE_SIZE_ON_HEAP] = 0; code[MI_NUM_BREAKPOINTS] = 0; code[MI_ON_LOAD_FUNCTION_PTR] = 0; @@ -5141,7 +5392,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - ptrs[i] = (Eterm) fp; + ptrs[i] = (BeamInstr) fp; #ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ #else @@ -5154,8 +5405,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the last pointer and the int_code_end instruction. */ - ptrs[i] = (Eterm) fp; - *fp++ = (Eterm) BeamOp(op_int_code_end); + ptrs[i] = (BeamInstr) fp; + *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* * Copy attributes and compilation information. @@ -5198,6 +5449,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (state.lambdas != state.def_lambdas) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); } + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); if (bin != NULL) { driver_free_binary(bin); } @@ -5209,9 +5463,18 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (code != NULL) { erts_free(ERTS_ALC_T_CODE, code); } + if (state.labels != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); + } if (state.lambdas != state.def_lambdas) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); } + if (state.atom != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); + } + if (state.export != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); + } if (bin != NULL) { driver_free_binary(bin); } @@ -5222,9 +5485,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) #undef WORDS_PER_FUNCTION -static int safe_mul(Uint a, Uint b, Uint* resp) +static int safe_mul(UWord a, UWord b, UWord* resp) { - Uint res = a * b; + Uint res = a * b; /* XXX:Pan - used in bit syntax, the multiplication has to be stored in Uint */ *resp = res; if (b == 0) { diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index c17844a553..26e3054c4b 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -44,13 +44,13 @@ extern void** beam_ops; #endif -extern Eterm beam_debug_apply[]; -extern Eterm* em_call_error_handler; -extern Eterm* em_apply_bif; -extern Eterm* em_call_traced_function; +extern BeamInstr beam_debug_apply[]; +extern BeamInstr* em_call_error_handler; +extern BeamInstr* em_apply_bif; +extern BeamInstr* em_call_traced_function; typedef struct { - Eterm* start; /* Pointer to start of module. */ - Eterm* end; /* Points one word beyond last function in module. */ + BeamInstr* start; /* Pointer to start of module. */ + BeamInstr* end; /* Points one word beyond last function in module. */ } Range; /* diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 1b670585a7..68b3350d7f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -368,7 +368,6 @@ static int demonitor(Process *c_p, Eterm ref) ErtsMonitor *mon = NULL; /* The monitor entry to delete */ Process *rp; /* Local target process */ Eterm to = NIL; /* Monitor link traget */ - Eterm ref_p; /* Pid of this end */ DistEntry *dep = NULL; /* Target's distribution entry */ int deref_de = 0; int res; @@ -381,7 +380,6 @@ static int demonitor(Process *c_p, Eterm ref) res = ERTS_DEMONITOR_BADARG; goto done; /* Cannot be this monitor's ref */ } - ref_p = c_p->id; mon = erts_lookup_monitor(c_p->monitors, ref); if (!mon) { @@ -616,13 +614,15 @@ local_name_monitor(Process *p, Eterm target_name) rp = erts_whereis_process(p, p_locks, target_name, ERTS_PROC_LOCK_LINK, ERTS_P2P_FLG_ALLOW_OTHER_X); if (!rp) { - Eterm lhp[3]; + DeclareTmpHeap(lhp,3,p); Eterm item; + UseTmpHeap(3,p); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname); erts_queue_monitor_message(p, &p_locks, mon_ref, am_process, item, am_noproc); + UnUseTmpHeap(3,p); } else if (rp != p) { erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, rp->id, @@ -811,7 +811,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + so.max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); so.scheduler = 0; /* @@ -1089,10 +1089,20 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) BIF_RETTYPE hibernate_3(BIF_ALIST_3) { /* - * hibernate/3 is implemented as an instruction; therefore - * this function will never be called. + * hibernate/3 is usually translated to an instruction; therefore + * this function is only called from HiPE or when the call could not + * be translated. */ - BIF_ERROR(BIF_P, BADARG); + Eterm reg[3]; + + if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) { + /* + * If hibernate succeeded, TRAP. The process will be suspended + * if status is P_WAITING or continue (if any message was in the queue). + */ + BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i); + } + return THE_NON_VALUE; } /**********************************************************************/ @@ -1130,6 +1140,34 @@ BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args) } /**********************************************************************/ +/* + * This is like exactly like error/1. The only difference is + * that Dialyzer thinks that it it will return an arbitrary term. + * It is useful in stub functions for NIFs. + */ + +BIF_RETTYPE nif_error_1(Process* p, Eterm term) +{ + p->fvalue = term; + BIF_ERROR(p, EXC_ERROR); +} + +/**********************************************************************/ +/* + * This is like exactly like error/2. The only difference is + * that Dialyzer thinks that it it will return an arbitrary term. + * It is useful in stub functions for NIFs. + */ + +BIF_RETTYPE nif_error_2(Process* p, Eterm value, Eterm args) +{ + Eterm* hp = HAlloc(p, 3); + + p->fvalue = TUPLE2(hp, value, args); + BIF_ERROR(p, EXC_ERROR_2); +} + +/**********************************************************************/ /* this is like throw/1 except that we set freason to EXC_EXIT */ BIF_RETTYPE exit_1(BIF_ALIST_1) @@ -1321,9 +1359,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) #ifdef ERTS_SMP if (rp == BIF_P) rp_locks &= ~ERTS_PROC_LOCK_MAIN; - else + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (rp != BIF_P) erts_smp_proc_dec_refc(rp); - erts_smp_proc_unlock(rp, rp_locks); #endif /* * We may have exited ourselves and may have to take action. @@ -2129,20 +2168,146 @@ BIF_RETTYPE tl_1(BIF_ALIST_1) /**********************************************************************/ /* return the size of an I/O list */ -BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +static Eterm +accumulate(Eterm acc, Uint size) { - Sint size = io_list_len(BIF_ARG_1); + if (is_non_value(acc)) { + /* + * There is no pre-existing accumulator. Allocate a + * bignum buffer with one extra word to be used if + * the bignum grows in the future. + */ + Eterm* hp = (Eterm *) erts_alloc(ERTS_ALC_T_TEMP_TERM, + (BIG_UINT_HEAP_SIZE+1) * + sizeof(Eterm)); + return uint_to_big(size, hp); + } else { + Eterm* big; + int need_heap; - if (size == -1) { - BIF_ERROR(BIF_P, BADARG); - } else if (IS_USMALL(0, (Uint) size)) { - BIF_RET(make_small(size)); + /* + * Add 'size' to 'acc' in place. There is always one + * extra word allocated in case the bignum grows by one word. + */ + big = big_val(acc); + need_heap = BIG_NEED_SIZE(BIG_SIZE(big)); + acc = big_plus_small(acc, size, big); + if (BIG_NEED_SIZE(big_size(acc)) > need_heap) { + /* + * The extra word has been consumed. Grow the + * allocation by one word. + */ + big = (Eterm *) erts_realloc(ERTS_ALC_T_TEMP_TERM, + big_val(acc), + (need_heap+1) * sizeof(Eterm)); + acc = make_big(big); + } + return acc; + } +} + +static Eterm +consolidate(Process* p, Eterm acc, Uint size) +{ + Eterm* hp; + + if (is_non_value(acc)) { + return erts_make_integer(size, p); } else { - Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); - BIF_RET(uint_to_big(size, hp)); + Eterm* big; + Uint sz; + Eterm res; + + acc = accumulate(acc, size); + big = big_val(acc); + sz = BIG_NEED_SIZE(BIG_SIZE(big)); + hp = HAlloc(p, sz); + res = make_big(hp); + while (sz--) { + *hp++ = *big++; + } + erts_free(ERTS_ALC_T_TEMP_TERM, (void *) big_val(acc)); + return res; } } +BIF_RETTYPE iolist_size_1(BIF_ALIST_1) +{ + Eterm obj, hd; + Eterm* objp; + Uint size = 0; + Uint cur_size; + Uint new_size; + Eterm acc = THE_NON_VALUE; + DECLARE_ESTACK(s); + + obj = BIF_ARG_1; + goto L_again; + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + hd = CAR(objp); + obj = CDR(objp); + /* Head */ + if (is_byte(hd)) { + size++; + if (size == 0) { + acc = accumulate(acc, (Uint) -1); + size = 1; + } + } else if (is_binary(hd) && binary_bitsize(hd) == 0) { + cur_size = binary_size(hd); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_list(hd)) { + ESTACK_PUSH(s, obj); + obj = hd; + goto L_iter_list; + } else if (is_not_nil(hd)) { + goto L_type_error; + } + /* Tail */ + if (is_list(obj)) { + goto L_iter_list; + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + cur_size = binary_size(obj); + if ((new_size = size + cur_size) >= size) { + size = new_size; + } else { + acc = accumulate(acc, size); + size = cur_size; + } + } else if (is_not_nil(obj)) { + goto L_type_error; + } + } + + DESTROY_ESTACK(s); + BIF_RET(consolidate(BIF_P, acc, size)); + + L_type_error: + DESTROY_ESTACK(s); + BIF_ERROR(BIF_P, BADARG); +} /**********************************************************************/ @@ -3176,20 +3341,32 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } - rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + if (BIF_P->id == BIF_ARG_1) + rp = BIF_P; + else { +#ifdef ERTS_SMP + rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_ARG_1, ERTS_PROC_LOCK_MAIN); - if (!rp) - BIF_RET(am_false); - if (rp == ERTS_PROC_LOCK_BUSY) - ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); + if (rp == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); +#else + rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); +#endif + if (!rp) + BIF_RET(am_false); + } /* The GC cost is taken for the process executing this BIF. */ FLAGS(rp) |= F_NEED_FULLSWEEP; reds = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - if (BIF_P != rp) +#ifdef ERTS_SMP + if (BIF_P != rp) { + erts_resume(rp, ERTS_PROC_LOCK_MAIN); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + } +#endif BIF_RET2(am_true, reds); } @@ -3231,6 +3408,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) Eterm* dead_ports; int alive, dead; Uint32 next_ss; + int i; /* To get a consistent snapshot... * We add alive ports from start of the buffer @@ -3239,27 +3417,25 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */ - erts_smp_atomic_set(&erts_dead_ports_ptr, (long) (port_buf + erts_max_ports)); + erts_smp_atomic_set(&erts_dead_ports_ptr, + (erts_aint_t) (port_buf + erts_max_ports)); - next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot); + next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot); - if (erts_smp_atomic_read(&erts_ports_alive) > 0) { - long i; - for (i = erts_max_ports-1; i >= 0; i--) { - Port* prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD) - && prt->snapshot != next_ss) { - ASSERT(prt->snapshot == next_ss - 1); - *pp++ = prt->id; - prt->snapshot = next_ss; /* Consumed by this snapshot */ - } - erts_smp_port_state_unlock(prt); + for (i = erts_max_ports-1; i >= 0; i--) { + Port* prt = &erts_port[i]; + erts_smp_port_state_lock(prt); + if (!(prt->status & ERTS_PORT_SFLGS_DEAD) + && prt->snapshot != next_ss) { + ASSERT(prt->snapshot == next_ss - 1); + *pp++ = prt->id; + prt->snapshot = next_ss; /* Consumed by this snapshot */ } + erts_smp_port_state_unlock(prt); } dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, - (long)NULL); + (erts_aint_t) NULL); erts_smp_mtx_unlock(&ports_snapshot_mtx); ASSERT(pp <= dead_ports); @@ -3270,7 +3446,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) ASSERT((alive+dead) <= erts_max_ports); if (alive+dead > 0) { - long i; + erts_aint_t i; Eterm *hp = HAlloc(BIF_P, (alive+dead)*2); for (i = 0; i < alive; i++) { @@ -3320,7 +3496,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1); if (pres < 0) - erl_exit(1, "Failed to convert term to string: %d (s)\n", + erl_exit(1, "Failed to convert term to string: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -3438,7 +3614,7 @@ term2list_dsprintf(Process *p, Eterm term) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%T", term); if (pres < 0) - erl_exit(1, "Failed to convert term to list: %d (s)\n", + erl_exit(1, "Failed to convert term to list: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -3466,9 +3642,16 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3) if (arity < 0) { goto error; } +#if HALFWORD_HEAP + hp = HAlloc(BIF_P, 3); + hp[0] = HEADER_EXPORT; + /* Yes, May be misaligned, but X86_64 will fix it... */ + *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); +#else hp = HAlloc(BIF_P, 2); hp[0] = HEADER_EXPORT; hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); +#endif BIF_RET(make_export(hp)); } @@ -3574,11 +3757,11 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); - etp->next = MSO(BIF_P).externals; + etp->next = MSO(BIF_P).first; etp->node = enp; etp->data.ui[0] = make_pid_data(c, b); - MSO(BIF_P).externals = etp; + MSO(BIF_P).first = (struct erl_off_heap_header*) etp; erts_deref_dist_entry(dep); BIF_RET(make_external_pid(etp)); } @@ -3759,7 +3942,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n); - oval = (Uint) erts_smp_atomic_xchg(&erts_max_gen_gcs, (long) nval); + oval = (Uint) erts_smp_atomic32_xchg(&erts_max_gen_gcs, + (erts_aint32_t) nval); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_heap_size) { int oval = H_MIN_SIZE; @@ -3884,7 +4068,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2) if ((range = signed_val(BIF_ARG_2)) <= 0) { /* [1..MAX_SMALL] */ BIF_ERROR(BIF_P, BADARG); } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (range > ((1L << 27) - 1)) BIF_ERROR(BIF_P, BADARG); #endif @@ -3956,7 +4140,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) /* * Return either a small or a big. Use the heap for bigs if there is room. */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP BIF_RET(make_small(final_hash)); #else if (IS_USMALL(0, final_hash)) { @@ -4102,7 +4286,7 @@ void erts_init_bif(void) erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); - erts_smp_atomic_init(&erts_dead_ports_ptr, (long)NULL); + erts_smp_atomic_init(&erts_dead_ports_ptr, (erts_aint_t) NULL); /* * bif_return_trap/1 is a hidden BIF that bifs that need to @@ -4118,8 +4302,8 @@ void erts_init_bif(void) #else bif_return_trap_export.code[2] = 1; #endif - bif_return_trap_export.code[3] = (Eterm) em_apply_bif; - bif_return_trap_export.code[4] = (Eterm) &bif_return_trap; + bif_return_trap_export.code[3] = (BeamInstr) em_apply_bif; + bif_return_trap_export.code[4] = (BeamInstr) &bif_return_trap; flush_monitor_message_trap = erts_export_put(am_erlang, am_flush_monitor_message, @@ -4134,54 +4318,6 @@ void erts_init_bif(void) await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); } -BIF_RETTYPE blocking_read_file_1(BIF_ALIST_1) -{ - Eterm bin; - Eterm* hp; - byte *buff; - int i, buff_size; - FILE *file; - struct stat file_info; - char *filename = NULL; - size_t size; - - i = list_length(BIF_ARG_1); - if (i < 0) { - BIF_ERROR(BIF_P, BADARG); - } - filename = erts_alloc(ERTS_ALC_T_TMP, i + 1); - if (intlist_to_buf(BIF_ARG_1, filename, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - filename[i] = '\0'; - - hp = HAlloc(BIF_P, 3); - - file = fopen(filename, "r"); - if(file == NULL){ - erts_free(ERTS_ALC_T_TMP, (void *) filename); - BIF_RET(TUPLE2(hp, am_error, am_nofile)); - } - - stat(filename, &file_info); - erts_free(ERTS_ALC_T_TMP, (void *) filename); - - buff_size = file_info.st_size; - buff = (byte *) erts_alloc_fnf(ERTS_ALC_T_TMP, buff_size); - if (!buff) { - fclose(file); - BIF_RET(TUPLE2(hp, am_error, am_allocator)); - } - size = fread(buff, 1, buff_size, file); - fclose(file); - if (size < 0) - size = 0; - else if (size > buff_size) - size = (size_t) buff_size; - bin = new_binary(BIF_P, buff, (int) size); - erts_free(ERTS_ALC_T_TMP, (void *) buff); - - BIF_RET(TUPLE2(hp, am_ok, bin)); -} #ifdef HARDDEBUG /* You'll need this line in bif.tab to be able to use this debug bif diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 05e9b78c28..8faa09feb8 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -125,7 +125,7 @@ do { \ #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ do { \ (Proc)->arity = 0; \ - (Proc)->def_arg_reg[3] = (Eterm) (Trap->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -134,7 +134,7 @@ do { \ do { \ (Proc)->arity = 1; \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -144,7 +144,7 @@ do { \ (Proc)->arity = 2; \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ (Proc)->def_arg_reg[1] = (Eterm) (A1); \ - (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -155,14 +155,14 @@ do { \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ (Proc)->def_arg_reg[1] = (Eterm) (A1); \ (Proc)->def_arg_reg[2] = (Eterm) (A2); \ - (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address); \ + *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) #define BIF_TRAP0(p, Trap_) do { \ (p)->arity = 0; \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -170,7 +170,7 @@ do { \ #define BIF_TRAP1(Trap_, p, A0) do { \ (p)->arity = 1; \ (p)->def_arg_reg[0] = (A0); \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -179,7 +179,7 @@ do { \ (p)->arity = 2; \ (p)->def_arg_reg[0] = (A0); \ (p)->def_arg_reg[1] = (A1); \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -189,14 +189,20 @@ do { \ (p)->def_arg_reg[0] = (A0); \ (p)->def_arg_reg[1] = (A1); \ (p)->def_arg_reg[2] = (A2); \ - (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) #define BIF_TRAP_CODE_PTR_0(p, Code_) do { \ (p)->arity = 0; \ - (p)->def_arg_reg[3] = (Eterm) (Code_); \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ + } while(0) + +#define BIF_TRAP_CODE_PTR_(p, Code_) do { \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index b6fa06354a..d9dd80fa8b 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -660,6 +660,7 @@ bif erts_debug:display/1 bif 'erl.system.debug':display/1 ebif_erts_debug_display_1 bif erts_debug:dist_ext_to_term/2 bif 'erl.system.debug':dist_ext_to_term/2 ebif_erts_debug_dist_ext_to_term_2 +bif erts_debug:instructions/0 # # Monitor testing bif's... @@ -686,8 +687,6 @@ bif 'erl.system.code':make_stub_module/3 ebif_code_make_stub_module_3 bif code:is_module_native/1 bif 'erl.system.code':is_native/1 ebif_code_is_native_1 code_is_module_native_1 -bif erlang:blocking_read_file/1 - # # New Bifs in R9C. # @@ -760,6 +759,50 @@ bif erlang:finish_after_on_load/2 bif erlang:binary_to_term/2 # +# The binary match bifs (New in R14A - EEP9) +# + +# +# The searching/splitting/substituting thingies +# +ubif erlang:binary_part/2 +ubif erlang:binary_part/3 + +bif binary:compile_pattern/1 +bif binary:match/2 +bif binary:match/3 +bif binary:matches/2 +bif binary:matches/3 +bif binary:longest_common_prefix/1 +bif binary:longest_common_suffix/1 +bif binary:first/1 +bif binary:last/1 +bif binary:at/2 +bif binary:part/2 binary_binary_part_2 +bif binary:part/3 binary_binary_part_3 +bif binary:bin_to_list/1 +bif binary:bin_to_list/2 +bif binary:bin_to_list/3 +bif binary:list_to_bin/1 +bif binary:copy/1 +bif binary:copy/2 +bif binary:referenced_byte_size/1 +bif binary:encode_unsigned/1 +bif binary:encode_unsigned/2 +bif binary:decode_unsigned/1 +bif binary:decode_unsigned/2 + +bif erlang:nif_error/1 +bif erlang:nif_error/2 + +# +# Helpers for unicode filenames +# +bif prim_file:internal_name2native/1 +bif prim_file:internal_native2name/1 +bif prim_file:internal_normalize_utf8/1 +bif file:native_name_encoding/0 +# # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 03c88da8c6..d18de9ae5d 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -1459,7 +1459,31 @@ Eterm uint_to_big(Uint x, Eterm *y) BIG_DIGIT(y, 0) = x; return make_big(y); } +/* +** convert UWord to bigint +** (must only be used if x is to big to be stored as a small) +** Allocation is tricky, the heap need has to be calculated +** with the macro BIG_UWORD_HEAP_SIZE(x) +*/ +Eterm uword_to_big(UWord x, Eterm *y) +{ +#if HALFWORD_HEAP + Uint upper = x >> 32; + Uint lower = x & 0xFFFFFFFFUL; + if (upper == 0) { + *y = make_pos_bignum_header(1); + } else { + *y = make_pos_bignum_header(2); + BIG_DIGIT(y, 1) = upper; + } + BIG_DIGIT(y, 0) = lower; +#else + *y = make_pos_bignum_header(1); + BIG_DIGIT(y, 0) = x; +#endif + return make_big(y); +} /* ** convert signed int to bigint @@ -1480,19 +1504,19 @@ Eterm small_to_big(Sint x, Eterm *y) Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) { Eterm *hp = *hpp; -#ifdef ARCH_32 +#if defined(ARCH_32) || HALFWORD_HEAP if (x >= (((Uint64) 1) << 32)) { *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); - *hpp += 2; + *hpp += 3; } else #endif { *hp = make_pos_bignum_header(1); BIG_DIGIT(hp, 0) = (Uint) x; - *hpp += 1; + *hpp += 2; } return make_big(hp); } @@ -1507,7 +1531,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) neg = 1; x = -x; } -#ifdef ARCH_32 +#if defined(ARCH_32) || HALFWORD_HEAP if (x >= (((Uint64) 1) << 32)) { if (neg) *hp = make_neg_bignum_header(2); @@ -1515,7 +1539,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); - *hpp += 2; + *hpp += 3; } else #endif @@ -1525,7 +1549,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) else *hp = make_pos_bignum_header(1); BIG_DIGIT(hp, 0) = (Uint) x; - *hpp += 1; + *hpp += 2; } return make_big(hp); } @@ -1534,7 +1558,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) ** Convert a bignum to a double float */ int -big_to_double(Eterm x, double* resp) +big_to_double(Wterm x, double* resp) { double d = 0.0; Eterm* xp = big_val(x); @@ -1564,7 +1588,7 @@ big_to_double(Eterm x, double* resp) /* ** Estimate the number of decimal digits (include sign) */ -int big_decimal_estimate(Eterm x) +int big_decimal_estimate(Wterm x) { Eterm* xp = big_val(x); int lg = I_lg(BIG_V(xp), BIG_SIZE(xp)); @@ -1578,7 +1602,7 @@ int big_decimal_estimate(Eterm x) ** Convert a bignum into a string of decimal numbers */ -static void write_big(Eterm x, void (*write_func)(void *, char), void *arg) +static void write_big(Wterm x, void (*write_func)(void *, char), void *arg) { Eterm* xp = big_val(x); ErtsDigit* dx = BIG_V(xp); @@ -1657,7 +1681,7 @@ write_string(void *arg, char c) *(--(*((char **) arg))) = c; } -char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz) +char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz) { char *big_str = buf + buf_sz - 1; *big_str = '\0'; @@ -1701,7 +1725,7 @@ static Eterm big_norm(Eterm *x, dsize_t xl, short sign) /* ** Compare bignums */ -int big_comp(Eterm x, Eterm y) +int big_comp(Wterm x, Wterm y) { Eterm* xp = big_val(x); Eterm* yp = big_val(y); @@ -1854,6 +1878,87 @@ term_to_Uint(Eterm term, Uint *up) } } +int +term_to_UWord(Eterm term, UWord *up) +{ +#if SIZEOF_VOID_P == ERTS_SIZEOF_ETERM + return term_to_Uint(term,up); +#else + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (UWord) i; + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + UWord uval = 0; + int n = 0; + + if (big_sign(term)) { + *up = BADARG; + return 0; + } else if (xl*D_EXP > sizeof(UWord)*8) { + *up = SYSTEM_LIMIT; + return 0; + } + while (xl-- > 0) { + uval |= ((UWord)(*xr++)) << n; + n += D_EXP; + } + *up = uval; + return 1; + } else { + *up = BADARG; + return 0; + } +#endif +} + +int +term_to_Uint64(Eterm term, Uint64 *up) +{ +#if SIZEOF_VOID_P == 8 + return term_to_UWord(term,up); +#else + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (Uint64) i; + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + Uint64 uval = 0; + int n = 0; + + if (big_sign(term)) { + *up = BADARG; + return 0; + } else if (xl*D_EXP > sizeof(Uint64)*8) { + *up = SYSTEM_LIMIT; + return 0; + } + while (xl-- > 0) { + uval |= ((Uint64)(*xr++)) << n; + n += D_EXP; + } + *up = uval; + return 1; + } else { + *up = BADARG; + return 0; + } +#endif +} + + int term_to_Sint(Eterm term, Sint *sp) { if (is_small(term)) { @@ -1888,6 +1993,47 @@ int term_to_Sint(Eterm term, Sint *sp) } } +#if HAVE_INT64 +int term_to_Sint64(Eterm term, Sint64 *sp) +{ +#if ERTS_SIZEOF_ETERM == 8 + return term_to_Sint(term, sp); +#else + if (is_small(term)) { + *sp = signed_val(term); + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + int sign = big_sign(term); + Uint64 uval = 0; + int n = 0; + + if (xl*D_EXP > sizeof(Uint64)*8) { + return 0; + } + while (xl-- > 0) { + uval |= ((Uint64)(*xr++)) << n; + n += D_EXP; + } + if (sign) { + uval = -uval; + if ((Sint64)uval > 0) + return 0; + } else { + if ((Sint64)uval < 0) + return 0; + } + *sp = uval; + return 1; + } else { + return 0; + } +#endif +} +#endif /* HAVE_INT64 */ + + /* ** Add and subtract */ @@ -1914,7 +2060,7 @@ static Eterm B_plus_minus(ErtsDigit *x, dsize_t xl, short xsgn, /* ** Add bignums */ -Eterm big_plus(Eterm x, Eterm y, Eterm *r) +Eterm big_plus(Wterm x, Wterm y, Eterm *r) { Eterm* xp = big_val(x); Eterm* yp = big_val(y); diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index b8e38d482c..2afc37004f 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef Uint ErtsDigit; -#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) +#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) /* Assume 32-bit machine with long long support */ typedef Uint64 ErtsDoubleDigit; typedef Uint16 ErtsHalfDigit; @@ -58,7 +58,7 @@ typedef Uint32 ErtsHalfDigit; typedef Uint dsize_t; /* Vector size type */ -#define D_EXP (SIZEOF_VOID_P*8) +#define D_EXP (ERTS_SIZEOF_ETERM*8) #define D_MASK ((ErtsDigit)(-1)) /* D_BASE-1 */ /* macros for bignum objects */ @@ -88,7 +88,13 @@ typedef Uint dsize_t; /* Vector size type */ #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ -#ifdef ARCH_32 +#if HALFWORD_HEAP +#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2) +#else +#define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE +#endif + +#if defined(ARCH_32) || HALFWORD_HEAP #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \ ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1)) @@ -108,13 +114,13 @@ typedef Uint dsize_t; /* Vector size type */ #endif -int big_decimal_estimate(Eterm); +int big_decimal_estimate(Wterm); Eterm erts_big_to_list(Eterm, Eterm**); -char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz); +char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz); Eterm small_times(Sint, Sint, Eterm*); -Eterm big_plus(Eterm, Eterm, Eterm*); +Eterm big_plus(Wterm, Wterm, Eterm*); Eterm big_minus(Eterm, Eterm, Eterm*); Eterm big_times(Eterm, Eterm, Eterm*); Eterm big_div(Eterm, Eterm, Eterm*); @@ -131,11 +137,12 @@ Eterm big_bxor(Eterm, Eterm, Eterm*); Eterm big_bnot(Eterm, Eterm*); Eterm big_lshift(Eterm, Sint, Eterm*); -int big_comp (Eterm, Eterm); +int big_comp (Wterm, Wterm); int big_ucomp (Eterm, Eterm); -int big_to_double(Eterm x, double* resp); +int big_to_double(Wterm x, double* resp); Eterm small_to_big(Sint, Eterm*); Eterm uint_to_big(Uint, Eterm*); +Eterm uword_to_big(UWord, Eterm*); Eterm erts_make_integer(Uint, Process *); dsize_t big_bytes(Eterm); @@ -143,7 +150,12 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*); byte* big_to_bytes(Eterm, byte*); int term_to_Uint(Eterm, Uint*); +int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); +#if HAVE_INT64 +int term_to_Uint64(Eterm, Uint64*); +int term_to_Sint64(Eterm, Sint64*); +#endif Uint32 big_to_uint32(Eterm b); int term_equals_2pow32(Eterm); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 08c64610a2..1fb39c6c67 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -32,17 +32,17 @@ #include "erl_bits.h" #ifdef DEBUG -static int list_to_bitstr_buf(Eterm obj, char* buf, int len); +static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len); #else static int list_to_bitstr_buf(Eterm obj, char* buf); #endif -static Sint bitstr_list_len(Eterm obj); +static int bitstr_list_len(Eterm obj, Uint* num_bytes); void erts_init_binary(void) { /* Verify Binary alignment... */ - if ((((Uint) &((Binary *) 0)->orig_bytes[0]) % ((Uint) 8)) != 0) { + if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) { /* I assume that any compiler should be able to optimize this away. If not, this test is not very expensive... */ erl_exit(ERTS_ABORT_EXIT, @@ -56,7 +56,7 @@ erts_init_binary(void) */ Eterm -new_binary(Process *p, byte *buf, int len) +new_binary(Process *p, byte *buf, Uint len) { ProcBin* pb; Binary* bptr; @@ -88,8 +88,8 @@ new_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len) /* * Miscellanous updates. Return the tagged binary. */ - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); } @@ -127,8 +127,8 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) /* * Miscellanous updates. Return the tagged binary. */ - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); } @@ -180,7 +180,7 @@ erts_realloc_binary(Eterm bin, size_t size) } byte* -erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) +erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t allocator, unsigned extra) { byte* bytes; Eterm* real_bin; @@ -208,7 +208,7 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) bytes = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + offs; } if (bit_offs) { - byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, byte_size + extra); + byte* buf = (byte *) erts_alloc(allocator, byte_size + extra); *base_ptr = buf; buf += extra; erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, byte_size*8); @@ -217,8 +217,8 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) return bytes; } -static Eterm -bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs) +Eterm +erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs) { if (bitoffs == 0) { while (size) { @@ -263,7 +263,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) Eterm* hp = HAlloc(BIF_P, 2 * size); byte* bytes = binary_bytes(real_bin)+offset; - BIF_RET(bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); } error: @@ -295,7 +295,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) } i = stop-start+1; hp = HAlloc(BIF_P, 2*i); - BIF_RET(bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); error: BIF_ERROR(BIF_P, BADARG); @@ -339,36 +339,50 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) previous = CONS(hp, make_binary(last), previous); hp += 2; } - BIF_RET(bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); } /* Turn a possibly deep list of ints (and binaries) into */ /* One large binary object */ -BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) +/* + * This bif also exists in the binary module, under the name + * binary:list_to_bin/1, why it's divided into interface and + * implementation. Also the backend for iolist_to_binary_1. + */ + +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; - int i; + Uint size; int offset; byte* bytes; - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); + + if (is_nil(arg)) { + BIF_RET(new_binary(p,(byte*)"",0)); } - if (is_not_list(BIF_ARG_1)) { + if (is_not_list(arg)) { goto error; } - if ((i = io_list_len(BIF_ARG_1)) < 0) { - goto error; + switch (erts_iolist_size(arg, &size)) { + case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT); + case ERTS_IOLIST_TYPE: goto error; + default: ; } - bin = new_binary(BIF_P, (byte *)NULL, i); + bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); - offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i); + offset = io_list_to_buf(arg, (char*) bytes, size); ASSERT(offset == 0); BIF_RET(bin); - error: - BIF_ERROR(BIF_P, BADARG); + error: + BIF_ERROR(p, BADARG); +} + +BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) +{ + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); } /* Turn a possibly deep list of ints (and binaries) into */ @@ -376,37 +390,17 @@ BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { - Eterm bin; - int i; - int offset; - byte* bytes; - if (is_binary(BIF_ARG_1)) { BIF_RET(BIF_ARG_1); } - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); - } - if (is_not_list(BIF_ARG_1)) { - goto error; - } - if ((i = io_list_len(BIF_ARG_1)) < 0) { - goto error; - } - bin = new_binary(BIF_P, (byte *)NULL, i); - bytes = binary_bytes(bin); - offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i); - ASSERT(offset == 0); - BIF_RET(bin); - - error: - BIF_ERROR(BIF_P, BADARG); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); } BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) { Eterm bin; - int i,offset; + Uint sz; + int offset; byte* bytes; ErlSubBin* sb1; Eterm* hp; @@ -415,15 +409,19 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) BIF_RET(new_binary(BIF_P,(byte*)"",0)); } if (is_not_list(BIF_ARG_1)) { - goto error; + error: + BIF_ERROR(BIF_P, BADARG); } - if ((i = bitstr_list_len(BIF_ARG_1)) < 0) { + switch (bitstr_list_len(BIF_ARG_1, &sz)) { + case ERTS_IOLIST_TYPE: goto error; + case ERTS_IOLIST_OVERFLOW: + BIF_ERROR(BIF_P, SYSTEM_LIMIT); } - bin = new_binary(BIF_P, (byte *)NULL, i); + bin = new_binary(BIF_P, (byte *)NULL, sz); bytes = binary_bytes(bin); #ifdef DEBUG - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, i); + offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz); #else offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes); #endif @@ -432,20 +430,16 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE); sb1 = (ErlSubBin *) hp; sb1->thing_word = HEADER_SUB_BIN; - sb1->size = i-1; + sb1->size = sz-1; sb1->offs = 0; sb1->orig = bin; sb1->bitoffs = 0; sb1->bitsize = offset; sb1->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; bin = make_binary(sb1); } BIF_RET(bin); - - error: - BIF_ERROR(BIF_P, BADARG); } BIF_RETTYPE split_binary_2(BIF_ALIST_2) @@ -497,16 +491,6 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } -void -erts_cleanup_mso(ProcBin* pb) -{ - while (pb != NULL) { - ProcBin* next = pb->next; - if (erts_refc_dectest(&pb->val->refc, 0) == 0) - erts_bin_free(pb->val); - pb = next; - } -} /* * Local functions. @@ -519,7 +503,7 @@ erts_cleanup_mso(ProcBin* pb) */ static int #ifdef DEBUG -list_to_bitstr_buf(Eterm obj, char* buf, int len) +list_to_bitstr_buf(Eterm obj, char* buf, Uint len) #else list_to_bitstr_buf(Eterm obj, char* buf) #endif @@ -622,8 +606,8 @@ list_to_bitstr_buf(Eterm obj, char* buf) return offset; } -static Sint -bitstr_list_len(Eterm obj) +static int +bitstr_list_len(Eterm obj, Uint* num_bytes) { Eterm* objp; Uint len = 0; @@ -631,6 +615,26 @@ bitstr_list_len(Eterm obj) DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + +#define SAFE_ADD_BITSIZE(Var, Bin) \ + do { \ + if (*binary_val(Bin) == HEADER_SUB_BIN) { \ + Uint valvar = ((ErlSubBin *) binary_val(Bin))->bitsize; \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -641,9 +645,12 @@ bitstr_list_len(Eterm obj) obj = CAR(objp); if (is_byte(obj)) { len++; + if (len == 0) { + goto L_overflow_error; + } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -655,23 +662,44 @@ bitstr_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj)) { - len += binary_size(obj); - offs += binary_bitsize(obj); + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD +#undef SAFE_ADD_BITSIZE DESTROY_ESTACK(s); - return (Sint) (len + (offs/8) + ((offs % 8) != 0)); + + /* + * Make sure that the number of bits in the bitstring will fit + * in an Uint to ensure that the binary can be matched using + * the binary syntax. + */ + if (len << 3 < len) { + goto L_overflow_error; + } + len += (offs >> 3) + ((offs & 7) != 0); + if (len << 3 < len) { + goto L_overflow_error; + } + *num_bytes = len; + return ERTS_IOLIST_OK; L_type_error: DESTROY_ESTACK(s); - return (Sint) -1; + return ERTS_IOLIST_TYPE; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; } + diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index cc69977b79..b8889e6206 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* This File contains functions which are called if a user hits ^C */ @@ -38,10 +38,6 @@ #include "erl_instrument.h" #include "erl_bif_timer.h" -#ifdef _OSE_ -#include "time.h" -#endif - /* Forward declarations -- should really appear somewhere else */ static void process_killer(void); void do_break(void); @@ -102,7 +98,7 @@ process_killer(void) switch(j) { case 'k': if (rp->status == P_WAITING) { - Uint32 rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; erts_smp_proc_inc_refc(rp); erts_smp_proc_lock(rp, rp_locks); (void) erts_send_exit_signal(NULL, @@ -262,17 +258,15 @@ print_process_info(int to, void *to_arg, Process *p) } { - long s = 0; int frags = 0; ErlHeapFragment *m = p->mbuf; while (m != NULL) { frags++; - s += m->size; m = m->next; } erts_print(to, to_arg, "Number of heap fragments: %d\n", frags); } - erts_print(to, to_arg, "Heap fragment data: %bpu\n", MBUF_SIZE(p)); + erts_print(to, to_arg, "Heap fragment data: %beu\n", MBUF_SIZE(p)); scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); if (scb) { @@ -319,15 +313,14 @@ print_process_info(int to, void *to_arg, Process *p) } /* print the number of reductions etc */ - erts_print(to, to_arg, "Reductions: %bpu\n", p->reds); + erts_print(to, to_arg, "Reductions: %beu\n", p->reds); - erts_print(to, to_arg, "Stack+heap: %bpu\n", p->heap_sz); + erts_print(to, to_arg, "Stack+heap: %beu\n", p->heap_sz); erts_print(to, to_arg, "OldHeap: %bpu\n", - (OLD_HEAP(p) == NULL) ? 0 : - (unsigned)(OLD_HEND(p) - OLD_HEAP(p)) ); + (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HEAP(p)) ); erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", - (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HEAP(p)) ); + (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); if (garbing) { print_garb_info(to, to_arg, p); @@ -381,7 +374,7 @@ loaded(int to, void *to_arg) int i; int old = 0; int cur = 0; - Eterm* code; + BeamInstr* code; /* * Calculate and print totals. @@ -564,7 +557,7 @@ do_break(void) #endif #ifdef DEBUG case 't': - p_slpq(); + erts_p_slpq(); return; case 'b': bin_check(); @@ -617,29 +610,29 @@ static void bin_check(void) { Process *rp; - ProcBin *bp; - int i, printed; + struct erl_off_heap_header* hdr; + int i, printed = 0; for (i=0; i < erts_max_processes; i++) { if ((rp = process_tab[i]) == NULL) continue; - if (!(bp = rp->off_heap.mso)) - continue; - printed = 0; - while (bp) { - if (printed == 0) { - erts_printf("Process %T holding binary data \n", rp->id); - printed = 1; + for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) { + if (hdr->thing_word == HEADER_PROC_BIN) { + ProcBin *bp = (ProcBin*) hdr; + if (!printed) { + erts_printf("Process %T holding binary data \n", rp->id); + printed = 1; + } + erts_printf("%p orig_size: %bpd, norefs = %bpd\n", + bp->val, + bp->val->orig_size, + erts_smp_atomic_read(&bp->val->refc)); } - erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n", - (unsigned long)bp->val, - (long)bp->val->orig_size, - erts_smp_atomic_read(&bp->val->refc)); - - bp = bp->next; } - if (printed == 1) + if (printed) { erts_printf("--------------------------------------\n"); + printed = 0; + } } /* db_bin_check() has to be rewritten for the AVL trees... */ /*db_bin_check();*/ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 0a5050b1fe..90201f3a90 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -37,6 +37,8 @@ MA_STACK_DECLARE(dst); MA_STACK_DECLARE(offset); #endif +static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*); + void init_copy(void) { @@ -70,8 +72,11 @@ copy_object(Eterm obj, Process* to) * Return the "flat" size of the object. */ -Uint -size_object(Eterm obj) +#if HALFWORD_HEAP +Uint size_object_rel(Eterm obj, Eterm* base) +#else +Uint size_object(Eterm obj) +#endif { Uint sum = 0; Eterm* ptr; @@ -82,24 +87,24 @@ size_object(Eterm obj) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: sum += 2; - ptr = list_val(obj); + ptr = list_val_rel(obj,base); obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); - } + } obj = *ptr; break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(obj); + Eterm hdr = *boxed_val_rel(obj,base); ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: - ptr = tuple_val(obj); + ptr = tuple_val_rel(obj,base); arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ - goto size_common; + goto pop_next; } while (arity-- > 1) { obj = *++ptr; @@ -111,11 +116,10 @@ size_object(Eterm obj) break; case FUN_SUBTAG: { - Eterm* bptr = fun_val(obj); + Eterm* bptr = fun_val_rel(obj,base); ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); - sum += 1 /* header */ + sz + eterms; bptr += 1 /* header */ + sz; while (eterms-- > 1) { @@ -135,7 +139,7 @@ size_object(Eterm obj) Uint bitoffs; Uint extra_bytes; Eterm hdr; - ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base); if ((bitsize + bitoffs) > 8) { sum += ERL_SUB_BIN_SIZE; extra_bytes = 2; @@ -145,13 +149,13 @@ size_object(Eterm obj) } else { extra_bytes = 0; } - hdr = *binary_val(real_bin); + hdr = *binary_val_rel(real_bin,base); if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { - sum += heap_bin_size(binary_size(obj)+extra_bytes); + sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes); } - goto size_common; + goto pop_next; } break; case BIN_MATCHSTATE_SUBTAG: @@ -159,18 +163,12 @@ size_object(Eterm obj) "size_object: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; - /* Fall through */ - size_common: - if (ESTACK_ISEMPTY(s)) { - DESTROY_ESTACK(s); - return sum; - } - obj = ESTACK_POP(s); - break; + goto pop_next; } } break; case TAG_PRIMARY_IMMED1: + pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); return sum; @@ -186,8 +184,12 @@ size_object(Eterm obj) /* * Copy a structure to a heap. */ -Eterm -copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#if HALFWORD_HEAP +Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, + Eterm* src_base, Eterm* dst_base) +#else +Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#endif { char* hstart; Uint hsize; @@ -219,7 +221,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) /* Copy the object onto the heap */ switch (primary_tag(obj)) { - case TAG_PRIMARY_LIST: argp = &res; goto L_copy_list; + case TAG_PRIMARY_LIST: + argp = &res; + objp = list_val_rel(obj,src_base); + goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: erl_exit(ERTS_ABORT_EXIT, @@ -236,32 +241,46 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) hp++; break; case TAG_PRIMARY_LIST: - objp = list_val(obj); + objp = list_val_rel(obj,src_base); + #if !HALFWORD_HEAP || defined(DEBUG) if (in_area(objp,hstart,hsize)) { + ASSERT(!HALFWORD_HEAP); hp++; break; } + #endif argp = hp++; /* Fall through */ L_copy_list: tailp = argp; - while (is_list(obj)) { - objp = list_val(obj); + for (;;) { tp = tailp; - elem = *objp; + elem = CAR(objp); if (IS_CONST(elem)) { - *(hbot-2) = elem; - tailp = hbot-1; hbot -= 2; + CAR(hbot) = elem; + tailp = &CDR(hbot); } else { - *htop = elem; - tailp = htop+1; + CAR(htop) = elem; + #if HALFWORD_HEAP + CDR(htop) = CDR(objp); + *tailp = make_list_rel(htop,dst_base); htop += 2; + goto L_copy; + #else + tailp = &CDR(htop); + htop += 2; + #endif + } + ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot); + *tp = make_list_rel(tailp - 1, dst_base); + obj = CDR(objp); + if (!is_list(obj)) { + break; } - *tp = make_list(tailp - 1); - obj = *(objp+1); + objp = list_val_rel(obj,src_base); } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -273,21 +292,24 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } case TAG_PRIMARY_BOXED: - if (in_area(boxed_val(obj),hstart,hsize)) { + #if !HALFWORD_HEAP || defined(DEBUG) + if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) { + ASSERT(!HALFWORD_HEAP); hp++; break; } + #endif argp = hp++; L_copy_boxed: - objp = boxed_val(obj); + objp = boxed_val_rel(obj, src_base); hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int const_flag = 1; /* assume constant tuple */ i = arityval(hdr); - *argp = make_tuple(htop); + *argp = make_tuple_rel(htop, dst_base); tp = htop; /* tp is pointer to new arity value */ *htop++ = *objp++; /* copy arity value */ while (i--) { @@ -316,13 +338,13 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) while (i--) { *tp++ = *objp++; } - *argp = make_binary(hbot); + *argp = make_binary_rel(hbot, dst_base); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; + pb->next = off_heap->first; pb->flags = 0; - off_heap->mso = pb; - off_heap->overhead += pb->size / sizeof(Eterm); + off_heap->first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } break; case SUB_BINARY_SUBTAG: @@ -343,7 +365,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) extra_bytes = 0; } real_size = size+extra_bytes; - objp = binary_val(real_bin); + objp = binary_val_rel(real_bin,src_base); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { ErlHeapBin* from = (ErlHeapBin *) objp; ErlHeapBin* to; @@ -368,12 +390,12 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) to->val = from->val; erts_refc_inc(&to->val->refc, 2); to->bytes = from->bytes + offset; - to->next = off_heap->mso; + to->next = off_heap->first; to->flags = 0; - off_heap->mso = to; - off_heap->overhead += to->size / sizeof(Eterm); + off_heap->first = (struct erl_off_heap_header*) to; + OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } - *argp = make_binary(hbot); + *argp = make_binary_rel(hbot, dst_base); if (extra_bytes != 0) { ErlSubBin* res; hbot -= ERL_SUB_BIN_SIZE; @@ -385,7 +407,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) res->offs = 0; res->is_writable = 0; res->orig = *argp; - *argp = make_binary(hbot); + *argp = make_binary_rel(hbot, dst_base); } break; } @@ -401,11 +423,11 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #ifndef HYBRID /* FIND ME! */ funp = (ErlFunThing *) tp; - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); #endif - *argp = make_fun(tp); + *argp = make_fun_rel(tp, dst_base); } break; case EXTERNAL_PID_SUBTAG: @@ -421,11 +443,11 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *htop++ = *objp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); - *argp = make_external(tp); + *argp = make_external_rel(tp, dst_base); } break; case BIN_MATCHSTATE_SUBTAG: @@ -435,7 +457,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) i = thing_arityval(hdr)+1; hbot -= i; tp = hbot; - *argp = make_boxed(hbot); + *argp = make_boxed_rel(hbot, dst_base); while (i--) { *tp++ = *objp++; } @@ -455,7 +477,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) if (htop != hbot) erl_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" - " htop=%p != hbot=%p (sz=%bpu)\n", + " htop=%p != hbot=%p (sz=%beu)\n", org_obj, htop, hbot, org_sz); #else if (htop > hbot) { @@ -655,9 +677,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } erts_refc_inc(&pb->val->refc, 2); - pb->next = erts_global_offheap.mso; - erts_global_offheap.mso = pb; - erts_global_offheap.overhead += pb->size / sizeof(Eterm); + pb->next = erts_global_offheap.first; + erts_global_offheap.first = pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); continue; } @@ -677,9 +699,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) while (i--) { *hp++ = *objp++; } -#ifndef HYBRID // FIND ME! - funp->next = erts_global_offheap.funs; - erts_global_offheap.funs = funp; +#ifndef HYBRID /* FIND ME! */ + funp->next = erts_global_offheap.first; + erts_global_offheap.first = funp; erts_refc_inc(&funp->fe->refc, 2); #endif for (i = k; i < j; i++) { @@ -723,8 +745,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } - etp->next = erts_global_offheap.externals; - erts_global_offheap.externals = etp; + etp->next = erts_global_offheap.first; + erts_global_offheap.first = etp; erts_refc_inc(&etp->node->refc, 2); continue; } @@ -780,9 +802,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) to_bin->size = real_size; to_bin->val = from_bin->val; to_bin->bytes = from_bin->bytes + sub_offset; - to_bin->next = erts_global_offheap.mso; - erts_global_offheap.mso = to_bin; - erts_global_offheap.overhead += to_bin->size / sizeof(Eterm); + to_bin->next = erts_global_offheap.first; + erts_global_offheap.first = to_bin; + OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm)); res_binary=make_binary(to_bin); hp += PROC_BIN_SIZE; } @@ -890,12 +912,21 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm -copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#if HALFWORD_HEAP +Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, + Eterm* src_base) +#else +Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#endif { Eterm* tp = ptr; Eterm* hp = *hpp; - Sint offs = hp - tp; + const Eterm res = make_tuple(hp); +#if HALFWORD_HEAP + const Sint offs = COMPRESS_POINTER(hp - (tp - src_base)); +#else + const Sint offs = (hp - tp) * sizeof(Eterm); +#endif while (sz--) { Eterm val = *tp++; @@ -906,7 +937,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - *hp++ = offset_ptr(val, offs); + *hp++ = byte_offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: *hp++ = val; @@ -915,57 +946,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case REFC_BINARY_SUBTAG: { - ProcBin* pb = (ProcBin *) (hp-1); - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } + ProcBin* pb = (ProcBin *) (tp-1); erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; - off_heap->mso = pb; - off_heap->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } - break; + goto off_heap_common; + case FUN_SUBTAG: { -#ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp = (ErlFunThing *) (hp-1); -#endif - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } -#ifndef HYBRID /* FIND ME! */ - funp->next = off_heap->funs; - off_heap->funs = funp; + ErlFunThing* funp = (ErlFunThing *) (tp-1); erts_refc_inc(&funp->fe->refc, 2); -#endif } - break; + goto off_heap_common; + case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: { - ExternalThing* etp = (ExternalThing *) (hp-1); + ExternalThing* etp = (ExternalThing *) (tp-1); + erts_refc_inc(&etp->node->refc, 2); + } + off_heap_common: + { + struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1); int tari = thing_arityval(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; - erts_refc_inc(&etp->node->refc, 2); + ohh->next = off_heap->first; + off_heap->first = ohh; } break; default: { int tari = header_arity(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; @@ -977,5 +994,95 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } *hpp = hp; - return make_tuple(ptr + offs); + + return res; +} + +/* Move all terms in heap fragments into heap. The terms must be guaranteed to + * be contained within the fragments. The source terms are destructed with + * move markers. + * Typically used to copy a multi-fragmented message (from NIF). + */ +void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs) +{ + ErlHeapFragment* bp; + Eterm* hp_start = *hpp; + Eterm* hp_end; + Eterm* hp; + unsigned i; + + for (bp=first; bp!=NULL; bp=bp->next) { + move_one_frag(hpp, bp->mem, bp->used_size, off_heap); + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + } + hp_end = *hpp; + for (hp=hp_start; hp<hp_end; ++hp) { + Eterm* ptr; + Eterm val; + Eterm gval = *hp; + switch (primary_tag(gval)) { + case TAG_PRIMARY_BOXED: + ptr = boxed_val(gval); + val = *ptr; + if (IS_MOVED_BOXED(val)) { + ASSERT(is_boxed(val)); + *hp = val; + } + break; + case TAG_PRIMARY_LIST: + ptr = list_val(gval); + val = *ptr; + if (IS_MOVED_CONS(val)) { + *hp = ptr[1]; + } + break; + case TAG_PRIMARY_HEADER: + if (header_is_thing(gval)) { + hp += thing_arityval(gval); + } + break; + } + } + for (i=0; i<nrefs; ++i) { + refs[i] = follow_moved(refs[i]); + } +} + +static void +move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) +{ + Eterm* ptr = src; + Eterm* end = ptr + src_sz; + Eterm dummy_ref; + Eterm* hp = *hpp; + + while (ptr != end) { + Eterm val; + ASSERT(ptr < end); + val = *ptr; + ASSERT(val != ERTS_HOLE_MARKER); + if (is_header(val)) { + struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; + ASSERT(ptr + header_arity(val) < end); + MOVE_BOXED(ptr, val, hp, &dummy_ref); + switch (val & _HEADER_SUBTAG_MASK) { + case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + hdr->next = off_heap->first; + off_heap->first = hdr; + break; + } + } + else { /* must be a cons cell */ + ASSERT(ptr+1 < end); + MOVE_CONS(ptr, val, hp, &dummy_ref); + ptr += 2; + } + } + *hpp = hp; } + diff --git a/erts/emulator/beam/decl.h b/erts/emulator/beam/decl.h deleted file mode 100644 index da1be29d53..0000000000 --- a/erts/emulator/beam/decl.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifndef __DECL_H__ -#define __DECL_H__ - -#if defined(__STDC__) || defined(_MSC_VER) -#define EXTERN_FUNCTION(t, f, x) extern t f x -#define FUNCTION(t, f, x) t f x -#define _DOTS_ ... -#define _VOID_ void -#elif defined(__cplusplus) -#define EXTERN_FUNCTION(f, x) extern "C" { f x } -#define FUNCTION(t, f, x) t f x -#define _DOTS_ ... -#define _VOID_ void -#else -#define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/) -#define FUNCTION(t, f, x) t f (/*x*/) -#define _DOTS_ -#define _VOID_ -#endif - -/* -** Example of declarations -** -** EXTERN_FUNCTION(void, foo, (int, int, char)); -** FUNCTION(void, bar, (int, char)); -** -** struct funcs { -** FUNCTION(int*, (*f1), (int, int)); -** FUNCTION(void, (*f2), (int, char)); -** FUNCTION(void, (*f3), (_VOID_)); -** FUNCTION(int, (*f4), (char*, _DOTS_)); -** }; -** -*/ - -#endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index e3094404e2..b1cdd0660a 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -97,6 +97,8 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) #define PASS_THROUGH 'p' /* This code should go */ int erts_is_alive; /* System must be blocked on change */ +int erts_dist_buf_busy_limit; + /* distribution trap functions */ Export* dsend2_trap = NULL; @@ -160,7 +162,7 @@ Uint erts_dist_cache_size(void) static ErtsProcList * get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&dep->qlock)); + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock)); dep->qflgs &= ~unset_qflgs; if (dep->qflgs & ERTS_DE_QFLG_EXIT) { /* No resume when exit has been scheduled */ @@ -228,6 +230,7 @@ int is_node_name_atom(Eterm a) typedef struct { DistEntry *dep; + Eterm *lhp; } NetExitsContext; /* @@ -253,8 +256,9 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) erts_destroy_monitor(rmon); } } else { - Eterm lhp[3]; + DeclareTmpHeapNoproc(lhp,3); Eterm watched; + UseTmpHeapNoproc(3); ASSERT(mon->type == MON_TARGET); rmon = erts_remove_monitor(&(rp->monitors),mon->ref); /* ASSERT(rmon != NULL); can happen during process exit */ @@ -271,6 +275,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) watched, am_noconnection); erts_destroy_monitor(rmon); } + UnUseTmpHeapNoproc(3); } erts_smp_proc_unlock(rp, rp_locks); done: @@ -450,17 +455,17 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) if (dep->status & ERTS_DE_SFLG_EXITING) { #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif } else { dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } erts_smp_de_links_lock(dep); @@ -574,7 +579,7 @@ static void clear_dist_entry(DistEntry *dep) erts_smp_de_links_unlock(dep); #endif - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (!dep->out_queue.last) obuf = dep->finalized_out_queue.first; @@ -590,7 +595,7 @@ static void clear_dist_entry(DistEntry *dep) dep->status = 0; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0); dep->send = NULL; erts_smp_de_rwunlock(dep); @@ -608,10 +613,10 @@ static void clear_dist_entry(DistEntry *dep) } if (obufsize) { - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } } @@ -632,19 +637,27 @@ static void clear_dist_entry(DistEntry *dep) int erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote) { - Eterm ctl_heap[4]; + DeclareTmpHeapNoproc(ctl_heap,4); Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_LINK), local, remote); + int res; + UseTmpHeapNoproc(4); - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(4); + return res; } int erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote) { - Eterm ctl_heap[4]; + DeclareTmpHeapNoproc(ctl_heap,4); Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_UNLINK), local, remote); + int res; - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UseTmpHeapNoproc(4); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(4); + return res; } @@ -656,7 +669,10 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref, Eterm reason) { Eterm ctl; - Eterm ctl_heap[6]; + DeclareTmpHeapNoproc(ctl_heap,6); + int res; + + UseTmpHeapNoproc(6); ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT), watched, watcher, ref, reason); @@ -667,7 +683,9 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, erts_smp_de_links_unlock(dsdp->dep); #endif - return dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + UnUseTmpHeapNoproc(6); + return res; } /* We want to monitor a process (named or unnamed) on another node, we send: @@ -678,13 +696,17 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref) { Eterm ctl; - Eterm ctl_heap[5]; + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + UseTmpHeapNoproc(5); ctl = TUPLE4(&ctl_heap[0], make_small(DOP_MONITOR_P), watcher, watched, ref); - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(5); + return res; } /* A local process monitoring a remote one wants to stop monitoring, either @@ -696,23 +718,29 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, Eterm ref, int force) { Eterm ctl; - Eterm ctl_heap[5]; + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + UseTmpHeapNoproc(5); ctl = TUPLE4(&ctl_heap[0], make_small(DOP_DEMONITOR_P), watcher, watched, ref); - return dsig_send(dsdp, ctl, THE_NON_VALUE, force); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, force); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) { Eterm ctl; - Eterm ctl_heap[5]; + DeclareTmpHeapNoproc(ctl_heap,5); Eterm token = NIL; Process *sender = dsdp->proc; + int res; + UseTmpHeapNoproc(5); if (SEQ_TRACE_TOKEN(sender) != NIL) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); @@ -724,17 +752,21 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) make_small(DOP_SEND_TT), am_Cookie, remote, token); else ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); - return dsig_send(dsdp, ctl, message, 0); + res = dsig_send(dsdp, ctl, message, 0); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) { Eterm ctl; - Eterm ctl_heap[6]; + DeclareTmpHeapNoproc(ctl_heap,6); Eterm token = NIL; Process *sender = dsdp->proc; + int res; + UseTmpHeapNoproc(6); if (SEQ_TRACE_TOKEN(sender) != NIL) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); @@ -747,7 +779,9 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) else ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND), sender->id, am_Cookie, remote_name); - return dsig_send(dsdp, ctl, message, 0); + res = dsig_send(dsdp, ctl, message, 0); + UnUseTmpHeapNoproc(6); + return res; } /* local has died, deliver the exit signal to remote */ @@ -756,8 +790,10 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason, Eterm token) { Eterm ctl; - Eterm ctl_heap[6]; + DeclareTmpHeapNoproc(ctl_heap,6); + int res; + UseTmpHeapNoproc(6); if (token != NIL) { seq_trace_update_send(dsdp->proc); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); @@ -767,38 +803,58 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); } /* forced, i.e ignore busy */ - return dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + UnUseTmpHeapNoproc(6); + return res; } int erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) { - Eterm ctl_heap[5]; - Eterm ctl = TUPLE4(&ctl_heap[0], - make_small(DOP_EXIT), local, remote, reason); + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + Eterm ctl; + + UseTmpHeapNoproc(5); + ctl = TUPLE4(&ctl_heap[0], + make_small(DOP_EXIT), local, remote, reason); /* forced, i.e ignore busy */ - return dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) { - Eterm ctl_heap[5]; - Eterm ctl = TUPLE4(&ctl_heap[0], - make_small(DOP_EXIT2), local, remote, reason); + DeclareTmpHeapNoproc(ctl_heap,5); + int res; + Eterm ctl; + + UseTmpHeapNoproc(5); + ctl = TUPLE4(&ctl_heap[0], + make_small(DOP_EXIT2), local, remote, reason); - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(5); + return res; } int erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) { - Eterm ctl_heap[4]; - Eterm ctl = TUPLE3(&ctl_heap[0], - make_small(DOP_GROUP_LEADER), leader, remote); + DeclareTmpHeapNoproc(ctl_heap,4); + int res; + Eterm ctl; + + UseTmpHeapNoproc(4); + ctl = TUPLE3(&ctl_heap[0], + make_small(DOP_GROUP_LEADER), leader, remote); - return dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + UnUseTmpHeapNoproc(4); + return res; } #if defined(PURIFY) @@ -808,11 +864,15 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) #include <valgrind/valgrind.h> #include <valgrind/memcheck.h> +#ifndef HAVE_VALGRIND_PRINTF_XML +#define VALGRIND_PRINTF_XML VALGRIND_PRINTF +#endif + # define PURIFY_MSG(msg) \ do { \ char buf__[1]; size_t bufsz__ = sizeof(buf__); \ if (erts_sys_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \ - VALGRIND_PRINTF("<erlang_error_log>" \ + VALGRIND_PRINTF_XML("<erlang_error_log>" \ "%s, line %d: %s</erlang_error_log>\n", \ __FILE__, __LINE__, msg); \ } else { \ @@ -832,6 +892,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) ** ** assert hlen == 0 !!! */ + int erts_net_message(Port *prt, DistEntry *dep, byte *hbuf, @@ -839,10 +900,10 @@ int erts_net_message(Port *prt, byte *buf, int len) { +#define DIST_CTL_DEFAULT_SIZE 64 ErtsDistExternal ede; byte *t; Sint ctl_len; - int orig_ctl_len; Eterm arg; Eterm from, to; Eterm watcher, watched; @@ -850,7 +911,7 @@ int erts_net_message(Port *prt, Eterm *tuple; Eterm reason; Process* rp; - Eterm ctl_default[64]; + DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; ErlOffHeap off_heap; Eterm* hp; @@ -859,29 +920,31 @@ int erts_net_message(Port *prt, Eterm token_size; ErtsMonitor *mon; ErtsLink *lnk; + Uint tuple_arity; int res; #ifdef ERTS_DIST_MSG_DBG int orig_len = len; #endif + UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); /* Thanks to Luke Gorrie */ - off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - off_heap.funs = NULL; -#endif + off_heap.first = NULL; off_heap.overhead = 0; - off_heap.externals = NULL; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (!erts_is_alive) + if (!erts_is_alive) { + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; + } if (hlen > 0) goto data_error; - if (len == 0) /* HANDLE TICK !!! */ + if (len == 0) { /* HANDLE TICK !!! */ + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; + } #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, "<< "); @@ -921,8 +984,8 @@ int erts_net_message(Port *prt, PURIFY_MSG("data error"); goto data_error; } - orig_ctl_len = ctl_len; - if (ctl_len > sizeof(ctl_default)/sizeof(ctl_default[0])) { + + if (ctl_len > DIST_CTL_DEFAULT_SIZE) { ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm)); } hp = ctl; @@ -943,29 +1006,23 @@ int erts_net_message(Port *prt, #endif if (is_not_tuple(arg) || - (tuple = tuple_val(arg), arityval(*tuple) < 1) || + (tuple = tuple_val(arg), (tuple_arity = arityval(*tuple)) < 1) || is_not_small(tuple[1])) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + goto invalid_message; } token_size = 0; switch (type = unsigned_val(tuple[1])) { case DOP_LINK: + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; to = tuple[3]; /* local proc to link to */ if (is_not_pid(from) || is_not_pid(to)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - PURIFY_MSG("data error"); - erts_dsprintf(dsbufp, - "Invalid DOP_LINK distribution message: %.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + goto invalid_message; } rp = erts_pid2proc_opt(NULL, 0, @@ -1004,8 +1061,14 @@ int erts_net_message(Port *prt, case DOP_UNLINK: { ErtsDistLinkData dld; + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; to = tuple[3]; + if (is_not_pid(from) || is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc_opt(NULL, 0, to, ERTS_PROC_LOCK_LINK, @@ -1032,11 +1095,19 @@ int erts_net_message(Port *prt, /* A remote process wants to monitor us, we get: {DOP_MONITOR_P, Remote pid, local pid or name, ref} */ Eterm name; + + if (tuple_arity != 4) { + goto invalid_message; + } watcher = tuple[2]; watched = tuple[3]; /* local proc to monitor */ ref = tuple[4]; + if (is_not_ref(ref)) { + goto invalid_message; + } + if (is_atom(watched)) { name = watched; rp = erts_whereis_process(NULL, 0, @@ -1078,10 +1149,17 @@ int erts_net_message(Port *prt, We get {DOP_DEMONITOR_P, Remote pid, Local pid or name, ref}, We need only the ref of course */ + if (tuple_arity != 4) { + goto invalid_message; + } /* watcher = tuple[2]; */ /* watched = tuple[3]; May be an atom in case of monitor name */ ref = tuple[4]; + if(is_not_ref(ref)) { + goto invalid_message; + } + erts_smp_de_links_lock(dep); mon = erts_remove_monitor(&(dep->monitors),ref); erts_smp_de_links_unlock(dep); @@ -1106,10 +1184,11 @@ int erts_net_message(Port *prt, erts_destroy_monitor(mon); break; - case DOP_NODE_LINK: /* XXX never sent ?? */ - break; - case DOP_REG_SEND_TT: + if (tuple_arity != 5) { + goto invalid_message; + } + token_size = size_object(tuple[5]); /* Fall through ... */ case DOP_REG_SEND: @@ -1120,12 +1199,19 @@ int erts_net_message(Port *prt, * There is intentionally no testing of the cookie (it is always '') * from R9B and onwards. */ + if (type != DOP_REG_SEND_TT && tuple_arity != 4) { + goto invalid_message; + } + #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif from = tuple[2]; to = tuple[4]; + if (is_not_pid(from) || is_not_atom(to)){ + goto invalid_message; + } rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); if (rp) { Uint xsize = (type == DOP_REG_SEND @@ -1157,6 +1243,10 @@ int erts_net_message(Port *prt, break; case DOP_SEND_TT: + if (tuple_arity != 4) { + goto invalid_message; + } + token_size = size_object(tuple[4]); /* Fall through ... */ case DOP_SEND: @@ -1167,8 +1257,13 @@ int erts_net_message(Port *prt, #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif - + if (type != DOP_SEND_TT && tuple_arity != 3) { + goto invalid_message; + } to = tuple[3]; + if (is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); if (rp) { Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); @@ -1202,15 +1297,23 @@ int erts_net_message(Port *prt, {DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */ - Eterm lhp[3]; + DeclareTmpHeapNoproc(lhp,3); Eterm sysname; ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK; + if (tuple_arity != 5) { + goto invalid_message; + } + /* watched = tuple[2]; */ /* remote proc which died */ /* watcher = tuple[3]; */ ref = tuple[4]; reason = tuple[5]; + if(is_not_ref(ref)) { + goto invalid_message; + } + erts_smp_de_links_lock(dep); sysname = dep->sysname; mon = erts_remove_monitor(&(dep->monitors), ref); @@ -1237,6 +1340,7 @@ int erts_net_message(Port *prt, erts_smp_proc_unlock(rp, rp_locks); break; } + UseTmpHeapNoproc(3); watched = (is_not_nil(mon->name) ? TUPLE2(&lhp[0], mon->name, sysname) @@ -1246,6 +1350,7 @@ int erts_net_message(Port *prt, ref, am_process, watched, reason); erts_smp_proc_unlock(rp, rp_locks); erts_destroy_monitor(mon); + UnUseTmpHeapNoproc(3); break; } @@ -1255,24 +1360,25 @@ int erts_net_message(Port *prt, ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; /* 'from', which 'to' is linked to, died */ if (type == DOP_EXIT) { - from = tuple[2]; - to = tuple[3]; - reason = tuple[4]; - token = NIL; + if (tuple_arity != 4) { + goto invalid_message; + } + + from = tuple[2]; + to = tuple[3]; + reason = tuple[4]; + token = NIL; } else { - from = tuple[2]; - to = tuple[3]; - token = tuple[4]; - reason = tuple[5]; + if (tuple_arity != 5) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + token = tuple[4]; + reason = tuple[5]; } - if (is_not_internal_pid(to)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - PURIFY_MSG("data error"); - erts_dsprintf(dsbufp, - "Invalid DOP_EXIT distribution message: %.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + if (is_not_pid(from) || is_not_internal_pid(to)) { + goto invalid_message; } rp = erts_pid2proc(NULL, 0, to, rp_locks); @@ -1319,15 +1425,24 @@ int erts_net_message(Port *prt, ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; /* 'from' is send an exit signal to 'to' */ if (type == DOP_EXIT2) { - from = tuple[2]; - to = tuple[3]; - reason = tuple[4]; - token = NIL; + if (tuple_arity != 4) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + reason = tuple[4]; + token = NIL; } else { - from = tuple[2]; - to = tuple[3]; - token = tuple[4]; - reason = tuple[5]; + if (tuple_arity != 5) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + token = tuple[4]; + reason = tuple[5]; + } + if (is_not_pid(from) || is_not_internal_pid(to)) { + goto invalid_message; } rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); @@ -1346,10 +1461,14 @@ int erts_net_message(Port *prt, break; } case DOP_GROUP_LEADER: + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; /* Group leader */ to = tuple[3]; /* new member */ - if (is_not_pid(from)) - break; + if (is_not_pid(from) || is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN); if (!rp) @@ -1358,57 +1477,39 @@ int erts_net_message(Port *prt, erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); break; - default: { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Illegal value in distribution dispatch switch: " - "%.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - PURIFY_MSG("data error"); - goto data_error; - } + default: + goto invalid_message; } - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); - } + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } #endif + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); ERTS_SMP_CHK_NO_PROC_LOCKS; return 0; - - data_error: - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); + invalid_message: + { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); + erts_send_error_to_logger_nogl(dsbufp); } + data_error: + PURIFY_MSG("data error"); + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } #endif + UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); erts_do_exit_port(prt, dep->cid, am_killed); ERTS_SMP_CHK_NO_PROC_LOCKS; return -1; } -#define ERTS_DE_BUSY_LIMIT (128*1024) - static int dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) { @@ -1492,18 +1593,18 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) } else { ErtsProcList *plp = NULL; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); dep->qsize += size_obuf(obuf); - if (dep->qsize >= ERTS_DE_BUSY_LIMIT) + if (dep->qsize >= erts_dist_buf_busy_limit) dep->qflgs |= ERTS_DE_QFLG_BUSY; if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); plp = erts_proclist_create(c_p); plp->next = NULL; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); suspended = 1; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); } /* Enqueue obuf on dist entry */ @@ -1529,7 +1630,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) } } - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_schedule_dist_command(NULL, dep); erts_smp_de_runlock(dep); @@ -1554,9 +1655,9 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) */ data_size >>= (10-4); -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP data_size &= 0x003fffffffffffff; -#elif defined(ARCH_32) +#elif defined(ARCH_32) || HALFWORD_HEAP data_size &= 0x003fffff; #else # error "Ohh come on ... !?!" @@ -1586,7 +1687,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) if (size > (Uint) INT_MAX) erl_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " - "(%bpu bytes) passed.\n", + "(%beu bytes) passed.\n", size); prt->caller = NIL; @@ -1613,7 +1714,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) if (size > (Uint) INT_MAX) erl_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " - "(%bpu bytes) passed.\n", + "(%beu bytes) passed.\n", size); iov[0].iov_base = NULL; @@ -1640,9 +1741,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) } -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL -#elif defined(ARCH_32) +#elif defined(ARCH_32) || HALFWORD_HEAP #define ERTS_PORT_REDS_MASK__ 0x003fffff #else # error "Ohh come on ... !?!" @@ -1662,10 +1763,8 @@ erts_dist_command(Port *prt, int reds_limit) { Sint reds = ERTS_PORT_REDS_DIST_CMD_START; int prt_busy; - int de_busy; Uint32 status; Uint32 flags; - Uint32 qflgs; Sint obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = prt->dist_entry; @@ -1700,13 +1799,12 @@ erts_dist_command(Port *prt, int reds_limit) * a mess. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); oq.first = dep->out_queue.first; oq.last = dep->out_queue.last; dep->out_queue.first = NULL; dep->out_queue.last = NULL; - qflgs = dep->qflgs; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); foq.first = dep->finalized_out_queue.first; foq.last = dep->finalized_out_queue.last; @@ -1717,17 +1815,8 @@ erts_dist_command(Port *prt, int reds_limit) goto preempted; prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY); - de_busy = (int) (qflgs & ERTS_DE_QFLG_BUSY); - if (prt_busy) { - if (!de_busy) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = 1; - } - } - else if (foq.first) { + if (!prt_busy && foq.first) { int preempt = 0; do { Uint size; @@ -1745,10 +1834,7 @@ erts_dist_command(Port *prt, int reds_limit) free_dist_obuf(fob); preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = prt_busy = 1; + prt_busy = 1; break; } } while (foq.first && !preempt); @@ -1831,10 +1917,7 @@ erts_dist_command(Port *prt, int reds_limit) free_dist_obuf(fob); preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = prt_busy = 1; + prt_busy = 1; if (oq.first && !preempt) goto finalize_only; } @@ -1861,22 +1944,23 @@ erts_dist_command(Port *prt, int reds_limit) * dist entry in a non-busy state and resume suspended * processes. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; obufsize = 0; - if (de_busy && !prt_busy && dep->qsize < ERTS_DE_BUSY_LIMIT) { + if (!prt_busy + && (dep->qflgs & ERTS_DE_QFLG_BUSY) + && dep->qsize < erts_dist_buf_busy_limit) { ErtsProcList *suspendees; int resumed; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); resumed = erts_resume_processes(suspendees); reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; - de_busy = 0; } else - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } ASSERT(!oq.first && !oq.last); @@ -1885,10 +1969,10 @@ erts_dist_command(Port *prt, int reds_limit) if (obufsize != 0) { ASSERT(obufsize > 0); - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } ASSERT(foq.first || !foq.last); @@ -1938,9 +2022,9 @@ erts_dist_command(Port *prt, int reds_limit) foq.last = NULL; #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize == obufsize); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif } else { @@ -1949,14 +2033,14 @@ erts_dist_command(Port *prt, int reds_limit) * Unhandle buffers need to be put back first * in out_queue. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); dep->qsize -= obufsize; obufsize = 0; oq.last->next = dep->out_queue.first; dep->out_queue.first = oq.first; if (!dep->out_queue.last) dep->out_queue.last = oq.last; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } erts_schedule_dist_command(prt, NULL); @@ -1980,10 +2064,10 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_schedule_dist_command(NULL, dep); } @@ -2354,13 +2438,13 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) ErtsProcList *plp = erts_proclist_create(BIF_P); plp->next = NULL; erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (dep->suspended.last) dep->suspended.last->next = plp; else dep->suspended.first = plp; dep->suspended.last = plp; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); goto yield; } @@ -2388,9 +2472,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) ASSERT(dep->send); #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize == 0); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); @@ -2547,12 +2631,15 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) int visible = 0; int hidden = 0; int this = 0; - Uint buf[2]; /* For one cons-cell */ + DeclareTmpHeap(buf,2,BIF_P); /* For one cons-cell */ DistEntry *dep; Eterm arg_list = BIF_ARG_1; #ifdef DEBUG Eterm* endp; #endif + + UseTmpHeap(2,BIF_P); + if (is_atom(BIF_ARG_1)) arg_list = CONS(buf, BIF_ARG_1, NIL); @@ -2563,13 +2650,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) case am_known: visible = hidden = not_connected = this = 1; break; case am_this: this = 1; break; case am_connected: visible = hidden = 1; break; - default: BIF_ERROR(BIF_P, BADARG); break; + default: goto error; break; } arg_list = CDR(list_val(arg_list)); } - if (is_not_nil(arg_list)) - BIF_ERROR(BIF_P, BADARG); + if (is_not_nil(arg_list)) { + goto error; + } length = 0; @@ -2591,7 +2679,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) if (length == 0) { erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - BIF_RET(result); + goto done; } hp = HAlloc(BIF_P, 2*length); @@ -2620,7 +2708,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) } ASSERT(endp == hp); erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + +done: + UnUseTmpHeap(2,BIF_P); BIF_RET(result); + +error: + UnUseTmpHeap(2,BIF_P); + BIF_ERROR(BIF_P,BADARG); } /**********************************************************************/ diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index ea1abcaeed..695a4fc3fe 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -38,6 +38,7 @@ #define DFLAG_UNICODE_IO 0x1000 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000 #define DFLAG_SMALL_ATOM_TAGS 0x4000 +#define DFLAGS_INTERNAL_TAGS 0x8000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -51,7 +52,7 @@ #define DOP_SEND 2 #define DOP_EXIT 3 #define DOP_UNLINK 4 -#define DOP_NODE_LINK 5 +/* Ancient DOP_NODE_LINK (5) was here, can be reused */ #define DOP_REG_SEND 6 #define DOP_GROUP_LEADER 7 #define DOP_EXIT2 8 @@ -68,7 +69,6 @@ /* distribution trap functions */ extern Export* dsend2_trap; extern Export* dsend3_trap; -/*extern Export* dsend_nosuspend_trap;*/ extern Export* dlink_trap; extern Export* dunlink_trap; extern Export* dmonitor_node_trap; @@ -99,7 +99,8 @@ typedef struct { #define ERTS_DE_IS_CONNECTED(DEP) \ (!ERTS_DE_IS_NOT_CONNECTED((DEP))) - +#define ERTS_DE_BUSY_LIMIT (1024*1024) +extern int erts_dist_buf_busy_limit; extern int erts_is_alive; /* @@ -153,10 +154,10 @@ erts_dsig_prepare(ErtsDSigData *dsdp, } if (no_suspend) { failure = ERTS_DSIG_PREP_CONNECTED; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (dep->qflgs & ERTS_DE_QFLG_BUSY) failure = ERTS_DSIG_PREP_WOULD_SUSPEND; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND) goto fail; } @@ -287,4 +288,5 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32); extern Uint erts_dist_cache_size(void); + #endif diff --git a/erts/emulator/beam/elib_malloc.c b/erts/emulator/beam/elib_malloc.c deleted file mode 100644 index b18c48d8d6..0000000000 --- a/erts/emulator/beam/elib_malloc.c +++ /dev/null @@ -1,2334 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* -** Description: Faster malloc(). -*/ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" - -#ifdef ENABLE_ELIB_MALLOC - -#undef THREAD_SAFE_ELIB_MALLOC -#ifdef USE_THREADS -#define THREAD_SAFE_ELIB_MALLOC 1 -#else -#define THREAD_SAFE_ELIB_MALLOC 0 -#endif - -#include "erl_driver.h" -#include "erl_threads.h" -#include "elib_stat.h" -#include <stdio.h> -#include <stdlib.h> - -/* To avoid clobbering of names becaure of reclaim on VxWorks, - we undefine all possible malloc, calloc etc. */ -#undef malloc -#undef calloc -#undef free -#undef realloc - -#define ELIB_INLINE /* inline all possible functions */ - -#ifndef ELIB_ALIGN -#define ELIB_ALIGN sizeof(double) -#endif - -#ifndef ELIB_HEAP_SIZE -#define ELIB_HEAP_SIZE (64*1024) /* Default 64K */ -#endif - -#ifndef ELIB_HEAP_INCREAMENT -#define ELIB_HEAP_INCREAMENT (32*1024) /* Default 32K */ -#endif - -#ifndef ELIB_FAILURE -#define ELIB_FAILURE abort() -#endif - -#undef ASSERT -#ifdef DEBUG -#define ASSERT(B) \ - ((void) ((B) ? 1 : (fprintf(stderr, "%s:%d: Assertion failed: %s\n", \ - __FILE__, __LINE__, #B), abort(), 0))) -#else -#define ASSERT(B) ((void) 1) -#endif - -#ifndef USE_RECURSIVE_MALLOC_MUTEX -#define USE_RECURSIVE_MALLOC_MUTEX 0 -#endif - -#if USE_RECURSIVE_MALLOC_MUTEX -static erts_mtx_t malloc_mutex = ERTS_REC_MTX_INITER; -#else /* #if USE_RECURSIVE_MALLOC_MUTEX */ -static erts_mtx_t malloc_mutex = ERTS_MTX_INITER; -#if THREAD_SAFE_ELIB_MALLOC -static erts_cnd_t malloc_cond = ERTS_CND_INITER; -#endif -#endif /* #if USE_RECURSIVE_MALLOC_MUTEX */ - -typedef unsigned long EWord; /* Assume 32-bit in this implementation */ -typedef unsigned short EHalfWord; /* Assume 16-bit in this implementation */ -typedef unsigned char EByte; /* Assume 8-bit byte */ - - -#define elib_printf fprintf -#define elib_putc fputc - - -#if defined(__STDC__) || defined(__WIN32__) -#define CONCAT(x,y) x##y -#else -#define CONCAT(x,y) x/**/y -#endif - - -#ifdef ELIB_DEBUG -#define ELIB_PREFIX(fun, args) CONCAT(elib__,fun) args -#else -#define ELIB_PREFIX(fun, args) CONCAT(elib_,fun) args -#endif - -#if defined(__STDC__) -void *ELIB_PREFIX(malloc, (size_t)); -void *ELIB_PREFIX(calloc, (size_t, size_t)); -void ELIB_PREFIX(cfree, (EWord *)); -void ELIB_PREFIX(free, (EWord *)); -void *ELIB_PREFIX(realloc, (EWord *, size_t)); -void* ELIB_PREFIX(memresize, (EWord *, int)); -void* ELIB_PREFIX(memalign, (int, int)); -void* ELIB_PREFIX(valloc, (int)); -void* ELIB_PREFIX(pvalloc, (int)); -int ELIB_PREFIX(memsize, (EWord *)); -/* Extern interfaces used by VxWorks */ -size_t elib_sizeof(void *); -void elib_init(EWord *, EWord); -void elib_force_init(EWord *, EWord); -#endif - -#if defined(__STDC__) -/* define prototypes for missing */ -void* memalign(size_t a, size_t s); -void* pvalloc(size_t nb); -void* memresize(void *p, int nb); -int memsize(void *p); -#endif - -/* bytes to pages */ -#define PAGES(x) (((x)+page_size-1) / page_size) -#define PAGE_ALIGN(p) ((char*)((((EWord)(p))+page_size-1)&~(page_size-1))) - -/* bytes to words */ -#define WORDS(x) (((x)+sizeof(EWord)-1) / sizeof(EWord)) - -/* Align an address */ -#define ALIGN(p) ((EWord*)((((EWord)(p)+ELIB_ALIGN-1)&~(ELIB_ALIGN-1)))) - -/* Calculate the size needed to keep alignment */ - -#define ALIGN_BSZ(nb) ((nb+sizeof(EWord)+ELIB_ALIGN-1) & ~(ELIB_ALIGN-1)) - -#define ALIGN_WSZ(nb) WORDS(ALIGN_BSZ(nb)) - -#define ALIGN_SIZE(nb) (ALIGN_WSZ(nb) - 1) - - -/* PARAMETERS */ - -#if defined(ELIB_HEAP_SBRK) - -#undef PAGE_SIZE - -/* Get the system page size (NEED MORE DEFINES HERE) */ -#ifdef _SC_PAGESIZE -#define PAGE_SIZE sysconf(_SC_PAGESIZE) -#elif defined(_MSC_VER) -# ifdef _M_ALPHA -# define PAGE_SIZE 0x2000 -# else -# define PAGE_SIZE 0x1000 -# endif -#else -#define PAGE_SIZE getpagesize() -#endif - -#define ELIB_EXPAND(need) expand_sbrk(need) -static FUNCTION(int, expand_sbrk, (EWord)); - -#elif defined(ELIB_HEAP_FIXED) - -#define PAGE_SIZE 1024 -#define ELIB_EXPAND(need) -1 -static EWord fix_heap[WORDS(ELIB_HEAP_SIZE)]; - -#elif defined(ELIB_HEAP_USER) - -#define PAGE_SIZE 1024 -#define ELIB_EXPAND(need) -1 - -#else - -#error "ELIB HEAP TYPE NOT SET" - -#endif - - -#define STAT_ALLOCED_BLOCK(SZ) \ -do { \ - tot_allocated += (SZ); \ - if (max_allocated < tot_allocated) \ - max_allocated = tot_allocated; \ -} while (0) - -#define STAT_FREED_BLOCK(SZ) \ -do { \ - tot_allocated -= (SZ); \ -} while (0) - -static int max_allocated = 0; -static int tot_allocated = 0; -static EWord* eheap; /* Align heap start */ -static EWord* eheap_top; /* Point to end of heap */ -EWord page_size = 0; /* Set by elib_init */ - -#if defined(ELIB_DEBUG) || defined(DEBUG) -#define ALIGN_CHECK(a, p) \ - do { \ - if ((EWord)(p) & (a-1)) { \ - elib_printf(stderr, \ - "RUNTIME ERROR: bad alignment (0x%lx:%d:%d)\n", \ - (unsigned long) (p), (int) a, __LINE__); \ - ELIB_FAILURE; \ - } \ - } while(0) -#define ELIB_ALIGN_CHECK(p) ALIGN_CHECK(ELIB_ALIGN, p) -#else -#define ALIGN_CHECK(a, p) -#define ELIB_ALIGN_CHECK(p) -#endif - -#define DYNAMIC 32 - -/* -** Free block layout -** 1 1 30 -** +--------------------------+ -** |F|P| Size | -** +--------------------------+ -** -** Where F is the free bit -** P is the free above bit -** Size is messured in words and does not include the hdr word -** -** If block is on the free list the size is also stored last in the block. -** -*/ -typedef struct _free_block FreeBlock; -struct _free_block { - EWord hdr; - Uint flags; - FreeBlock* parent; - FreeBlock* left; - FreeBlock* right; - EWord v[1]; -}; - -typedef struct _allocated_block { - EWord hdr; - EWord v[5]; -} AllocatedBlock; - - -/* - * Interface to tree routines. - */ -typedef Uint Block_t; - -static Block_t* get_free_block(Uint); -static void link_free_block(Block_t *); -static void unlink_free_block(Block_t *del); - -#define FREE_BIT 0x80000000 -#define FREE_ABOVE_BIT 0x40000000 -#define SIZE_MASK 0x3fffffff /* 2^30 words = 2^32 bytes */ - -/* Work on both FreeBlock and AllocatedBlock */ -#define SIZEOF(p) ((p)->hdr & SIZE_MASK) -#define IS_FREE(p) (((p)->hdr & FREE_BIT) != 0) -#define IS_FREE_ABOVE(p) (((p)->hdr & FREE_ABOVE_BIT) != 0) - -/* Given that we have a free block above find its size */ -#define SIZEOF_ABOVE(p) *(((EWord*) (p)) - 1) - -#define MIN_BLOCK_SIZE (sizeof(FreeBlock)/sizeof(EWord)) -#define MIN_WORD_SIZE (MIN_BLOCK_SIZE-1) -#define MIN_BYTE_SIZE (sizeof(FreeBlock)-sizeof(EWord)) - -#define MIN_ALIGN_SIZE ALIGN_SIZE(MIN_BYTE_SIZE) - - -static AllocatedBlock* heap_head = 0; -static AllocatedBlock* heap_tail = 0; -static EWord eheap_size = 0; - -static int heap_locked; - -static int elib_need_init = 1; -#if THREAD_SAFE_ELIB_MALLOC -static int elib_is_initing = 0; -#endif - -typedef FreeBlock RBTree_t; - -static RBTree_t* root = NULL; - - -static FUNCTION(void, deallocate, (AllocatedBlock*, int)); - -/* - * Unlink a free block - */ - -#define mark_allocated(p, szp) do { \ - (p)->hdr = ((p)->hdr & FREE_ABOVE_BIT) | (szp); \ - (p)->v[szp] &= ~FREE_ABOVE_BIT; \ - } while(0) - -#define mark_free(p, szp) do { \ - (p)->hdr = FREE_BIT | (szp); \ - ((FreeBlock *)p)->v[szp-sizeof(FreeBlock)/sizeof(EWord)+1] = (szp); \ - } while(0) - -#if 0 -/* Help macros to log2 */ -#define LOG_1(x) (((x) > 1) ? 1 : 0) -#define LOG_2(x) (((x) > 3) ? 2+LOG_1((x) >> 2) : LOG_1(x)) -#define LOG_4(x) (((x) > 15) ? 4+LOG_2((x) >> 4) : LOG_2(x)) -#define LOG_8(x) (((x) > 255) ? 8+LOG_4((x)>>8) : LOG_4(x)) -#define LOG_16(x) (((x) > 65535) ? 16+LOG_8((x)>>16) : LOG_8(x)) - -#define log2(x) LOG_16(x) -#endif - -/* - * Split a block to be allocated. - * Mark block as ALLOCATED and clear - * FREE_ABOVE_BIT on next block - * - * nw is SIZE aligned and szp is SIZE aligned + 1 - */ -static void -split_block(FreeBlock* p, EWord nw, EWord szp) -{ - EWord szq; - FreeBlock* q; - - szq = szp - nw; - /* Preserve FREE_ABOVE bit in p->hdr !!! */ - - if (szq >= MIN_ALIGN_SIZE+1) { - szq--; - p->hdr = (p->hdr & FREE_ABOVE_BIT) | nw; - - q = (FreeBlock*) (((EWord*) p) + nw + 1); - mark_free(q, szq); - link_free_block((Block_t *) q); - - q = (FreeBlock*) (((EWord*) q) + szq + 1); - q->hdr |= FREE_ABOVE_BIT; - } - else { - mark_allocated((AllocatedBlock*)p, szp); - } -} - -/* - * Find a free block - */ -static FreeBlock* -alloc_block(EWord nw) -{ - for (;;) { - FreeBlock* p = (FreeBlock *) get_free_block(nw); - - if (p != NULL) { - return p; - } else if (ELIB_EXPAND(nw+MIN_WORD_SIZE)) { - return 0; - } - } -} - - -size_t elib_sizeof(void *p) -{ - AllocatedBlock* pp; - - if (p != 0) { - pp = (AllocatedBlock*) (((char *)p)-1); - return SIZEOF(pp); - } - return 0; -} - -static void locked_elib_init(EWord*, EWord); -static void init_elib_malloc(EWord*, EWord); - -/* -** Initialize the elib -** The addr and sz is only used when compiled with EXPAND_ADDR -*/ -/* Not static, this is used by VxWorks */ -void elib_init(EWord* addr, EWord sz) -{ - if (!elib_need_init) - return; - erts_mtx_lock(&malloc_mutex); - locked_elib_init(addr, sz); - erts_mtx_unlock(&malloc_mutex); -} - -static void locked_elib_init(EWord* addr, EWord sz) -{ - if (!elib_need_init) - return; - -#if THREAD_SAFE_ELIB_MALLOC - -#if !USE_RECURSIVE_MALLOC_MUTEX - { - static erts_tid_t initer_tid; - - if(elib_is_initing) { - - if(erts_equal_tids(initer_tid, erts_thr_self())) - return; - - /* Wait until initializing thread is done with initialization */ - - while(elib_need_init) - erts_cnd_wait(&malloc_cond, &malloc_mutex); - - return; - } - else { - initer_tid = erts_thr_self(); - elib_is_initing = 1; - } - } -#else - if(elib_is_initing) - return; - elib_is_initing = 1; -#endif - -#endif /* #if THREAD_SAFE_ELIB_MALLOC */ - - /* Do the actual initialization of the malloc implementation */ - init_elib_malloc(addr, sz); - -#if THREAD_SAFE_ELIB_MALLOC - -#if !USE_RECURSIVE_MALLOC_MUTEX - erts_mtx_unlock(&malloc_mutex); -#endif - - /* Recursive calls to malloc are allowed here... */ - erts_mtx_set_forksafe(&malloc_mutex); - -#if !USE_RECURSIVE_MALLOC_MUTEX - erts_mtx_lock(&malloc_mutex); - elib_is_initing = 0; -#endif - -#endif /* #if THREAD_SAFE_ELIB_MALLOC */ - - elib_need_init = 0; - -#if THREAD_SAFE_ELIB_MALLOC && !USE_RECURSIVE_MALLOC_MUTEX - erts_cnd_broadcast(&malloc_cond); -#endif - -} - -static void init_elib_malloc(EWord* addr, EWord sz) -{ - int i; - FreeBlock* freep; - EWord tmp_sz; -#ifdef ELIB_HEAP_SBRK - char* top; - EWord n; -#endif - - max_allocated = 0; - tot_allocated = 0; - root = NULL; - - /* Get the page size (may involve system call!!!) */ - page_size = PAGE_SIZE; - -#if defined(ELIB_HEAP_SBRK) - sz = PAGES(ELIB_HEAP_SIZE)*page_size; - - if ((top = (char*) sbrk(0)) == (char*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(0)"); - ELIB_FAILURE; - } - n = PAGE_ALIGN(top) - top; - if ((top = (char*) sbrk(n)) == (char*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(n)"); - ELIB_FAILURE; - } - if ((eheap = (EWord*) sbrk(sz)) == (EWord*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(SIZE)"); - ELIB_FAILURE; - } - sz = WORDS(ELIB_HEAP_SIZE); -#elif defined(ELIB_HEAP_FIXED) - eheap = fix_heap; - sz = WORDS(ELIB_HEAP_SIZE); -#elif defined(ELIB_HEAP_USER) - eheap = addr; - sz = WORDS(sz); -#else - return -1; -#endif - eheap_size = 0; - - /* Make sure that the first word of the heap_head is aligned */ - addr = ALIGN(eheap+1); - sz -= ((addr - 1) - eheap); /* Subtract unusable size */ - eheap_top = eheap = addr - 1; /* Set new aligned heap start */ - - eheap_top[sz-1] = 0; /* Heap stop mark */ - - addr = eheap; - heap_head = (AllocatedBlock*) addr; - heap_head->hdr = MIN_ALIGN_SIZE; - for (i = 0; i < MIN_ALIGN_SIZE; i++) - heap_head->v[i] = 0; - - addr += (MIN_ALIGN_SIZE+1); - freep = (FreeBlock*) addr; - tmp_sz = sz - (((MIN_ALIGN_SIZE+1) + MIN_BLOCK_SIZE) + 1 + 1); - mark_free(freep, tmp_sz); - link_free_block((Block_t *) freep); - - /* No need to align heap tail */ - heap_tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1]; - heap_tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE; - heap_tail->v[0] = 0; - heap_tail->v[1] = 0; - heap_tail->v[2] = 0; - - eheap_top += sz; - eheap_size += sz; - - heap_locked = 0; -} - -#ifdef ELIB_HEAP_USER -void elib_force_init(EWord* addr, EWord sz) -{ - elib_need_init = 1; - elib_init(addr,sz); -} -#endif - -#ifdef ELIB_HEAP_SBRK - -/* -** need in number of words (should include head and tail words) -*/ -static int expand_sbrk(EWord sz) -{ - EWord* p; - EWord bytes = sz * sizeof(EWord); - EWord size; - AllocatedBlock* tail; - - if (bytes < ELIB_HEAP_SIZE) - size = PAGES(ELIB_HEAP_INCREAMENT)*page_size; - else - size = PAGES(bytes)*page_size; - - if ((p = (EWord*) sbrk(size)) == ((EWord*) -1)) - return -1; - - if (p != eheap_top) { - elib_printf(stderr, "panic: sbrk moved\n"); - ELIB_FAILURE; - } - - sz = WORDS(size); - - /* Set new endof heap marker and a new heap tail */ - eheap_top[sz-1] = 0; - - tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1]; - tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE; - tail->v[0] = 0; - tail->v[1] = 0; - tail->v[2] = 0; - - /* Patch old tail with new appended size */ - heap_tail->hdr = (heap_tail->hdr & FREE_ABOVE_BIT) | - (MIN_WORD_SIZE+1+(sz-MIN_BLOCK_SIZE-1)); - deallocate(heap_tail, 0); - - heap_tail = tail; - - eheap_size += sz; - eheap_top += sz; - - return 0; -} - -#endif /* ELIB_HEAP_SBRK */ - - -/* -** Scan heap and check for corrupted heap -*/ -int elib_check_heap(void) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - if (heap_locked) { - elib_printf(stderr, "heap is locked no info avaiable\n"); - return 0; - } - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - if (p->v[sz-1] != sz) { - elib_printf(stderr, "panic: heap corrupted\r\n"); - ELIB_FAILURE; - } - p = (AllocatedBlock*) (p->v + sz); - if (!IS_FREE_ABOVE(p)) { - elib_printf(stderr, "panic: heap corrupted\r\n"); - ELIB_FAILURE; - } - } - else - p = (AllocatedBlock*) (p->v + sz); - } - return 1; -} - -/* -** Load the byte vector pointed to by v of length vsz -** with a heap image -** The scale is defined by vsz and the current heap size -** free = 0, full = 255 -** -** -*/ -int elib_heap_map(EByte* v, int vsz) -{ - AllocatedBlock* p = heap_head; - EWord sz; - int gsz = eheap_size / vsz; /* The granuality used */ - int fsz = 0; - int usz = 0; - - if (gsz == 0) - return -1; /* too good reolution */ - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - fsz += sz; - if ((fsz + usz) > gsz) { - *v++ = (255*usz)/gsz; - fsz -= (gsz - usz); - usz = 0; - while(fsz >= gsz) { - *v++ = 0; - fsz -= gsz; - } - } - } - else { - usz += sz; - if ((fsz + usz) > gsz) { - *v++ = 255 - (255*fsz)/gsz; - usz -= (gsz - fsz); - fsz = 0; - while(usz >= gsz) { - *v++ = 255; - usz -= gsz; - } - } - } - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Generate a histogram of free/allocated blocks -** Count granuality of 10 gives -** (0-10],(10-100],(100-1000],(1000-10000] ... -** (0-2], (2-4], (4-8], (8-16], .... -*/ -static int i_logb(EWord size, int base) -{ - int lg = 0; - while(size >= base) { - size /= base; - lg++; - } - return lg; -} - -int elib_histo(EWord* vf, EWord* va, int vsz, int base) -{ - AllocatedBlock* p = heap_head; - EWord sz; - int i; - int linear; - - if ((vsz <= 1) || (vf == 0 && va == 0)) - return -1; - - if (base < 0) { - linear = 1; - base = -base; - } - else - linear = 0; - - if (base <= 1) - return -1; - - if (vf != 0) { - for (i = 0; i < vsz; i++) - vf[i] = 0; - } - if (va != 0) { - for (i = 0; i < vsz; i++) - va[i] = 0; - } - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - if (vf != 0) { - int val; - if (linear) - val = sz / base; - else - val = i_logb(sz, base); - if (val >= vsz) - vf[vsz-1]++; - else - vf[val]++; - } - } - else { - if (va != 0) { - int val; - if (linear) - val = sz / base; - else - val = i_logb(sz, base); - if (val >= vsz) - va[vsz-1]++; - else - va[val]++; - } - } - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Fill the info structure with actual values -** Total -** Allocated -** Free -** maxMaxFree -*/ -void elib_stat(struct elib_stat* info) -{ - EWord blks = 0; - EWord sz_free = 0; - EWord sz_alloc = 0; - EWord sz_max_free = 0; - EWord sz_min_used = 0x7fffffff; - EWord sz; - EWord num_free = 0; - AllocatedBlock* p = heap_head; - - info->mem_total = eheap_size; - - p = (AllocatedBlock*) (p->v + SIZEOF(p)); - - while((sz = SIZEOF(p)) != 0) { - blks++; - if (IS_FREE(p)) { - if (sz > sz_max_free) - sz_max_free = sz; - sz_free += sz; - ++num_free; - } - else { - if (sz < sz_min_used) - sz_min_used = sz; - sz_alloc += sz; - } - p = (AllocatedBlock*) (p->v + sz); - } - info->mem_blocks = blks; - info->free_blocks = num_free; - info->mem_alloc = sz_alloc; - info->mem_free = sz_free; - info->min_used = sz_min_used; - info->max_free = sz_max_free; - info->mem_max_alloc = max_allocated; - ASSERT(sz_alloc == tot_allocated); -} - -/* -** Dump the heap -*/ -void elib_heap_dump(char* label) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - elib_printf(stderr, "HEAP DUMP (%s)\n", label); - if (!elib_check_heap()) - return; - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - elib_printf(stderr, "%p: FREE, size = %d\n", p, (int) sz); - } - else { - elib_printf(stderr, "%p: USED, size = %d %s\n", p, (int) sz, - IS_FREE_ABOVE(p)?"(FREE ABOVE)":""); - } - p = (AllocatedBlock*) (p->v + sz); - } -} - -/* -** Scan heaps and count: -** free_size, allocated_size, max_free_block -*/ -void elib_statistics(void* to) -{ - struct elib_stat info; - EWord frag; - - if (!elib_check_heap()) - return; - - elib_stat(&info); - - frag = 1000 - ((1000 * info.max_free) / info.mem_free); - - elib_printf(to, "Heap Statistics: total(%d), blocks(%d), frag(%d.%d%%)\n", - info.mem_total, info.mem_blocks, - (int) frag/10, (int) frag % 10); - - elib_printf(to, " allocated(%d), free(%d), " - "free_blocks(%d)\n", - info.mem_alloc, info.mem_free,info.free_blocks); - elib_printf(to, " max_free(%d), min_used(%d)\n", - info.max_free, info.min_used); -} - -/* -** Allocate a least nb bytes with alignment a -** Algorithm: -** 1) Try locate a block which match exacly among the by direct index. -** 2) Try using a fix block of greater size -** 3) Try locate a block by searching in lists where block sizes -** X may vary between 2^i < X <= 2^(i+1) -** -** Reset memory to zero if clear is true -*/ -static AllocatedBlock* allocate(EWord nb, EWord a, int clear) -{ - FreeBlock* p; - EWord nw; - - if (a == ELIB_ALIGN) { - /* - * Common case: Called by malloc(), realloc(), calloc(). - */ - nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb); - - if ((p = alloc_block(nw)) == 0) - return NULL; - } else { - /* - * Special case: Called by memalign(). - */ - EWord asz, szp, szq, tmpsz; - FreeBlock *q; - - if ((p = alloc_block((1+MIN_ALIGN_SIZE)*sizeof(EWord)+a-1+nb)) == 0) - return NULL; - - asz = a - ((EWord) ((AllocatedBlock *)p)->v) % a; - - if (asz != a) { - /* Enforce the alignment requirement by cutting of a free - block at the beginning of the block. */ - - if (asz < (1+MIN_ALIGN_SIZE)*sizeof(EWord) && !IS_FREE_ABOVE(p)) { - /* Not enough room to cut of a free block; - increase align size */ - asz += (((1+MIN_ALIGN_SIZE)*sizeof(EWord) + a - 1)/a)*a; - } - - szq = ALIGN_SIZE(asz - sizeof(EWord)); - szp = SIZEOF(p) - szq - 1; - - q = p; - p = (FreeBlock*) (((EWord*) q) + szq + 1); - p->hdr = FREE_ABOVE_BIT | FREE_BIT | szp; - - if (IS_FREE_ABOVE(q)) { /* This should not be possible I think, - but just in case... */ - tmpsz = SIZEOF_ABOVE(q) + 1; - szq += tmpsz; - q = (FreeBlock*) (((EWord*) q) - tmpsz); - unlink_free_block((Block_t *) q); - q->hdr = (q->hdr & FREE_ABOVE_BIT) | FREE_BIT | szq; - } - mark_free(q, szq); - link_free_block((Block_t *) q); - - } /* else already had the correct alignment */ - - nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb); - } - - split_block(p, nw, SIZEOF(p)); - - STAT_ALLOCED_BLOCK(SIZEOF(p)); - - if (clear) { - EWord* pp = ((AllocatedBlock*)p)->v; - - while(nw--) - *pp++ = 0; - } - - return (AllocatedBlock*) p; -} - - -/* -** Deallocate memory pointed to by p -** 1. Merge with block above if this block is free -** 2. Merge with block below if this block is free -** Link the block to the correct free list -** -** p points to the block header! -** -*/ -static void deallocate(AllocatedBlock* p, int stat_count) -{ - FreeBlock* q; - EWord szq; - EWord szp; - - szp = SIZEOF(p); - - if (stat_count) - STAT_FREED_BLOCK(SIZEOF(p)); - - if (IS_FREE_ABOVE(p)) { - szq = SIZEOF_ABOVE(p); - q = (FreeBlock*) ( ((EWord*) p) - szq - 1); - unlink_free_block((Block_t *) q); - - p = (AllocatedBlock*) q; - szp += (szq + 1); - } - q = (FreeBlock*) (p->v + szp); - if (IS_FREE(q)) { - szq = SIZEOF(q); - unlink_free_block((Block_t *) q); - szp += (szq + 1); - } - else - q->hdr |= FREE_ABOVE_BIT; - - /* The block above p can NEVER be free !!! */ - p->hdr = FREE_BIT | szp; - p->v[szp-1] = szp; - - link_free_block((Block_t *) p); -} - -/* -** Reallocate memory -** If preserve is true then data is moved if neccesary -*/ -static AllocatedBlock* reallocate(AllocatedBlock* p, EWord nb, int preserve) -{ - EWord szp; - EWord szq; - EWord sz; - EWord nw; - FreeBlock* q; - - if (nb < MIN_BYTE_SIZE) - nw = MIN_ALIGN_SIZE; - else - nw = ALIGN_SIZE(nb); - - sz = szp = SIZEOF(p); - - STAT_FREED_BLOCK(szp); - - /* Merge with block below */ - q = (FreeBlock*) (p->v + szp); - if (IS_FREE(q)) { - szq = SIZEOF(q); - unlink_free_block((Block_t *) q); - szp += (szq + 1); - } - - if (nw <= szp) { - split_block((FreeBlock *) p, nw, szp); - STAT_ALLOCED_BLOCK(SIZEOF(p)); - return p; - } - else { - EWord* dp = p->v; - AllocatedBlock* npp; - - if (IS_FREE_ABOVE(p)) { - szq = SIZEOF_ABOVE(p); - if (szq + szp + 1 >= nw) { - q = (FreeBlock*) (((EWord*) p) - szq - 1); - unlink_free_block((Block_t * )q); - szp += (szq + 1); - p = (AllocatedBlock*) q; - - if (preserve) { - EWord* pp = p->v; - while(sz--) - *pp++ = *dp++; - } - split_block((FreeBlock *) p, nw, szp); - STAT_ALLOCED_BLOCK(SIZEOF(p)); - return p; - } - } - - /* - * Update p so that allocate() and deallocate() works. - * (Note that allocate() may call expand_sbrk(), which in - * in turn calls deallocate().) - */ - - p->hdr = (p->hdr & FREE_ABOVE_BIT) | szp; - p->v[szp] &= ~FREE_ABOVE_BIT; - - npp = allocate(nb, ELIB_ALIGN, 0); - if(npp == NULL) - return NULL; - if (preserve) { - EWord* pp = npp->v; - while(sz--) - *pp++ = *dp++; - } - deallocate(p, 0); - return npp; - } -} - -/* -** What malloc() and friends should do (and return) when the heap is -** exhausted. [sverkerw] -*/ -static void* heap_exhausted(void) -{ - /* Choose behaviour */ -#if 0 - /* Crash-and-burn --- leave a usable corpse (hopefully) */ - abort(); -#endif - /* The usual ANSI-compliant behaviour */ - return NULL; -} - -/* -** Allocate size bytes of memory -*/ -void* ELIB_PREFIX(malloc, (size_t nb)) -{ - void *res; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (nb == 0) - res = NULL; - else if ((p = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - - -void* ELIB_PREFIX(calloc, (size_t nelem, size_t size)) -{ - void *res; - int nb; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if ((nb = nelem * size) == 0) - res = NULL; - else if ((p = allocate(nb, ELIB_ALIGN, 1)) != 0) { - ELIB_ALIGN_CHECK(p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -/* -** Free memory allocated by malloc -*/ - -void ELIB_PREFIX(free, (EWord* p)) -{ - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) - deallocate((AllocatedBlock*)(p-1), 1); - - erts_mtx_unlock(&malloc_mutex); -} - -void ELIB_PREFIX(cfree, (EWord* p)) -{ - ELIB_PREFIX(free, (p)); -} - - -/* -** Realloc the memory allocated in p to nb number of bytes -** -*/ - -void* ELIB_PREFIX(realloc, (EWord* p, size_t nb)) -{ - void *res = NULL; - AllocatedBlock* pp; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) { - pp = (AllocatedBlock*) (p-1); - if (nb > 0) { - if ((pp = reallocate(pp, nb, 1)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - } - else - deallocate(pp, 1); - } - else if (nb > 0) { - if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - else - res = heap_exhausted(); - } - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -/* -** Resize the memory area pointed to by p with nb number of bytes -*/ -void* ELIB_PREFIX(memresize, (EWord* p, int nb)) -{ - void *res = NULL; - AllocatedBlock* pp; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) { - pp = (AllocatedBlock*) (p-1); - if (nb > 0) { - if ((pp = reallocate(pp, nb, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - } - else - deallocate(pp, 1); - } - else if (nb > 0) { - if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - else - res = heap_exhausted(); - } - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - - -/* Create aligned memory a must be a power of 2 !!! */ - -void* ELIB_PREFIX(memalign, (int a, int nb)) -{ - void *res; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (nb == 0 || a <= 0) - res = NULL; - else if ((p = allocate(nb, a, 0)) != 0) { - ALIGN_CHECK(a, p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -void* ELIB_PREFIX(valloc, (int nb)) -{ - return ELIB_PREFIX(memalign, (page_size, nb)); -} - - -void* ELIB_PREFIX(pvalloc, (int nb)) -{ - return ELIB_PREFIX(memalign, (page_size, PAGES(nb)*page_size)); -} -/* Return memory size for pointer p in bytes */ - -int ELIB_PREFIX(memsize, (p)) -EWord* p; -{ - return SIZEOF((AllocatedBlock*)(p-1))*4; -} - - -/* -** -------------------------------------------------------------------------- -** DEBUG LIBRARY -** -------------------------------------------------------------------------- -*/ - -#ifdef ELIB_DEBUG - -#define IN_HEAP(p) (((p) >= (char*) eheap) && (p) < (char*) eheap_top) -/* -** ptr_to_block: return the pointer to heap block pointed into by ptr -** Returns 0 if not pointing into a block -*/ - -static EWord* ptr_to_block(char* ptr) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - while((sz = SIZEOF(p)) != 0) { - if ((ptr >= (char*) p->v) && (ptr < (char*)(p->v+sz))) - return p->v; - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Validate a pointer -** returns: -** 0 - if points to start of a block -** 1 - if points outsize heap -** -1 - if points inside block -** -*/ -static int check_pointer(char* ptr) -{ - if (IN_HEAP(ptr)) { - if (ptr_to_block(ptr) == 0) - return 1; - return 0; - } - return -1; -} - -/* -** Validate a memory area -** returns: -** 0 - if area is included in a block -** -1 - if area overlap a heap block -** 1 - if area is outside heap -*/ -static int check_area(char* ptr, int n) -{ - if (IN_HEAP(ptr)) { - if (IN_HEAP(ptr+n-1)) { - EWord* p1 = ptr_to_block(ptr); - EWord* p2 = ptr_to_block(ptr+n-1); - - if (p1 == p2) - return (p1 == 0) ? -1 : 0; - return -1; - } - } - else if (IN_HEAP(ptr+n-1)) - return -1; - return 1; -} - -/* -** Check if a block write will overwrite heap block -*/ -static void check_write(char* ptr, int n, char* file, int line, char* fun) -{ - if (check_area(ptr, n) == -1) { - elib_printf(stderr, "RUNTIME ERROR: %s heap overwrite\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } -} - -/* -** Check if a pointer is an allocated object -*/ -static void check_allocated_block(char* ptr, char* file, int line, char* fun) -{ - EWord* q; - - if (!IN_HEAP(ptr) || ((q=ptr_to_block(ptr)) == 0) || (ptr != (char*) q)) { - elib_printf(stderr, "RUNTIME ERROR: %s non heap pointer\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } - - if (IS_FREE((AllocatedBlock*)(q-1))) { - elib_printf(stderr, "RUNTIME ERROR: %s free pointer\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } - -} - -/* -** -------------------------------------------------------------------------- -** DEBUG VERSIONS (COMPILED WITH THE ELIB.H) -** -------------------------------------------------------------------------- -*/ - -void* elib_dbg_malloc(int n, char* file, int line) -{ - return elib__malloc(n); -} - -void* elib_dbg_calloc(int n, int s, char* file, int line) -{ - return elib__calloc(n, s); -} - -void* elib_dbg_realloc(EWord* p, int n, char* file, int line) -{ - if (p == 0) - return elib__malloc(n); - check_allocated_block(p, file, line, "elib_realloc"); - return elib__realloc(p, n); -} - -void elib_dbg_free(EWord* p, char* file, int line) -{ - if (p == 0) - return; - check_allocated_block(p, file, line, "elib_free"); - elib__free(p); -} - -void elib_dbg_cfree(EWord* p, char* file, int line) -{ - if (p == 0) - return; - check_allocated_block(p, file, line, "elib_free"); - elib__cfree(p); -} - -void* elib_dbg_memalign(int a, int n, char* file, int line) -{ - return elib__memalign(a, n); -} - -void* elib_dbg_valloc(int n, char* file, int line) -{ - return elib__valloc(n); -} - -void* elib_dbg_pvalloc(int n, char* file, int line) -{ - return elib__pvalloc(n); -} - -void* elib_dbg_memresize(EWord* p, int n, char* file, int line) -{ - if (p == 0) - return elib__malloc(n); - check_allocated_block(p, file, line, "elib_memresize"); - return elib__memresize(p, n); -} - -int elib_dbg_memsize(void* p, char* file, int line) -{ - check_allocated_block(p, file, line, "elib_memsize"); - return elib__memsize(p); -} - -/* -** -------------------------------------------------------------------------- -** LINK TIME FUNCTIONS (NOT COMPILED CALLS) -** -------------------------------------------------------------------------- -*/ - -void* elib_malloc(int n) -{ - return elib_dbg_malloc(n, "", -1); -} - -void* elib_calloc(int n, int s) -{ - return elib_dbg_calloc(n, s, "", -1); -} - -void* elib_realloc(EWord* p, int n) -{ - return elib_dbg_realloc(p, n, "", -1); -} - -void elib_free(EWord* p) -{ - elib_dbg_free(p, "", -1); -} - -void elib_cfree(EWord* p) -{ - elib_dbg_cfree(p, "", -1); -} - -void* elib_memalign(int a, int n) -{ - return elib_dbg_memalign(a, n, "", -1); -} - -void* elib_valloc(int n) -{ - return elib_dbg_valloc(n, "", -1); -} - -void* elib_pvalloc(int n) -{ - return elib_dbg_pvalloc(n, "", -1); -} - -void* elib_memresize(EWord* p, int n) -{ - return elib_dbg_memresize(p, n, "", -1); -} - - -int elib_memsize(EWord* p) -{ - return elib_dbg_memsize(p, "", -1); -} - -#endif /* ELIB_DEBUG */ - -/* -** -------------------------------------------------------------------------- -** Map c library functions to elib -** -------------------------------------------------------------------------- -*/ - -#if defined(ELIB_ALLOC_IS_CLIB) -void* malloc(size_t nb) -{ - return elib_malloc(nb); -} - -void* calloc(size_t nelem, size_t size) -{ - return elib_calloc(nelem, size); -} - - -void free(void *p) -{ - elib_free(p); -} - -void cfree(void *p) -{ - elib_cfree(p); -} - -void* realloc(void* p, size_t nb) -{ - return elib_realloc(p, nb); -} - - -void* memalign(size_t a, size_t s) -{ - return elib_memalign(a, s); -} - -void* valloc(size_t nb) -{ - return elib_valloc(nb); -} - -void* pvalloc(size_t nb) -{ - return elib_pvalloc(nb); -} - -#if 0 -void* memresize(void* p, int nb) -{ - return elib_memresize(p, nb); -} - -int memsize(void* p) -{ - return elib_memsize(p); -} -#endif -#endif /* ELIB_ALLOC_IS_CLIB */ - -#endif /* ENABLE_ELIB_MALLOC */ - -void elib_ensure_initialized(void) -{ -#ifdef ENABLE_ELIB_MALLOC -#ifndef ELIB_DONT_INITIALIZE - elib_init(NULL, 0); -#endif -#endif -} - -#ifdef ENABLE_ELIB_MALLOC -/** - ** A Slightly modified version of the "address order best fit" algorithm - ** used in erl_bestfit_alloc.c. Comments refer to that implementation. - **/ - -/* - * Description: A combined "address order best fit"/"best fit" allocator - * based on a Red-Black (binary search) Tree. The search, - * insert, and delete operations are all O(log n) operations - * on a Red-Black Tree. In the "address order best fit" case - * n equals number of free blocks, and in the "best fit" case - * n equals number of distinct sizes of free blocks. Red-Black - * Trees are described in "Introduction to Algorithms", by - * Thomas H. Cormen, Charles E. Leiserson, and - * Ronald L. Riverest. - * - * This module is a callback-module for erl_alloc_util.c - * - * Author: Rickard Green - */ - -#ifdef DEBUG -#if 0 -#define HARD_DEBUG -#endif -#else -#undef HARD_DEBUG -#endif - -#define SZ_MASK SIZE_MASK -#define FLG_MASK (~(SZ_MASK)) - -#define BLK_SZ(B) (*((Block_t *) (B)) & SZ_MASK) - -#define TREE_NODE_FLG (((Uint) 1) << 0) -#define RED_FLG (((Uint) 1) << 1) -#ifdef HARD_DEBUG -# define LEFT_VISITED_FLG (((Uint) 1) << 2) -# define RIGHT_VISITED_FLG (((Uint) 1) << 3) -#endif - -#define IS_TREE_NODE(N) (((RBTree_t *) (N))->flags & TREE_NODE_FLG) -#define IS_LIST_ELEM(N) (!IS_TREE_NODE(((RBTree_t *) (N)))) - -#define SET_TREE_NODE(N) (((RBTree_t *) (N))->flags |= TREE_NODE_FLG) -#define SET_LIST_ELEM(N) (((RBTree_t *) (N))->flags &= ~TREE_NODE_FLG) - -#define IS_RED(N) (((RBTree_t *) (N)) \ - && ((RBTree_t *) (N))->flags & RED_FLG) -#define IS_BLACK(N) (!IS_RED(((RBTree_t *) (N)))) - -#define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG) -#define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG) - -#undef ASSERT -#define ASSERT ASSERT_EXPR - -#if 1 -#define RBT_ASSERT ASSERT -#else -#define RBT_ASSERT(x) -#endif - - -#ifdef HARD_DEBUG -static RBTree_t * check_tree(Uint); -#endif - -#ifdef ERTS_INLINE -# ifndef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 1 -# endif -#else -# if defined(__GNUC__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline__ -# elif defined(__WIN32__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline -# else -# define ERTS_CAN_INLINE 0 -# define ERTS_INLINE -# endif -#endif - -/* Types... */ -#if 0 -typedef struct RBTree_t_ RBTree_t; - -struct RBTree_t_ { - Block_t hdr; - Uint flags; - RBTree_t *parent; - RBTree_t *left; - RBTree_t *right; -}; -#endif - -#if 0 -typedef struct { - RBTree_t t; - RBTree_t *next; -} RBTreeList_t; - -#define LIST_NEXT(N) (((RBTreeList_t *) (N))->next) -#define LIST_PREV(N) (((RBTreeList_t *) (N))->t.parent) -#endif - -#ifdef DEBUG - -/* Destroy all tree fields */ -#define DESTROY_TREE_NODE(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(RBTree_t) - sizeof(Block_t))) - -/* Destroy all tree and list fields */ -#define DESTROY_LIST_ELEM(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(RBTreeList_t) - sizeof(Block_t))) - -#else - -#define DESTROY_TREE_NODE(N) -#define DESTROY_LIST_ELEM(N) - -#endif - - -/* - * Red-Black Tree operations needed - */ - -static ERTS_INLINE void -left_rotate(RBTree_t **root, RBTree_t *x) -{ - RBTree_t *y = x->right; - x->right = y->left; - if (y->left) - y->left->parent = x; - y->parent = x->parent; - if (!y->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->left) - x->parent->left = y; - else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; - } - y->left = x; - x->parent = y; -} - -static ERTS_INLINE void -right_rotate(RBTree_t **root, RBTree_t *x) -{ - RBTree_t *y = x->left; - x->left = y->right; - if (y->right) - y->right->parent = x; - y->parent = x->parent; - if (!y->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->right) - x->parent->right = y; - else { - RBT_ASSERT(x == x->parent->left); - x->parent->left = y; - } - y->right = x; - x->parent = y; -} - - -/* - * Replace node x with node y - * NOTE: block header of y is not changed - */ -static ERTS_INLINE void -replace(RBTree_t **root, RBTree_t *x, RBTree_t *y) -{ - - if (!x->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->left) - x->parent->left = y; - else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; - } - if (x->left) { - RBT_ASSERT(x->left->parent == x); - x->left->parent = y; - } - if (x->right) { - RBT_ASSERT(x->right->parent == x); - x->right->parent = y; - } - - y->flags = x->flags; - y->parent = x->parent; - y->right = x->right; - y->left = x->left; - - DESTROY_TREE_NODE(x); - -} - -static void -tree_insert_fixup(RBTree_t *blk) -{ - RBTree_t *x = blk, *y; - - /* - * Rearrange the tree so that it satisfies the Red-Black Tree properties - */ - - RBT_ASSERT(x != root && IS_RED(x->parent)); - do { - - /* - * x and its parent are both red. Move the red pair up the tree - * until we get to the root or until we can separate them. - */ - - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(x->parent->parent); - - if (x->parent == x->parent->parent->left) { - y = x->parent->parent->right; - if (IS_RED(y)) { - SET_BLACK(y); - x = x->parent; - SET_BLACK(x); - x = x->parent; - SET_RED(x); - } - else { - - if (x == x->parent->right) { - x = x->parent; - left_rotate(&root, x); - } - - RBT_ASSERT(x == x->parent->parent->left->left); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(IS_BLACK(y)); - - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - right_rotate(&root, x->parent->parent); - - RBT_ASSERT(x == x->parent->left); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->right)); - RBT_ASSERT(IS_BLACK(x->parent)); - break; - } - } - else { - RBT_ASSERT(x->parent == x->parent->parent->right); - y = x->parent->parent->left; - if (IS_RED(y)) { - SET_BLACK(y); - x = x->parent; - SET_BLACK(x); - x = x->parent; - SET_RED(x); - } - else { - - if (x == x->parent->left) { - x = x->parent; - right_rotate(&root, x); - } - - RBT_ASSERT(x == x->parent->parent->right->right); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(IS_BLACK(y)); - - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - left_rotate(&root, x->parent->parent); - - RBT_ASSERT(x == x->parent->right); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->left)); - RBT_ASSERT(IS_BLACK(x->parent)); - break; - } - } - } while (x != root && IS_RED(x->parent)); - - SET_BLACK(root); -} - -static void -unlink_free_block(Block_t *del) -{ - Uint spliced_is_black; - RBTree_t *x, *y, *z = (RBTree_t *) del; - RBTree_t null_x; /* null_x is used to get the fixup started when we - splice out a node without children. */ - - null_x.parent = NULL; - -#ifdef HARD_DEBUG - check_tree(0); -#endif - - /* Remove node from tree... */ - - /* Find node to splice out */ - if (!z->left || !z->right) - y = z; - else - /* Set y to z:s successor */ - for(y = z->right; y->left; y = y->left); - /* splice out y */ - x = y->left ? y->left : y->right; - spliced_is_black = IS_BLACK(y); - if (x) { - x->parent = y->parent; - } - else if (!x && spliced_is_black) { - x = &null_x; - x->flags = 0; - SET_BLACK(x); - x->right = x->left = NULL; - x->parent = y->parent; - y->left = x; - } - - if (!y->parent) { - RBT_ASSERT(root == y); - root = x; - } - else if (y == y->parent->left) - y->parent->left = x; - else { - RBT_ASSERT(y == y->parent->right); - y->parent->right = x; - } - if (y != z) { - /* We spliced out the successor of z; replace z by the successor */ - replace(&root, z, y); - } - - if (spliced_is_black) { - /* We removed a black node which makes the resulting tree - violate the Red-Black Tree properties. Fixup tree... */ - - while (IS_BLACK(x) && x->parent) { - - /* - * x has an "extra black" which we move up the tree - * until we reach the root or until we can get rid of it. - * - * y is the sibbling of x - */ - - if (x == x->parent->left) { - y = x->parent->right; - RBT_ASSERT(y); - if (IS_RED(y)) { - RBT_ASSERT(y->right); - RBT_ASSERT(y->left); - SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - left_rotate(&root, x->parent); - y = x->parent->right; - } - RBT_ASSERT(y); - RBT_ASSERT(IS_BLACK(y)); - if (IS_BLACK(y->left) && IS_BLACK(y->right)) { - SET_RED(y); - x = x->parent; - } - else { - if (IS_BLACK(y->right)) { - SET_BLACK(y->left); - SET_RED(y); - right_rotate(&root, y); - y = x->parent->right; - } - RBT_ASSERT(y); - if (IS_RED(x->parent)) { - - SET_BLACK(x->parent); - SET_RED(y); - } - RBT_ASSERT(y->right); - SET_BLACK(y->right); - left_rotate(&root, x->parent); - x = root; - break; - } - } - else { - RBT_ASSERT(x == x->parent->right); - y = x->parent->left; - RBT_ASSERT(y); - if (IS_RED(y)) { - RBT_ASSERT(y->right); - RBT_ASSERT(y->left); - SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - right_rotate(&root, x->parent); - y = x->parent->left; - } - RBT_ASSERT(y); - RBT_ASSERT(IS_BLACK(y)); - if (IS_BLACK(y->right) && IS_BLACK(y->left)) { - SET_RED(y); - x = x->parent; - } - else { - if (IS_BLACK(y->left)) { - SET_BLACK(y->right); - SET_RED(y); - left_rotate(&root, y); - y = x->parent->left; - } - RBT_ASSERT(y); - if (IS_RED(x->parent)) { - SET_BLACK(x->parent); - SET_RED(y); - } - RBT_ASSERT(y->left); - SET_BLACK(y->left); - right_rotate(&root, x->parent); - x = root; - break; - } - } - } - SET_BLACK(x); - - if (null_x.parent) { - if (null_x.parent->left == &null_x) - null_x.parent->left = NULL; - else { - RBT_ASSERT(null_x.parent->right == &null_x); - null_x.parent->right = NULL; - } - RBT_ASSERT(!null_x.left); - RBT_ASSERT(!null_x.right); - } - else if (root == &null_x) { - root = NULL; - RBT_ASSERT(!null_x.left); - RBT_ASSERT(!null_x.right); - } - } - - - DESTROY_TREE_NODE(del); - -#ifdef HARD_DEBUG - check_tree(0); -#endif - -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * "Address order best fit" specific callbacks. * -\* */ - -static void -link_free_block(Block_t *block) -{ - RBTree_t *blk = (RBTree_t *) block; - Uint blk_sz = BLK_SZ(blk); - - blk->flags = 0; - blk->left = NULL; - blk->right = NULL; - - if (!root) { - blk->parent = NULL; - SET_BLACK(blk); - root = blk; - } else { - RBTree_t *x = root; - while (1) { - Uint size; - - size = BLK_SZ(x); - - if (blk_sz < size || (blk_sz == size && blk < x)) { - if (!x->left) { - blk->parent = x; - x->left = blk; - break; - } - x = x->left; - } - else { - if (!x->right) { - blk->parent = x; - x->right = blk; - break; - } - x = x->right; - } - - } - - /* Insert block into size tree */ - RBT_ASSERT(blk->parent); - - SET_RED(blk); - if (IS_RED(blk->parent)) { - tree_insert_fixup(blk); - } - } - -#ifdef HARD_DEBUG - check_tree(0); -#endif -} - - -static Block_t * -get_free_block(Uint size) -{ - RBTree_t *x = root; - RBTree_t *blk = NULL; - Uint blk_sz; - - while (x) { - blk_sz = BLK_SZ(x); - if (blk_sz < size) { - x = x->right; - } - else { - blk = x; - x = x->left; - } - } - - if (!blk) - return NULL; - -#ifdef HARD_DEBUG - ASSERT(blk == check_tree(size)); -#endif - - unlink_free_block((Block_t *) blk); - - return (Block_t *) blk; -} - - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * Debug functions * -\* */ - - -#ifdef HARD_DEBUG - -#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) -#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) - -#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG) -#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG) - -#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG) -#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG) - - -#if 0 -# define PRINT_TREE -#else -# undef PRINT_TREE -#endif - -#ifdef PRINT_TREE -static void print_tree(void); -#endif - -/* - * Checks that the order between parent and children are correct, - * and that the Red-Black Tree properies are satisfied. if size > 0, - * check_tree() returns a node that satisfies "best fit" resp. - * "address order best fit". - * - * The Red-Black Tree properies are: - * 1. Every node is either red or black. - * 2. Every leaf (NIL) is black. - * 3. If a node is red, then both its children are black. - * 4. Every simple path from a node to a descendant leaf - * contains the same number of black nodes. - */ - -static RBTree_t * -check_tree(Uint size) -{ - RBTree_t *res = NULL; - Sint blacks; - Sint curr_blacks; - RBTree_t *x; - -#ifdef PRINT_TREE - print_tree(); -#endif - - if (!root) - return res; - - x = root; - ASSERT(IS_BLACK(x)); - ASSERT(!x->parent); - curr_blacks = 1; - blacks = -1; - - while (x) { - if (!IS_LEFT_VISITED(x)) { - SET_LEFT_VISITED(x); - if (x->left) { - x = x->left; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - ASSERT(blacks == curr_blacks); - } - } - - if (!IS_RIGHT_VISITED(x)) { - SET_RIGHT_VISITED(x); - if (x->right) { - x = x->right; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - ASSERT(blacks == curr_blacks); - } - } - - - if (IS_RED(x)) { - ASSERT(IS_BLACK(x->right)); - ASSERT(IS_BLACK(x->left)); - } - - ASSERT(x->parent || x == root); - - if (x->left) { - ASSERT(x->left->parent == x); - ASSERT(BLK_SZ(x->left) < BLK_SZ(x) - || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x)); - } - - if (x->right) { - ASSERT(x->right->parent == x); - ASSERT(BLK_SZ(x->right) > BLK_SZ(x) - || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x)); - } - - if (size && BLK_SZ(x) >= size) { - if (!res - || BLK_SZ(x) < BLK_SZ(res) - || (BLK_SZ(x) == BLK_SZ(res) && x < res)) - res = x; - } - - UNSET_LEFT_VISITED(x); - UNSET_RIGHT_VISITED(x); - if (IS_BLACK(x)) - curr_blacks--; - x = x->parent; - - } - - ASSERT(curr_blacks == 0); - - UNSET_LEFT_VISITED(root); - UNSET_RIGHT_VISITED(root); - - return res; - -} - - -#ifdef PRINT_TREE -#define INDENT_STEP 2 - -#include <stdio.h> - -static void -print_tree_aux(RBTree_t *x, int indent) -{ - int i; - - if (!x) { - for (i = 0; i < indent; i++) { - putc(' ', stderr); - } - fprintf(stderr, "BLACK: nil\r\n"); - } - else { - print_tree_aux(x->right, indent + INDENT_STEP); - for (i = 0; i < indent; i++) { - putc(' ', stderr); - } - fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n", - IS_BLACK(x) ? "BLACK" : "RED", - BLK_SZ(x), - (Uint) x); - print_tree_aux(x->left, indent + INDENT_STEP); - } -} - - -static void -print_tree(void) -{ - fprintf(stderr, " --- Size-Adress tree begin ---\r\n"); - print_tree_aux(root, 0); - fprintf(stderr, " --- Size-Adress tree end ---\r\n"); -} - -#endif - -#endif - -#endif /* ENABLE_ELIB_MALLOC */ diff --git a/erts/emulator/beam/elib_stat.h b/erts/emulator/beam/elib_stat.h deleted file mode 100644 index d8c7f31737..0000000000 --- a/erts/emulator/beam/elib_stat.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* -** Interface to elib statistics -** -*/ -#ifndef __ELIB_STAT_H__ -#define __ELIB_STAT_H__ - -struct elib_stat { - int mem_total; /* Number of heap words */ - int mem_blocks; /* Number of block */ - int mem_alloc; /* Number of words in use */ - int mem_free; /* Number of words free */ - int min_used; /* Size of the smallest block used */ - int max_free; /* Size of the largest free block */ - int free_blocks; /* Number of fragments in free list */ - int mem_max_alloc;/* Max number of words in use */ -}; - -EXTERN_FUNCTION(void, elib_statistics, (void*)); -EXTERN_FUNCTION(int, elib_check_heap, (_VOID_)); -EXTERN_FUNCTION(void, elib_heap_dump, (char*)); -EXTERN_FUNCTION(void, elib_stat, (struct elib_stat*)); -EXTERN_FUNCTION(int, elib_heap_map, (unsigned char*, int)); -EXTERN_FUNCTION(int, elib_histo, (unsigned long*, unsigned long*, int, int)); - -#endif diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index b853ec0f01..cda404af5e 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -38,9 +38,6 @@ #include "erl_bits.h" #include "erl_instrument.h" #include "erl_mseg.h" -#ifdef ELIB_ALLOC_IS_CLIB -#include "erl_version.h" -#endif #include "erl_monitors.h" #include "erl_bif_timer.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) @@ -64,8 +61,15 @@ #ifdef DEBUG static Uint install_debug_functions(void); +#if 0 +#define HARD_DEBUG +#ifdef __GNUC__ +#warning "* * * * * * * * * * * * * *" +#warning "* HARD DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * *" +#endif +#endif #endif -extern void elib_ensure_initialized(void); ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; @@ -86,6 +90,10 @@ typedef union { static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; +#if HALFWORD_HEAP +static ErtsAllocatorState_t std_alloc_low_state; +static ErtsAllocatorState_t ll_alloc_low_state; +#endif static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; static ErtsAllocatorState_t binary_alloc_state; @@ -162,6 +170,10 @@ typedef struct { struct au_init binary_alloc; struct au_init ets_alloc; struct au_init driver_alloc; +#if HALFWORD_HEAP + struct au_init std_alloc_low; + struct au_init ll_alloc_low; +#endif } erts_alc_hndl_args_init_t; #define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} @@ -189,6 +201,10 @@ set_default_sl_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; +#if HALFWORD_HEAP + ip->init.util.low_mem = 1; +#endif + } static void @@ -220,7 +236,7 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; ip->init.util.mmmbc = 0; - ip->init.util.sbct = ~((Uint) 0); + ip->init.util.sbct = ~((UWord) 0); ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY @@ -252,6 +268,9 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY; ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; +#if HALFWORD_HEAP + ip->init.util.low_mem = 1; +#endif } static void @@ -271,6 +290,9 @@ set_default_eheap_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; +#if HALFWORD_HEAP + ip->init.util.low_mem = 1; +#endif } static void @@ -391,10 +413,14 @@ refuse_af_strategy(struct au_init *init) static void init_thr_ix(int static_ixs); +#ifdef HARD_DEBUG +static void hdbg_init(void); +#endif + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { - Uint extra_block_size = 0; + UWord extra_block_size = 0; int i; erts_alc_hndl_args_init_t init = { 0, @@ -406,6 +432,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) ERTS_DEFAULT_ALCU_INIT }; +#ifdef HARD_DEBUG + hdbg_init(); +#endif + erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); @@ -519,6 +549,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1; +#if HALFWORD_HEAP + /* Init low memory variants by cloning */ + init.std_alloc_low = init.std_alloc; + init.std_alloc_low.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; + init.std_alloc_low.init.util.low_mem = 1; + + init.ll_alloc_low = init.ll_alloc; + init.ll_alloc_low.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; + init.ll_alloc_low.init.util.low_mem = 1; + + set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low); + set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low); +#endif /* HALFWORD */ + set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc); @@ -542,7 +586,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold); sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad); - if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled) erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR; else @@ -565,7 +608,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, &ll_alloc_state); - +#if HALFWORD_HEAP + start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, + &init.ll_alloc_low, + &ll_alloc_low_state); + start_au_allocator(ERTS_ALC_A_STANDARD_LOW, + &init.std_alloc_low, + &std_alloc_low_state); +#endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, &eheap_alloc_state); @@ -601,11 +651,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_set_fix_size(ERTS_ALC_T_PROC, sizeof(Process)); erts_set_fix_size(ERTS_ALC_T_DB_TABLE, sizeof(DbTable)); erts_set_fix_size(ERTS_ALC_T_ATOM, sizeof(Atom)); - erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); + erts_set_fix_size(ERTS_ALC_T_MODULE, sizeof(Module)); erts_set_fix_size(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); - erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); - erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY, sizeof(ErlFunEntry)); #ifdef ERTS_ALC_T_DRV_EV_D_STATE erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE, @@ -615,6 +663,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE, sizeof(ErtsDrvSelectDataState)); #endif +#if !HALFWORD_HEAP + erts_set_fix_size(ERTS_ALC_T_EXPORT, sizeof(Export)); + erts_set_fix_size(ERTS_ALC_T_MONITOR_SH, ERTS_MONITOR_SH_SIZE*sizeof(Uint)); + erts_set_fix_size(ERTS_ALC_T_NLINK_SH, ERTS_LINK_SH_SIZE*sizeof(Uint)); +#endif #endif #endif @@ -627,6 +680,15 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init) ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; +#if HALFWORD_HEAP + /* If halfword heap, silently ignore any disabling of internal + * allocators for low memory + */ + if (init->init.util.low_mem) { + init->enable = 1; + } +#endif + if (!init->enable) { af->alloc = erts_sys_alloc; af->realloc = erts_sys_realloc; @@ -719,8 +781,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1); - states = ((((Uint) states) & ERTS_CACHE_LINE_MASK) - ? (void *) ((((Uint) states) & ~ERTS_CACHE_LINE_MASK) + states = ((((UWord) states) & ERTS_CACHE_LINE_MASK) + ? (void *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE) : (void *) states); tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL; @@ -1337,7 +1399,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[j++] = argv[i]; } *argc = j; - } static char *type_no_str(ErtsAlcType_t n) @@ -1393,6 +1454,33 @@ void erts_alloc_reg_scheduler_id(Uint id) erts_tsd_set(thr_ix_key, (void *)(long) ix); } +static void +no_verify(Allctr_t *allctr) +{ + +} + +erts_alloc_verify_func_t +erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr) +{ + if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util + && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) { + ErtsAllocatorThrSpec_t *tspec; + tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY]; + if (!tspec->all_thr_safe) { + int ix = erts_alc_get_thr_ix(); + + if (ix < tspec->size) { + *allctr = tspec->allctr[ix]; + return erts_alcu_verify_unused; + } + } + } + + *allctr = NULL; + return no_verify; +} + __decl_noreturn void erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) { @@ -1483,10 +1571,10 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size); } -static ERTS_INLINE Uint +static ERTS_INLINE UWord alcu_size(ErtsAlcType_t ai) { - Uint res = 0; + UWord res = 0; ASSERT(erts_allctrs_info[ai].enabled); ASSERT(erts_allctrs_info[ai].alloc_util); @@ -1518,6 +1606,49 @@ alcu_size(ErtsAlcType_t ai) return res; } +#if HALFWORD_HEAP +static ERTS_INLINE int +alcu_is_low(ErtsAlcType_t ai) +{ + int is_low = 0; + ASSERT(erts_allctrs_info[ai].enabled); + ASSERT(erts_allctrs_info[ai].alloc_util); + + if (!erts_allctrs_info[ai].thr_spec) { + Allctr_t *allctr = erts_allctrs_info[ai].extra; + is_low = allctr->mseg_opt.low_mem; + } + else { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; + int i; +# ifdef DEBUG + int found_one = 0; +# endif + + ASSERT(tspec->all_thr_safe); + ASSERT(tspec->enabled); + + for (i = tspec->size - 1; i >= 0; i--) { + Allctr_t *allctr = tspec->allctr[i]; + if (allctr) { +# ifdef DEBUG + if (!found_one) { + is_low = allctr->mseg_opt.low_mem; + found_one = 1; + } + else ASSERT(is_low == allctr->mseg_opt.low_mem); +# else + is_low = allctr->mseg_opt.low_mem; + break; +# endif + } + } + ASSERT(found_one); + } + return is_low; +} +#endif /* HALFWORD */ + Eterm erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) { @@ -1534,23 +1665,28 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) int code; int ets; int maximum; +#if HALFWORD_HEAP + int low; +#endif } want = {0}; struct { - Uint total; - Uint processes; - Uint processes_used; - Uint system; - Uint atom; - Uint atom_used; - Uint binary; - Uint code; - Uint ets; - Uint maximum; + UWord total; + UWord processes; + UWord processes_used; + UWord system; + UWord atom; + UWord atom_used; + UWord binary; + UWord code; + UWord ets; + UWord maximum; +#if HALFWORD_HEAP + UWord low; +#endif } size = {0}; - Eterm atoms[sizeof(size)/sizeof(Uint)]; - Uint *uintps[sizeof(size)/sizeof(Uint)]; - Eterm euints[sizeof(size)/sizeof(Uint)]; - int need_atom; + Eterm atoms[sizeof(size)/sizeof(UWord)]; + UWord *uintps[sizeof(size)/sizeof(UWord)]; + Eterm euints[sizeof(size)/sizeof(UWord)]; int want_tot_or_sys; int length; Eterm res = THE_NON_VALUE; @@ -1602,15 +1738,20 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } - +#if HALFWORD_HEAP + want.low = 1; + atoms[length] = am_low; + uintps[length++] = &size.low; +#endif } else { - Eterm tmp_heap[2]; + DeclareTmpHeapNoproc(tmp_heap,2); Eterm wanted_list; if (is_nil(earg)) return NIL; + UseTmpHeapNoproc(2); if (is_not_atom(earg)) wanted_list = earg; else { @@ -1690,15 +1831,27 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } - } - else + } else { + UnUseTmpHeapNoproc(2); return am_badarg; + } + break; +#if HALFWORD_HEAP + case am_low: + if (!want.low) { + want.low = 1; + atoms[length] = am_low; + uintps[length++] = &size.low; + } break; +#endif default: + UnUseTmpHeapNoproc(2); return am_badarg; } wanted_list = CDR(list_val(wanted_list)); } + UnUseTmpHeapNoproc(2); if (is_not_nil(wanted_list)) return am_badarg; } @@ -1721,7 +1874,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) ASSERT(length <= sizeof(atoms)/sizeof(Eterm)); ASSERT(length <= sizeof(euints)/sizeof(Eterm)); - ASSERT(length <= sizeof(uintps)/sizeof(Uint)); + ASSERT(length <= sizeof(uintps)/sizeof(UWord)); if (proc) { @@ -1734,15 +1887,14 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) /* Calculate values needed... */ want_tot_or_sys = want.total || want.system; - need_atom = ERTS_MEM_NEED_ALL_ALCU || want.atom; if (ERTS_MEM_NEED_ALL_ALCU) { size.total = 0; for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { if (erts_allctrs_info[ai].alloc_util) { - Uint *save; - Uint asz; + UWord *save; + UWord asz; switch (ai) { case ERTS_ALC_A_TEMPORARY: /* @@ -1767,6 +1919,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (save) *save = asz; size.total += asz; +#if HALFWORD_HEAP + if (alcu_is_low(ai)) { + size.low += asz; + } +#endif } } } @@ -1774,7 +1931,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (want_tot_or_sys || want.processes || want.processes_used) { - Uint tmp; + UWord tmp; if (ERTS_MEM_NEED_ALL_ALCU) tmp = size.processes; @@ -1789,6 +1946,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) size.processes = size.processes_used = tmp; +#if HALFWORD_HEAP + /* BUG: We ignore link and monitor memory */ +#else erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; @@ -1796,6 +1956,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi); size.processes += efi.total; size.processes_used += efi.used; +#endif erts_fix_info(ERTS_ALC_T_PROC, &efi); size.processes += efi.total; @@ -1832,8 +1993,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) erts_fix_info(ERTS_ALC_T_MODULE, &efi); size.code += efi.used; size.code += export_table_sz(); +#if HALFWORD_HEAP + size.code += export_list_size() * sizeof(Export); +#else erts_fix_info(ERTS_ALC_T_EXPORT, &efi); size.code += efi.used; +#endif size.code += erts_fun_table_sz(); erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi); size.code += efi.used; @@ -1879,9 +2044,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (only_one_value) { ASSERT(length == 1); hsz = 0; - erts_bld_uint(NULL, &hsz, *uintps[0]); + erts_bld_uword(NULL, &hsz, *uintps[0]); hp = hsz ? HAlloc((Process *) proc, hsz) : NULL; - res = erts_bld_uint(&hp, NULL, *uintps[0]); + res = erts_bld_uword(&hp, NULL, *uintps[0]); } else { Uint **hpp = NULL; @@ -1891,7 +2056,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) while (1) { int i; for (i = 0; i < length; i++) - euints[i] = erts_bld_uint(hpp, hszp, *uintps[i]); + euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]); res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints); if (hpp) break; @@ -2060,11 +2225,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) for (i = 0; i < length; i++) { switch (values[i].arity) { case 2: - erts_print(to, arg, "%s: %bpu\n", + erts_print(to, arg, "%s: %beu\n", values[i].name, values[i].ui[0]); break; case 3: - erts_print(to, arg, "%s: %bpu %bpu\n", + erts_print(to, arg, "%s: %beu %beu\n", values[i].name, values[i].ui[0], values[i].ui[1]); break; default: @@ -2285,8 +2450,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) SysAllocStat sas; Eterm opts_am; Eterm opts; - Eterm as[4]; - Eterm ts[4]; + Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ + Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */ int l; if (only_sz) @@ -2302,13 +2467,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) l = 0; as[l] = am_atom_put("e", 1); ts[l++] = am_true; -#ifdef ELIB_ALLOC_IS_CLIB - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("elib", 4); -#else as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); -#endif if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, @@ -2462,11 +2622,7 @@ erts_allocator_info(int to, void *arg) case ERTS_ALC_A_SYSTEM: { SysAllocStat sas; erts_print(to, arg, "option e: true\n"); -#ifdef ELIB_ALLOC_IS_CLIB - erts_print(to, arg, "option m: elib\n"); -#else erts_print(to, arg, "option m: libc\n"); -#endif sys_alloc_stat(&sas); if(sas.trim_threshold >= 0) erts_print(to, arg, "option tt: %d\n", sas.trim_threshold); @@ -2570,13 +2726,8 @@ erts_allocator_options(void *proc) switch (a) { case ERTS_ALC_A_SYSTEM: -#ifdef ELIB_ALLOC_IS_CLIB - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("elib", 4); -#else as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); -#endif if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, @@ -2647,23 +2798,7 @@ erts_allocator_options(void *proc) features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; -#if defined(ELIB_ALLOC_IS_CLIB) - { - Eterm version; - int i; - int ver[5]; - i = sscanf(ERLANG_VERSION, - "%d.%d.%d.%d.%d", - &ver[0], &ver[1], &ver[2], &ver[3], &ver[4]); - - version = NIL; - for(i--; i >= 0; i--) - version = erts_bld_cons(hpp, szp, make_small(ver[i]), version); - - res = erts_bld_tuple(hpp, szp, 4, - am_elib_malloc, version, features, settings); - } -#elif defined(__GLIBC__) +#if defined(__GLIBC__) { Eterm AM_glibc = am_atom_put("glibc", 5); Eterm version; @@ -2859,12 +2994,10 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0a: - if (ethr_mutex_lock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_lock((ethr_mutex *) a1); break; case 0xf0b: - if (ethr_mutex_unlock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); @@ -2880,31 +3013,21 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0e: - if (ethr_cond_broadcast((ethr_cond *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_cond_broadcast((ethr_cond *) a1); break; case 0xf0f: { int res; do { res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2); } while (res == EINTR); - if (res != 0) - ERTS_ALC_TEST_ABORT; break; } case 0xf10: { ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid)); -#ifdef ERTS_ENABLE_LOCK_COUNT - if (erts_lcnt_thr_create(tid, - (void * (*)(void *)) a1, - (void *) a2, - NULL) != 0) -#else if (ethr_thr_create(tid, (void * (*)(void *)) a1, (void *) a2, NULL) != 0) -#endif ERTS_ALC_TEST_ABORT; return (unsigned long) tid; } @@ -2943,10 +3066,13 @@ unsigned long erts_alc_test(unsigned long op, #undef PRINT_OPS #endif +#ifdef HARD_DEBUG +#define FENCE_SZ (4*sizeof(UWord)) +#else +#define FENCE_SZ (3*sizeof(UWord)) +#endif -#define FENCE_SZ (3*sizeof(Uint)) - -#ifdef ARCH_64 +#if defined(ARCH_64) #define FENCE_PATTERN 0xABCDEF97ABCDEF97 #else #define FENCE_PATTERN 0xABCDEF97 @@ -2956,7 +3082,7 @@ unsigned long erts_alc_test(unsigned long op, #define TYPE_PATTERN_SHIFT 16 #define FIXED_FENCE_PATTERN_MASK \ - (~((Uint) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT))) + (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT))) #define FIXED_FENCE_PATTERN \ (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK) @@ -2966,22 +3092,170 @@ unsigned long erts_alc_test(unsigned long op, #define GET_TYPE_OF_PATTERN(P) \ (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK) +#ifdef HARD_DEBUG + +#define ERL_ALC_HDBG_MAX_MBLK 100000 +#define ERTS_ALC_O_CHECK -1 + +typedef struct hdbg_mblk_ hdbg_mblk; +struct hdbg_mblk_ { + hdbg_mblk *next; + hdbg_mblk *prev; + void *p; + Uint s; + ErtsAlcType_t n; +}; + +static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK]; + +static hdbg_mblk *free_hdbg_mblks; +static hdbg_mblk *used_hdbg_mblks; +static erts_mtx_t hdbg_mblk_mtx; + +static void +hdbg_init(void) +{ + int i; + for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++) + hdbg_mblks[i].next = &hdbg_mblks[i+1]; + hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL; + free_hdbg_mblks = &hdbg_mblks[0]; + used_hdbg_mblks = NULL; + erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug"); +} + +static void *check_memory_fence(void *ptr, + Uint *size, + ErtsAlcType_t n, + int func); +void erts_hdbg_chk_blks(void); + +void +erts_hdbg_chk_blks(void) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) { + Uint sz; + check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK); + ASSERT(sz == mblk->s); + } + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +static hdbg_mblk * +hdbg_alloc(void *p, Uint s, ErtsAlcType_t n) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + mblk = free_hdbg_mblks; + if (!mblk) { + erts_fprintf(stderr, + "Ran out of debug blocks; please increase " + "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n", + ERL_ALC_HDBG_MAX_MBLK); + abort(); + } + free_hdbg_mblks = mblk->next; + + mblk->p = p; + mblk->s = s; + mblk->n = n; + + mblk->next = used_hdbg_mblks; + mblk->prev = NULL; + if (used_hdbg_mblks) + used_hdbg_mblks->prev = mblk; + used_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); + return (void *) mblk; +} + +static void +hdbg_free(hdbg_mblk *mblk) +{ + erts_mtx_lock(&hdbg_mblk_mtx); + if (mblk->next) + mblk->next->prev = mblk->prev; + if (mblk->prev) + mblk->prev->next = mblk->next; + else + used_hdbg_mblks = mblk->next; + + mblk->next = free_hdbg_mblks; + free_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +#endif + +#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG +static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func); + +void check_allocated_block( Uint type, void *blk) +{ + Uint dummy; + check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE); +} + +void check_allocators(void) +{ + int i; + if (!erts_initialized) + return; + for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) { + if (erts_allctrs_info[i].alloc_util) { + ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra; + Allctr_t *allctr = real_af->extra; + Carrier_t *ct; +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + if (allctr->check_mbc) { + for (ct = allctr->mbc_list.first; ct; ct = ct->next) { + fprintf(stderr,"Checking allocator %d\r\n",i); + allctr->check_mbc(allctr,ct); + } + } +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif + } + } +} +#endif static void * set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) { - Uint *ui_ptr; - Uint pattern; + UWord *ui_ptr; + UWord pattern; +#ifdef HARD_DEBUG + hdbg_mblk **mblkpp; +#endif if (!ptr) return NULL; - ui_ptr = (Uint *) ptr; + ui_ptr = (UWord *) ptr; pattern = MK_PATTERN(n); - + +#ifdef HARD_DEBUG + mblkpp = (hdbg_mblk **) ui_ptr++; +#endif + *(ui_ptr++) = sz; *(ui_ptr++) = pattern; - memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(Uint)); + memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); + +#ifdef HARD_DEBUG + *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); +#endif return (void *) ui_ptr; } @@ -2991,16 +3265,22 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) { Uint sz; Uint found_type; - Uint pre_pattern; - Uint post_pattern; - Uint *ui_ptr; + UWord pre_pattern; + UWord post_pattern; + UWord *ui_ptr; +#ifdef HARD_DEBUG + hdbg_mblk *mblk; +#endif if (!ptr) return NULL; - ui_ptr = (Uint *) ptr; + ui_ptr = (UWord *) ptr; pre_pattern = *(--ui_ptr); *size = sz = *(--ui_ptr); +#ifdef HARD_DEBUG + mblk = (hdbg_mblk *) *(--ui_ptr); +#endif found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { @@ -3011,7 +3291,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (unsigned long) ptr); } - memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(Uint)); + memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); if (post_pattern != MK_PATTERN(n) || pre_pattern != post_pattern) { @@ -3056,6 +3336,17 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype); } +#ifdef HARD_DEBUG + switch (func) { + case ERTS_ALC_O_REALLOC: + case ERTS_ALC_O_FREE: + hdbg_free(mblk); + break; + default: + break; + } +#endif + return (void *) ui_ptr; } @@ -3068,6 +3359,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint dsize; void *res; +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; res = (*real_af->alloc)(n, real_af->extra, dsize); @@ -3097,13 +3392,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) dsize = size + FENCE_SZ; dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC); +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + if (old_size > size) sys_memset((void *) (((char *) ptr) + size), 0xf, sizeof(Uint) + old_size - size); res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); - + res = set_memory_fence(res, size, n); #ifdef PRINT_OPS @@ -3133,6 +3432,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); #endif +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + } static Uint diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index e7a203002f..ce792d4d17 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -172,9 +172,17 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size); void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); +void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size); + #endif /* #if !ERTS_ALC_DO_INLINE */ +#ifndef ERTS_CACHE_LINE_SIZE +/* Assume a cache line size of 64 bytes */ +# define ERTS_CACHE_LINE_SIZE ((UWord) 64) +# define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) +#endif + #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) ERTS_ALC_INLINE @@ -234,13 +242,25 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) size); } +ERTS_ALC_INLINE +void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size) +{ + UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)); + + if (v & ERTS_CACHE_LINE_MASK) { + v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + } + ASSERT((v & ERTS_CACHE_LINE_MASK) == 0); + return (void*)v; +} + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ -#ifndef ERTS_CACHE_LINE_SIZE -/* Assume a cache line size of 64 bytes */ -# define ERTS_CACHE_LINE_SIZE ((Uint) 64) -# define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) -#endif +typedef void (*erts_alloc_verify_func_t)(Allctr_t *); + +erts_alloc_verify_func_t +erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr); + #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \ (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE) @@ -486,9 +506,9 @@ init_##NAME##_alloc(void) \ qa_data_##NAME##__ = erts_alloc(ERTS_ALC_T_PRE_ALLOC_DATA,tot_size);\ chunk_start = (((char *) qa_data_##NAME##__) \ + sizeof(erts_sched_pref_quick_alloc_data_t)); \ - if ((((Uint) chunk_start) & ERTS_CACHE_LINE_MASK) != ((Uint) 0)) \ + if ((((UWord) chunk_start) & ERTS_CACHE_LINE_MASK) != ((UWord) 0)) \ chunk_start = ((char *) \ - ((((Uint) chunk_start) & ~ERTS_CACHE_LINE_MASK) \ + ((((UWord) chunk_start) & ~ERTS_CACHE_LINE_MASK) \ + ERTS_CACHE_LINE_SIZE)); \ qa_data_##NAME##__->chunks_mem_size = chunk_mem_size; \ qa_data_##NAME##__->start = (void *) chunk_start; \ @@ -553,7 +573,7 @@ NAME##_free(TYPE *p) \ } #ifdef DEBUG -#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((Uint *) (PTR)) - 2)) +#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2)) #endif /* #ifdef DEBUG */ #undef ERTS_ALC_INLINE diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index f701f71c7d..c6cc0e1fac 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 2003-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 2003-2011. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -75,6 +75,11 @@ allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc ++if halfword +allocator LONG_LIVED_LOW true ll_alloc_low +allocator STANDARD_LOW true std_alloc_low ++endif + +else # Non smp build allocator TEMPORARY false temp_alloc @@ -85,12 +90,18 @@ allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc ++if halfword +allocator LONG_LIVED_LOW false ll_alloc_low +allocator STANDARD_LOW false std_alloc_low ++endif + +endif allocator BINARY true binary_alloc allocator DRIVER true driver_alloc + # --- Class declarations ----------------------------------------------------- # # Syntax: class <CLASS> <DESCRIPTION> @@ -125,19 +136,15 @@ class SYSTEM system_data type PROC FIXED_SIZE PROCESSES proc type ATOM FIXED_SIZE ATOM atom_entry -type EXPORT FIXED_SIZE CODE export_entry type MODULE FIXED_SIZE CODE module_entry type REG_PROC FIXED_SIZE PROCESSES reg_proc type LINK_LH STANDARD PROCESSES link_lh -type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh -type MONITOR_LH STANDARD PROCESSES monitor_lh -type NLINK_SH FIXED_SIZE PROCESSES nlink_sh -type NLINK_LH STANDARD PROCESSES nlink_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list type FUN_ENTRY FIXED_SIZE CODE fun_entry type ATOM_TXT LONG_LIVED ATOM atom_text +type BEAM_REGISTER EHEAP PROCESSES beam_register type HEAP EHEAP PROCESSES heap type OLD_HEAP EHEAP PROCESSES old_heap type HEAP_FRAG EHEAP PROCESSES heap_frag @@ -174,7 +181,6 @@ type DRIVER STANDARD SYSTEM driver type NIF DRIVER SYSTEM nif_internal type BINARY BINARY BINARIES binary type NBIF_TABLE SYSTEM SYSTEM nbif_tab -type CODE LONG_LIVED CODE code type ARG_REG STANDARD PROCESSES arg_reg type PROC_DICT STANDARD PROCESSES proc_dict type CALLS_BUF STANDARD PROCESSES calls_buf @@ -194,7 +200,6 @@ type DB_TABLES LONG_LIVED ETS db_tabs type DB_NTAB_ENT STANDARD ETS db_named_table_entry type DB_TMP TEMPORARY ETS db_tmp type DB_MC_STK TEMPORARY ETS db_mc_stack -type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type DB_MS_RUN_HEAP SHORT_LIVED ETS db_match_spec_run_heap type DB_MS_CMPL_HEAP TEMPORARY ETS db_match_spec_cmpl_heap type DB_SEG ETS ETS db_segment @@ -211,9 +216,8 @@ type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur -type SCHDLR_DATA LONG_LIVED PROCESSES scheduler_data +type SCHDLR_SLP_INFO LONG_LIVED SYSTEM scheduler_sleep_info type RUNQS LONG_LIVED SYSTEM run_queues -type DDLL_PROCESS STANDARD SYSTEM ddll_processes type DDLL_HANDLE STANDARD SYSTEM ddll_handle type DDLL_ERRCODES LONG_LIVED SYSTEM ddll_errcodes type DDLL_TMP_BUF TEMPORARY SYSTEM ddll_tmp_buf @@ -231,6 +235,7 @@ type RE_SUBJECT SHORT_LIVED SYSTEM re_subject type RE_HEAP STANDARD SYSTEM re_heap type RE_STACK SHORT_LIVED SYSTEM re_stack type UNICODE_BUFFER SHORT_LIVED SYSTEM unicode_buffer +type BINARY_BUFFER SHORT_LIVED SYSTEM binary_buffer type PRE_ALLOC_DATA LONG_LIVED SYSTEM pre_alloc_data type DRV_THR_OPTS DRIVER SYSTEM driver_thread_opts type DRV_TID DRIVER SYSTEM driver_tid @@ -244,6 +249,7 @@ type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib +type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -259,6 +265,8 @@ type XPORTS_LIST SHORT_LIVED SYSTEM extra_port_list type PROC_LCK_WTR LONG_LIVED SYSTEM proc_lock_waiter type PROC_LCK_QS LONG_LIVED SYSTEM proc_lock_queues type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q +type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work +endif # @@ -267,7 +275,9 @@ type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +if threads -type ETHR_INTERNAL SYSTEM SYSTEM ethread_internal +type ETHR_STD STANDARD SYSTEM ethread_standard +type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived +type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived +ifnot smp @@ -318,10 +328,45 @@ type SSB SHORT_LIVED PROCESSES ssb +endif ++if halfword + +type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes +type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh +type NLINK_LH STANDARD_LOW PROCESSES nlink_lh +type CODE LONG_LIVED_LOW CODE code +type DB_HEIR_DATA STANDARD_LOW ETS db_heir_data +type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc +type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data +type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term + +# no FIXED_SIZE for low memory +type EXPORT STANDARD_LOW CODE export_entry +type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh +type NLINK_SH STANDARD_LOW PROCESSES nlink_sh + ++else # "fullword" + +type DDLL_PROCESS STANDARD SYSTEM ddll_processes +type MONITOR_LH STANDARD PROCESSES monitor_lh +type NLINK_LH STANDARD PROCESSES nlink_lh +type CODE LONG_LIVED CODE code +type DB_HEIR_DATA STANDARD ETS db_heir_data +type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc +type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data +type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term + +type EXPORT FIXED_SIZE CODE export_entry +type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh +type NLINK_SH FIXED_SIZE PROCESSES nlink_sh + ++endif + + # # Types used by system specific code # +type TEMP_TERM TEMPORARY SYSTEM temp_term type DRV_TAB LONG_LIVED SYSTEM drv_tab type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 9b7bc24c1c..cc04ef65bf 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -69,14 +69,14 @@ static int initialized = 0; #if HAVE_ERTS_MSEG -#define INV_MSEG_UNIT_MASK ((Uint) (mseg_unit_size - 1)) +#define INV_MSEG_UNIT_MASK ((UWord) (mseg_unit_size - 1)) #define MSEG_UNIT_MASK (~INV_MSEG_UNIT_MASK) #define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK) #define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + INV_MSEG_UNIT_MASK) #endif -#define INV_SYS_ALLOC_CARRIER_MASK ((Uint) (sys_alloc_carrier_size - 1)) +#define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1)) #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_FLOOR(X) ((X) & SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_CEILING(X) \ @@ -85,7 +85,7 @@ static int initialized = 0; #undef ASSERT #define ASSERT ASSERT_EXPR -#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((Uint) 1) +#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((UWord) 1) #if 0 /* Can be useful for debugging */ @@ -114,12 +114,12 @@ static Uint mseg_unit_size; /* Blocks ... */ -#define SBC_BLK_FTR_FLG (((Uint) 1) << 0) -#define UNUSED1_BLK_FTR_FLG (((Uint) 1) << 1) -#define UNUSED2_BLK_FTR_FLG (((Uint) 1) << 2) +#define SBC_BLK_FTR_FLG (((UWord) 1) << 0) +#define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1) +#define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2) #define ABLK_HDR_SZ (sizeof(Block_t)) -#define FBLK_FTR_SZ (sizeof(Uint)) +#define FBLK_FTR_SZ (sizeof(UWord)) #define UMEMSZ2BLKSZ(AP, SZ) \ (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size \ @@ -130,14 +130,14 @@ static Uint mseg_unit_size; #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ)) #define PREV_BLK_SZ(B) \ - ((Uint) (*(((Uint *) (B)) - 1) & SZ_MASK)) + ((UWord) (*(((UWord *) (B)) - 1) & SZ_MASK)) #define SET_BLK_SZ_FTR(B, SZ) \ - (*((Uint *) (((char *) (B)) + (SZ) - sizeof(Uint))) = (SZ)) + (*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ)) -#define THIS_FREE_BLK_HDR_FLG (((Uint) 1) << 0) -#define PREV_FREE_BLK_HDR_FLG (((Uint) 1) << 1) -#define LAST_BLK_HDR_FLG (((Uint) 1) << 2) +#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) +#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) +#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) #define SET_BLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ @@ -156,11 +156,11 @@ static Uint mseg_unit_size; (*((Block_t *) (B)) &= ~LAST_BLK_HDR_FLG) #define SBH_THIS_FREE THIS_FREE_BLK_HDR_FLG -#define SBH_THIS_ALLOCED ((Uint) 0) +#define SBH_THIS_ALLOCED ((UWord) 0) #define SBH_PREV_FREE PREV_FREE_BLK_HDR_FLG -#define SBH_PREV_ALLOCED ((Uint) 0) +#define SBH_PREV_ALLOCED ((UWord) 0) #define SBH_LAST_BLK LAST_BLK_HDR_FLG -#define SBH_NOT_LAST_BLK ((Uint) 0) +#define SBH_NOT_LAST_BLK ((UWord) 0) #define SET_BLK_HDR(B, Sz, F) \ (ASSERT(((Sz) & FLG_MASK) == 0), *((Block_t *) (B)) = ((Sz) | (F))) @@ -200,7 +200,7 @@ static Uint mseg_unit_size; ((FTR) = 0) #define IS_SBC_BLK(B) \ - (IS_PREV_BLK_FREE((B)) && (((Uint *) (B))[-1] & SBC_BLK_FTR_FLG)) + (IS_PREV_BLK_FREE((B)) && (((UWord *) (B))[-1] & SBC_BLK_FTR_FLG)) #define IS_MBC_BLK(B) \ (!IS_SBC_BLK((B))) @@ -211,8 +211,8 @@ static Uint mseg_unit_size; /* Carriers ... */ -#define MSEG_CARRIER_HDR_FLAG (((Uint) 1) << 0) -#define SBC_CARRIER_HDR_FLAG (((Uint) 1) << 1) +#define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) +#define SBC_CARRIER_HDR_FLAG (((UWord) 1) << 1) #define SCH_SYS_ALLOC 0 #define SCH_MSEG MSEG_CARRIER_HDR_FLAG @@ -407,18 +407,18 @@ do { \ /* Debug stuff... */ #ifdef DEBUG -static Uint carrier_alignment; +static UWord carrier_alignment; #define DEBUG_SAVE_ALIGNMENT(C) \ do { \ - Uint algnmnt__ = sizeof(Unit_t) - (((Uint) (C)) % sizeof(Unit_t)); \ + UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t)); \ carrier_alignment = MIN(carrier_alignment, algnmnt__); \ - ASSERT(((Uint) (C)) % sizeof(Uint) == 0); \ + ASSERT(((UWord) (C)) % sizeof(UWord) == 0); \ } while (0) #define DEBUG_CHECK_ALIGNMENT(P) \ do { \ - ASSERT(sizeof(Unit_t) - (((Uint) (P)) % sizeof(Unit_t)) \ + ASSERT(sizeof(Unit_t) - (((UWord) (P)) % sizeof(Unit_t)) \ >= carrier_alignment); \ - ASSERT(((Uint) (P)) % sizeof(Uint) == 0); \ + ASSERT(((UWord) (P)) % sizeof(UWord) == 0); \ } while (0) #else @@ -610,7 +610,7 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } -static Block_t *create_carrier(Allctr_t *, Uint, Uint); +static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *); /* Multi block carrier alloc/realloc/free ... */ @@ -630,6 +630,11 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0); +#if HALFWORD_HEAP + if (!blk) { + blk = create_carrier(allctr, *blk_szp, CFLG_MBC|CFLG_FORCE_MSEG); + } +#else if (!blk) { blk = create_carrier(allctr, *blk_szp, CFLG_MBC); if (!blk) { @@ -640,6 +645,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC); } } +#endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { @@ -656,14 +662,14 @@ static ERTS_INLINE void mbc_alloc_finalize(Allctr_t *allctr, Block_t *blk, Uint org_blk_sz, - Uint flags, + UWord flags, Uint want_blk_sz, int valid_blk_info) { Uint blk_sz; Uint nxt_blk_sz; Block_t *nxt_blk; - Uint prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; + UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG; ASSERT(org_blk_sz >= want_blk_sz); ASSERT(blk); @@ -853,7 +859,7 @@ mbc_free(Allctr_t *allctr, void *p) } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs) { void *new_p; Uint old_blk_sz; @@ -1144,7 +1150,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs) } else { Uint new_blk_sz; - Uint new_blk_flgs; + UWord new_blk_flgs; Uint prev_blk_sz; Uint blk_cpy_sz; @@ -1239,7 +1245,7 @@ do { \ static Block_t * -create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) +create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) { Block_t *blk; Carrier_t *crr; @@ -1283,8 +1289,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) if (crr_sz < allctr->mbc_header_size + blk_sz) crr_sz = allctr->mbc_header_size + blk_sz; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz += sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz += sizeof(UWord); #endif } crr_sz = MSEG_UNIT_CEILING(crr_sz); @@ -1324,8 +1330,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) && bcrr_sz < allctr->smallest_mbc_size) bcrr_sz = allctr->smallest_mbc_size; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - bcrr_sz += sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + bcrr_sz += sizeof(UWord); #endif } @@ -1360,7 +1366,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) blk = SBC2BLK(allctr, crr); - SET_SBC_BLK_FTR(((Uint *) blk)[-1]); + SET_SBC_BLK_FTR(((UWord *) blk)[-1]); SET_BLK_HDR(blk, blk_sz, SBH_THIS_ALLOCED|SBH_PREV_FREE|SBH_LAST_BLK); link_carrier(&allctr->sbc_list, crr); @@ -1379,13 +1385,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) blk = MBC2FBLK(allctr, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz -= sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz -= sizeof(UWord); #endif blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); - SET_MBC_BLK_FTR(((Uint *) blk)[-1]); + SET_MBC_BLK_FTR(((UWord *) blk)[-1]); SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -1400,13 +1406,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) link_carrier(&allctr->mbc_list, crr); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz += sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz += sizeof(UWord); #endif CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (sizeof(Unit_t) == sizeof(Uint)) - crr_sz -= sizeof(Uint); + if (sizeof(Unit_t) == sizeof(UWord)) + crr_sz -= sizeof(UWord); #endif if (allctr->creating_mbc) (*allctr->creating_mbc)(allctr, crr); @@ -1418,11 +1424,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags) } static Block_t * -resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, Uint flags) +resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) { Block_t *new_blk; Carrier_t *new_crr, *old_crr; - Uint create_flags, old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz; + UWord create_flags; + Uint old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz; Uint new_bcrr_sz; if (flags & CFLG_MBC) { @@ -1632,6 +1639,9 @@ static struct { Eterm e; Eterm t; Eterm ramv; +#if HALFWORD_HEAP + Eterm low; +#endif Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; @@ -1717,6 +1727,9 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); +#if HALFWORD_HEAP + AM_INIT(low); +#endif AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); @@ -1870,7 +1883,7 @@ sz_info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers size: %bpu %bpu %bpu\n", + "%scarriers size: %beu %bpu %bpu\n", prefix, curr_size, cs->max.size, @@ -1926,7 +1939,7 @@ info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers: %bpu %bpu %bpu\n", + "%scarriers: %beu %bpu %bpu\n", prefix, curr_no, cs->max.no, @@ -1945,7 +1958,7 @@ info_carriers(Allctr_t *allctr, cs->curr_sys_alloc.no); erts_print(to, arg, - "%scarriers size: %bpu %bpu %bpu\n", + "%scarriers size: %beu %bpu %bpu\n", prefix, curr_size, cs->max.size, @@ -2046,15 +2059,15 @@ info_calls(Allctr_t *allctr, #define PRINT_CC_4(TO, TOA, NAME, CC) \ if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s calls: %bpu\n", NAME, CC.no); \ + erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no); \ else \ - erts_print(TO, TOA, "%s calls: %bpu%09lu\n", NAME, CC.giga_no, CC.no) + erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no) #define PRINT_CC_5(TO, TOA, PRFX, NAME, CC) \ if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s%s calls: %bpu\n",PRFX,NAME,CC.no); \ + erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no); \ else \ - erts_print(TO, TOA, "%s%s calls: %bpu%09lu\n",PRFX,NAME,CC.giga_no,CC.no) + erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no) char *prefix = allctr->name_prefix; int to = *print_to_p; @@ -2161,23 +2174,29 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" - "option sbct: %bpu\n" +#if HALFWORD_HEAP + "option low: %s\n" +#endif + "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" "option rsbcst: %bpu\n" #endif - "option rsbcmt: %bpu\n" - "option rmbcmt: %bpu\n" - "option mmbcs: %bpu\n" + "option rsbcmt: %beu\n" + "option rmbcmt: %beu\n" + "option mmbcs: %beu\n" #if HAVE_ERTS_MSEG - "option mmsbc: %bpu\n" - "option mmmbc: %bpu\n" + "option mmsbc: %beu\n" + "option mmmbc: %beu\n" #endif - "option lmbcs: %bpu\n" - "option smbcs: %bpu\n" - "option mbcgs: %bpu\n", + "option lmbcs: %beu\n" + "option smbcs: %beu\n" + "option mbcgs: %beu\n", topt, allctr->ramv ? "true" : "false", +#if HALFWORD_HEAP + allctr->mseg_opt.low_mem ? "true" : "false", +#endif allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -2236,6 +2255,9 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); +#if HALFWORD_HEAP + add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); +#endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? bld_uint(hpp, szp, (Uint) allctr->t) @@ -2285,9 +2307,9 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, erts_print(*print_to_p, print_to_arg, #if HAVE_ERTS_MSEG - "option mmc: %bpu\n" + "option mmc: %beu\n" #endif - "option ycs: %bpu\n", + "option ycs: %beu\n", #if HAVE_ERTS_MSEG max_mseg_carriers, #endif @@ -2522,7 +2544,12 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); if (size >= allctr->sbc_threshold) { +#if HALFWORD_HEAP + Block_t *blk = create_carrier(allctr, size, + CFLG_SBC | CFLG_FORCE_MSEG); +#else Block_t *blk = create_carrier(allctr, size, CFLG_SBC); +#endif res = blk ? BLK2UMEM(blk) : NULL; } else @@ -2594,16 +2621,16 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) Allctr_t *allctr; void *res; - ASSERT(sizeof(Uint) == sizeof(Allctr_t *)); + ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(ix > 0); if (ix >= tspec->size) ix = (ix % (tspec->size - 1)) + 1; allctr = tspec->allctr[ix]; erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_alloc(type, allctr, size + sizeof(Uint)); + res = do_erts_alcu_alloc(type, allctr, size + sizeof(UWord)); if (res) { *((Allctr_t **) res) = allctr; - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); } erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -2681,7 +2708,7 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t type, void *unused, void *p) { if (p) { - void *ptr = (void *) (((char *) p) - sizeof(Uint)); + void *ptr = (void *) (((char *) p) - sizeof(UWord)); Allctr_t *allctr = *((Allctr_t **) ptr); erts_mtx_lock(&allctr->mutex); do_erts_alcu_free(type, allctr, ptr); @@ -2698,7 +2725,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - Uint flgs) + UWord flgs) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -2780,13 +2807,21 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Block_t *new_blk; if(IS_SBC_BLK(blk)) { do_carrier_resize: +#if HALFWORD_HEAP + new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG); +#else new_blk = resize_carrier(allctr, blk, size, CFLG_SBC); +#endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; else { +#if HALFWORD_HEAP + new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG); +#else new_blk = create_carrier(allctr, size, CFLG_SBC); +#endif if (new_blk) { res = BLK2UMEM(new_blk); sys_memcpy((void *) res, @@ -2962,7 +2997,7 @@ erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - ptr = (void *) (((char *) p) - sizeof(Uint)); + ptr = (void *) (((char *) p) - sizeof(UWord)); used_allctr = *((Allctr_t **) ptr); ix = erts_alc_get_thr_ix(); @@ -2976,32 +3011,32 @@ erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size) res = do_erts_alcu_realloc(type, used_allctr, ptr, - size + sizeof(Uint), + size + sizeof(UWord), (pref_allctr != used_allctr ? ERTS_ALCU_FLG_FAIL_REALLOC_MOVE : 0)); erts_mtx_unlock(&used_allctr->mutex); if (res) { ASSERT(used_allctr == *((Allctr_t **) res)); - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); DEBUG_CHECK_ALIGNMENT(res); } else { erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint)); + res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); erts_mtx_unlock(&pref_allctr->mutex); if (res) { Block_t *blk; size_t cpy_size; *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); DEBUG_CHECK_ALIGNMENT(res); erts_mtx_lock(&used_allctr->mutex); blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint); + cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); @@ -3026,7 +3061,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - ptr = (void *) (((char *) p) - sizeof(Uint)); + ptr = (void *) (((char *) p) - sizeof(UWord)); used_allctr = *((Allctr_t **) ptr); ix = erts_alc_get_thr_ix(); @@ -3037,7 +3072,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, ASSERT(used_allctr && pref_allctr); erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint)); + res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); if (!res) { erts_mtx_unlock(&pref_allctr->mutex); res = erts_alcu_realloc_thr_pref(type, extra, p, size); @@ -3048,7 +3083,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, Allctr_t *allctr; *((Allctr_t **) res) = pref_allctr; - res = (void *) (((char *) res) + sizeof(Uint)); + res = (void *) (((char *) res) + sizeof(UWord)); DEBUG_CHECK_ALIGNMENT(res); @@ -3061,7 +3096,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, } blk = UMEM2BLK(ptr); - cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint); + cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(UWord); if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); @@ -3085,13 +3120,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; #if HAVE_ERTS_MSEG - { - ErtsMsegOpt_t mseg_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER; - - sys_memcpy((void *) &allctr->mseg_opt, - (void *) &mseg_opt, - sizeof(ErtsMsegOpt_t)); - } + sys_memcpy((void *) &allctr->mseg_opt, + (void *) &erts_mseg_default_opt, + sizeof(ErtsMsegOpt_t)); +# if HALFWORD_HEAP + allctr->mseg_opt.low_mem = init->low_mem; +# endif #endif allctr->name_prefix = init->name_prefix; @@ -3138,11 +3172,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->min_block_size < ABLK_HDR_SZ) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size - + sizeof(Uint)); + + sizeof(UWord)); #if HAVE_ERTS_MSEG - if (allctr->mseg_opt.abs_shrink_th > ~((Uint) 0) / 100) - allctr->mseg_opt.abs_shrink_th = ~((Uint) 0) / 100; + if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) + allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; #endif #ifdef USE_THREADS @@ -3182,15 +3216,15 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + FBLK_FTR_SZ + ABLK_HDR_SZ - + sizeof(Uint)) + + sizeof(UWord)) - ABLK_HDR_SZ - - sizeof(Uint)); + - sizeof(UWord)); allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t) + FBLK_FTR_SZ + ABLK_HDR_SZ - + sizeof(Uint)) + + sizeof(UWord)) - ABLK_HDR_SZ - - sizeof(Uint)); + - sizeof(UWord)); } else #endif @@ -3208,12 +3242,21 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->main_carrier_size) { Block_t *blk; +#if HALFWORD_HEAP + blk = create_carrier(allctr, + allctr->main_carrier_size, + CFLG_MBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_MSEG + | CFLG_MAIN_CARRIER); +#else blk = create_carrier(allctr, allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE | CFLG_FORCE_SYS_ALLOC | CFLG_MAIN_CARRIER); +#endif if (!blk) goto error; @@ -3339,6 +3382,38 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) * Debug functions * \* */ +void +erts_alcu_verify_unused(Allctr_t *allctr) +{ + UWord no; + + no = allctr->sbcs.curr_mseg.no; + no += allctr->sbcs.curr_sys_alloc.no; + no += allctr->mbcs.blocks.curr.no; + + if (no) { + UWord sz = allctr->sbcs.blocks.curr.size; + sz += allctr->mbcs.blocks.curr.size; + erl_exit(ERTS_ABORT_EXIT, + "%salloc() used when expected to be unused!\n" + "Total amount of blocks allocated: %bpu\n" + "Total amount of bytes allocated: %bpu\n", + allctr->name_prefix, no, sz); + } +} + +void +erts_alcu_verify_unused_ts(Allctr_t *allctr) +{ +#ifdef USE_THREADS + erts_mtx_lock(&allctr->mutex); +#endif + erts_alcu_verify_unused(allctr); +#ifdef USE_THREADS + erts_mtx_unlock(&allctr->mutex); +#endif +} + #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void @@ -3409,7 +3484,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) is_free_blk = (int) IS_FREE_BLK(blk); if(is_free_blk) { if (IS_NOT_LAST_BLK(blk)) - ASSERT(*((Uint *) (((char *) blk)+blk_sz-sizeof(Uint))) + ASSERT(*((UWord *) (((char *) blk)+blk_sz-sizeof(UWord))) == blk_sz); } @@ -3417,7 +3492,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) (*allctr->check_block)(allctr, blk, (int) is_free_blk); if (IS_LAST_BLK(blk)) { - carrier_end = ((char *) NXT_BLK(blk)) + sizeof(Uint); + carrier_end = ((char *) NXT_BLK(blk)) + sizeof(UWord); mbc = *((Carrier_t **) NXT_BLK(blk)); prev_blk = NULL; blk = MBC2FBLK(allctr, mbc); @@ -3433,7 +3508,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT((((char *) mbc) + allctr->mbc_header_size + tot_blk_sz - + sizeof(Uint)) == carrier_end); + + sizeof(UWord)) == carrier_end); ASSERT(((char *) mbc) + CARRIER_SZ(mbc) == carrier_end); if (allctr->check_mbc) diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 10b11661e6..ddf84c086c 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -27,8 +27,8 @@ typedef struct Allctr_t_ Allctr_t; typedef struct { - Uint ycs; - Uint mmc; + UWord ycs; + UWord mmc; } AlcUInit_t; typedef struct { @@ -38,22 +38,23 @@ typedef struct { int tspec; int tpref; int ramv; - Uint sbct; - Uint asbcst; - Uint rsbcst; - Uint rsbcmt; - Uint rmbcmt; - Uint mmbcs; - Uint mmsbc; - Uint mmmbc; - Uint lmbcs; - Uint smbcs; - Uint mbcgs; + int low_mem; /* HALFWORD only */ + UWord sbct; + UWord asbcst; + UWord rsbcst; + UWord rsbcmt; + UWord rmbcmt; + UWord mmbcs; + UWord mmsbc; + UWord mmmbc; + UWord lmbcs; + UWord smbcs; + UWord mbcgs; } AllctrInit_t; typedef struct { - Uint blocks; - Uint carriers; + UWord blocks; + UWord carriers; } AllctrSize_t; #ifndef SMALL_MEMORY @@ -70,6 +71,7 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) low_mem: HALFWORD only */\ 512*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -97,6 +99,7 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ + 0, /* (bool) low_mem: HALFWORD only */\ 64*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -163,19 +166,19 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *); #define CEILING(X, I) ((((X) - 1)/(I) + 1)*(I)) #undef WORD_MASK -#define INV_WORD_MASK ((Uint) (sizeof(Uint) - 1)) +#define INV_WORD_MASK ((UWord) (sizeof(UWord) - 1)) #define WORD_MASK (~INV_WORD_MASK) #define WORD_FLOOR(X) ((X) & WORD_MASK) #define WORD_CEILING(X) WORD_FLOOR((X) + INV_WORD_MASK) #undef UNIT_MASK -#define INV_UNIT_MASK ((Uint) (sizeof(Unit_t) - 1)) +#define INV_UNIT_MASK ((UWord) (sizeof(Unit_t) - 1)) #define UNIT_MASK (~INV_UNIT_MASK) #define UNIT_FLOOR(X) ((X) & UNIT_MASK) #define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK) -#define SZ_MASK (~((Uint) 0) << 3) +#define SZ_MASK (~((UWord) 0) << 3) #define FLG_MASK (~(SZ_MASK)) @@ -189,7 +192,7 @@ typedef union {char c[8]; long l; double d;} Unit_t; typedef struct Carrier_t_ Carrier_t; struct Carrier_t_ { - Uint chdr; + UWord chdr; Carrier_t *next; Carrier_t *prev; }; @@ -199,17 +202,17 @@ typedef struct { Carrier_t *last; } CarrierList_t; -typedef Uint Block_t; -typedef Uint FreeBlkFtr_t; +typedef UWord Block_t; +typedef UWord FreeBlkFtr_t; typedef struct { - Uint giga_no; - Uint no; + UWord giga_no; + UWord no; } CallCounter_t; typedef struct { - Uint no; - Uint size; + UWord no; + UWord size; } StatValues_t; typedef struct { @@ -333,6 +336,9 @@ struct Allctr_t_ { int erts_alcu_start(Allctr_t *, AllctrInit_t *); void erts_alcu_stop(Allctr_t *); +void erts_alcu_verify_unused(Allctr_t *); +void erts_alcu_verify_unused_ts(Allctr_t *allctr); + unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long); diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 126ec7cc73..64fad9fe0e 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -41,6 +41,16 @@ # define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif +#if !HEAP_ON_C_STACK +# define DECLARE_TMP(VariableName,N,P) \ + Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N)) +#else +# define DECLARE_TMP(VariableName,N,P) \ + Eterm VariableName[2] +#endif +# define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) + + static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right); static ERTS_INLINE void maybe_shrink(Process* p, Eterm* hp, Eterm res, Uint alloc) @@ -169,7 +179,7 @@ shift(Process* p, Eterm arg1, Eterm arg2, int right) { Sint i; Sint ires; - Eterm tmp_big1[2]; + DECLARE_TMP(tmp_big1,0,p); Eterm* bigp; Uint need; @@ -312,8 +322,8 @@ BIF_RETTYPE bnot_1(BIF_ALIST_1) Eterm erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm res; Eterm hdr; FloatDef f1, f2; @@ -458,8 +468,8 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) Eterm erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -602,8 +612,8 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) Eterm erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -627,8 +637,8 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) } else if (arg2 == SMALL_ONE) { return(arg1); } else { - Eterm big_res[3]; - + DeclareTmpHeap(big_res,3,p); + UseTmpHeap(3,p); /* * The following code is optimized for the case that * result is small (which should be the most common case @@ -636,6 +646,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) */ res = small_times(signed_val(arg1), signed_val(arg2), big_res); if (is_small(res)) { + UnUseTmpHeap(3,p); return res; } else { /* @@ -657,6 +668,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2) if (arity > 1) { *hp = big_res[2]; } + UnUseTmpHeap(3,p); return res; } } @@ -915,8 +927,8 @@ erts_mixed_div(Process* p, Eterm arg1, Eterm arg2) Eterm erts_int_div(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; switch (NUMBER_CODE(arg1, arg2)) { @@ -967,8 +979,8 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2) Eterm erts_int_rem(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; switch (NUMBER_CODE(arg1, arg2)) { @@ -979,7 +991,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2) if (arg1 != make_small(MIN_SMALL)) { return arg1; } else { - Eterm tmp = small_to_big(signed_val(arg1), tmp_big1); + Eterm tmp; + tmp = small_to_big(signed_val(arg1), tmp_big1); if ((ires = big_ucomp(tmp, arg2)) == 0) { return SMALL_ZERO; } else { @@ -1013,8 +1026,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2) Eterm erts_band(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm* hp; int need; @@ -1041,8 +1054,8 @@ Eterm erts_band(Process* p, Eterm arg1, Eterm arg2) Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm* hp; int need; @@ -1069,8 +1082,8 @@ Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2) Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2) { - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm* hp; int need; @@ -1148,8 +1161,8 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm res; Eterm hdr; FloatDef f1, f2; @@ -1237,10 +1250,10 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live) need_heap = BIG_NEED_SIZE(sz); if (ERTS_NEED_GC(p, need_heap)) { erts_garbage_collect(p, need_heap, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1316,8 +1329,8 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -1394,10 +1407,10 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live) need_heap = BIG_NEED_SIZE(sz); if (ERTS_NEED_GC(p, need_heap)) { erts_garbage_collect(p, need_heap, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1482,8 +1495,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); Eterm hdr; Eterm res; FloatDef f1, f2; @@ -1509,7 +1522,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) } else if (arg2 == SMALL_ONE) { return(arg1); } else { - Eterm big_res[3]; + DeclareTmpHeap(big_res,3,p); + UseTmpHeap(3,p); /* * The following code is optimized for the case that @@ -1519,6 +1533,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) res = small_times(signed_val(arg1), signed_val(arg2), big_res); if (is_small(res)) { + UnUseTmpHeap(3,p); return res; } else { /* @@ -1546,6 +1561,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) if (arity > 1) { *hp = big_res[2]; } + UnUseTmpHeap(3,p); return res; } } @@ -1609,17 +1625,17 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live) need_heap = BIG_NEED_SIZE(sz); if (ERTS_NEED_GC(p, need_heap)) { erts_garbage_collect(p, need_heap, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } hp = p->htop; p->htop += need_heap; res = big_times(arg1, arg2, hp); - trim_heap(p, hp, res); + trim_heap(p, hp, res); /* * Note that the result must be big in this case, since @@ -1828,8 +1844,8 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; arg1 = reg[live]; @@ -1866,10 +1882,10 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live) need = BIG_NEED_SIZE(i-ires+1) + BIG_NEED_SIZE(i); if (ERTS_NEED_GC(p, need)) { erts_garbage_collect(p, need, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1894,8 +1910,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live) { Eterm arg1; Eterm arg2; - Eterm tmp_big1[2]; - Eterm tmp_big2[2]; + DECLARE_TMP(tmp_big1,0,p); + DECLARE_TMP(tmp_big2,1,p); int ires; arg1 = reg[live]; @@ -1908,7 +1924,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live) if (arg1 != make_small(MIN_SMALL)) { return arg1; } else { - Eterm tmp = small_to_big(signed_val(arg1), tmp_big1); + Eterm tmp; + tmp = small_to_big(signed_val(arg1), tmp_big1); if ((ires = big_ucomp(tmp, arg2)) == 0) { return SMALL_ZERO; } else { @@ -1928,10 +1945,10 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live) if (ERTS_NEED_GC(p, need)) { erts_garbage_collect(p, need, reg, live+2); - if (arg1 != make_big(tmp_big1)) { + if (ARG_IS_NOT_TMP(arg1,tmp_big1)) { arg1 = reg[live]; } - if (arg2 != make_big(tmp_big2)) { + if (ARG_IS_NOT_TMP(arg2,tmp_big2)) { arg2 = reg[live+1]; } } @@ -1956,8 +1973,8 @@ Eterm erts_gc_##func(Process* p, Eterm* reg, Uint live) \ { \ Eterm arg1; \ Eterm arg2; \ - Eterm tmp_big1[2]; \ - Eterm tmp_big2[2]; \ + DECLARE_TMP(tmp_big1,0,p); \ + DECLARE_TMP(tmp_big2,1,p); \ Eterm* hp; \ int need; \ \ diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index b090564649..91b64411d4 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifdef HAVE_CONFIG_H @@ -70,7 +70,6 @@ static ErlAsync* async_ready_list = NULL; /* Detach from driver */ static void async_detach(DE_Handle* dh) { - /* XXX:PaN what should happen here? we want to unload the driver or??? */ return; } @@ -176,7 +175,6 @@ int exit_async() static void async_add(ErlAsync* a, AsyncQueue* q) { - /* XXX:PaN Is this still necessary when ports lock drivers? */ if (is_internal_port(a->port)) { ERTS_LC_ASSERT(erts_drvportid2port(a->port)); /* make sure the driver will stay around */ @@ -253,6 +251,7 @@ static int async_del(long id) erts_free(ERTS_ALC_T_ASYNC, a); return 1; } + a = a->next; } erts_mtx_unlock(&async_q[i].mtx); } diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c new file mode 100644 index 0000000000..684fa5d12f --- /dev/null +++ b/erts/emulator/beam/erl_bif_binary.c @@ -0,0 +1,2939 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * NOTE: This file contains the BIF's for the *module* binary in stdlib. + * other BIF's concerning binaries are in binary.c. + */ + + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "erl_process.h" +#include "error.h" +#include "bif.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bits.h" + + +/* + * The native implementation functions for the module binary. + * Searching is implemented using either Boyer-Moore or Aho-Corasick + * depending on number of searchstrings (BM if one, AC if more than one). + * Native implementation is mostly for efficiency, nothing + * (except binary:referenced_byte_size) really *needs* to be implemented + * in native code. + */ + +/* #define HARDDEBUG */ + +/* Init and local variables */ + +static Export binary_match_trap_export; +static BIF_RETTYPE binary_match_trap(BIF_ALIST_3); +static Export binary_matches_trap_export; +static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3); +static Export binary_longest_prefix_trap_export; +static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); +static Export binary_longest_suffix_trap_export; +static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); +static Export binary_bin_to_list_trap_export; +static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); +static Export binary_copy_trap_export; +static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); +static Uint max_loop_limit; + + +void erts_init_bif_binary(void) +{ + sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export)); + binary_match_trap_export.address = &binary_match_trap_export.code[3]; + binary_match_trap_export.code[0] = am_erlang; + binary_match_trap_export.code[1] = am_binary_match_trap; + binary_match_trap_export.code[2] = 3; + binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap; + + sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export)); + binary_matches_trap_export.address = &binary_matches_trap_export.code[3]; + binary_matches_trap_export.code[0] = am_erlang; + binary_matches_trap_export.code[1] = am_binary_matches_trap; + binary_matches_trap_export.code[2] = 3; + binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap; + + sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export)); + binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3]; + binary_longest_prefix_trap_export.code[0] = am_erlang; + binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap; + binary_longest_prefix_trap_export.code[2] = 3; + binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap; + + sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export)); + binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3]; + binary_longest_suffix_trap_export.code[0] = am_erlang; + binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap; + binary_longest_suffix_trap_export.code[2] = 3; + binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap; + + sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export)); + binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3]; + binary_bin_to_list_trap_export.code[0] = am_erlang; + binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap; + binary_bin_to_list_trap_export.code[2] = 3; + binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap; + sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export)); + binary_copy_trap_export.address = &binary_copy_trap_export.code[3]; + binary_copy_trap_export.code[0] = am_erlang; + binary_copy_trap_export.code[1] = am_binary_copy_trap; + binary_copy_trap_export.code[2] = 2; + binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap; + + max_loop_limit = 0; + return; +} + +/* + * Setting the loop_limit for searches for debugging + */ +Sint erts_binary_set_loop_limit(Sint limit) +{ + Sint save = (Sint) max_loop_limit; + if (limit <= 0) { + max_loop_limit = 0; + } else { + max_loop_limit = (Uint) limit; + } + + return save; +} + +static Uint get_reds(Process *p, int loop_factor) +{ + Uint reds = ERTS_BIF_REDS_LEFT(p) * loop_factor; + Uint tmp = max_loop_limit; + if (tmp != 0 && tmp < reds) { + return tmp; + } + if (!reds) { + reds = 1; + } + return reds; +} + +/* + * A micro allocator used when building search structures, just a convenience + * for building structures inside a pre-allocated magic binary using + * conventional malloc-like interface. + */ + +#define MYALIGN(Size) (SIZEOF_VOID_P * (((Size) / SIZEOF_VOID_P) + \ + !!(((Size) % SIZEOF_VOID_P)))) + +#ifdef DEBUG +#define CHECK_ALLOCATOR(My) ASSERT((My).current <= ((My).mem + (My).size)) +#else +#define CHECK_ALLOCATOR(My) /* nothing */ +#endif + +typedef struct _my_allocator { + Uint size; + byte *current; + byte *mem; +} MyAllocator; + +static void init_my_allocator(MyAllocator *my, Uint siz, byte *array) +{ + ASSERT((siz % SIZEOF_VOID_P) == 0); + my->size = siz; + my->mem = array; + my->current = my->mem; +} + +static void *my_alloc(MyAllocator *my, Uint size) +{ + void *ptr = my->current; + my->current += MYALIGN(size); + return ptr; +} + +/* + * The search functionality. + * + * The search is byte oriented, which works nicely for UTF-8 as well as + * latin1 data + */ + +#define ALPHABET_SIZE 256 + +typedef struct _ac_node { +#ifdef HARDDEBUG + Uint32 id; /* To identify h pointer targets when + dumping */ +#endif + Uint32 d; /* Depth in trie, also represents the + length (-1) of the matched string if + in final set */ + Sint32 final; /* Members in final set represent + * matches. + * The set representation is scattered + * among the nodes in this way: + * >0 -> this represents a member of + * the final set, <0 -> member of + * final set somewhere in the failure + * chain, + * 0 -> not member of the final set */ + struct _ac_node *h; /* h(Hode) is the failure function */ + struct _ac_node *g[ALPHABET_SIZE]; /* g(Node,Character) is the + transition function */ +} ACNode; + +typedef struct _ac_trie { +#ifdef HARDDEBUG + Uint32 idc; +#endif + Uint32 counter; /* Number of added patterns */ + ACNode *root; /* pointer to the root state */ +} ACTrie; + +typedef struct _bm_data { + byte *x; + Sint len; + Sint *goodshift; + Sint badshift[ALPHABET_SIZE]; +} BMData; + +#ifdef HARDDEBUG +static void dump_bm_data(BMData *bm); +static void dump_ac_trie(ACTrie *act); +static void dump_ac_node(ACNode *node, int indent, int ch); +#endif + +/* + * The needed size of binary data for a search structure - given the + * accumulated string lengths. + */ +#define BM_SIZE(StrLen) /* StrLen: length of searchstring */ \ +((MYALIGN(sizeof(Sint) * (StrLen))) + /* goodshift array */ \ + MYALIGN(StrLen) + /* searchstring saved */ \ + (MYALIGN(sizeof(BMData)))) /* Structure */ + +#define AC_SIZE(StrLens) /* StrLens: sum of all searchstring lengths */ \ +((MYALIGN(sizeof(ACNode)) * \ +((StrLens)+1)) + /* The actual nodes (including rootnode) */ \ + MYALIGN(sizeof(ACTrie))) /* Structure */ + + +#ifndef MAX +#define MAX(A,B) (((A) > (B)) ? (A) : (B)) +#endif + +#ifndef MIN +#define MIN(A,B) (((A) > (B)) ? (B) : (A)) +#endif +/* + * Callback for the magic binary + */ +static void cleanup_my_data_ac(Binary *bp) +{ + return; +} +static void cleanup_my_data_bm(Binary *bp) +{ + return; +} + +/* + * Initiate a (allocated) micro allocator and fill in the base + * for an Aho-Corasick search trie, given the accumulated length of the search + * strings. + */ +static ACTrie *create_acdata(MyAllocator *my, Uint len, + ACNode ***qbuff /* out */, + Binary **the_bin /* out */) +{ + Uint datasize = AC_SIZE(len); + ACTrie *act; + ACNode *acn; + Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_ac); + byte *data = ERTS_MAGIC_BIN_DATA(mb); + + init_my_allocator(my, datasize, data); + act = my_alloc(my, sizeof(ACTrie)); /* Important that this is the first + allocation */ + act->counter = 0; + act->root = acn = my_alloc(my, sizeof(ACNode)); + acn->d = 0; + acn->final = 0; + acn->h = NULL; + memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); +#ifdef HARDDEBUG + act->idc = 0; + acn->id = 0; +#endif + *qbuff = erts_alloc(ERTS_ALC_T_TMP, sizeof(ACNode *) * len); + *the_bin = mb; + return act; +} + +/* + * The same initialization of allocator and basic data for Boyer-Moore. + */ +static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, + Binary **the_bin /* out */) +{ + Uint datasize = BM_SIZE(len); + BMData *bmd; + Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_bm); + byte *data = ERTS_MAGIC_BIN_DATA(mb); + init_my_allocator(my, datasize, data); + bmd = my_alloc(my, sizeof(BMData)); + bmd->x = my_alloc(my,len); + memcpy(bmd->x,x,len); + bmd->len = len; + bmd->goodshift = my_alloc(my,sizeof(Uint) * len); + *the_bin = mb; + return bmd; +} + +/* + * Compilation of search structures + */ + +/* + * Aho Corasick - Build a Trie and fill in the failure functions + * when all strings are added. + * The algorithm is nicely described by Dieter B�hler of University of + * T�bingen: + * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html + */ + +/* + * Helper called once for each search pattern + */ +static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len) +{ + ACNode *acn = act->root; + Uint32 n = ++act->counter; /* Always increase counter, even if it's a + duplicate as this may identify the pattern + in the final set (not in current interface + though) */ + Uint i = 0; + + while(i < len) { + if (acn->g[x[i]] != NULL) { + /* node exists, continue */ + acn = acn->g[x[i]]; + ++i; + } else { + /* allocate a new node */ + ACNode *nn = my_alloc(my,sizeof(ACNode)); +#ifdef HARDDEBUG + nn->id = ++(act->idc); +#endif + nn->d = i+1; + nn->h = act->root; + nn->final = 0; + memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); + acn->g[x[i]] = nn; + ++i; + acn = nn; + } + } + if (acn->final == 0) { /* New pattern, add to final set */ + acn->final = n; + } +} + +/* + * Called when all search patterns are added. + */ +static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff) +{ + ACNode *root = act->root; + ACNode *parent; + int i; + int qh = 0,qt = 0; + ACNode *child, *r; + + /* Set all children of the root to have the root as failure function */ + for (i = 0; i < ALPHABET_SIZE; ++i) { + if (root->g[i] != NULL) { + root->g[i]->h = root; + /* Add to que for later traversal */ + qbuff[qt++] = root->g[i]; + } + } + + /* So, now we've handled children of the root state, traverse the + rest of the trie BF... */ + while (qh < qt) { + parent = qbuff[qh++]; + for (i = 0; i < ALPHABET_SIZE; ++ i) { + if ((child = parent->g[i]) != NULL) { + /* Visit this node to */ + qbuff[qt++] = child; + /* Search for correct failure function, follow the parent's + failure function until you find a similar transition + funtion to this child's */ + r = parent->h; + while (r != NULL && r->g[i] == NULL) { + r = r->h; + } + if (r == NULL) { + /* Replace NULL failures with the root as we go */ + child->h = (root->g[i] == NULL) ? root : root->g[i]; + } else { + child->h = r->g[i]; + /* + * The "final" set is scattered among the nodes. When + * the failure function points to a member of the final + * set, we have a match, but we might not see it in the + * current node if we dont mark it as a special type of + * final, i.e. foolow the failure function and you will + * find a real member of final set. This is marked with + * a negative string id and only done if this node does + * not represent a member in the final set. + */ + if (!(child->final) && (child->h->final)) { + child->final = -1; + } + } + } + } + } + /* Finally the failure function of the root should point to itself */ + root->h = root; +} + + +/* + * The actual searching for needles in the haystack... + * Find first match using Aho-Coracick Trie + * return pattern number and fill in mpos + mlen if found, otherwise return 0 + * Return the matching pattern that *starts* first, and ends + * last (difference when overlapping), hence the candidate thing. + * Basic AC finds the first end before the first start... + * + */ +typedef struct { + ACNode *q; + Uint pos; + Uint len; + ACNode *candidate; + Uint candidate_start; +} ACFindFirstState; + + +static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len) +{ + state->q = act->root; + state->pos = startpos; + state->len = len; + state->candidate = NULL; + state->candidate_start = 0; +} +#define AC_OK 0 +#define AC_NOT_FOUND -1 +#define AC_RESTART -2 + +#define AC_LOOP_FACTOR 10 + +static int ac_find_first_match(ACFindFirstState *state, byte *haystack, + Uint *mpos, Uint *mlen, Uint *reductions) +{ + ACNode *q = state->q; + Uint i = state->pos; + ACNode *candidate = state->candidate, *r; + Uint len = state->len; + Uint candidate_start = state->candidate_start; + Uint rstart; + register Uint reds = *reductions; + + while (i < len) { + if (--reds == 0) { + state->q = q; + state->pos = i; + state->len = len; + state->candidate = candidate; + state->candidate_start = candidate_start; + return AC_RESTART; + } + + while (q->g[haystack[i]] == NULL && q->h != q) { + q = q->h; + } + if (q->g[haystack[i]] != NULL) { + q = q->g[haystack[i]]; + } +#ifdef HARDDEBUG + erts_printf("ch = %c, Current: %u\n", (int) haystack[i], (unsigned) q->id); +#endif + ++i; + if (candidate != NULL && (i - q->d) > candidate_start) { + break; + } + if (q->final) { + r = q; + while (r->final < 0) + r = r->h; + rstart = i - r->d; + if (candidate == NULL || rstart < candidate_start || + (rstart == candidate_start && candidate->d < q->d)) { + candidate_start = rstart; + candidate = r; + } + } + } + *reductions = reds; + if (!candidate) { + return AC_NOT_FOUND; + } +#ifdef HARDDEBUG + dump_ac_node(candidate,0,'?'); +#endif + *mpos = candidate_start; + *mlen = candidate->d; + return AC_OK; +} + +typedef struct _findall_data { + Uint pos; + Uint len; +#ifdef HARDDEBUG + Uint id; +#endif + Eterm epos; + Eterm elen; +} FindallData; + +typedef struct { + ACNode *q; + Uint pos; + Uint len; + Uint m; + Uint allocated; + FindallData *out; +} ACFindAllState; + +static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len) +{ + state->q = act->root; + state->pos = startpos; + state->len = len; + state->m = 0; + state->allocated = 0; + state->out = NULL; +} + +static void ac_restore_find_all(ACFindAllState *state, char *buff) +{ + memcpy(state,buff,sizeof(ACFindAllState)); + if (state->allocated > 0) { + state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); + memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m); + } else { + state->out = NULL; + } +} + +static void ac_serialize_find_all(ACFindAllState *state, char *buff) +{ + memcpy(buff,state,sizeof(ACFindAllState)); + memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m); +} + +static void ac_clean_find_all(ACFindAllState *state) +{ + if (state->out != NULL) { + erts_free(ERTS_ALC_T_TMP, state->out); + } +#ifdef HARDDEBUG + state->out = NULL; + state->allocated = 0; +#endif +} + +#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \ + (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m)) + +/* + * Differs to the find_first function in that it stores all matches and the values + * arte returned only in the state. + */ +static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, + Uint *reductions) +{ + ACNode *q = state->q; + Uint i = state->pos; + Uint rstart; + ACNode *r; + Uint len = state->len; + Uint m = state->m, save_m; + Uint allocated = state->allocated; + FindallData *out = state->out; + register Uint reds = *reductions; + + + while (i < len) { + if (--reds == 0) { + state->q = q; + state->pos = i; + state->len = len; + state->m = m; + state->allocated = allocated; + state->out = out; + return AC_RESTART; + } + while (q->g[haystack[i]] == NULL && q->h != q) { + q = q->h; + } + if (q->g[haystack[i]] != NULL) { + q = q->g[haystack[i]]; + } + ++i; + if (q->final) { + r = q; + while (r->final) { + while (r->final < 0) + r = r->h; +#ifdef HARDDEBUG + erts_printf("Trying to add %u\n",(unsigned) r->final); +#endif + rstart = i - r->d; + save_m = m; + while (m > 0 && (out[m-1].pos > rstart || + (out[m-1].pos == rstart && + out[m-1].len < r->d))) { +#ifdef HARDDEBUG + erts_printf("Popping %u\n",(unsigned) out[m-1].id); +#endif + --m; + } +#ifdef HARDDEBUG + if (m > 0) { + erts_printf("Pos %u\n",out[m-1].pos); + erts_printf("Len %u\n",out[m-1].len); + } + erts_printf("Rstart %u\n",rstart); +#endif + if (m == 0 || out[m-1].pos + out[m-1].len <= rstart) { + if (m >= allocated) { + if (!allocated) { + allocated = 10; + out = erts_alloc(ERTS_ALC_T_TMP, + sizeof(FindallData) * allocated); + } else { + allocated *= 2; + out = erts_realloc(ERTS_ALC_T_TMP, out, + sizeof(FindallData) * + allocated); + } + } + out[m].pos = rstart; + out[m].len = r->d; +#ifdef HARDDEBUG + out[m].id = r->final; +#endif + ++m; +#ifdef HARDDEBUG + erts_printf("Pushing %u\n",(unsigned) out[m-1].id); +#endif + } else { +#ifdef HARDDEBUG + erts_printf("Backtracking %d steps\n",save_m - m); +#endif + m = save_m; + } + r = r->h; + } + } + } + *reductions = reds; + state->m = m; + state->out = out; + return (m == 0) ? AC_NOT_FOUND : AC_OK; +} + +/* + * Boyer Moore - most obviously implemented more or less exactly as + * Christian Charras and Thierry Lecroq describe it in "Handbook of + * Exact String-Matching Algorithms" + * http://www-igm.univ-mlv.fr/~lecroq/string/ + */ + +/* + * Call this to compute badshifts array + */ +static void compute_badshifts(BMData *bmd) +{ + Sint i; + Sint m = bmd->len; + + for (i = 0; i < ALPHABET_SIZE; ++i) { + bmd->badshift[i] = m; + } + for (i = 0; i < m - 1; ++i) { + bmd->badshift[bmd->x[i]] = m - i - 1; + } +} + +/* Helper for "compute_goodshifts" */ +static void compute_suffixes(byte *x, Sint m, Sint *suffixes) +{ + int f,g,i; + + suffixes[m - 1] = m; + + f = 0; /* To avoid use before set warning */ + + g = m - 1; + + for (i = m - 2; i >= 0; --i) { + if (i > g && suffixes[i + m - 1 - f] < i - g) { + suffixes[i] = suffixes[i + m - 1 - f]; + } else { + if (i < g) { + g = i; + } + f = i; + while ( g >= 0 && x[g] == x[g + m - 1 - f] ) { + --g; + } + suffixes[i] = f - g; + } + } +} + +/* + * Call this to compute goodshift array + */ +static void compute_goodshifts(BMData *bmd) +{ + Sint m = bmd->len; + byte *x = bmd->x; + Sint i, j; + Sint *suffixes = erts_alloc(ERTS_ALC_T_TMP, m * sizeof(Sint)); + + compute_suffixes(x, m, suffixes); + + for (i = 0; i < m; ++i) { + bmd->goodshift[i] = m; + } + + j = 0; + + for (i = m - 1; i >= -1; --i) { + if (i == -1 || suffixes[i] == i + 1) { + while (j < m - 1 - i) { + if (bmd->goodshift[j] == m) { + bmd->goodshift[j] = m - 1 - i; + } + ++j; + } + } + } + for (i = 0; i <= m - 2; ++i) { + bmd->goodshift[m - 1 - suffixes[i]] = m - 1 - i; + } + erts_free(ERTS_ALC_T_TMP, suffixes); +} + +typedef struct { + Sint pos; + Sint len; +} BMFindFirstState; + +#define BM_OK 0 /* used only for find_all */ +#define BM_NOT_FOUND -1 +#define BM_RESTART -2 +#define BM_LOOP_FACTOR 10 /* Should we have a higher value? */ + +static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos, + Uint len) +{ + state->pos = startpos; + state->len = (Sint) len; +} + + +static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd, + byte *haystack, Uint *reductions) +{ + Sint blen = bmd->len; + Sint len = state->len; + Sint *gs = bmd->goodshift; + Sint *bs = bmd->badshift; + byte *needle = bmd->x; + Sint i; + Sint j = state->pos; + register Uint reds = *reductions; + + while (j <= len - blen) { + if (--reds == 0) { + state->pos = j; + return BM_RESTART; + } + for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + ; + if (i < 0) { /* found */ + *reductions = reds; + return j; + } + j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); + } + *reductions = reds; + return BM_NOT_FOUND; +} + +typedef struct { + Sint pos; + Sint len; + Uint m; + Uint allocated; + FindallData *out; +} BMFindAllState; + +static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len) +{ + state->pos = startpos; + state->len = (Sint) len; + state->m = 0; + state->allocated = 0; + state->out = NULL; +} + +static void bm_restore_find_all(BMFindAllState *state, char *buff) +{ + memcpy(state,buff,sizeof(BMFindAllState)); + if (state->allocated > 0) { + state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * + (state->allocated)); + memcpy(state->out,buff+sizeof(BMFindAllState), + sizeof(FindallData)*state->m); + } else { + state->out = NULL; + } +} + +static void bm_serialize_find_all(BMFindAllState *state, char *buff) +{ + memcpy(buff,state,sizeof(BMFindAllState)); + memcpy(buff+sizeof(BMFindAllState),state->out, + sizeof(FindallData)*state->m); +} + +static void bm_clean_find_all(BMFindAllState *state) +{ + if (state->out != NULL) { + erts_free(ERTS_ALC_T_TMP, state->out); + } +#ifdef HARDDEBUG + state->out = NULL; + state->allocated = 0; +#endif +} + +#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \ + (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m)) + +/* + * Differs to the find_first function in that it stores all matches and the + * values are returned only in the state. + */ +static Sint bm_find_all_non_overlapping(BMFindAllState *state, + BMData *bmd, byte *haystack, + Uint *reductions) +{ + Sint blen = bmd->len; + Sint len = state->len; + Sint *gs = bmd->goodshift; + Sint *bs = bmd->badshift; + byte *needle = bmd->x; + Sint i; + Sint j = state->pos; + Uint m = state->m; + Uint allocated = state->allocated; + FindallData *out = state->out; + register Uint reds = *reductions; + + while (j <= len - blen) { + if (--reds == 0) { + state->pos = j; + state->m = m; + state->allocated = allocated; + state->out = out; + return BM_RESTART; + } + for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + ; + if (i < 0) { /* found */ + if (m >= allocated) { + if (!allocated) { + allocated = 10; + out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated); + } else { + allocated *= 2; + out = erts_realloc(ERTS_ALC_T_TMP, out, + sizeof(FindallData) * allocated); + } + } + out[m].pos = j; + out[m].len = blen; + ++m; + j += blen; + } else { + j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); + } + } + state->m = m; + state->out = out; + *reductions = reds; + return (m == 0) ? BM_NOT_FOUND : BM_OK; +} + +/* + * Interface functions (i.e. "bif's") + */ + +/* + * Search functionality interfaces + */ + +static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp) +{ + Eterm t, b, comp_term = NIL; + Uint characters; + Uint words; + + characters = 0; + words = 0; + + if (is_list(argument)) { + t = argument; + while (is_list(t)) { + b = CAR(list_val(t)); + t = CDR(list_val(t)); + if (!is_binary(b)) { + goto badarg; + } + if (binary_bitsize(b) != 0) { + goto badarg; + } + ++words; + characters += binary_size(b); + } + if (is_not_nil(t)) { + goto badarg; + } + if (words > 1) { + comp_term = argument; + } else { + comp_term = CAR(list_val(argument)); + } + } else if (is_binary(argument)) { + if (binary_bitsize(argument) != 0) { + goto badarg; + } + words = 1; + comp_term = argument; + characters = binary_size(argument); + } + + if (characters == 0) { + goto badarg; + } + ASSERT(words > 0); + + if (words == 1) { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + MyAllocator my; + BMData *bmd; + Binary *bin; + + ERTS_GET_BINARY_BYTES(comp_term, bytes, bitoffs, bitsize); + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(comp_term, &temp_alloc); + } + bmd = create_bmdata(&my, bytes, characters, &bin); + compute_badshifts(bmd); + compute_goodshifts(bmd); + erts_free_aligned_binary_bytes(temp_alloc); + CHECK_ALLOCATOR(my); + *tag = am_bm; + *binp = bin; + return 0; + } else { + ACTrie *act; + MyAllocator my; + ACNode **qbuff; + Binary *bin; + + act = create_acdata(&my, characters, &qbuff, &bin); + t = comp_term; + while (is_list(t)) { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + b = CAR(list_val(t)); + t = CDR(list_val(t)); + ERTS_GET_BINARY_BYTES(b, bytes, bitoffs, bitsize); + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(b, &temp_alloc); + } + ac_add_one_pattern(&my,act,bytes,binary_size(b)); + erts_free_aligned_binary_bytes(temp_alloc); + } + ac_compute_failure_functions(act,qbuff); + CHECK_ALLOCATOR(my); + erts_free(ERTS_ALC_T_TMP,qbuff); + *tag = am_ac; + *binp = bin; + return 0; + } + badarg: + return -1; +} + +BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) +{ + Binary *bin; + Eterm tag, ret; + Eterm *hp; + + if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) { + BIF_ERROR(BIF_P,BADARG); + } + hp = HAlloc(BIF_P, PROC_BIN_SIZE+3); + ret = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + ret = TUPLE2(hp, tag, ret); + BIF_RET(ret); +} + +#define DO_BIN_MATCH_OK 0 +#define DO_BIN_MATCH_BADARG -1 +#define DO_BIN_MATCH_RESTART -2 + +static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend, + Eterm type, Binary *bin, Eterm state_term, + Eterm *res_term) +{ + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + Eterm *ptr = big_val(state_term); + type = ptr[1]; + } + + if (type == am_bm) { + BMData *bm; + Sint pos; + Eterm ret; + Eterm *hp; + BMFindFirstState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_first_match(&state, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + memcpy(&state,ptr+2,sizeof(state)); + } +#ifdef HARDDEBUG + erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, + state.len); +#endif + pos = bm_find_first_match(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + ret = am_nomatch; + } else if (pos == BM_RESTART) { + int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) + + !!(sizeof(BMFindFirstState) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + memcpy(hp+2,&state,sizeof(state)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + Eterm erlen = erts_make_integer((Uint) bm->len, p); + ret = erts_make_integer(pos,p); + hp = HAlloc(p,3); + ret = TUPLE2(hp, ret, erlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } else if (type == am_ac) { + ACTrie *act; + Uint pos, rlen; + int acr; + ACFindFirstState state; + Eterm ret; + Eterm *hp; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_first_match(&state, act, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + memcpy(&state,ptr+2,sizeof(state)); + } + acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); + if (acr == AC_NOT_FOUND) { + ret = am_nomatch; + } else if (acr == AC_RESTART) { + int x = (sizeof(state) / sizeof(Eterm)) + + !!(sizeof(ACFindFirstState) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + memcpy(hp+2,&state,sizeof(state)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + Eterm epos = erts_make_integer(pos+hsstart,p); + Eterm erlen = erts_make_integer(rlen,p); + hp = HAlloc(p,3); + ret = TUPLE2(hp, epos, erlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } + badarg: + return DO_BIN_MATCH_BADARG; +} + +static int do_binary_matches(Process *p, Eterm subject, Uint hsstart, + Uint hsend, Eterm type, Binary *bin, + Eterm state_term, Eterm *res_term) +{ + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + Eterm *ptr = big_val(state_term); + type = ptr[1]; + } + + if (type == am_bm) { + BMData *bm; + Sint pos; + Eterm ret,tpl; + Eterm *hp; + BMFindAllState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_all(&state, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + bm_restore_find_all(&state,(char *) (ptr+2)); + } + + pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + ret = NIL; + } else if (pos == BM_RESTART) { + int x = + (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + bm_serialize_find_all(&state, (char *) (hp+2)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + FindallData *fad = state.out; + int i; + for (i = 0; i < state.m; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos,p); + fad[i].elen = erts_make_integer(fad[i].len,p); + } + hp = HAlloc(p,state.m * (3 + 2)); + ret = NIL; + for (i = state.m - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp +=3; + ret = CONS(hp,tpl,ret); + hp += 2; + } + } + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } else if (type == am_ac) { + ACTrie *act; + int acr; + ACFindAllState state; + Eterm ret,tpl; + Eterm *hp; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_all(&state, act, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + ac_restore_find_all(&state,(char *) (ptr+2)); + } + acr = ac_find_all_non_overlapping(&state, bytes, &reds); + if (acr == AC_NOT_FOUND) { + ret = NIL; + } else if (acr == AC_RESTART) { + int x = + (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + ac_serialize_find_all(&state, (char *) (hp+2)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + FindallData *fad = state.out; + int i; + for (i = 0; i < state.m; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos,p); + fad[i].elen = erts_make_integer(fad[i].len,p); + } + hp = HAlloc(p,state.m * (3 + 2)); + ret = NIL; + for (i = state.m - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp +=3; + ret = CONS(hp,tpl,ret); + hp += 2; + } + } + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } + badarg: + return DO_BIN_MATCH_BADARG; +} + +static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) +{ + Eterm *tp; + Uint pos; + Sint len; + if (l == ((Eterm) 0) || l == NIL) { + /* Invalid term or NIL, we're called from binary_match(es)_2 or + have no options*/ + *posp = 0; + *endp = binary_size(bin); + return 0; + } else if (is_list(l)) { + while(is_list(l)) { + Eterm t = CAR(list_val(l)); + Uint orig_size; + if (!is_tuple(t)) { + goto badarg; + } + tp = tuple_val(t); + if (arityval(*tp) != 2) { + goto badarg; + } + if (tp[1] != am_scope || is_not_tuple(tp[2])) { + goto badarg; + } + tp = tuple_val(tp[2]); + if (arityval(*tp) != 2) { + goto badarg; + } + if (!term_to_Uint(tp[1], &pos)) { + goto badarg; + } + if (!term_to_Sint(tp[2], &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + *endp = len + pos; + *posp = pos; + if ((orig_size = binary_size(bin)) < pos || + orig_size < (*endp)) { + goto badarg; + } + l = CDR(list_val(l)); + } + return 0; + } else { + badarg: + return 1; + } +} + +static BIF_RETTYPE binary_match_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, + BIF_ARG_3); + } +} + +static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + BIF_ARG_3); + } +} + +BIF_RETTYPE binary_match_3(BIF_ALIST_3) +{ + Uint hsstart; + Uint hsend; + Eterm *tp; + Eterm type; + Binary *bin; + Eterm bin_term = NIL; + int runres; + Eterm result; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + goto badarg; + } + if (hsend == 0) { + BIF_RET(am_nomatch); + } + if (is_tuple(BIF_ARG_2)) { + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + goto badarg; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + goto badarg; + } + type = tp[1]; + bin = ((ProcBin *) binary_val(tp[2]))->val; + if (type == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { + goto badarg; + } + if (type == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { + goto badarg; + } + bin_term = tp[2]; + } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + goto badarg; + } + runres = do_binary_match(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin,NIL,&result); + if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { + Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + } else if (bin_term == NIL) { + erts_bin_free(bin); + } + switch (runres) { + case DO_BIN_MATCH_OK: + BIF_RET(result); + case DO_BIN_MATCH_RESTART: + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, bin_term); + default: + goto badarg; + } + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_matches_3(BIF_ALIST_3) +{ + Uint hsstart, hsend; + Eterm *tp; + Eterm type; + Binary *bin; + Eterm bin_term = NIL; + int runres; + Eterm result; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + goto badarg; + } + if (hsend == 0) { + BIF_RET(NIL); + } + if (is_tuple(BIF_ARG_2)) { + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + goto badarg; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + goto badarg; + } + type = tp[1]; + bin = ((ProcBin *) binary_val(tp[2]))->val; + if (type == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { + goto badarg; + } + if (type == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { + goto badarg; + } + bin_term = tp[2]; + } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + goto badarg; + } + runres = do_binary_matches(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin, + NIL,&result); + if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { + Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + } else if (bin_term == NIL) { + erts_bin_free(bin); + } + switch (runres) { + case DO_BIN_MATCH_OK: + BIF_RET(result); + case DO_BIN_MATCH_RESTART: + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + bin_term); + default: + goto badarg; + } + badarg: + BIF_ERROR(BIF_P,BADARG); +} + + +BIF_RETTYPE binary_match_2(BIF_ALIST_2) +{ + return binary_match_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); +} + + +BIF_RETTYPE binary_matches_2(BIF_ALIST_2) +{ + return binary_matches_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); +} + + +BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen) +{ + Uint pos; + Sint len; + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + Eterm* hp; + ErlSubBin* sb; + + if (is_not_binary(binary)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)){ + goto badarg; + } + if ((orig_size = binary_size(binary)) < pos || + orig_size < (pos + len)) { + goto badarg; + } + + + + hp = HAlloc(p, ERL_SUB_BIN_SIZE); + + ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size); + sb = (ErlSubBin *) hp; + sb->thing_word = HEADER_SUB_BIN; + sb->size = len; + sb->offs = offset + pos; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + + BIF_RET(make_binary(sb)); + + badarg: + BIF_ERROR(p, BADARG); +} + +#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need)) + +BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple) +{ + Uint pos; + Sint len; + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + Eterm* hp; + ErlSubBin* sb; + Eterm binary; + Eterm *tp; + Eterm epos, elen; + int extra_args; + + + if (range_is_tuple) { + Eterm tpl = reg[live]; + extra_args = 1; + if (is_not_tuple(tpl)) { + goto badarg; + } + tp = tuple_val(tpl); + if (arityval(*tp) != 2) { + goto badarg; + } + + epos = tp[1]; + elen = tp[2]; + } else { + extra_args = 2; + epos = reg[live-1]; + elen = reg[live]; + } + binary = reg[live-extra_args]; + + if (is_not_binary(binary)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + if ((orig_size = binary_size(binary)) < pos || + orig_size < (pos + len)) { + goto badarg; + } + + if (ERTS_NEED_GC(p, ERL_SUB_BIN_SIZE)) { + erts_garbage_collect(p, ERL_SUB_BIN_SIZE, reg, live+1-extra_args); /* I don't need the tuple + or indices any more */ + binary = reg[live-extra_args]; + } + + hp = p->htop; + p->htop += ERL_SUB_BIN_SIZE; + + ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size); + + sb = (ErlSubBin *) hp; + sb->thing_word = HEADER_SUB_BIN; + sb->size = len; + sb->offs = offset + pos; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + + BIF_RET(make_binary(sb)); + + badarg: + BIF_ERROR(p, BADARG); +} +/************************************************************* + * The actual guard BIFs are in erl_bif_guard.c + * but the implementation of both the non-gc and the gc + * variants are here. Note that the functions are named so that they do + * not clash with the guard bif's erlang:binary_part/2,3 + *************************************************************/ + +BIF_RETTYPE binary_binary_part_3(BIF_ALIST_3) +{ + return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE binary_binary_part_2(BIF_ALIST_2) +{ + Eterm *tp; + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +typedef struct { + int type; /* CL_TYPE_XXX */ + byte *temp_alloc; /* Used for erts_get/free_aligned, i.e. CL_TYPE_ALIGNED */ + unsigned char *buff; /* Used for all types, malloced if CL_TYPE_HEAP */ + Uint bufflen; /* The length (in bytes) of buffer */ +} CommonData; + +#define COMMON_LOOP_FACTOR 10 + +#define DIRECTION_PREFIX 0 +#define DIRECTION_SUFFIX 1 + +#define CL_OK 0 +#define CL_RESTART 1 + +/* The type field in the above structure */ +#define CL_TYPE_EMPTY 0 /* End of array */ +#define CL_TYPE_HEAP 1 +#define CL_TYPE_ALIGNED 2 +#define CL_TYPE_COMMON 3 /* emacsulated */ +#define CL_TYPE_HEAP_NOALLOC 4 /* Will need allocating when trapping */ + + +static int do_search_forward(CommonData *cd, Uint *posp, Uint *redsp) +{ + Uint pos = *posp; + Sint reds = (Sint) *redsp; + int i; + unsigned char current = 0; + + for(;;) { + for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) { + if (pos >= cd[i].bufflen) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + if (i == 0) { + current = cd[i].buff[pos]; + } else { + if (cd[i].buff[pos] != current) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + } + --reds; + } + ++pos; + if (reds <= 0) { + *posp = pos; + *redsp = 0; + return CL_RESTART; + } + } +} +static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp) +{ + Uint pos = *posp; + Sint reds = (Sint) *redsp; + int i; + unsigned char current = 0; + + for(;;) { + for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) { + if (pos >= cd[i].bufflen) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + if (i == 0) { + current = cd[i].buff[cd[i].bufflen - 1 - pos]; + } else { + if (cd[i].buff[cd[i].bufflen - 1 - pos] != current) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + } + --reds; + } + ++pos; + if (reds <= 0) { + *posp = pos; + *redsp = 0; + return CL_RESTART; + } + } +} + +static void cleanup_common_data(Binary *bp) +{ + int i; + CommonData *cd; + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bp); + for (i=0;cd[i].type != CL_TYPE_EMPTY;++i) { + switch (cd[i].type) { + case CL_TYPE_HEAP: + erts_free(ERTS_ALC_T_BINARY_BUFFER,cd[i].buff); + break; + case CL_TYPE_ALIGNED: + erts_free_aligned_binary_bytes_extra(cd[i].temp_alloc, ERTS_ALC_T_BINARY_BUFFER); + break; + default: + break; + } + } + return; +} + +static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) +{ + Eterm l = list; + int n = 0; + Binary *mb; + CommonData *cd; + int i = 0; + Uint reds = get_reds(p, COMMON_LOOP_FACTOR); + Uint save_reds = reds; + int res; + Export *trapper; + Uint pos; + Eterm epos; + Eterm *hp; + Eterm bin_term; + Eterm b; + + /* First just count the number of binaries */ + while (is_list(l)) { + b = CAR(list_val(l)); + if (!is_binary(b)) { + goto badarg; + } + ++n; + l = CDR(list_val(l)); + } + if (l != NIL || n == 0) { + goto badarg; + } + + /* OK, now create a buffer of the right size, we can do a magic binary right away, + that's not too costly. */ + mb = erts_create_magic_binary((n+1)*sizeof(CommonData),cleanup_common_data); + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(mb); + l = list; + while (is_list(l)) { + Uint bitoffs; + Uint bitsize; + Uint offset; + Eterm real_bin; + ProcBin* pb; + + cd[i].type = CL_TYPE_EMPTY; + b = CAR(list_val(l)); + ERTS_GET_REAL_BIN(b, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + erts_bin_free(mb); + goto badarg; + } + cd[i].bufflen = binary_size(b); + cd[i].temp_alloc = NULL; + if (*(binary_val(real_bin)) == HEADER_PROC_BIN) { + pb = (ProcBin *) binary_val(real_bin); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc), + ERTS_ALC_T_BINARY_BUFFER,0); + cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_COMMON; + } else { /* Heap binary */ + cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc), + ERTS_ALC_T_BINARY_BUFFER,0); + /* CL_TYPE_HEAP_NOALLOC means you have to copy if trapping */ + cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_HEAP_NOALLOC; + } + ++i; + l = CDR(list_val(l)); + } + cd[i].type = CL_TYPE_EMPTY; +#if defined(DEBUG) || defined(VALGRIND) + cd[i].temp_alloc = NULL; + cd[i].buff = NULL; + cd[i].bufflen = 0; +#endif + + pos = 0; + if (direction == DIRECTION_PREFIX) { + trapper = &binary_longest_prefix_trap_export; + res = do_search_forward(cd,&pos,&reds); + } else { + ASSERT(direction == DIRECTION_SUFFIX); + trapper = &binary_longest_suffix_trap_export; + res = do_search_backward(cd,&pos,&reds); + } + epos = erts_make_integer(pos,p); + if (res == CL_OK) { + erts_bin_free(mb); + BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR); + BIF_RET(epos); + } else { + ASSERT(res == CL_RESTART); + /* Copy all heap binaries that are not already copied (aligned) */ + for(i = 0; i < n; ++i) { + if (cd[i].type == CL_TYPE_HEAP_NOALLOC) { + unsigned char *tmp = cd[i].buff; + cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen); + memcpy(cd[i].buff,tmp,cd[i].bufflen); + cd[i].type = CL_TYPE_HEAP; + } + } + hp = HAlloc(p, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + BUMP_ALL_REDS(p); + BIF_TRAP3(trapper, p, bin_term, epos,list); + } + badarg: + BIF_ERROR(p,BADARG); +} + +static BIF_RETTYPE do_longest_common_trap(Process *p, Eterm bin_term, Eterm current_pos, + Eterm orig_list, int direction) +{ + Uint reds = get_reds(p, COMMON_LOOP_FACTOR); + Uint save_reds = reds; + Uint pos; + Binary *bin; + CommonData *cd; + int res; + Eterm epos; + Export *trapper; + +#ifdef DEBUG + int r; + r = term_to_Uint(current_pos, &pos); + ASSERT(r != 0); +#else + term_to_Uint(current_pos, &pos); +#endif + ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin_term)); + bin = ((ProcBin *) binary_val(bin_term))->val; + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bin); + if (direction == DIRECTION_PREFIX) { + trapper = &binary_longest_prefix_trap_export; + res = do_search_forward(cd,&pos,&reds); + } else { + ASSERT(direction == DIRECTION_SUFFIX); + trapper = &binary_longest_suffix_trap_export; + res = do_search_backward(cd,&pos,&reds); + } + epos = erts_make_integer(pos,p); + if (res == CL_OK) { + BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR); + BIF_RET(epos); + } else { + ASSERT(res == CL_RESTART); + /* Copy all heap binaries that are not already copied (aligned) */ + BUMP_ALL_REDS(p); + BIF_TRAP3(trapper, p, bin_term, epos, orig_list); + } +} + +static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3) +{ + return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_PREFIX); +} + +static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3) +{ + return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_SUFFIX); +} + +BIF_RETTYPE binary_longest_common_prefix_1(BIF_ALIST_1) +{ + return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_PREFIX); +} + +BIF_RETTYPE binary_longest_common_suffix_1(BIF_ALIST_1) +{ + return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_SUFFIX); +} + +BIF_RETTYPE binary_first_1(BIF_ALIST_1) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[0]) << bit_offs) | (((Uint) bytes[1]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[0]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_last_1(BIF_ALIST_1) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[byte_size-1]) << bit_offs) | + (((Uint) bytes[byte_size]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[byte_size-1]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_at_2(BIF_ALIST_2) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + Uint index; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + if (!term_to_Uint(BIF_ARG_2, &index)) { + goto badarg; + } + if (index >= byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[index]) << bit_offs) | + (((Uint) bytes[index+1]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[index]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +#define BIN_TO_LIST_OK 0 +#define BIN_TO_LIST_TRAP 1 +/* No badarg, checked before call */ + +#define BIN_TO_LIST_LOOP_FACTOR 10 + +static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, + Uint start, Sint *lenp, Eterm *termp) +{ + Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ + Uint len = *lenp; + Uint loops; + Eterm *hp; + Eterm term = *termp; + Uint n; + + ASSERT(reds > 0); + + loops = MIN(reds,len); + + BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); + + hp = HAlloc(p,2*loops); + while (loops--) { + --len; + if (bit_offs) { + n = ((((Uint) bytes[start+len]) << bit_offs) | + (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; + } else { + n = bytes[start+len]; + } + + term = CONS(hp,make_small(n),term); + hp +=2; + } + *termp = term; + *lenp = len; + if (len) { + BUMP_ALL_REDS(p); + return BIN_TO_LIST_TRAP; + } + return BIN_TO_LIST_OK; +} + + +static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, + Uint start, Sint len, Eterm sofar) +{ + Eterm *hp; + Eterm blob; + + hp = HAlloc(p,3); + hp[0] = make_pos_bignum_header(2); + hp[1] = start; + hp[2] = (Uint) len; + blob = make_big(hp); + BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); +} + +static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) +{ + Eterm *ptr; + Uint start; + Sint len; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = BIF_ARG_3; + + ptr = big_val(BIF_ARG_2); + start = ptr[1]; + len = (Sint) ptr[2]; + + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); +} + +static BIF_RETTYPE binary_bin_to_list_common(Process *p, + Eterm bin, + Eterm epos, + Eterm elen) +{ + Uint pos; + Sint len; + size_t sz; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = NIL; + + if (is_not_binary(bin)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + sz = binary_size(bin); + + if (pos+len > sz) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(p,bin,pos,len,res); + + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) +{ + return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); +} + +BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) +{ + Eterm *tp; + + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) +{ + Uint pos = 0; + Sint len; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = NIL; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + len = binary_size(BIF_ARG_1); + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +/* + * Ok, erlang:list_to_binary does not interrupt, and we really don't want + * an alternative implementation for the exact same thing, why we + * have descided to use the old non-restarting implementation for now. + * In reality, there are seldom many iterations involved in doing this, so the + * problem of long-running bifs is not really that big in this case. + * So, for now we use the old implementation also in the module binary. + */ + +BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) +{ + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); +} + +typedef struct { + Uint times_left; + Uint source_size; + int source_type; + byte *source; + byte *temp_alloc; + Uint result_pos; + Binary *result; +} CopyBinState; + +#define BC_TYPE_EMPTY 0 +#define BC_TYPE_HEAP 1 +#define BC_TYPE_ALIGNED 2 /* May or may not point to (emasculated) binary, temp_alloc field is set + so that erts_free_aligned_binary_bytes_extra can handle either */ + + +#define BINARY_COPY_LOOP_FACTOR 100 + +static void cleanup_copy_bin_state(Binary *bp) +{ + CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp); + if (cbs->result != NULL) { + erts_bin_free(cbs->result); + cbs->result = NULL; + } + switch (cbs->source_type) { + case BC_TYPE_HEAP: + erts_free(ERTS_ALC_T_BINARY_BUFFER,cbs->source); + break; + case BC_TYPE_ALIGNED: + erts_free_aligned_binary_bytes_extra(cbs->temp_alloc, + ERTS_ALC_T_BINARY_BUFFER); + break; + default: + /* otherwise do nothing */ + break; + } + cbs->source_type = BC_TYPE_EMPTY; +} + +/* + * Binary *erts_bin_nrml_alloc(Uint size); + * Binary *erts_bin_realloc(Binary *bp, Uint size); + * void erts_bin_free(Binary *bp); + */ +static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) +{ + Uint n; + byte *bytes; + Uint bit_offs; + Uint bit_size; + size_t size; + Uint reds = get_reds(p, BINARY_COPY_LOOP_FACTOR); + Uint target_size; + byte *t; + Uint pos; + + + if (is_not_binary(bin)) { + goto badarg; + } + if (!term_to_Uint(en, &n)) { + goto badarg; + } + if (!n) { + Eterm res_term = erts_new_heap_binary(p,NULL,0,&bytes); + BIF_RET(res_term); + } + ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + + size = binary_size(bin); + target_size = size * n; + + if ((target_size - size) >= reds) { + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + CopyBinState *cbs; + Eterm *hp; + Eterm trap_term; + int i; + + /* We will trap, set up the structure for trapping right away */ + Binary *mb = erts_create_magic_binary(sizeof(CopyBinState), + cleanup_copy_bin_state); + cbs = ERTS_MAGIC_BIN_DATA(mb); + + cbs->temp_alloc = NULL; + cbs->source = NULL; + + ERTS_GET_REAL_BIN(bin, orig, offset, bit_offset, bit_size); + if (*(binary_val(orig)) == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin *) binary_val(orig); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + cbs->source = + erts_get_aligned_binary_bytes_extra(bin, + &(cbs->temp_alloc), + ERTS_ALC_T_BINARY_BUFFER, + 0); + cbs->source_type = BC_TYPE_ALIGNED; + } else { /* Heap binary */ + cbs->source = + erts_get_aligned_binary_bytes_extra(bin, + &(cbs->temp_alloc), + ERTS_ALC_T_BINARY_BUFFER, + 0); + if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */ + byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size); + memcpy(tmp,cbs->source,size); + cbs->source = tmp; + cbs->source_type = BC_TYPE_HEAP; + } else { + cbs->source_type = BC_TYPE_ALIGNED; + } + } + cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap + if trapping */ + cbs->result->flags = 0; + cbs->result->orig_size = target_size; + erts_refc_init(&(cbs->result->refc), 1); + t = (byte *) cbs->result->orig_bytes; /* No offset or anything */ + pos = 0; + i = 0; + while (pos < reds) { + memcpy(t+pos,cbs->source, size); + pos += size; + ++i; + } + cbs->source_size = size; + cbs->result_pos = pos; + cbs->times_left = n-i; + hp = HAlloc(p,PROC_BIN_SIZE); + trap_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + BUMP_ALL_REDS(p); + BIF_TRAP2(&binary_copy_trap_export, p, bin, trap_term); + } else { + Eterm res_term; + byte *temp_alloc = NULL; + byte *source = + erts_get_aligned_binary_bytes(bin, + &temp_alloc); + if (target_size <= ERL_ONHEAP_BIN_LIMIT) { + res_term = erts_new_heap_binary(p,NULL,target_size,&t); + } else { + res_term = erts_new_mso_binary(p,NULL,target_size); + t = ((ProcBin *) binary_val(res_term))->bytes; + } + pos = 0; + while (pos < target_size) { + memcpy(t+pos,source, size); + pos += size; + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p,pos / BINARY_COPY_LOOP_FACTOR); + BIF_RET(res_term); + } + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) +{ + Uint n; + size_t size; + Uint reds = get_reds(BIF_P, BINARY_COPY_LOOP_FACTOR); + byte *t; + Uint pos; + Binary *mb = ((ProcBin *) binary_val(BIF_ARG_2))->val; + CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(mb); + Uint opos; + + /* swapout... */ + n = cbs->times_left; + size = cbs->source_size; + opos = pos = cbs->result_pos; + t = (byte *) cbs->result->orig_bytes; /* "well behaved" binary */ + if ((n-1) * size >= reds) { + Uint i = 0; + while ((pos - opos) < reds) { + memcpy(t+pos,cbs->source, size); + pos += size; + ++i; + } + cbs->result_pos = pos; + cbs->times_left -= i; + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&binary_copy_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2); + } else { + Binary *save; + ProcBin* pb; + Uint target_size = cbs->result->orig_size; + while (pos < target_size) { + memcpy(t+pos,cbs->source, size); + pos += size; + } + save = cbs->result; + cbs->result = NULL; + cleanup_copy_bin_state(mb); /* now cbs is dead */ + pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = target_size; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = save; + pb->bytes = t; + pb->flags = 0; + + OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm)); + BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR); + + BIF_RET(make_binary(pb)); + } +} + + +BIF_RETTYPE binary_copy_1(BIF_ALIST_1) +{ + return do_binary_copy(BIF_P,BIF_ARG_1,make_small(1)); +} + +BIF_RETTYPE binary_copy_2(BIF_ALIST_2) +{ + return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) +{ + ErlSubBin *sb; + ProcBin *pb; + Eterm res; + Eterm bin = BIF_ARG_1; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + sb = (ErlSubBin *) binary_val(bin); + if (sb->thing_word == HEADER_SUB_BIN) { + bin = sb->orig; + } + pb = (ProcBin *) binary_val(bin); + if (pb->thing_word == HEADER_PROC_BIN) { + /* XXX:PaN - Halfword - orig_size is a long, we should handle that */ + res = erts_make_integer((Uint) pb->val->orig_size, BIF_P); + } else { /* heap binary */ + res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P); + } + BIF_RET(res); +} + +#define END_BIG 0 +#define END_SMALL 1 + +#ifdef WORDS_BIGENDIAN +#define END_NATIVE END_BIG +#else +#define END_NATIVE END_SMALL +#endif + +static int get_need(Uint u) { +#if defined(ARCH_64) && !HALFWORD_HEAP + if (u > 0xFFFFFFFFUL) { + if (u > 0xFFFFFFFFFFFFUL) { + if (u > 0xFFFFFFFFFFFFFFUL) { + return 8; + } + return 7; + } + if (u > 0xFFFFFFFFFFUL) { + return 6; + } + return 5; + } +#endif + if (u > 0xFFFFUL) { + if (u > 0xFFFFFFUL) { + return 4; + } + return 3; + } + if (u > 0xFFUL) { + return 2; + } + return 1; +} + +static BIF_RETTYPE do_encode_unsigned(Process *p, Eterm uns, Eterm endianess) +{ + Eterm res; + if ((is_not_small(uns) && is_not_big(uns)) || is_not_atom(endianess) || + (endianess != am_big && endianess != am_little)) { + goto badarg; + } + if (is_small(uns)) { + Sint x = signed_val(uns); + Uint u; + int n,i; + byte *b; + + if (x < 0) { + goto badarg; + } + + u = (Uint) x; + n = get_need(u); + ASSERT(n <= ERL_ONHEAP_BIN_LIMIT); + res = erts_new_heap_binary(p, NULL, n, &b); + if (endianess == am_big) { + for(i=n-1;i>=0;--i) { + b[i] = u & 0xFF; + u >>= 8; + } + } else { + for(i=0;i<n;++i) { + b[i] = u & 0xFF; + u >>= 8; + } + } + BIF_RET(res); + } else { + /* Big */ + Eterm *bigp = big_val(uns); + Uint n; + dsize_t num_parts = BIG_SIZE(bigp); + Eterm res; + byte *b; + ErtsDigit d; + + if(BIG_SIGN(bigp)) { + goto badarg; + } + n = (num_parts-1)*sizeof(ErtsDigit)+get_need(BIG_DIGIT(bigp,(num_parts-1))); + if (n <= ERL_ONHEAP_BIN_LIMIT) { + res = erts_new_heap_binary(p,NULL,n,&b); + } else { + res = erts_new_mso_binary(p,NULL,n); + b = ((ProcBin *) binary_val(res))->bytes; + } + + if (endianess == am_big) { + Sint i,j; + j = 0; + d = BIG_DIGIT(bigp,0); + for (i=n-1;i>=0;--i) { + b[i] = d & 0xFF; + if (!((++j) % sizeof(ErtsDigit))) { + d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit)); + } else { + d >>= 8; + } + } + } else { + Sint i,j; + j = 0; + d = BIG_DIGIT(bigp,0); + for (i=0;i<n;++i) { + b[i] = d & 0xFF; + if (!((++j) % sizeof(ErtsDigit))) { + d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit)); + } else { + d >>= 8; + } + } + + } + BIF_RET(res); + } + badarg: + BIF_ERROR(p,BADARG); +} + +static BIF_RETTYPE do_decode_unsigned(Process *p, Eterm uns, Eterm endianess) +{ + byte *bytes; + Uint bitoffs, bitsize; + Uint size; + Eterm res; + + if (is_not_binary(uns) || is_not_atom(endianess) || + (endianess != am_big && endianess != am_little)) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(uns, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + /* align while rolling */ + size = binary_size(uns); + if (bitoffs) { + if (endianess == am_big) { + while (size && (((((Uint) bytes[0]) << bitoffs) | + (((Uint) bytes[1]) >> (8-bitoffs))) & 0xFF) == 0) { + ++bytes; + --size; + } + } else { + while(size && + (((((Uint) bytes[size-1]) << bitoffs) | + (((Uint) bytes[size]) >> (8-bitoffs))) & 0xFF) == 0) { + --size; + } + } + } else { + if (endianess == am_big) { + while (size && *bytes == 0) { + ++bytes; + --size; + } + } else { + while(size && bytes[size-1] == 0) { + --size; + } + } + } + if (!size) { + BIF_RET(make_small(0)); + } + + if (size <= sizeof(Uint)) { + Uint u = 0; + Sint i; + + if (endianess == am_big) { + if (bitoffs) { + for(i=0;i<size;++i) { + u <<=8; + u |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + } + } else { + for(i=0;i<size;++i) { + u <<=8; + u |= bytes[i]; + } + } + } else { + + if (bitoffs) { + for(i=size-1;i>=0;--i) { + u <<=8; + u |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + } + } else { + for(i=size-1;i>=0;--i) { + u <<=8; + u |= bytes[i]; + } + } + } + res = erts_make_integer(u,p); + BIF_RET(res); + } else { + /* Assume big, as we stripped away all zeroes from the MSB part of the binary */ + dsize_t num_parts = size / sizeof(ErtsDigit) + !!(size % sizeof(ErtsDigit)); + Eterm *bigp; + + bigp = HAlloc(p, BIG_NEED_SIZE(num_parts)); + *bigp = make_pos_bignum_header(num_parts); + res = make_big(bigp); + + if (endianess == am_big) { + Sint i,j; + ErtsDigit *d; + j = size; + d = &(BIG_DIGIT(bigp,num_parts - 1)); + *d = 0; + i = 0; + if(bitoffs) { + for (;;){ + (*d) <<= 8; + (*d) |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + if (++i >= size) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } else { + for (;;){ + (*d) <<= 8; + (*d) |= bytes[i]; + if (++i >= size) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } + } else { + Sint i,j; + ErtsDigit *d; + j = size; + d = &(BIG_DIGIT(bigp,num_parts - 1)); + *d = 0; + i = size-1; + if (bitoffs) { + for (;;){ + (*d) <<= 8; + (*d) |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + if (--i < 0) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } else { + for (;;){ + (*d) <<= 8; + (*d) |= bytes[i]; + if (--i < 0) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } + } + BIF_RET(res); + } + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_encode_unsigned_1(BIF_ALIST_1) +{ + return do_encode_unsigned(BIF_P,BIF_ARG_1,am_big); +} + +BIF_RETTYPE binary_encode_unsigned_2(BIF_ALIST_2) +{ + return do_encode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +BIF_RETTYPE binary_decode_unsigned_1(BIF_ALIST_1) +{ + return do_decode_unsigned(BIF_P,BIF_ARG_1,am_big); +} + +BIF_RETTYPE binary_decode_unsigned_2(BIF_ALIST_2) +{ + return do_decode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +/* + * Hard debug functions (dump) for the search structures + */ + +#ifdef HARDDEBUG +static void dump_bm_data(BMData *bm) +{ + int i,j; + erts_printf("Dumping Boyer-Moore structure.\n"); + erts_printf("=============================\n"); + erts_printf("Searchstring [%ld]:\n", bm->len); + erts_printf("<<"); + for (i = 0; i < bm->len; ++i) { + if (i > 0) { + erts_printf(", "); + } + erts_printf("%d", (int) bm->x[i]); + if (bm->x[i] >= 'A') { + erts_printf(" ($%c)",(char) bm->x[i]); + } + } + erts_printf(">>\n"); + erts_printf("GoodShift array:\n"); + for (i = 0; i < bm->len; ++i) { + erts_printf("GoodShift[%d]: %ld\n", i, bm->goodshift[i]); + } + erts_printf("BadShift array:\n"); + j = 0; + for (i = 0; i < ALPHABET_SIZE; i += j) { + for (j = 0; i + j < ALPHABET_SIZE && j < 6; ++j) { + erts_printf("BS[%03d]:%02ld, ", i+j, bm->badshift[i+j]); + } + erts_printf("\n"); + } +} + +static void dump_ac_node(ACNode *node, int indent, int ch) { + int i; + char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1); + memset(spaces,' ',10*indent); + spaces[10*indent] = '\0'; + erts_printf("%s-> %c\n",spaces,ch); + erts_printf("%sId: %u\n",spaces,(unsigned) node->id); + erts_printf("%sD: %u\n",spaces,(unsigned)node->d); + erts_printf("%sFinal: %d\n",spaces,(int)node->final); + erts_printf("%sFail: %u\n",spaces,(unsigned)node->h->id); + erts_free(ERTS_ALC_T_TMP,spaces); + for(i=0;i<ALPHABET_SIZE;++i) { + if (node->g[i] != NULL && node->g[i] != node) { + dump_ac_node(node->g[i],indent+1,i); + } + } +} + + +static void dump_ac_trie(ACTrie *act) +{ + erts_printf("Aho Corasick Trie dump.\n"); + erts_printf("=======================\n"); + erts_printf("Node counter: %u\n", (unsigned) act->idc); + erts_printf("Searchstring counter: %u\n", (unsigned) act->counter); + erts_printf("Trie:\n"); + dump_ac_node(act->root, 0, '0'); + return; +} +#endif diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 445ba00ca7..06b7ffdf32 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -49,9 +49,9 @@ void erts_init_bif_chksum(void) chksum_md5_2_exp.code[1] = am_atom_put("md5_trap",8); chksum_md5_2_exp.code[2] = 2; chksum_md5_2_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; chksum_md5_2_exp.code[4] = - (Eterm) &md5_2; + (BeamInstr) &md5_2; } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 9d5f0d9c02..9631fb50db 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -146,7 +146,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, Eterm name_term, Eterm options) { char *path = NULL; - int path_len; + Uint path_len; char *name = NULL; DE_Handle *dh; erts_driver_t *drv; @@ -221,9 +221,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto error; } - path_len = io_list_len(path_term); - - if (path_len <= 0) { + if (erts_iolist_size(path_term, &path_len)) { goto error; } path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1); @@ -1193,7 +1191,7 @@ int erts_ddll_driver_ok(DE_Handle *dh) static void ddll_no_more_references(void *vdh) { DE_Handle *dh = (DE_Handle *) vdh; - int x; + erts_aint_t x; lock_drv_list(); @@ -1604,7 +1602,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) erts_sys_ddll_close(dh->handle); return ERL_DE_LOAD_ERROR_BAD_NAME; } - erts_smp_atomic_init(&(dh->refc), (long) 0); + erts_smp_atomic_init(&(dh->refc), (erts_aint_t) 0); dh->port_count = 0; dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); @@ -1646,7 +1644,8 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name) if (save_name != NULL) { *save_name = mkatom(q->name); } - /* XXX:PaN Future locking problems? Don't dare to let go of the diver_list lock here!*/ + /* Future locking problems? Don't dare to let go of the + diver_list lock here!*/ if (q->finish) { int fpe_was_unmasked = erts_block_fpe(); (*(q->finish))(); @@ -1671,7 +1670,7 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name) dh->handle = NULL; dh->procs = NULL; dh->port_count = 0; - erts_refc_init(&(dh->refc), (long) 0); + erts_refc_init(&(dh->refc), (erts_aint_t) 0); dh->status = -1; dh->reload_full_path = NULL; dh->reload_driver_name = NULL; @@ -1877,7 +1876,7 @@ static Eterm mkatom(char *str) static char *pick_list_or_atom(Eterm name_term) { char *name = NULL; - int name_len; + Uint name_len; if (is_atom(name_term)) { Atom *ap = atom_tab(atom_val(name_term)); if (ap->len == 0) { @@ -1889,8 +1888,7 @@ static char *pick_list_or_atom(Eterm name_term) memcpy(name,ap->name,ap->len); name[ap->len] = '\0'; } else { - name_len = io_list_len(name_term); - if (name_len <= 0) { + if (erts_iolist_size(name_term, &name_len)) { goto error; } name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1); diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 440b0b4f14..01e6977a2c 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -314,6 +314,30 @@ double_to_integer(Process* p, double x) return res; } +/******************************************************************************** + * binary_part guards. The actual implementation is in erl_bif_binary.c + ********************************************************************************/ +BIF_RETTYPE binary_part_3(BIF_ALIST_3) +{ + return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE binary_part_2(BIF_ALIST_2) +{ + Eterm *tp; + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + + /* * The following code is used when a guard that may build on the * heap is called directly. They must not use HAlloc(), but must @@ -630,3 +654,16 @@ gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live) } return res; } + +/******************************************************************************** + * binary_part guards. The actual implementation is in erl_bif_binary.c + ********************************************************************************/ +Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live) +{ + return erts_gc_binary_part(p,reg,live,0); +} + +Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live) +{ + return erts_gc_binary_part(p,reg,live,1); +} diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index a34d400ed8..f264bf44df 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -38,9 +38,7 @@ #include "erl_instrument.h" #include "dist.h" #include "erl_gc.h" -#ifdef ELIB_ALLOC_IS_CLIB -#include "elib_stat.h" -#endif +#include "erl_cpu_topology.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -59,16 +57,23 @@ /* Keep erts_system_version as a global variable for easy access from a core */ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE " (erts-" ERLANG_VERSION ")" +#if !HEAP_ON_C_STACK && !HALFWORD_HEAP + " [no-c-stack-objects]" +#endif #ifndef OTP_RELEASE " [source]" #endif #ifdef ARCH_64 +#if HALFWORD_HEAP + " [64-bit halfword]" +#else " [64-bit]" #endif +#endif #ifdef ERTS_SMP - " [smp:%bpu:%bpu]" + " [smp:%beu:%beu]" #endif - " [rq:%bpu]" + " [rq:%beu]" #ifdef USE_THREADS " [async-threads:%d]" #endif @@ -115,22 +120,26 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #endif static Eterm -bld_bin_list(Uint **hpp, Uint *szp, ProcBin* pb) +bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) { + struct erl_off_heap_header* ohh; Eterm res = NIL; Eterm tuple; - for (; pb; pb = pb->next) { - Eterm val = erts_bld_uint(hpp, szp, (Uint) pb->val); - Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); - - if (szp) - *szp += 4+2; - if (hpp) { - Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); - tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); - res = CONS(*hpp + 4, tuple, res); - *hpp += 4+2; + for (ohh = oh->first; ohh; ohh = ohh->next) { + if (ohh->thing_word == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin*) ohh; + Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val); + Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); + + if (szp) + *szp += 4+2; + if (hpp) { + Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); + tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); + res = CONS(*hpp + 4, tuple, res); + *hpp += 4+2; + } } } return res; @@ -169,10 +178,10 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) Eterm tup; Eterm r = (IS_CONST(mon->ref) ? mon->ref - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->ref)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref)); Eterm p = (IS_CONST(mon->pid) ? mon->pid - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->pid)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid)); tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name); pmlc->hp += 6; pmlc->res = CONS(pmlc->hp, tup, pmlc->res); @@ -233,7 +242,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc) Eterm old_res, targets = NIL; Eterm p = (IS_CONST(lnk->pid) ? lnk->pid - : STORE_NC(&(pllc->hp), &MSO(pllc->p).externals, lnk->pid)); + : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid)); if (lnk->type == LINK_NODE) { targets = make_small(ERTS_LINK_REFC(lnk)); } else if (ERTS_LINK_ROOT(lnk) != NULL) { @@ -624,12 +633,18 @@ static Eterm pi_1_keys[] = { #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm)) static Eterm pi_1_keys_list; -static Uint pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS]; +#if HEAP_ON_C_STACK +static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS]; +#endif static void process_info_init(void) { +#if HEAP_ON_C_STACK Eterm *hp = &pi_1_keys_list_heap[0]; +#else + Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS); +#endif int i; pi_1_keys_list = NIL; @@ -998,7 +1013,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3); res = am_undefined; } else { - Eterm* current; + BeamInstr* current; if (rp->current[0] == am_erlang && rp->current[1] == am_process_info && @@ -1127,9 +1142,9 @@ process_info_aux(Process *BIF_P, } else { /* Make our copy of the message */ - ASSERT(size_object(msg) == hfp->size); + ASSERT(size_object(msg) == hfp->used_size); msg = copy_struct(msg, - hfp->size, + hfp->used_size, &hp, &MSO(BIF_P)); } @@ -1212,7 +1227,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1245,9 +1260,7 @@ process_info_aux(Process *BIF_P, else { /* Monitor by pid. Build {process, Pid} and cons it. */ Eterm t; - Eterm pid = STORE_NC(&hp, - &MSO(BIF_P).externals, - mic.mi[i].entity); + Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, pid); hp += 3; res = CONS(hp, t, res); @@ -1269,7 +1282,7 @@ process_info_aux(Process *BIF_P, res = NIL; for (i = 0; i < mic.mi_i; ++i) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1478,7 +1491,7 @@ process_info_aux(Process *BIF_P, case am_group_leader: { int sz = NC_HEAP_SIZE(rp->group_leader); hp = HAlloc(BIF_P, 3 + sz); - res = STORE_NC(&hp, &MSO(BIF_P).externals, rp->group_leader); + res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader); break; } @@ -1503,9 +1516,9 @@ process_info_aux(Process *BIF_P, case am_binary: { Uint sz = 3; - (void) bld_bin_list(NULL, &sz, MSO(rp).mso); + (void) bld_bin_list(NULL, &sz, &MSO(rp)); hp = HAlloc(BIF_P, sz); - res = bld_bin_list(&hp, NULL, MSO(rp).mso); + res = bld_bin_list(&hp, NULL, &MSO(rp)); break; } @@ -1532,7 +1545,7 @@ process_info_aux(Process *BIF_P, case am_backtrace: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp); - res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len); + res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hp = HAlloc(BIF_P, 3); break; @@ -1622,6 +1635,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ if (sel == am_allocator_sizes && arity == 2) { return erts_allocator_info_term(BIF_P, *tp, 1); + } else if (sel == am_wordsize && arity == 2) { + if (tp[0] == am_internal) { + return make_small(sizeof(Eterm)); + } + if (tp[0] == am_external) { + return make_small(sizeof(UWord)); + } + goto badarg; } else if (sel == am_allocated) { if (arity == 2) { Eterm res = THE_NON_VALUE; @@ -1667,6 +1688,8 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return erts_get_cpu_topology_term(BIF_P, *tp); } else if (ERTS_IS_ATOM_STR("cpu_topology", sel) && arity == 2) { Eterm res = erts_get_cpu_topology_term(BIF_P, *tp); + if (res == THE_NON_VALUE) + goto badarg; ERTS_BIF_PREP_TRAP1(ret, erts_format_cpu_topology_trap, BIF_P, res); return ret; #if defined(PURIFY) || defined(VALGRIND) @@ -1700,17 +1723,23 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } else if (is_list(*tp)) { #if defined(PURIFY) #define ERTS_ERROR_CHECKER_PRINTF purify_printf +#define ERTS_ERROR_CHECKER_PRINTF_XML purify_printf #elif defined(VALGRIND) #define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF +# ifndef HAVE_VALGRIND_PRINTF_XML +# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF +# else +# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML +# endif #endif - int buf_size = 8*1024; /* Try with 8KB first */ + Uint buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); if (r < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); - buf_size = io_list_len(*tp); - if (buf_size < 0) + if (erts_iolist_size(*tp, &buf_size)) { goto badarg; + } buf_size++; buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); @@ -1718,8 +1747,8 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } buf[buf_size - 1 - r] = '\0'; if (check_if_xml()) { - ERTS_ERROR_CHECKER_PRINTF("<erlang_info_log>" - "%s</erlang_info_log>\n", buf); + ERTS_ERROR_CHECKER_PRINTF_XML("<erlang_info_log>" + "%s</erlang_info_log>\n", buf); } else { ERTS_ERROR_CHECKER_PRINTF("%s\n", buf); } @@ -1871,6 +1900,37 @@ c_compiler_used(Eterm **hpp, Uint *szp) } +static int is_snif_term(Eterm module_atom) { + int i; + Atom *a = atom_tab(atom_val(module_atom)); + char *aname = (char *) a->name; + + /* if a->name has a '.' then the bif (snif) is bogus i.e a package */ + for (i = 0; i < a->len; i++) { + if (aname[i] == '.') + return 0; + } + + return 1; +} + +static Eterm build_snif_term(Eterm **hpp, Uint *szp, int ix, Eterm res) { + Eterm tup; + tup = erts_bld_tuple(hpp, szp, 3, bif_table[ix].module, bif_table[ix].name, make_small(bif_table[ix].arity)); + res = erts_bld_cons( hpp, szp, tup, res); + return res; +} + +static Eterm build_snifs_term(Eterm **hpp, Uint *szp, Eterm res) { + int i; + for (i = 0; i < BIF_SIZE; i++) { + if (is_snif_term(bif_table[i].module)) { + res = build_snif_term(hpp, szp, i, res); + } + } + return res; +} + BIF_RETTYPE system_info_1(BIF_ALIST_1) { Eterm res; @@ -1904,6 +1964,35 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) : am_enabled); } #endif + } else if (BIF_ARG_1 == am_build_type) { +#if defined(DEBUG) + ERTS_DECL_AM(debug); + BIF_RET(AM_debug); +#elif defined(PURIFY) + ERTS_DECL_AM(purify); + BIF_RET(AM_purify); +#elif defined(QUANTIFY) + ERTS_DECL_AM(quantify); + BIF_RET(AM_quantify); +#elif defined(PURECOV) + ERTS_DECL_AM(purecov); + BIF_RET(AM_purecov); +#elif defined(ERTS_GCOV) + ERTS_DECL_AM(gcov); + BIF_RET(AM_gcov); +#elif defined(VALGRIND) + ERTS_DECL_AM(valgrind); + BIF_RET(AM_valgrind); +#elif defined(GPROF) + ERTS_DECL_AM(gprof); + BIF_RET(AM_gprof); +#elif defined(ERTS_ENABLE_LOCK_COUNT) + ERTS_DECL_AM(lcnt); + BIF_RET(AM_lcnt); +#else + BIF_RET(am_opt); +#endif + BIF_RET(res); } else if (BIF_ARG_1 == am_allocated_areas) { res = erts_allocated_areas(NULL, NULL, BIF_P); BIF_RET(res); @@ -1919,6 +2008,17 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(db_get_trace_control_word_0(BIF_P)); } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) { BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("ets_always_compress", BIF_ARG_1)) { + BIF_RET((erts_ets_always_compress) ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("snifs", BIF_ARG_1)) { + Uint size = 0; + Uint *szp; + + szp = &size; + build_snifs_term(NULL, szp, NIL); + hp = HAlloc(BIF_P, size); + res = build_snifs_term(&hp, NULL, NIL); + BIF_RET(res); } else if (BIF_ARG_1 == am_sequential_tracer) { val = erts_get_system_seq_tracer(); ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false) @@ -1926,7 +2026,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection){ - Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); Eterm tup; hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); @@ -1941,7 +2041,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ - Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_fullsweep_after, make_small(val)); BIF_RET(res); @@ -1980,7 +2080,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); ASSERT(dsbufp && dsbufp->str); - res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len); + res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_info_dsbuf(dsbufp); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) { @@ -2063,86 +2163,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_alloc_util_allocators((void *) BIF_P)); } else if (BIF_ARG_1 == am_elib_malloc) { -#ifdef ELIB_ALLOC_IS_CLIB - struct elib_stat stat; - DECL_AM(heap_size); - DECL_AM(max_alloced_size); - DECL_AM(alloced_size); - DECL_AM(free_size); - DECL_AM(no_alloced_blocks); - DECL_AM(no_free_blocks); - DECL_AM(smallest_alloced_block); - DECL_AM(largest_free_block); - Eterm atoms[8]; - Eterm ints[8]; - Uint **hpp; - Uint sz; - Uint *szp; - int length; -#ifdef DEBUG - Uint *endp; -#endif - - elib_stat(&stat); - - /* First find out the heap size needed ... */ - hpp = NULL; - szp = &sz; - sz = 0; - - build_elib_malloc_term: - length = 0; - atoms[length] = AM_heap_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_total*sizeof(Uint)); - atoms[length] = AM_max_alloced_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_max_alloc*sizeof(Uint)); - atoms[length] = AM_alloced_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_alloc*sizeof(Uint)); - atoms[length] = AM_free_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_free*sizeof(Uint)); - atoms[length] = AM_no_alloced_blocks; - ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.mem_blocks); - atoms[length] = AM_no_free_blocks; - ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.free_blocks); - atoms[length] = AM_smallest_alloced_block; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.min_used*sizeof(Uint)); - atoms[length] = AM_largest_free_block; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.max_free*sizeof(Uint)); - - - - ASSERT(length <= sizeof(atoms)/sizeof(Eterm)); - ASSERT(length <= sizeof(ints)/sizeof(Eterm)); - - res = erts_bld_2tup_list(hpp, szp, length, atoms, ints); - - if (szp) { - /* ... and then build the term */ - hp = HAlloc(BIF_P, sz); -#ifdef DEBUG - endp = hp + sz; -#endif - - szp = NULL; - hpp = &hp; - goto build_elib_malloc_term; - } - -#ifdef DEBUG - ASSERT(endp == hp); -#endif - -#else /* #ifdef ELIB_ALLOC_IS_CLIB */ - res = am_false; -#endif /* #ifdef ELIB_ALLOC_IS_CLIB */ - - BIF_RET(res); + /* To be removed in R15 */ + BIF_RET(am_false); } else if (BIF_ARG_1 == am_os_version) { int major, minor, build; @@ -2253,6 +2275,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("cpu_topology", BIF_ARG_1)) { res = erts_get_cpu_topology_term(BIF_P, am_used); BIF_TRAP1(erts_format_cpu_topology_trap, BIF_P, res); + } else if (ERTS_IS_ATOM_STR("update_cpu_info", BIF_ARG_1)) { + if (erts_update_cpu_info()) { + ERTS_DECL_AM(changed); + BIF_RET(AM_changed); + } + else { + ERTS_DECL_AM(unchanged); + BIF_RET(AM_unchanged); + } #if defined(__GNUC__) && defined(HAVE_SOLARIS_SPARC_PERFMON) } else if (ERTS_IS_ATOM_STR("ultrasparc_read_tick1", BIF_ARG_1)) { register unsigned high asm("%l0"); @@ -2324,7 +2355,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } /* Arguments that are unusual follow ... */ else if (ERTS_IS_ATOM_STR("logical_processors", BIF_ARG_1)) { - int no = erts_get_cpu_configured(erts_cpuinfo); + int no; + erts_get_logical_processors(&no, NULL, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2333,7 +2365,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } } else if (ERTS_IS_ATOM_STR("logical_processors_online", BIF_ARG_1)) { - int no = erts_get_cpu_online(erts_cpuinfo); + int no; + erts_get_logical_processors(NULL, &no, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2342,7 +2375,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } } else if (ERTS_IS_ATOM_STR("logical_processors_available", BIF_ARG_1)) { - int no = erts_get_cpu_available(erts_cpuinfo); + int no; + erts_get_logical_processors(NULL, NULL, &no); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2502,6 +2536,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_sched_stat_term(BIF_P, 1)); } else if (ERTS_IS_ATOM_STR("taints", BIF_ARG_1)) { BIF_RET(erts_nif_taints(BIF_P)); + } else if (ERTS_IS_ATOM_STR("reader_groups_map", BIF_ARG_1)) { + BIF_RET(erts_get_reader_groups_map(BIF_P)); + } else if (ERTS_IS_ATOM_STR("dist_buf_busy_limit", BIF_ARG_1)) { + Uint hsz = 0; + + (void) erts_bld_uint(NULL, &hsz, erts_dist_buf_busy_limit); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); + BIF_RET(res); } BIF_ERROR(BIF_P, BADARG); @@ -2617,7 +2660,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -2637,7 +2680,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) res = NIL; for (i = 0; i < mic.mi_i; i++) { Eterm t; - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, item); hp += 3; res = CONS(hp, t, res); @@ -2694,7 +2737,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) erts_doforall_links(prt->nlinks, &one_link_size, &size); for (bp = prt->bp; bp; bp = bp->next) - size += sizeof(ErlHeapFragment) + (bp->size - 1)*sizeof(Eterm); + size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm); if (prt->linebuf) size += sizeof(LineBuf) + prt->linebuf->ovsiz; @@ -2817,7 +2860,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what) goto error; } } else if (is_export(fun)) { - Export* exp = (Export *) (export_val(fun))[1]; + Export* exp = (Export *) ((UWord) (export_val(fun))[1]); switch (what) { case am_type: hp = HAlloc(p, 3); @@ -3010,11 +3053,11 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = erts_run_queues_len(NULL); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { - Uint w1, w2; + UWord w1, w2; Eterm b1, b2; wall_clock_elapsed_time_both(&w1, &w2); - b1 = erts_make_integer(w1,BIF_P); - b2 = erts_make_integer(w2,BIF_P); + b1 = erts_make_integer((Uint) w1,BIF_P); + b2 = erts_make_integer((Uint) w2,BIF_P); hp = HAlloc(BIF_P,3); res = TUPLE2(hp, b1, b2); BIF_RET(res); @@ -3365,6 +3408,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("fake_scheduler_bindings", tp[1])) { return erts_fake_scheduler_bindings(BIF_P, tp[2]); } + else if (ERTS_IS_ATOM_STR("reader_groups_map", tp[1])) { + Sint groups; + if (is_not_small(tp[2])) + BIF_ERROR(BIF_P, BADARG); + groups = signed_val(tp[2]); + if (groups < (Sint) 1 || groups > (Sint) INT_MAX) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); + } break; } default: @@ -3383,8 +3436,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) */ if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1) && (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) { - long on = (long) (BIF_ARG_2 == am_true); - long prev_on = erts_smp_atomic_xchg(&available_internal_state, on); + erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true); + erts_aint_t prev_on = erts_smp_atomic_xchg(&available_internal_state, on); if (on) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Process %T ", BIF_P->id); @@ -3546,6 +3599,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } } } + else if (ERTS_IS_ATOM_STR("binary_loop_limit", BIF_ARG_1)) { + /* Used by binary_module_SUITE (stdlib) */ + Uint max_loops; + if (is_atom(BIF_ARG_2) && ERTS_IS_ATOM_STR("default", BIF_ARG_2)) { + max_loops = erts_binary_set_loop_limit(-1); + BIF_RET(make_small(max_loops)); + } else if (term_to_Uint(BIF_ARG_2, &max_loops) != 0) { + max_loops = erts_binary_set_loop_limit(max_loops); + BIF_RET(make_small(max_loops)); + } + } else if (ERTS_IS_ATOM_STR("re_loop_limit", BIF_ARG_1)) { /* Used by re_SUITE (stdlib) */ Uint max_loops; @@ -3570,7 +3634,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) { /* Used by hipe test suites */ - long flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); + erts_aint_t flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); if (!flag && BIF_ARG_2 != am_false) { erts_smp_atomic_set(&hipe_test_reschedule_flag, 1); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); @@ -3645,7 +3709,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) #ifdef ERTS_ENABLE_LOCK_COUNT static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) { - unsigned long tries = 0, colls = 0; + Uint tries = 0, colls = 0; unsigned long timer_s = 0, timer_ns = 0, timer_n = 0; unsigned int line = 0; @@ -3658,8 +3722,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] */ - ethr_atomic_read(&stats->tries, (long *)&tries); - ethr_atomic_read(&stats->colls, (long *)&colls); + tries = (Uint) ethr_atomic_read(&stats->tries); + colls = (Uint) ethr_atomic_read(&stats->colls); line = stats->line; timer_s = stats->timer.s; diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index a9e8dd86f7..47c48e74d6 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -89,13 +89,14 @@ BIF_RETTYPE append_2(BIF_ALIST_2) BIF_RET(copy); } +#define SMALL_VEC_SIZE 10 BIF_RETTYPE subtract_2(BIF_ALIST_2) { Eterm list; Eterm* hp; Uint need; Eterm res; - Eterm small_vec[10]; /* Preallocated memory for small lists */ + Eterm small_vec[SMALL_VEC_SIZE]; /* Preallocated memory for small lists */ Eterm* vec_p; Eterm* vp; int i; @@ -115,7 +116,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) BIF_RET(BIF_ARG_1); /* allocate element vector */ - if (n <= sizeof(small_vec)/sizeof(small_vec[0])) + if (n <= SMALL_VEC_SIZE) vec_p = small_vec; else vec_p = (Eterm*) erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm)); @@ -377,7 +378,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) Eterm *tuple_ptr = tuple_val(term); if (pos <= arityval(*tuple_ptr)) { Eterm element = tuple_ptr[pos]; - if (cmp(Key, element) == 0) { + if (CMP(Key, element) == 0) { return term; } } diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index 6da72dcef9..deda7adc1f 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -251,7 +251,7 @@ BIF_RETTYPE is_function_2(BIF_ALIST_2) BIF_RET(am_true); } } else if (is_export(BIF_ARG_1)) { - Export* exp = (Export *) (export_val(BIF_ARG_1))[1]; + Export* exp = (Export *) EXPAND_POINTER((export_val(BIF_ARG_1))[1]); if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index f454f2e12d..3fd35dd963 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -21,10 +21,6 @@ # include "config.h" #endif -#ifdef _OSE_ -# include "ose.h" -#endif - #include <ctype.h> #define ERTS_WANT_EXTERNAL_TAGS @@ -583,8 +579,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) if (prt->bp == NULL) { /* MUST be CONST! */ res = prt->data; } else { - Eterm* hp = HAlloc(BIF_P, prt->bp->size); - res = copy_struct(prt->data, prt->bp->size, &hp, &MSO(BIF_P)); + Eterm* hp = HAlloc(BIF_P, prt->bp->used_size); + res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P)); } erts_smp_port_unlock(prt); BIF_RET(res); @@ -614,6 +610,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) int binary_io; int soft_eof; Sint linebuf; + Eterm edir = NIL; byte dir[MAXPATHLEN]; /* These are the defaults */ @@ -690,19 +687,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) } else if (option == am_arg0) { char *a0; - int n; - if (is_nil(*tp)) { - n = 0; - } else if( (n = is_string(*tp)) == 0) { + + if ((a0 = erts_convert_filename_to_native(*tp, ERTS_ALC_T_TMP, 1)) == NULL) { goto badarg; } - a0 = (char *) erts_alloc(ERTS_ALC_T_TMP, - (n + 1) * sizeof(byte)); - if (intlist_to_buf(*tp, a0, n) != n) { - erl_exit(1, "%s:%d: Internal error\n", - __FILE__, __LINE__); - } - a0[n] = '\0'; if (opts.argv == NULL) { opts.argv = erts_alloc(ERTS_ALC_T_TMP, 2 * sizeof(char **)); @@ -715,20 +703,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) opts.argv[0] = a0; } } else if (option == am_cd) { - Eterm iolist; - Eterm heap[4]; - int r; - - heap[0] = *tp; - heap[1] = make_list(heap+2); - heap[2] = make_small(0); - heap[3] = NIL; - iolist = make_list(heap); - r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); - if (r < 0) { - goto badarg; - } - opts.wd = (char *) dir; + edir = *tp; } else { goto badarg; } @@ -840,19 +815,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) goto badarg; } name = tp[1]; - if (is_atom(name)) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, - atom_tab(atom_val(name))->len+1); - sys_memcpy((void *) name_buf, - (void *) atom_tab(atom_val(name))->name, - atom_tab(atom_val(name))->len); - name_buf[atom_tab(atom_val(name))->len] = '\0'; - } else if ((i = is_string(name))) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); - if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - name_buf[i] = '\0'; - } else { + if ((name_buf = erts_convert_filename_to_native(name,ERTS_ALC_T_TMP,0)) == NULL) { goto badarg; } opts.spawn_type = ERTS_SPAWN_EXECUTABLE; @@ -894,7 +857,33 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) /* Argument vector only if explicit spawn_executable */ goto badarg; } - + + if (edir != NIL) { + /* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles + for spawn_executable... */ + if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) { + Eterm iolist; + DeclareTmpHeap(heap,4,p); + int r; + + UseTmpHeap(4,p); + heap[0] = edir; + heap[1] = make_list(heap+2); + heap[2] = make_small(0); + heap[3] = NIL; + iolist = make_list(heap); + r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); + UnUseTmpHeap(4,p); + if (r < 0) { + goto badarg; + } + opts.wd = (char *) dir; + } else { + if ((opts.wd = erts_convert_filename_to_native(edir,ERTS_ALC_T_TMP,0)) == NULL) { + goto badarg; + } + } + } if (driver != &spawn_driver && opts.exit_status) { goto badarg; @@ -943,6 +932,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) if (opts.argv) { free_args(opts.argv); } + if (opts.wd && opts.wd != ((char *)dir)) { + erts_free(ERTS_ALC_T_TMP, (void *) opts.wd); + } return port_num; badarg: @@ -952,6 +944,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) #undef OPEN_PORT_ERROR } +/* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */ static char **convert_args(Eterm l) { char **pp; @@ -968,22 +961,14 @@ static char **convert_args(Eterm l) pp[i++] = erts_default_arg0; while (is_list(l)) { str = CAR(list_val(l)); - - if (is_nil(str)) { - n = 0; - } else if( (n = is_string(str)) == 0) { - /* Not a string... */ + if ((b = erts_convert_filename_to_native(str,ERTS_ALC_T_TMP,1)) == NULL) { int j; for (j = 1; j < i; ++j) erts_free(ERTS_ALC_T_TMP, pp[j]); erts_free(ERTS_ALC_T_TMP, pp); return NULL; - } - b = (char *) erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(byte)); - pp[i++] = (char *) b; - if (intlist_to_buf(str, b, n) != n) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - b[n] = '\0'; + } + pp[i++] = b; l = CDR(list_val(l)); } pp[i] = NULL; @@ -1011,6 +996,7 @@ static byte* convert_environment(Process* p, Eterm env) Eterm* hp; Uint heap_size; int n; + Uint size; byte* bytes; if ((n = list_length(env)) < 0) { @@ -1054,15 +1040,15 @@ static byte* convert_environment(Process* p, Eterm env) if (is_not_nil(env)) { goto done; } - if ((n = io_list_len(all)) < 0) { + if (erts_iolist_size(all, &size)) { goto done; } /* * Put the result in a binary (no risk for a memory leak that way). */ - (void) erts_new_heap_binary(p, NULL, n, &bytes); - io_list_to_buf(all, (char*)bytes, n); + (void) erts_new_heap_binary(p, NULL, size, &bytes); + io_list_to_buf(all, (char*)bytes, size); done: erts_free(ERTS_ALC_T_TMP, temp_heap); @@ -1077,27 +1063,33 @@ struct packet_callback_args Eterm res; /* Out */ int string_as_bin; /* return strings as binaries (http_bin): */ byte* aligned_ptr; + Uint bin_sz; Eterm orig; Uint bin_offs; byte bin_bitoffs; }; +#define in_area(ptr,start,nbytes) \ + ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes)) + static Eterm http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp, const char *str, Sint len) { Eterm res = THE_NON_VALUE; Uint size; + int make_subbin; if (pca->string_as_bin) { size = heap_bin_size(len); - + make_subbin = (size > ERL_SUB_BIN_SIZE + && in_area(str, pca->aligned_ptr, pca->bin_sz)); if (szp) { - *szp += (size > ERL_SUB_BIN_SIZE) ? ERL_SUB_BIN_SIZE : size; + *szp += make_subbin ? ERL_SUB_BIN_SIZE : size; } if (hpp) { res = make_binary(*hpp); - if (size > ERL_SUB_BIN_SIZE) { + if (make_subbin) { ErlSubBin* bin = (ErlSubBin*) *hpp; bin->thing_word = HEADER_SUB_BIN; bin->size = len; @@ -1328,7 +1320,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) int packet_sz; /*-------Binaries involved: ------------------*/ byte* bin_ptr; /*| orig: original binary */ byte bin_bitsz; /*| bin: BIF_ARG_2, may be sub-binary of orig */ - Uint bin_sz; /*| packet: prefix of bin */ + /*| packet: prefix of bin */ char* body_ptr; /*| body: part of packet to return */ int body_sz; /*| rest: bin without packet */ struct packet_callback_args pca; @@ -1389,18 +1381,18 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) } - bin_sz = binary_size(BIF_ARG_2); + pca.bin_sz = binary_size(BIF_ARG_2); ERTS_GET_BINARY_BYTES(BIF_ARG_2, bin_ptr, pca.bin_bitoffs, bin_bitsz); if (pca.bin_bitoffs != 0) { - pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, bin_sz); - erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, bin_sz*8); + pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, pca.bin_sz); + erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, pca.bin_sz*8); } else { pca.aligned_ptr = bin_ptr; } - packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, bin_sz, + packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz, max_plen, trunc_len, &http_state); - if (!(packet_sz > 0 && packet_sz <= bin_sz)) { + if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) { if (packet_sz < 0) { goto error; } @@ -1456,7 +1448,7 @@ error: rest = (ErlSubBin *) hp; rest->thing_word = HEADER_SUB_BIN; - rest->size = bin_sz - packet_sz; + rest->size = pca.bin_sz - packet_sz; rest->offs = pca.bin_offs + packet_sz; rest->orig = pca.orig; rest->bitoffs = pca.bin_bitoffs; diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index c027cd5984..26891c4348 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -76,8 +76,8 @@ void erts_init_bif_re(void) re_exec_trap_export.code[0] = am_erlang; re_exec_trap_export.code[1] = am_re_run_trap; re_exec_trap_export.code[2] = 3; - re_exec_trap_export.code[3] = (Eterm) em_apply_bif; - re_exec_trap_export.code[4] = (Eterm) &re_exec_trap; + re_exec_trap_export.code[3] = (BeamInstr) em_apply_bif; + re_exec_trap_export.code[4] = (BeamInstr) &re_exec_trap; grun_trap_exportp = erts_export_put(am_re,am_grun,3); urun_trap_exportp = erts_export_put(am_re,am_urun,3); @@ -103,7 +103,7 @@ Sint erts_re_set_loop_limit(Sint limit) static int term_to_int(Eterm term, int *sp) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (is_small(term)) { Uint x = signed_val(term); @@ -154,7 +154,7 @@ static int term_to_int(Eterm term, int *sp) static Eterm make_signed_integer(int x, Process *p) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP return make_small(x); #else Eterm* hp; @@ -417,7 +417,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con BIF_RETTYPE re_compile_2(BIF_ALIST_2) { - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -444,7 +444,7 @@ re_compile_2(BIF_ALIST_2) BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2); } - if ((slen = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slen)) { BIF_ERROR(BIF_P,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); @@ -795,8 +795,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) memcpy(tmpb,ap->name,ap->len); tmpb[ap->len] = '\0'; } else { - int slen = io_list_len(val); - if (slen < 0) { + Uint slen; + if (erts_iolist_size(val, &slen)) { goto error; } if ((slen + 1) > tmpbsiz) { @@ -851,7 +851,7 @@ re_run_3(BIF_ALIST_3) const pcre *code_tmp; RestartContext restart; byte *temp_alloc = NULL; - int slength; + Uint slength; int startoffset = 0; int options = 0, comp_options = 0; int ovsize; @@ -875,7 +875,7 @@ re_run_3(BIF_ALIST_3) if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) { if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { /* Compile from textual RE */ - int slen; + Uint slen; char *expr; pcre *result; int errcode = 0; @@ -889,7 +889,7 @@ re_run_3(BIF_ALIST_3) BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } - if ((slen = io_list_len(BIF_ARG_2)) < 0) { + if (erts_iolist_size(BIF_ARG_2, &slen)) { BIF_ERROR(BIF_P,BADARG); } @@ -1027,7 +1027,7 @@ re_run_3(BIF_ALIST_3) restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { handle_iolist: - if ((slength = io_list_len(BIF_ARG_1)) < 0) { + if (erts_iolist_size(BIF_ARG_1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); if (restart.ret_info != NULL) { diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index 172bb37952..db771bd216 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -155,7 +155,7 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len) erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP hp[0] = make_ref_thing_header(len/2 + 1); datap = (Uint32 *) &hp[1]; *(datap++) = len; @@ -173,13 +173,13 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len) static int eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define MAX_REF_HEAP_SZ (1+(ERTS_MAX_REF_NUMBERS/2+1)) #else #define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS) #endif - Uint r1_hp[MAX_REF_HEAP_SZ]; - Uint r2_hp[MAX_REF_HEAP_SZ]; + DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2)); + Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ; return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2)); #undef MAX_REF_HEAP_SZ @@ -357,7 +357,7 @@ bif_timer_timeout(ErtsBifTimer* btm) rp, &rp_locks); } else { - Eterm old_size = bp->size; + Eterm old_size = bp->used_size; bp = erts_resize_message_buffer(bp, old_size + wrap_size, &message, 1); hp = &bp->mem[0] + old_size; @@ -398,7 +398,7 @@ setup_bif_timer(Uint32 xflags, if (!term_to_Uint(time, &timeout)) return THE_NON_VALUE; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if ((timeout >> 32) != 0) return THE_NON_VALUE; #endif @@ -478,7 +478,7 @@ setup_bif_timer(Uint32 xflags, tab_insert(btm); ASSERT(btm == tab_find(ref)); btm->tm.active = 0; /* MUST be initalized */ - erl_set_timer(&btm->tm, + erts_set_timer(&btm->tm, (ErlTimeoutProc) bif_timer_timeout, (ErlCancelProc) bif_timer_cleanup, (void *) btm, @@ -550,7 +550,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) res = am_false; } else { - Uint left = time_left(&btm->tm); + Uint left = erts_time_left(&btm->tm); if (!(btm->flags & BTM_FLG_BYNAME)) { erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ); unlink_proc(btm); @@ -558,7 +558,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) } tab_remove(btm); ASSERT(!tab_find(BIF_ARG_1)); - erl_cancel_timer(&btm->tm); + erts_cancel_timer(&btm->tm); erts_smp_btm_rwunlock(); res = erts_make_integer(left, BIF_P); } @@ -587,7 +587,7 @@ BIF_RETTYPE read_timer_1(BIF_ALIST_1) res = am_false; } else { - Uint left = time_left(&btm->tm); + Uint left = erts_time_left(&btm->tm); res = erts_make_integer(left, BIF_P); } @@ -613,7 +613,8 @@ erts_print_bif_timer_info(int to, void *to_arg) : btm->receiver.proc.ess->id); erts_print(to, to_arg, "=timer:%T\n", receiver); erts_print(to, to_arg, "Message: %T\n", btm->message); - erts_print(to, to_arg, "Time left: %d ms\n", time_left(&btm->tm)); + erts_print(to, to_arg, "Time left: %u ms\n", + erts_time_left(&btm->tm)); } } @@ -640,7 +641,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) tab_remove(btm); tmp_btm = btm; btm = btm->receiver.proc.next; - erl_cancel_timer(&tmp_btm->tm); + erts_cancel_timer(&tmp_btm->tm); } p->bif_timers = NULL; diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 7dff5e0eeb..0509e51a6f 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -40,8 +40,7 @@ #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) -static erts_smp_mtx_t trace_pattern_mutex; -const struct trace_pattern_flags erts_trace_pattern_flags_off = {0, 0, 0, 0}; +const struct trace_pattern_flags erts_trace_pattern_flags_off = {0, 0, 0, 0, 0}; static int erts_default_trace_pattern_is_on; static Binary *erts_default_match_spec; static Binary *erts_default_meta_match_spec; @@ -65,7 +64,6 @@ static void clear_trace_bif(int bif_index); void erts_bif_trace_init(void) { - erts_smp_mtx_init(&trace_pattern_mutex, "trace_pattern"); erts_default_trace_pattern_is_on = 0; erts_default_match_spec = NULL; erts_default_meta_match_spec = NULL; @@ -86,7 +84,7 @@ trace_pattern_2(Process* p, Eterm MFA, Eterm Pattern) Eterm trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { - Eterm mfa[3]; + DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; int matches = 0; int specified = 0; @@ -101,6 +99,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_block_system(0); + UseTmpHeap(3,p); /* * Check and compile the match specification. */ @@ -185,6 +184,14 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) flags.breakpoint = 1; flags.call_count = 1; break; + case am_call_time: + if (is_global) { + goto error; + } + flags.breakpoint = 1; + flags.call_time = 1; + break; + default: goto error; } @@ -194,8 +201,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) goto error; } - if (match_prog_set && !flags.local && !flags.meta && flags.call_count) { - /* A match prog is not allowed with just call_count */ + if (match_prog_set && !flags.local && !flags.meta && (flags.call_count || flags.call_time)) { + /* A match prog is not allowed with just call_count or call_time*/ goto error; } @@ -234,6 +241,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) |= flags.meta; erts_default_trace_pattern_flags.call_count |= (on == 1) ? flags.call_count : 0; + erts_default_trace_pattern_flags.call_time + |= (on == 1) ? flags.call_time : 0; } else { erts_default_trace_pattern_flags.local &= ~flags.local; @@ -241,10 +250,13 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) &= ~flags.meta; erts_default_trace_pattern_flags.call_count &= ~flags.call_count; + erts_default_trace_pattern_flags.call_time + &= ~flags.call_time; if (! (erts_default_trace_pattern_flags.breakpoint = erts_default_trace_pattern_flags.local | erts_default_trace_pattern_flags.meta | - erts_default_trace_pattern_flags.call_count)) { + erts_default_trace_pattern_flags.call_count | + erts_default_trace_pattern_flags.call_time)) { erts_default_trace_pattern_is_on = !!on; /* i.e off */ } } @@ -266,8 +278,9 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) if (on) { if (on != 1) { flags.call_count = 0; + flags.call_time = 0; } - flags.breakpoint = flags.local | flags.meta | flags.call_count; + flags.breakpoint = flags.local | flags.meta | flags.call_count | flags.call_time; erts_default_trace_pattern_flags = flags; /* Struct copy */ erts_default_trace_pattern_is_on = !!flags.breakpoint; } @@ -312,7 +325,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetUnref(match_prog_set); done: - + UnUseTmpHeap(3,p); erts_smp_release_system(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); @@ -322,6 +335,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetUnref(match_prog_set); + UnUseTmpHeap(3,p); erts_smp_release_system(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_ERROR(p, BADARG); @@ -334,7 +348,6 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, struct trace_pattern_flags *trace_pattern_flags, Eterm *meta_tracer_pid) { - erts_smp_mtx_lock(&trace_pattern_mutex); if (trace_pattern_is_on) *trace_pattern_is_on = erts_default_trace_pattern_is_on; if (match_spec) @@ -345,12 +358,10 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, *trace_pattern_flags = erts_default_trace_pattern_flags; if (meta_tracer_pid) *meta_tracer_pid = erts_default_meta_tracer_pid; - erts_smp_mtx_unlock(&trace_pattern_mutex); } - Uint erts_trace_flag2bit(Eterm flag) { @@ -378,7 +389,7 @@ erts_trace_flag2bit(Eterm flag) default: return 0; } } - + /* Scan the argument list and sort out the trace flags. ** ** Returns !0 on success, 0 on failure. @@ -929,6 +940,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) #define FUNC_TRACE_LOCAL_TRACE (1<<2) #define FUNC_TRACE_META_TRACE (1<<3) #define FUNC_TRACE_COUNT_TRACE (1<<4) +#define FUNC_TRACE_TIME_TRACE (1<<5) /* * Returns either FUNC_TRACE_NOEXIST, FUNC_TRACE_UNTRACED, * FUNC_TRACE_GLOBAL_TRACE, or, @@ -943,16 +955,18 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) * * If the return value contains FUNC_TRACE_COUNT_TRACE, *count is set. */ -static int function_is_traced(Eterm mfa[3], - Binary **ms, /* out */ - Binary **ms_meta, /* out */ +static int function_is_traced(Process *p, + Eterm mfa[3], + Binary **ms, /* out */ + Binary **ms_meta, /* out */ Eterm *tracer_pid_meta, /* out */ - Sint *count) /* out */ + Sint *count, /* out */ + Eterm *call_time) /* out */ { Export e; Export* ep; int i; - Uint *code; + BeamInstr *code; /* First look for an export entry */ e.code[0] = mfa[0]; @@ -960,12 +974,12 @@ static int function_is_traced(Eterm mfa[3], e.code[2] = mfa[2]; if ((ep = export_get(&e)) != NULL) { if (ep->address == ep->code+3 && - ep->code[3] != (Uint) em_call_error_handler) { - if (ep->code[3] == (Uint) em_call_traced_function) { + ep->code[3] != (BeamInstr) em_call_error_handler) { + if (ep->code[3] == (BeamInstr) em_call_traced_function) { *ms = ep->match_prog_set; return FUNC_TRACE_GLOBAL_TRACE; } - if (ep->code[3] == (Uint) em_apply_bif) { + if (ep->code[3] == (BeamInstr) em_apply_bif) { for (i = 0; i < BIF_SIZE; ++i) { if (bif_export[i] == ep) { int r = 0; @@ -978,10 +992,13 @@ static int function_is_traced(Eterm mfa[3], r |= FUNC_TRACE_LOCAL_TRACE; *ms = ep->match_prog_set; } - if (erts_is_mtrace_bif(ep->code+3, ms_meta, + if (erts_is_mtrace_break(ep->code+3, ms_meta, tracer_pid_meta)) { r |= FUNC_TRACE_META_TRACE; } + if (erts_is_time_break(p, ep->code+3, call_time)) { + r |= FUNC_TRACE_TIME_TRACE; + } } return r ? r : FUNC_TRACE_UNTRACED; } @@ -999,7 +1016,9 @@ static int function_is_traced(Eterm mfa[3], | (erts_is_mtrace_break(code, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) | (erts_is_count_break(code, count) - ? FUNC_TRACE_COUNT_TRACE : 0); + ? FUNC_TRACE_COUNT_TRACE : 0) + | (erts_is_time_break(p, code, call_time) + ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; } @@ -1011,15 +1030,19 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) { Eterm* tp; Eterm* hp; - Eterm mfa[3]; + DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ Binary *ms = NULL, *ms_meta = NULL; Sint count = 0; Eterm traced = am_false; Eterm match_spec = am_false; Eterm retval = am_false; Eterm meta = am_false; + Eterm call_time = NIL; int r; + + UseTmpHeap(3,p); + if (!is_tuple(func_spec)) { goto error; } @@ -1034,12 +1057,29 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) mfa[1] = tp[2]; mfa[2] = signed_val(tp[3]); - r = function_is_traced(mfa, &ms, &ms_meta, &meta, &count); +#ifdef ERTS_SMP + if ( (key == am_call_time) || (key == am_all)) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_block_system(0); + } +#endif + + r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); + +#ifdef ERTS_SMP + if ( (key == am_call_time) || (key == am_all)) { + erts_smp_release_system(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + } +#endif + switch (r) { case FUNC_TRACE_NOEXIST: + UnUseTmpHeap(3,p); hp = HAlloc(p, 3); return TUPLE2(hp, key, am_undefined); case FUNC_TRACE_UNTRACED: + UnUseTmpHeap(3,p); hp = HAlloc(p, 3); return TUPLE2(hp, key, am_false); case FUNC_TRACE_GLOBAL_TRACE: @@ -1085,8 +1125,13 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_make_integer(count, p); } break; + case am_call_time: + if (r & FUNC_TRACE_TIME_TRACE) { + retval = call_time; + } + break; case am_all: { - Eterm match_spec_meta = am_false, c = am_false, t; + Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false; if (ms) { match_spec = MatchSetGetSource(ms); @@ -1104,10 +1149,15 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_make_integer(-count-1, p) : erts_make_integer(count, p); } - hp = HAlloc(p, (3+2)*5); + if (r & FUNC_TRACE_TIME_TRACE) { + ct = call_time; + } + hp = HAlloc(p, (3+2)*6); retval = NIL; t = TUPLE2(hp, am_call_count, c); hp += 3; retval = CONS(hp, t, retval); hp += 2; + t = TUPLE2(hp, am_call_time, ct); hp += 3; + retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3; retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_meta, meta); hp += 3; @@ -1120,10 +1170,12 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) default: goto error; } + UnUseTmpHeap(3,p); hp = HAlloc(p, 3); return TUPLE2(hp, key, retval); error: + UnUseTmpHeap(3,p); BIF_ERROR(p, BADARG); } @@ -1201,6 +1253,13 @@ trace_info_on_load(Process* p, Eterm key) } else { return TUPLE2(hp, key, am_false); } + case am_call_time: + hp = HAlloc(p, 3); + if (erts_default_trace_pattern_flags.call_time) { + return TUPLE2(hp, key, am_true); + } else { + return TUPLE2(hp, key, am_false); + } case am_all: { Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t; @@ -1275,6 +1334,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { /* Empty loop body */ } + if (j == specified) { if (on) { if (! flags.breakpoint) @@ -1312,7 +1372,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) { ASSERT(ExportIsBuiltIn(bif_export[i])); erts_clear_mtrace_bif - ((Uint *)bif_export[i]->code + 3); + ((BeamInstr *)bif_export[i]->code + 3); erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META; } set_trace_bif(i, match_prog_set); @@ -1341,12 +1401,18 @@ erts_set_trace_pattern(Eterm* mfa, int specified, } if (flags.meta) { erts_set_mtrace_bif - ((Uint *)bif_export[i]->code + 3, + ((BeamInstr *)bif_export[i]->code + 3, meta_match_prog_set, meta_tracer_pid); erts_bif_trace_flags[i] |= BIF_TRACE_AS_META; erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL; m = 1; } + if (flags.call_time) { + erts_set_time_trace_bif(bif_export[i]->code + 3, on); + /* I don't want to remove any other tracers */ + erts_bif_trace_flags[i] |= BIF_TRACE_AS_CALL_TIME; + m = 1; + } if (erts_bif_trace_flags[i]) { setup_bif_trace(i); } @@ -1361,11 +1427,16 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.meta) { if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) { erts_clear_mtrace_bif - ((Uint *)bif_export[i]->code + 3); + ((BeamInstr *)bif_export[i]->code + 3); erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META; } m = 1; } + if (flags.call_time) { + erts_clear_time_trace_bif(bif_export[i]->code + 3); + erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_CALL_TIME; + m = 1; + } if (! erts_bif_trace_flags[i]) { reset_bif_trace(i); } @@ -1383,6 +1454,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, erts_clear_trace_break(mfa, specified); erts_clear_mtrace_break(mfa, specified); erts_clear_count_break(mfa, specified); + erts_clear_time_break(mfa, specified); } else { int m = 0; if (flags.local) { @@ -1396,6 +1468,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.call_count) { m = erts_set_count_break(mfa, specified, on); } + if (flags.call_time) { + m = erts_set_time_break(mfa, specified, on); + } /* All assignments to 'm' above should give the same value, * so just use the last */ matches += m; @@ -1411,6 +1486,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.call_count) { m = erts_clear_count_break(mfa, specified); } + if (flags.call_time) { + m = erts_clear_time_break(mfa, specified); + } /* All assignments to 'm' above should give the same value, * so just use the last */ matches += m; @@ -1430,9 +1508,9 @@ static int setup_func_trace(Export* ep, void* match_prog) { if (ep->address == ep->code+3) { - if (ep->code[3] == (Uint) em_call_error_handler) { + if (ep->code[3] == (BeamInstr) em_call_error_handler) { return 0; - } else if (ep->code[3] == (Uint) em_call_traced_function) { + } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { MatchSetUnref(ep->match_prog_set); ep->match_prog_set = match_prog; MatchSetRef(ep->match_prog_set); @@ -1452,8 +1530,8 @@ setup_func_trace(Export* ep, void* match_prog) return 0; } - ep->code[3] = (Uint) em_call_traced_function; - ep->code[4] = (Uint) ep->address; + ep->code[3] = (BeamInstr) em_call_traced_function; + ep->code[4] = (BeamInstr) ep->address; ep->address = ep->code+3; ep->match_prog_set = match_prog; MatchSetRef(ep->match_prog_set); @@ -1465,7 +1543,7 @@ static void setup_bif_trace(int bif_index) { ASSERT(ExportIsBuiltIn(ep)); ASSERT(ep->code[4]); - ep->code[4] = (Uint) bif_table[bif_index].traced; + ep->code[4] = (BeamInstr) bif_table[bif_index].traced; } static void set_trace_bif(int bif_index, void* match_prog) { @@ -1492,9 +1570,9 @@ static int reset_func_trace(Export* ep) { if (ep->address == ep->code+3) { - if (ep->code[3] == (Uint) em_call_error_handler) { + if (ep->code[3] == (BeamInstr) em_call_error_handler) { return 0; - } else if (ep->code[3] == (Uint) em_call_traced_function) { + } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { ep->address = (Uint *) ep->code[4]; MatchSetUnref(ep->match_prog_set); ep->match_prog_set = NULL; @@ -1527,8 +1605,8 @@ static void reset_bif_trace(int bif_index) { ASSERT(ExportIsBuiltIn(ep)); ASSERT(ep->code[4]); ASSERT(! ep->match_prog_set); - ASSERT(! erts_is_mtrace_bif((Uint *)ep->code+3, NULL, NULL)); - ep->code[4] = (Uint) bif_table[bif_index].f; + ASSERT(! erts_is_mtrace_break((BeamInstr *)ep->code+3, NULL, NULL)); + ep->code[4] = (BeamInstr) bif_table[bif_index].f; } static void clear_trace_bif(int bif_index) { @@ -2083,7 +2161,7 @@ trace_delivered_1(BIF_ALIST_1) #ifdef ERTS_SMP bp = new_message_buffer(REF_THING_SIZE + 4); hp = &bp->mem[0]; - msg_ref = STORE_NC(&hp, &bp->off_heap.externals, ref); + msg_ref = STORE_NC(&hp, &bp->off_heap, ref); #else hp = HAlloc(BIF_P, 4); msg_ref = ref; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 1f948a9684..506c4813fa 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,7 @@ #define __ERL_BINARY_H #include "erl_threads.h" +#include "bif.h" /* * Maximum number of bytes to place in a heap binary. @@ -70,6 +71,7 @@ typedef struct erl_heap_bin { */ #define binary_size(Bin) (binary_val(Bin)[1]) +#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1]) #define binary_bitsize(Bin) \ ((*binary_val(Bin) == HEADER_SUB_BIN) ? \ @@ -92,9 +94,12 @@ typedef struct erl_heap_bin { * Bitsize: output variable (Uint) */ -#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ +#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ + ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL) + +#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr) \ do { \ - Eterm* _real_bin = binary_val(Bin); \ + Eterm* _real_bin = binary_val_rel(Bin,BasePtr); \ Uint _offs = 0; \ Bitoffs = Bitsize = 0; \ if (*_real_bin == HEADER_SUB_BIN) { \ @@ -102,7 +107,7 @@ do { \ _offs = _sb->offs; \ Bitoffs = _sb->bitoffs; \ Bitsize = _sb->bitsize; \ - _real_bin = binary_val(_sb->orig); \ + _real_bin = binary_val_rel(_sb->orig,BasePtr); \ } \ if (*_real_bin == HEADER_PROC_BIN) { \ Bytep = ((ProcBin *) _real_bin)->bytes + _offs; \ @@ -124,9 +129,12 @@ do { \ * BitSize: Extra bit size (Uint) */ -#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \ +#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \ + ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL) + +#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \ do { \ - ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin); \ + ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr); \ if (_sb->thing_word == HEADER_SUB_BIN) { \ RealBin = _sb->orig; \ ByteOffset = _sb->offs; \ @@ -150,7 +158,18 @@ do { \ void erts_init_binary(void); -byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra); +byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra); +/* Used by unicode module */ +Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs); + +/* + * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1 + */ + +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg); +BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple); +BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); + #if defined(__i386__) || !defined(__GNUC__) /* @@ -164,10 +183,11 @@ byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra); #endif #define ERTS_CHK_BIN_ALIGNMENT(B) \ - do { ASSERT(!(B) || (((Uint) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((Uint) 0)) } while(0) + do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)) } while(0) ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr); ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf); +ERTS_GLB_INLINE void erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t); ERTS_GLB_INLINE Binary *erts_bin_drv_alloc_fnf(Uint size); ERTS_GLB_INLINE Binary *erts_bin_drv_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); @@ -184,17 +204,23 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr) { - return erts_get_aligned_binary_bytes_extra(bin, base_ptr, 0); + return erts_get_aligned_binary_bytes_extra(bin, base_ptr, ERTS_ALC_T_TMP, 0); } ERTS_GLB_INLINE void -erts_free_aligned_binary_bytes(byte* buf) +erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t allocator) { if (buf) { - erts_free(ERTS_ALC_T_TMP, (void *) buf); + erts_free(allocator, (void *) buf); } } +ERTS_GLB_INLINE void +erts_free_aligned_binary_bytes(byte* buf) +{ + erts_free_aligned_binary_bytes_extra(buf,ERTS_ALC_T_TMP); +} + /* Explicit extra bytes allocated to counter buggy drivers. ** These extra bytes where earlier (< R13B04) added by an alignment-bug ** in this code. Do we dare remove this in some major release (R14?) maybe? diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index e4f5d50ddf..e56084b9cb 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -177,7 +177,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff byte* LSB; byte* MSB; Uint* hp; - Uint* hp_end; Uint words_needed; Uint actual; Uint v32; @@ -255,7 +254,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff * Simply shift whole bytes into the result. */ switch (BYTE_OFFSET(n)) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP case 7: w = (w << 8) | *bp++; case 6: w = (w << 8) | *bp++; case 5: w = (w << 8) | *bp++; @@ -360,7 +359,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff case 3: v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); goto big_small; -#if !defined(ARCH_64) +#if !defined(ARCH_64) || HALFWORD_HEAP case 4: v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24)); if (!IS_USMALL(sgn, v32)) { @@ -405,7 +404,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff default: words_needed = 1+WSIZE(bytes); hp = HeapOnlyAlloc(p, words_needed); - hp_end = hp + words_needed; res = bytes_to_big(LSB, bytes, sgn, hp); if (is_small(res)) { p->htop = hp; @@ -425,7 +423,6 @@ Eterm erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb) { ErlSubBin* sb; - size_t num_bytes; /* Number of bytes in binary. */ if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; @@ -435,7 +432,6 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe * From now on, we can't fail. */ - num_bytes = NBYTES(num_bits); sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE); sb->thing_word = HEADER_SUB_BIN; @@ -555,10 +551,11 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags) { unsigned long offs; - ASSERT(size != 0); offs = BIT_OFFSET(size); if (is_small(val)) { Sint v = signed_val(val); + + ASSERT(size != 0); /* Tested by caller */ if (flags & BSF_LITTLE) { /* Little endian */ sz--; COPY_VAL(buf,1,v,sz); @@ -578,6 +575,9 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags) ErtsDigit* dp = big_v(val); int n = MIN(sz,ds); + if (size == 0) { + return 0; + } if (flags & BSF_LITTLE) { sz -= n; /* pad with this amount */ if (sign) { @@ -729,15 +729,13 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag Uint b; byte *iptr; - if (num_bits == 0) { - return 1; - } - bit_offset = BIT_OFFSET(bin_offset); if (is_small(arg)) { Uint rbits = 8 - bit_offset; - if (bit_offset + num_bits <= 8) { + if (num_bits == 0) { + return 1; + } else if (bit_offset + num_bits <= 8) { /* * All bits are in the same byte. */ @@ -1335,12 +1333,12 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = used_size_in_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; - MSO(c_p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); /* * Now allocate the sub binary and set its size to include the @@ -1506,12 +1504,12 @@ erts_bs_init_writable(Process* p, Eterm sz) hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); /* * Now allocate the sub binary. @@ -1555,7 +1553,6 @@ Uint32 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) { Uint bytes; - Uint bits; Uint offs; byte bigbuf[4]; byte* LSB; @@ -1565,7 +1562,6 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) ASSERT(mb->size - mb->offset >= 32); bytes = 4; - bits = 8; offs = 0; LSB = bigbuf; diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index e3f8e0b679..0f67733fa4 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -63,7 +63,7 @@ typedef struct erl_bin_match_struct{ #define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1) #define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms)) -#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*)(_Ms - TAG_PRIMARY_BOXED))->mb) +#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb) #if defined(ERTS_SMP) diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c new file mode 100644 index 0000000000..bcf8bcf270 --- /dev/null +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -0,0 +1,2361 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: CPU topology and related functionality + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <ctype.h> + +#include "global.h" +#include "error.h" +#include "bif.h" +#include "erl_cpu_topology.h" + +#define ERTS_MAX_READER_GROUPS 8 + +/* + * Cpu topology hierarchy. + */ +#define ERTS_TOPOLOGY_NODE 0 +#define ERTS_TOPOLOGY_PROCESSOR 1 +#define ERTS_TOPOLOGY_PROCESSOR_NODE 2 +#define ERTS_TOPOLOGY_CORE 3 +#define ERTS_TOPOLOGY_THREAD 4 +#define ERTS_TOPOLOGY_LOGICAL 5 + +#define ERTS_TOPOLOGY_MAX_DEPTH 6 + +typedef struct { + int bind_id; + int bound_id; +} ErtsCpuBindData; + +static erts_cpu_info_t *cpuinfo; + +static int max_main_threads; +static int reader_groups; + +static ErtsCpuBindData *scheduler2cpu_map; +static erts_smp_rwmtx_t cpuinfo_rwmtx; + +typedef enum { + ERTS_CPU_BIND_UNDEFINED, + ERTS_CPU_BIND_SPREAD, + ERTS_CPU_BIND_PROCESSOR_SPREAD, + ERTS_CPU_BIND_THREAD_SPREAD, + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD, + ERTS_CPU_BIND_NO_SPREAD, + ERTS_CPU_BIND_NONE +} ErtsCpuBindOrder; + +#define ERTS_CPU_BIND_DEFAULT_BIND \ + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + +static int no_cpu_groups_callbacks; +static ErtsCpuBindOrder cpu_bind_order; + +static erts_cpu_topology_t *user_cpudata; +static int user_cpudata_size; +static erts_cpu_topology_t *system_cpudata; +static int system_cpudata_size; + +typedef struct { + int level[ERTS_TOPOLOGY_MAX_DEPTH+1]; +} erts_avail_cput; + +typedef struct { + int id; + int sub_levels; + int cpu_groups; +} erts_cpu_groups_count_t; + +typedef struct { + int logical; + int cpu_group; +} erts_cpu_groups_map_array_t; + +typedef struct erts_cpu_groups_callback_list_t_ erts_cpu_groups_callback_list_t; +struct erts_cpu_groups_callback_list_t_ { + erts_cpu_groups_callback_list_t *next; + erts_cpu_groups_callback_t callback; + void *arg; +}; + +typedef struct erts_cpu_groups_map_t_ erts_cpu_groups_map_t; +struct erts_cpu_groups_map_t_ { + erts_cpu_groups_map_t *next; + int groups; + erts_cpu_groups_map_array_t *array; + int size; + int logical_processors; + erts_cpu_groups_callback_list_t *callback_list; +}; + +typedef struct { + erts_cpu_groups_callback_t callback; + int ix; + void *arg; +} erts_cpu_groups_callback_call_t; + +static erts_cpu_groups_map_t *cpu_groups_maps; + +static erts_cpu_groups_map_t *reader_groups_map; + +#define ERTS_TOPOLOGY_CG ERTS_TOPOLOGY_MAX_DEPTH + +#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) + +#ifdef ERTS_SMP +static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq); +static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); +#endif + +static void reader_groups_callback(int, ErtsSchedulerData *, int, void *); +static erts_cpu_groups_map_t *add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg); +static void update_cpu_groups_maps(void); +static void make_cpu_groups_map(erts_cpu_groups_map_t *map, int test); +static int cpu_groups_lookup(erts_cpu_groups_map_t *map, + ErtsSchedulerData *esdp); + +static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, + int *cpudata_size); +static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata); + +static int +int_cmp(const void *vx, const void *vy) +{ + return *((int *) vx) - *((int *) vy); +} + +static int +cpu_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + return 0; +} + +static int +cpu_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_no_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +static ERTS_INLINE void +make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node) +{ + int ix; + int node = -1; + int processor = -1; + int processor_node = -1; + int processor_node_node = -1; + int core = -1; + int thread = -1; + int old_node = -1; + int old_processor = -1; + int old_processor_node = -1; + int old_core = -1; + int old_thread = -1; + + for (ix = 0; ix < size; ix++) { + if (!no_node || cpudata[ix].node >= 0) { + if (old_node == cpudata[ix].node) + cpudata[ix].node = node; + else { + old_node = cpudata[ix].node; + old_processor = processor = -1; + if (!no_node) + old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + if (no_node || cpudata[ix].node >= 0) + cpudata[ix].node = ++node; + } + } + if (old_processor == cpudata[ix].processor) + cpudata[ix].processor = processor; + else { + old_processor = cpudata[ix].processor; + if (!no_node) + processor_node_node = old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + cpudata[ix].processor = ++processor; + } + if (no_node && cpudata[ix].processor_node < 0) + old_processor_node = -1; + else { + if (old_processor_node == cpudata[ix].processor_node) { + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = node; + else { + if (processor_node_node >= 0) + cpudata[ix].node = processor_node_node; + cpudata[ix].processor_node = processor_node; + } + } + else { + old_processor_node = cpudata[ix].processor_node; + old_core = core = -1; + old_thread = thread = -1; + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = ++node; + else { + cpudata[ix].node = processor_node_node = ++node; + cpudata[ix].processor_node = ++processor_node; + } + } + } + if (!no_node && cpudata[ix].processor_node < 0) + cpudata[ix].processor_node = 0; + if (old_core == cpudata[ix].core) + cpudata[ix].core = core; + else { + old_core = cpudata[ix].core; + old_thread = thread = -1; + cpudata[ix].core = ++core; + } + if (old_thread == cpudata[ix].thread) + cpudata[ix].thread = thread; + else + old_thread = cpudata[ix].thread = ++thread; + } +} + +static void +cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq) +{ + if (size > 1) { + int no_node = 0; + int (*cmp_func)(const void *, const void *); + switch (bind_order) { + case ERTS_CPU_BIND_SPREAD: + cmp_func = cpu_spread_order_cmp; + break; + case ERTS_CPU_BIND_PROCESSOR_SPREAD: + cmp_func = cpu_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_SPREAD: + cmp_func = cpu_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_thread_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_SPREAD: + cmp_func = cpu_no_spread_order_cmp; + break; + default: + cmp_func = NULL; + erl_exit(ERTS_ABORT_EXIT, + "Bad cpu bind type: %d\n", + (int) cpu_bind_order); + break; + } + + if (mk_seq) + make_cpudata_id_seq(cpudata, size, no_node); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func); + } +} + +static int +processor_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +#ifdef ERTS_SMP +void +erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp) +{ + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; + int cgcc_ix; + + /* Unbind from cpu */ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + if (scheduler2cpu_map[esdp->no].bound_id >= 0 + && erts_unbind_from_cpu(cpuinfo) == 0) { + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + } + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(1, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(0, 0); + +} + +void +erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue)); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); + + /* Make sure we check if we should bind to a cpu or not... */ + if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) + erts_smp_atomic32_set(&esdp->chk_cpu_bind, 1); + else + esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; +} + +#endif + +void +erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) +{ + int res, cpu_id, cgcc_ix; + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; +#ifdef ERTS_SMP + if (erts_common_run_queue) + erts_smp_atomic32_set(&esdp->chk_cpu_bind, 0); + else { + esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; + } +#endif + erts_smp_runq_unlock(esdp->run_queue); + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + cpu_id = scheduler2cpu_map[esdp->no].bind_id; + if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { + res = erts_bind_to_cpu(cpuinfo, cpu_id); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id; + else { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + if (scheduler2cpu_map[esdp->no].bound_id >= 0) + goto unbind; + } + } + else if (cpu_id < 0) { + unbind: + /* Get rid of old binding */ + res = erts_unbind_from_cpu(cpuinfo); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + else if (res != -ENOTSUP) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + } + } + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(0, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + erts_smp_runq_lock(esdp->run_queue); +} + +#ifdef ERTS_SMP +void +erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp) +{ + int cgcc_ix; + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; + + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(0, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); +} +#endif + +static void +write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) +{ + int s_ix = 1; + int cpu_ix; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) { + + cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1); + + for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++) + if (erts_is_cpu_available(cpuinfo, cpudata[cpu_ix].logical)) + scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical; + } + + if (s_ix <= erts_no_schedulers) + for (; s_ix <= erts_no_schedulers; s_ix++) + scheduler2cpu_map[s_ix].bind_id = -1; +} + +int +erts_init_scheduler_bind_type_string(char *how) +{ + if (sys_strcmp(how, "u") == 0) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) + return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; + else if (!system_cpudata && !user_cpudata) + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; + else if (sys_strcmp(how, "db") == 0) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (sys_strcmp(how, "s") == 0) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (sys_strcmp(how, "ps") == 0) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "ts") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (sys_strcmp(how, "tnnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnts") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (sys_strcmp(how, "ns") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; + return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; +} + +static Eterm +bound_schedulers_term(ErtsCpuBindOrder order) +{ + switch (order) { + case ERTS_CPU_BIND_SPREAD: { + ERTS_DECL_AM(spread); + return AM_spread; + } + case ERTS_CPU_BIND_PROCESSOR_SPREAD: { + ERTS_DECL_AM(processor_spread); + return AM_processor_spread; + } + case ERTS_CPU_BIND_THREAD_SPREAD: { + ERTS_DECL_AM(thread_spread); + return AM_thread_spread; + } + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(thread_no_node_processor_spread); + return AM_thread_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(no_node_processor_spread); + return AM_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: { + ERTS_DECL_AM(no_node_thread_spread); + return AM_no_node_thread_spread; + } + case ERTS_CPU_BIND_NO_SPREAD: { + ERTS_DECL_AM(no_spread); + return AM_no_spread; + } + case ERTS_CPU_BIND_NONE: { + ERTS_DECL_AM(unbound); + return AM_unbound; + } + default: + ASSERT(0); + return THE_NON_VALUE; + } +} + +Eterm +erts_bound_schedulers_term(Process *c_p) +{ + ErtsCpuBindOrder order; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + order = cpu_bind_order; + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return bound_schedulers_term(order); +} + +Eterm +erts_bind_schedulers(Process *c_p, Eterm how) +{ + int notify = 0; + Eterm res; + erts_cpu_topology_t *cpudata; + int cpudata_size; + ErtsCpuBindOrder old_cpu_bind_order; + + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + + if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) { + if (cpu_bind_order == ERTS_CPU_BIND_NONE + && ERTS_IS_ATOM_STR("unbound", how)) { + res = bound_schedulers_term(ERTS_CPU_BIND_NONE); + goto done; + } + ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP); + } + else { + + old_cpu_bind_order = cpu_bind_order; + + if (ERTS_IS_ATOM_STR("default_bind", how)) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + + if (!cpudata) { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + write_schedulers_bind_change(cpudata, cpudata_size); + notify = 1; + + destroy_tmp_cpu_topology_copy(cpudata); + + res = bound_schedulers_term(old_cpu_bind_order); + } + + done: + + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + if (notify) + erts_sched_notify_check_cpu_bind(); + + return res; +} + +int +erts_sched_bind_atthrcreate_prepare(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + return esdp != NULL && erts_is_scheduler_bound(esdp); +} + +int +erts_sched_bind_atthrcreate_child(int unbind) +{ + int res = 0; + if (unbind) { + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + res = erts_unbind_from_cpu(cpuinfo); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + } + return res; +} + +void +erts_sched_bind_atthrcreate_parent(int unbind) +{ + +} + +int +erts_sched_bind_atfork_prepare(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int unbind = esdp != NULL && erts_is_scheduler_bound(esdp); + if (unbind) + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + return unbind; +} + +int +erts_sched_bind_atfork_child(int unbind) +{ + if (unbind) { + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + return erts_unbind_from_cpu(cpuinfo); + } + return 0; +} + +char * +erts_sched_bind_atvfork_child(int unbind) +{ + if (unbind) { + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + return erts_get_unbind_from_cpu_str(cpuinfo); + } + return "false"; +} + +void +erts_sched_bind_atfork_parent(int unbind) +{ + if (unbind) + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); +} + +Eterm +erts_fake_scheduler_bindings(Process *p, Eterm how) +{ + ErtsCpuBindOrder fake_cpu_bind_order; + erts_cpu_topology_t *cpudata; + int cpudata_size; + Eterm res; + + if (ERTS_IS_ATOM_STR("default_bind", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + ERTS_BIF_PREP_ERROR(res, p, BADARG); + return res; + } + + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + + if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) + ERTS_BIF_PREP_RET(res, am_false); + else { + int i; + Eterm *hp; + + cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1); + +#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA + + erts_fprintf(stderr, "node: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].node); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "processor: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor); + erts_fprintf(stderr, "\n"); + if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) { + erts_fprintf(stderr, "processor_node:"); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor_node); + erts_fprintf(stderr, "\n"); + } + erts_fprintf(stderr, "core: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].core); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "thread: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].thread); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "logical: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].logical); + erts_fprintf(stderr, "\n"); +#endif + + hp = HAlloc(p, cpudata_size+1); + ERTS_BIF_PREP_RET(res, make_tuple(hp)); + *hp++ = make_arityval((Uint) cpudata_size); + for (i = 0; i < cpudata_size; i++) + *hp++ = make_small((Uint) cpudata[i].logical); + } + + destroy_tmp_cpu_topology_copy(cpudata); + + return res; +} + +Eterm +erts_get_schedulers_binds(Process *c_p) +{ + int ix; + ERTS_DECL_AM(unbound); + Eterm *hp = HAlloc(c_p, erts_no_schedulers+1); + Eterm res = make_tuple(hp); + + *(hp++) = make_arityval(erts_no_schedulers); + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + for (ix = 1; ix <= erts_no_schedulers; ix++) + *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 + ? make_small(scheduler2cpu_map[ix].bound_id) + : AM_unbound); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +/* + * CPU topology + */ + +typedef struct { + int *id; + int used; + int size; +} ErtsCpuTopIdSeq; + +typedef struct { + ErtsCpuTopIdSeq logical; + ErtsCpuTopIdSeq thread; + ErtsCpuTopIdSeq core; + ErtsCpuTopIdSeq processor_node; + ErtsCpuTopIdSeq processor; + ErtsCpuTopIdSeq node; +} ErtsCpuTopEntry; + +static void +init_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + int size = 10; + cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->logical.size = size; + cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->thread.size = size; + cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->core.size = size; + cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor_node.size = size; + cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor.size = size; + cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->node.size = size; +} + +static void +destroy_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id); +} + +static int +get_cput_value_or_range(int *v, int *vr, char **str) +{ + long l; + char *c = *str; + errno = 0; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + *v = (int) l; + if (*c == '-') { + c++; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + *vr = (int) l; + } + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str) +{ + int ix = 0; + int need_size = 0; + char *c = *str; + + while (1) { + int res; + int val; + int nids; + int val_range = -1; + res = get_cput_value_or_range(&val, &val_range, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + if (val_range < 0 || val_range == val) + nids = 1; + else { + if (val_range > val) + nids = val_range - val + 1; + else + nids = val - val_range + 1; + } + need_size += nids; + if (need_size > idseq->size) { + idseq->size = need_size + 10; + idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS, + idseq->id, + sizeof(int)*idseq->size); + } + if (nids == 1) + idseq->id[ix++] = val; + else if (val_range > val) { + for (; val <= val_range; val++) + idseq->id[ix++] = val; + } + else { + for (; val >= val_range; val--) + idseq->id[ix++] = val; + } + if (*c != ',') + break; + c++; + } + *str = c; + idseq->used = ix; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_entry(ErtsCpuTopEntry *cput, char **str) +{ + int h; + char *c = *str; + + cput->logical.used = 0; + cput->thread.id[0] = 0; + cput->thread.used = 1; + cput->core.id[0] = 0; + cput->core.used = 1; + cput->processor_node.id[0] = -1; + cput->processor_node.used = 1; + cput->processor.id[0] = 0; + cput->processor.used = 1; + cput->node.id[0] = -1; + cput->node.used = 1; + + h = ERTS_TOPOLOGY_MAX_DEPTH; + while (*c != ':' && *c != '\0') { + int res; + ErtsCpuTopIdSeq *idseqp; + switch (*c++) { + case 'L': + if (h <= ERTS_TOPOLOGY_LOGICAL) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->logical; + h = ERTS_TOPOLOGY_LOGICAL; + break; + case 't': + case 'T': + if (h <= ERTS_TOPOLOGY_THREAD) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->thread; + h = ERTS_TOPOLOGY_THREAD; + break; + case 'c': + case 'C': + if (h <= ERTS_TOPOLOGY_CORE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->core; + h = ERTS_TOPOLOGY_CORE; + break; + case 'p': + case 'P': + if (h <= ERTS_TOPOLOGY_PROCESSOR) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor; + h = ERTS_TOPOLOGY_PROCESSOR; + break; + case 'n': + case 'N': + if (h <= ERTS_TOPOLOGY_PROCESSOR) { + do_node: + if (h <= ERTS_TOPOLOGY_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->node; + h = ERTS_TOPOLOGY_NODE; + } + else { + int p_node = 0; + char *p_chk = c; + while (*p_chk != '\0' && *p_chk != ':') { + if (*p_chk == 'p' || *p_chk == 'P') { + p_node = 1; + break; + } + p_chk++; + } + if (!p_node) + goto do_node; + if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor_node; + h = ERTS_TOPOLOGY_PROCESSOR_NODE; + } + break; + default: + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE; + } + res = get_cput_id_seq(idseqp, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + } + + if (cput->logical.used < 1) + return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID; + + if (*c == ':') { + c++; + } + + if (cput->thread.used != 1 + && cput->thread.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->core.used != 1 + && cput->core.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor_node.used != 1 + && cput->processor_node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor.used != 1 + && cput->processor.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->node.used != 1 + && cput->node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +verify_topology(erts_cpu_topology_t *cpudata, int size) +{ + if (size > 0) { + int *logical; + int node, processor, no_nodes, i; + + /* Verify logical ids */ + logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); + + for (i = 0; i < size; i++) + logical[i] = cpudata[i].logical; + + qsort(logical, size, sizeof(int), int_cmp); + for (i = 0; i < size-1; i++) { + if (logical[i] == logical[i+1]) { + erts_free(ERTS_ALC_T_TMP, logical); + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; + } + } + + erts_free(ERTS_ALC_T_TMP, logical); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp); + + /* Verify unique entities */ + + for (i = 1; i < size; i++) { + if (cpudata[i-1].processor == cpudata[i].processor + && cpudata[i-1].node == cpudata[i].node + && (cpudata[i-1].processor_node + == cpudata[i].processor_node) + && cpudata[i-1].core == cpudata[i].core + && cpudata[i-1].thread == cpudata[i].thread) { + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; + } + } + + /* Verify numa nodes */ + node = cpudata[0].node; + processor = cpudata[0].processor; + no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0; + for (i = 1; i < size; i++) { + if (no_nodes) { + if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + else { + if (cpudata[i].processor == processor && cpudata[i].node != node) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + node = cpudata[i].node; + processor = cpudata[i].processor; + if (node >= 0 && cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + if (node < 0 && cpudata[i].processor_node < 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + } + } + + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +int +erts_init_cpu_topology_string(char *topology_str) +{ + ErtsCpuTopEntry cput; + int need_size; + char *c; + int ix; + int error = ERTS_INIT_CPU_TOPOLOGY_OK; + + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 10; + + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + + init_cpu_top_entry(&cput); + + ix = 0; + need_size = 0; + + c = topology_str; + if (*c == '\0') { + error = ERTS_INIT_CPU_TOPOLOGY_MISSING; + goto fail; + } + do { + int r; + error = get_cput_entry(&cput, &c); + if (error != ERTS_INIT_CPU_TOPOLOGY_OK) + goto fail; + need_size += cput.logical.used; + if (user_cpudata_size < need_size) { + user_cpudata_size = need_size + 10; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + ASSERT(cput.thread.used == 1 + || cput.thread.used == cput.logical.used); + ASSERT(cput.core.used == 1 + || cput.core.used == cput.logical.used); + ASSERT(cput.processor_node.used == 1 + || cput.processor_node.used == cput.logical.used); + ASSERT(cput.processor.used == 1 + || cput.processor.used == cput.logical.used); + ASSERT(cput.node.used == 1 + || cput.node.used == cput.logical.used); + + for (r = 0; r < cput.logical.used; r++) { + user_cpudata[ix].logical = cput.logical.id[r]; + user_cpudata[ix].thread = + cput.thread.id[cput.thread.used == 1 ? 0 : r]; + user_cpudata[ix].core = + cput.core.id[cput.core.used == 1 ? 0 : r]; + user_cpudata[ix].processor_node = + cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r]; + user_cpudata[ix].processor = + cput.processor.id[cput.processor.used == 1 ? 0 : r]; + user_cpudata[ix].node = + cput.node.id[cput.node.used == 1 ? 0 : r]; + ix++; + } + } while (*c != '\0'); + + if (user_cpudata_size != ix) { + user_cpudata_size = ix; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + error = verify_topology(user_cpudata, user_cpudata_size); + if (error == ERTS_INIT_CPU_TOPOLOGY_OK) { + destroy_cpu_top_entry(&cput); + return ERTS_INIT_CPU_TOPOLOGY_OK; + } + + fail: + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 0; + destroy_cpu_top_entry(&cput); + return error; +} + +#define ERTS_GET_CPU_TOPOLOGY_ERROR -1 +#define ERTS_GET_USED_CPU_TOPOLOGY 0 +#define ERTS_GET_DETECTED_CPU_TOPOLOGY 1 +#define ERTS_GET_DEFINED_CPU_TOPOLOGY 2 + +static Eterm get_cpu_topology_term(Process *c_p, int type); + +Eterm +erts_set_cpu_topology(Process *c_p, Eterm term) +{ + erts_cpu_topology_t *cpudata = NULL; + int cpudata_size = 0; + Eterm res; + + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); + if (term == am_undefined) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = NULL; + user_cpudata_size = 0; + + if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) { + cpudata_size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + } + else if (is_not_list(term)) { + error: + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + res = THE_NON_VALUE; + goto done; + } + else { + Eterm list = term; + int ix = 0; + + cpudata_size = 100; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + while (is_list(list)) { + Eterm *lp = list_val(list); + Eterm cpu = CAR(lp); + Eterm* tp; + Sint id; + + if (is_not_tuple(cpu)) + goto error; + + tp = tuple_val(cpu); + + if (arityval(tp[0]) != 7 || tp[1] != am_cpu) + goto error; + + if (ix >= cpudata_size) { + cpudata_size += 100; + cpudata = erts_realloc(ERTS_ALC_T_TMP, + cpudata, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + } + + id = signed_val(tp[2]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].node = (int) id; + + id = signed_val(tp[3]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor = (int) id; + + id = signed_val(tp[4]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor_node = (int) id; + + id = signed_val(tp[5]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].core = (int) id; + + id = signed_val(tp[6]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].thread = (int) id; + + id = signed_val(tp[7]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].logical = (int) id; + + list = CDR(lp); + ix++; + } + + if (is_not_nil(list)) + goto error; + + cpudata_size = ix; + + if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size)) + goto error; + + if (user_cpudata_size != cpudata_size) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + sizeof(erts_cpu_topology_t)*cpudata_size); + user_cpudata_size = cpudata_size; + } + + sys_memcpy((void *) user_cpudata, + (void *) cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + + update_cpu_groups_maps(); + + write_schedulers_bind_change(cpudata, cpudata_size); + + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_sched_notify_check_cpu_bind(); + + done: + + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +static void +create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size) +{ + if (user_cpudata) { + *cpudata_size = user_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) user_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else if (system_cpudata) { + *cpudata_size = system_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else { + *cpudata = NULL; + *cpudata_size = 0; + } +} + +static void +destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata) +{ + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); +} + + +static Eterm +bld_topology_term(Eterm **hpp, + Uint *hszp, + erts_cpu_topology_t *cpudata, + int size) +{ + Eterm res = NIL; + int i; + + if (size == 0) + return am_undefined; + + for (i = size-1; i >= 0; i--) { + res = erts_bld_cons(hpp, + hszp, + erts_bld_tuple(hpp, + hszp, + 7, + am_cpu, + make_small(cpudata[i].node), + make_small(cpudata[i].processor), + make_small(cpudata[i].processor_node), + make_small(cpudata[i].core), + make_small(cpudata[i].thread), + make_small(cpudata[i].logical)), + res); + } + return res; +} + +static Eterm +get_cpu_topology_term(Process *c_p, int type) +{ +#ifdef DEBUG + Eterm *hp_end; +#endif + Eterm *hp; + Uint hsz; + Eterm res = THE_NON_VALUE; + erts_cpu_topology_t *cpudata = NULL; + int size = 0; + + switch (type) { + case ERTS_GET_USED_CPU_TOPOLOGY: + if (user_cpudata) + goto defined; + else + goto detected; + case ERTS_GET_DETECTED_CPU_TOPOLOGY: + detected: + if (!system_cpudata) + res = am_undefined; + else { + size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * size)); + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*size); + } + break; + case ERTS_GET_DEFINED_CPU_TOPOLOGY: + defined: + if (!user_cpudata) + res = am_undefined; + else { + size = user_cpudata_size; + cpudata = user_cpudata; + } + break; + default: + erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + break; + } + + if (res == am_undefined) { + ASSERT(!cpudata); + return res; + } + + hsz = 0; + + bld_topology_term(NULL, &hsz, + cpudata, size); + + hp = HAlloc(c_p, hsz); + +#ifdef DEBUG + hp_end = hp + hsz; +#endif + + res = bld_topology_term(&hp, NULL, + cpudata, size); + + ASSERT(hp_end == hp); + + if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +Eterm +erts_get_cpu_topology_term(Process *c_p, Eterm which) +{ + Eterm res; + int type; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + if (ERTS_IS_ATOM_STR("used", which)) + type = ERTS_GET_USED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("detected", which)) + type = ERTS_GET_DETECTED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("defined", which)) + type = ERTS_GET_DEFINED_CPU_TOPOLOGY; + else + type = ERTS_GET_CPU_TOPOLOGY_ERROR; + if (type == ERTS_GET_CPU_TOPOLOGY_ERROR) + res = THE_NON_VALUE; + else + res = get_cpu_topology_term(c_p, type); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +static void +get_logical_processors(int *conf, int *onln, int *avail) +{ + if (conf) + *conf = erts_get_cpu_configured(cpuinfo); + if (onln) + *onln = erts_get_cpu_online(cpuinfo); + if (avail) + *avail = erts_get_cpu_available(cpuinfo); +} + +void +erts_get_logical_processors(int *conf, int *onln, int *avail) +{ + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + get_logical_processors(conf, onln, avail); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); +} + +void +erts_pre_early_init_cpu_topology(int *max_rg_p, + int *conf_p, + int *onln_p, + int *avail_p) +{ + cpu_groups_maps = NULL; + no_cpu_groups_callbacks = 0; + *max_rg_p = ERTS_MAX_READER_GROUPS; + cpuinfo = erts_cpu_info_create(); + get_logical_processors(conf_p, onln_p, avail_p); +} + +void +erts_early_init_cpu_topology(int no_schedulers, + int *max_main_threads_p, + int max_reader_groups, + int *reader_groups_p) +{ + user_cpudata = NULL; + user_cpudata_size = 0; + + system_cpudata_size = erts_get_cpu_topology_size(cpuinfo); + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + cpu_bind_order = ERTS_CPU_BIND_UNDEFINED; + + if (!erts_get_cpu_topology(cpuinfo, system_cpudata) + || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, + system_cpudata_size)) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } + + max_main_threads = erts_get_cpu_configured(cpuinfo); + if (max_main_threads > no_schedulers) + max_main_threads = no_schedulers; + *max_main_threads_p = max_main_threads; + + reader_groups = max_main_threads; + if (reader_groups <= 1 || max_reader_groups <= 1) + reader_groups = 0; + if (reader_groups > max_reader_groups) + reader_groups = max_reader_groups; + *reader_groups_p = reader_groups; +} + +void +erts_init_cpu_topology(void) +{ + int ix; + + erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info"); + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + + scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(ErtsCpuBindData) + * (erts_no_schedulers+1))); + for (ix = 1; ix <= erts_no_schedulers; ix++) { + scheduler2cpu_map[ix].bind_id = -1; + scheduler2cpu_map[ix].bound_id = -1; + } + + if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) { + int ncpus = erts_get_cpu_configured(cpuinfo); + if (ncpus < 1 || erts_no_schedulers < ncpus) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else + cpu_bind_order = ((system_cpudata || user_cpudata) + && (erts_bind_to_cpu(cpuinfo, -1) != -ENOTSUP) + ? ERTS_CPU_BIND_DEFAULT_BIND + : ERTS_CPU_BIND_NONE); + } + + reader_groups_map = add_cpu_groups(reader_groups, + reader_groups_callback, + NULL); + + if (cpu_bind_order == ERTS_CPU_BIND_NONE) + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + else { + erts_cpu_topology_t *cpudata; + int cpudata_size; + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + write_schedulers_bind_change(cpudata, cpudata_size); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_sched_notify_check_cpu_bind(); + destroy_tmp_cpu_topology_copy(cpudata); + } +} + +int +erts_update_cpu_info(void) +{ + int changed; + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + changed = erts_cpu_info_update(cpuinfo); + if (changed) { + erts_cpu_topology_t *cpudata; + int cpudata_size; + + if (system_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + + system_cpudata_size = erts_get_cpu_topology_size(cpuinfo); + if (!system_cpudata_size) + system_cpudata = NULL; + else { + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + if (!erts_get_cpu_topology(cpuinfo, system_cpudata) + || (ERTS_INIT_CPU_TOPOLOGY_OK + != verify_topology(system_cpudata, + system_cpudata_size))) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } + } + + update_cpu_groups_maps(); + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + write_schedulers_bind_change(cpudata, cpudata_size); + destroy_tmp_cpu_topology_copy(cpudata); + } + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + if (changed) + erts_sched_notify_check_cpu_bind(); + return changed; +} + +/* + * reader groups map + */ + +void +reader_groups_callback(int suspending, + ErtsSchedulerData *esdp, + int group, + void *unused) +{ + if (reader_groups && esdp->no <= max_main_threads) + erts_smp_rwmtx_set_reader_group(suspending ? 0 : group+1); +} + +static Eterm get_cpu_groups_map(Process *c_p, + erts_cpu_groups_map_t *map, + int offset); +Eterm +erts_debug_reader_groups_map(Process *c_p, int groups) +{ + Eterm res; + erts_cpu_groups_map_t test; + + test.array = NULL; + test.groups = groups; + make_cpu_groups_map(&test, 1); + if (!test.array) + res = NIL; + else { + res = get_cpu_groups_map(c_p, &test, 1); + erts_free(ERTS_ALC_T_TMP, test.array); + } + return res; +} + + +Eterm +erts_get_reader_groups_map(Process *c_p) +{ + Eterm res; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + res = get_cpu_groups_map(c_p, reader_groups_map, 1); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +/* + * CPU groups + */ + +static Eterm +get_cpu_groups_map(Process *c_p, + erts_cpu_groups_map_t *map, + int offset) +{ +#ifdef DEBUG + Eterm *endp; +#endif + Eterm res = NIL, tuple; + Eterm *hp; + int i; + + hp = HAlloc(c_p, map->logical_processors*(2+3)); +#ifdef DEBUG + endp = hp + map->logical_processors*(2+3); +#endif + for (i = map->size - 1; i >= 0; i--) { + if (map->array[i].logical >= 0) { + tuple = TUPLE2(hp, + make_small(map->array[i].logical), + make_small(map->array[i].cpu_group + offset)); + hp += 3; + res = CONS(hp, tuple, res); + hp += 2; + } + } + ASSERT(hp == endp); + return res; +} + +static void +make_available_cpu_topology(erts_avail_cput *no, + erts_avail_cput *avail, + erts_cpu_topology_t *cpudata, + int *size, + int test) +{ + int len = *size; + erts_cpu_topology_t last; + int a, i, j; + + no->level[ERTS_TOPOLOGY_NODE] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1; + no->level[ERTS_TOPOLOGY_CORE] = -1; + no->level[ERTS_TOPOLOGY_THREAD] = -1; + no->level[ERTS_TOPOLOGY_LOGICAL] = -1; + + last.node = INT_MIN; + last.processor = INT_MIN; + last.processor_node = INT_MIN; + last.core = INT_MIN; + last.thread = INT_MIN; + last.logical = INT_MIN; + + a = 0; + + for (i = 0; i < len; i++) { + + if (!test && !erts_is_cpu_available(cpuinfo, cpudata[i].logical)) + continue; + + if (last.node != cpudata[i].node) + goto node; + if (last.processor != cpudata[i].processor) + goto processor; + if (last.processor_node != cpudata[i].processor_node) + goto processor_node; + if (last.core != cpudata[i].core) + goto core; + ASSERT(last.thread != cpudata[i].thread); + goto thread; + + node: + no->level[ERTS_TOPOLOGY_NODE]++; + processor: + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + processor_node: + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + core: + no->level[ERTS_TOPOLOGY_CORE]++; + thread: + no->level[ERTS_TOPOLOGY_THREAD]++; + + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++) + avail[a].level[j] = no->level[j]; + + avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical; + avail[a].level[ERTS_TOPOLOGY_CG] = 0; + + ASSERT(last.logical != cpudata[i].logical); + + last = cpudata[i]; + a++; + } + + no->level[ERTS_TOPOLOGY_NODE]++; + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + no->level[ERTS_TOPOLOGY_CORE]++; + no->level[ERTS_TOPOLOGY_THREAD]++; + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + *size = a; +} + +static void +cpu_group_insert(erts_cpu_groups_map_t *map, + int logical, int cpu_group) +{ + int start = logical % map->size; + int ix = start; + + do { + if (map->array[ix].logical < 0) { + map->array[ix].logical = logical; + map->array[ix].cpu_group = cpu_group; + return; + } + ix++; + if (ix == map->size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); +} + + +static int +sub_levels(erts_cpu_groups_count_t *cgc, int level, int aix, + int avail_sz, erts_avail_cput *avail) +{ + int sub_level = level+1; + int last = -1; + cgc->sub_levels = 0; + + do { + if (last != avail[aix].level[sub_level]) { + cgc->sub_levels++; + last = avail[aix].level[sub_level]; + } + aix++; + } + while (aix < avail_sz && cgc->id == avail[aix].level[level]); + cgc->cpu_groups = 0; + return aix; +} + +static int +write_cpu_groups(int *cgp, erts_cpu_groups_count_t *cgcp, + int level, int a, + int avail_sz, erts_avail_cput *avail) +{ + int cg = *cgp; + int sub_level = level+1; + int sl_per_gr = cgcp->sub_levels / cgcp->cpu_groups; + int xsl = cgcp->sub_levels % cgcp->cpu_groups; + int sls = 0; + int last = -1; + int xsl_cg_lim = (cgcp->cpu_groups - xsl) + cg + 1; + + ASSERT(level < 0 || avail[a].level[level] == cgcp->id); + + do { + if (last != avail[a].level[sub_level]) { + if (!sls) { + sls = sl_per_gr; + cg++; + if (cg >= xsl_cg_lim) + sls++; + } + last = avail[a].level[sub_level]; + sls--; + } + avail[a].level[ERTS_TOPOLOGY_CG] = cg; + a++; + } while (a < avail_sz && (level < 0 + || avail[a].level[level] == cgcp->id)); + + ASSERT(cgcp->cpu_groups == cg - *cgp); + + *cgp = cg; + + return a; +} + +static int +cg_count_sub_levels_compare(const void *vx, const void *vy) +{ + erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx; + erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy; + if (x->sub_levels != y->sub_levels) + return y->sub_levels - x->sub_levels; + return x->id - y->id; +} + +static int +cg_count_id_compare(const void *vx, const void *vy) +{ + erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx; + erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy; + return x->id - y->id; +} + +static void +make_cpu_groups_map(erts_cpu_groups_map_t *map, int test) +{ + int i, spread_level, avail_sz; + erts_avail_cput no, *avail; + erts_cpu_topology_t *cpudata; + ErtsAlcType_t alc_type = (test + ? ERTS_ALC_T_TMP + : ERTS_ALC_T_CPU_GRPS_MAP); + + if (map->array) + erts_free(alc_type, map->array); + + map->array = NULL; + map->logical_processors = 0; + map->size = 0; + + if (!map->groups) + return; + + create_tmp_cpu_topology_copy(&cpudata, &avail_sz); + + if (!cpudata) + return; + + cpu_bind_order_sort(cpudata, + avail_sz, + ERTS_CPU_BIND_NO_SPREAD, + 1); + + avail = erts_alloc(ERTS_ALC_T_TMP, + sizeof(erts_avail_cput)*avail_sz); + + make_available_cpu_topology(&no, avail, cpudata, + &avail_sz, test); + + destroy_tmp_cpu_topology_copy(cpudata); + + map->size = avail_sz*2+1; + + map->array = erts_alloc(alc_type, + (sizeof(erts_cpu_groups_map_array_t) + * map->size));; + map->logical_processors = avail_sz; + + for (i = 0; i < map->size; i++) { + map->array[i].logical = -1; + map->array[i].cpu_group = -1; + } + + spread_level = ERTS_TOPOLOGY_CORE; + for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) { + if (no.level[i] > map->groups) { + spread_level = i; + break; + } + } + + if (no.level[spread_level] <= map->groups) { + int a, cg, last = -1; + cg = -1; + ASSERT(spread_level == ERTS_TOPOLOGY_CORE); + for (a = 0; a < avail_sz; a++) { + if (last != avail[a].level[spread_level]) { + cg++; + last = avail[a].level[spread_level]; + } + cpu_group_insert(map, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + cg); + } + } + else { /* map->groups < no.level[spread_level] */ + erts_cpu_groups_count_t *cg_count; + int a, cg, tl, toplevels; + + tl = spread_level-1; + + if (spread_level == ERTS_TOPOLOGY_NODE) + toplevels = 1; + else + toplevels = no.level[tl]; + + cg_count = erts_alloc(ERTS_ALC_T_TMP, + toplevels*sizeof(erts_cpu_groups_count_t)); + + if (toplevels == 1) { + cg_count[0].id = 0; + cg_count[0].sub_levels = no.level[spread_level]; + cg_count[0].cpu_groups = map->groups; + } + else { + int cgs_per_tl, cgs; + cgs = map->groups; + cgs_per_tl = cgs / toplevels; + + a = 0; + for (i = 0; i < toplevels; i++) { + cg_count[i].id = avail[a].level[tl]; + a = sub_levels(&cg_count[i], tl, a, avail_sz, avail); + } + + qsort(cg_count, + toplevels, + sizeof(erts_cpu_groups_count_t), + cg_count_sub_levels_compare); + + for (i = 0; i < toplevels; i++) { + if (cg_count[i].sub_levels < cgs_per_tl) { + cg_count[i].cpu_groups = cg_count[i].sub_levels; + cgs -= cg_count[i].sub_levels; + } + else { + cg_count[i].cpu_groups = cgs_per_tl; + cgs -= cgs_per_tl; + } + } + + while (cgs > 0) { + for (i = 0; i < toplevels; i++) { + if (cg_count[i].sub_levels == cg_count[i].cpu_groups) + break; + else { + cg_count[i].cpu_groups++; + if (--cgs == 0) + break; + } + } + } + + qsort(cg_count, + toplevels, + sizeof(erts_cpu_groups_count_t), + cg_count_id_compare); + } + + a = i = 0; + cg = -1; + while (a < avail_sz) { + a = write_cpu_groups(&cg, &cg_count[i], tl, + a, avail_sz, avail); + i++; + } + + ASSERT(map->groups == cg + 1); + + for (a = 0; a < avail_sz; a++) + cpu_group_insert(map, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + avail[a].level[ERTS_TOPOLOGY_CG]); + + erts_free(ERTS_ALC_T_TMP, cg_count); + } + + erts_free(ERTS_ALC_T_TMP, avail); +} + +static erts_cpu_groups_map_t * +add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg) +{ + int use_groups = groups; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_map_t *cgm; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (use_groups > max_main_threads) + use_groups = max_main_threads; + + if (!use_groups) + return NULL; + + no_cpu_groups_callbacks++; + cgcl = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP, + sizeof(erts_cpu_groups_callback_list_t)); + cgcl->callback = callback; + cgcl->arg = arg; + + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + if (cgm->groups == use_groups) { + cgcl->next = cgm->callback_list; + cgm->callback_list = cgcl; + return cgm; + } + } + + + cgm = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP, + sizeof(erts_cpu_groups_map_t)); + cgm->next = cpu_groups_maps; + cgm->groups = use_groups; + cgm->array = NULL; + cgm->size = 0; + cgm->logical_processors = 0; + cgm->callback_list = cgcl; + + cgcl->next = NULL; + + make_cpu_groups_map(cgm, 0); + + cpu_groups_maps = cgm; + + return cgm; +} + +static void +remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) +{ + erts_cpu_groups_map_t *prev_cgm, *cgm; + erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + no_cpu_groups_callbacks--; + + prev_cgm = NULL; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + prev_cgcl = NULL; + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + if (cgcl->callback == callback && cgcl->arg == arg) { + if (prev_cgcl) + prev_cgcl->next = cgcl->next; + else + cgm->callback_list = cgcl->next; + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl); + if (!cgm->callback_list) { + if (prev_cgm) + prev_cgm->next = cgm->next; + else + cpu_groups_maps = cgm->next; + if (cgm->array) + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array); + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm); + } + return; + } + prev_cgcl = cgcl; + } + prev_cgm = cgm; + } + + erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); +} + +static int +cpu_groups_lookup(erts_cpu_groups_map_t *map, + ErtsSchedulerData *esdp) +{ + int start, logical, ix; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (esdp->cpu_id < 0) + return (((int) esdp->no) - 1) % map->groups; + + logical = esdp->cpu_id; + start = logical % map->size; + ix = start; + + do { + if (map->array[ix].logical == logical) { + int group = map->array[ix].cpu_group; + ASSERT(0 <= group && group < map->groups); + return group; + } + ix++; + if (ix == map->size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); +} + +static void +update_cpu_groups_maps(void) +{ + erts_cpu_groups_map_t *cgm; + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) + make_cpu_groups_map(cgm, 0); +} + +void +erts_add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg) +{ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + add_cpu_groups(groups, callback, arg); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); +} + +void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, + void *arg) +{ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + remove_cpu_groups(callback, arg); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); +} diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h new file mode 100644 index 0000000000..c5a9520b61 --- /dev/null +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -0,0 +1,105 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: CPU topology and related functionality + * + * Author: Rickard Green + */ + +#ifndef ERL_CPU_TOPOLOGY_H__ +#define ERL_CPU_TOPOLOGY_H__ + +void erts_pre_early_init_cpu_topology(int *max_rg_p, + int *conf_p, + int *onln_p, + int *avail_p); +void erts_early_init_cpu_topology(int no_schedulers, + int *max_main_threads_p, + int max_reader_groups, + int *reader_groups_p); +void erts_init_cpu_topology(void); + + +#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS 0 +#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED 1 +#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY 2 +#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE 3 + +int erts_init_scheduler_bind_type_string(char *how); + + +#define ERTS_INIT_CPU_TOPOLOGY_OK 0 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID 1 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE 2 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY 3 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE 4 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES 5 +#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID 6 +#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS 7 +#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES 8 +#define ERTS_INIT_CPU_TOPOLOGY_MISSING 9 + +int erts_init_cpu_topology_string(char *topology_str); + +void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp); +#ifdef ERTS_SMP +void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp); +void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp); +void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp); +#endif + +int erts_update_cpu_info(void); + +Eterm erts_bind_schedulers(Process *c_p, Eterm how); +Eterm erts_get_schedulers_binds(Process *c_p); + +Eterm erts_get_reader_groups_map(Process *c_p); + +Eterm erts_set_cpu_topology(Process *c_p, Eterm term); +Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); + +int erts_update_cpu_info(void); +void erts_get_logical_processors(int *conf, int *onln, int *avail); + +int erts_sched_bind_atthrcreate_prepare(void); +int erts_sched_bind_atthrcreate_child(int unbind); +void erts_sched_bind_atthrcreate_parent(int unbind); + +int erts_sched_bind_atfork_prepare(void); +int erts_sched_bind_atfork_child(int unbind); +char *erts_sched_bind_atvfork_child(int unbind); +void erts_sched_bind_atfork_parent(int unbind); + +Eterm erts_fake_scheduler_bindings(Process *p, Eterm how); +Eterm erts_debug_cpu_groups_map(Process *c_p, int groups); + + +typedef void (*erts_cpu_groups_callback_t)(int, + ErtsSchedulerData *, + int, + void *); + +void erts_add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg); +void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, + void *arg); + +#endif diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 15b1c6bb56..e0a6aa05c6 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -78,16 +78,24 @@ enum DbIterSafety { ** The main meta table, containing all ets tables. */ #ifdef ERTS_SMP -# define META_MAIN_TAB_LOCK_CNT 16 -static union { - erts_smp_spinlock_t lck; - byte _cache_line_alignment[64]; -}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT]; + +#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8 +#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS) +#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1) + +typedef union { + erts_smp_rwmtx_t rwmtx; + byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(erts_smp_rwmtx_t))]; +} erts_meta_main_tab_lock_t; + +static erts_meta_main_tab_lock_t *meta_main_tab_locks; + #endif static struct { union { DbTable *tb; /* Only directly readable if slot is ALIVE */ - Uint next_free; /* (index<<2)|1 if slot is FREE */ + UWord next_free; /* (index<<2)|1 if slot is FREE */ }u; } *meta_main_tab; @@ -104,17 +112,13 @@ static struct { #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ -static ERTS_INLINE void meta_main_tab_lock(unsigned slot) -{ -#ifdef ERTS_SMP - erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); -#endif -} - -static ERTS_INLINE void meta_main_tab_unlock(unsigned slot) +static ERTS_INLINE erts_smp_rwmtx_t * +get_meta_main_tab_lock(unsigned slot) { #ifdef ERTS_SMP - erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); + return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx; +#else + return NULL; #endif } @@ -166,7 +170,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, typedef enum { LCK_READ=1, /* read only access */ LCK_WRITE=2, /* exclusive table write access */ - LCK_WRITE_REC=3 /* record write access */ + LCK_WRITE_REC=3, /* record write access */ + LCK_NONE=4 } db_lock_kind_t; extern DbTableMethod db_hash; @@ -174,6 +179,7 @@ extern DbTableMethod db_tree; int user_requested_db_max_tabs; int erts_ets_realloc_always_moves; +int erts_ets_always_compress; static int db_max_tabs; static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */ static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */ @@ -187,7 +193,7 @@ static Eterm ms_delete_all_buff[8]; /* To compare with for deletion static void fix_table_locked(Process* p, DbTable* tb); static void unfix_table_locked(Process* p, DbTable* tb, db_lock_kind_t* kind); -static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data); +static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data); static void free_heir_data(DbTable*); static void free_fixations_locked(DbTable *tb); @@ -213,61 +219,87 @@ Export ets_select_continue_exp; * Static traps */ static Export ets_delete_continue_exp; - -static ERTS_INLINE DbTable* db_ref(DbTable* tb) -{ - if (tb != NULL) { - erts_refc_inc(&tb->common.ref, 2); - } - return tb; -} - -static ERTS_INLINE DbTable* db_unref(DbTable* tb) + +static void +free_dbtable(DbTable* tb) { - if (!erts_refc_dectest(&tb->common.ref, 0)) { #ifdef HARDDEBUG if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { - erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n", - erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), + erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", + erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), tb->common.fixations); } - erts_fprintf(stderr, "ets: db_unref(%T) deleted!!!\r\n", + erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n", tb->common.id); - erts_fprintf(stderr, "ets: db_unref: meta_pid_to_tab common.memory_size = %ld\n", + erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n", erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab); - erts_fprintf(stderr, "ets: db_unref: meta_pid_to_fixed_tab common.memory_size = %ld\n", + erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n", erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab); - #endif #ifdef ERTS_SMP erts_smp_rwmtx_destroy(&tb->common.rwlock); erts_smp_mtx_destroy(&tb->common.fixlock); #endif ASSERT(is_immed(tb->common.heir_data)); - erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); + erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); - return NULL; - } - return tb; } -static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname) +#ifdef ERTS_SMP +static void +chk_free_dbtable(void *vtb) +{ + DbTable * tb = (DbTable *) vtb; + ERTS_THR_MEMORY_BARRIER; + if (erts_refc_dectest(&tb->common.ref, 0) == 0) + free_dbtable(tb); +} +#endif + +static void schedule_free_dbtable(DbTable* tb) { - erts_refc_init(&tb->common.ref, 1); - erts_refc_init(&tb->common.fixref, 0); + /* + * NON-SMP case: Caller is *not* allowed to access the *tb + * structure after this function has returned! + * SMP case: Caller is allowed to access the *tb structure + * until the bif has returned (we typically + * need to unlock the table lock after this + * function has returned). + */ #ifdef ERTS_SMP - erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name); + int scheds = erts_get_max_no_executing_schedulers(); + ASSERT(scheds >= 1); + ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); + erts_refc_init(&tb->common.ref, scheds); + ERTS_THR_MEMORY_BARRIER; + erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb); +#else + free_dbtable(tb); +#endif +} + +static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, + char *rwname, char* fixname) +{ +#ifdef ERTS_SMP + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + if (use_frequent_read_lock) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; +#endif +#ifdef ERTS_SMP + erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, + rwname, tb->common.the_name); erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name); tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED); #endif } -static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) +static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { #ifdef ERTS_SMP ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); @@ -295,16 +327,13 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) #endif } -static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) -{ - (void) db_ref(tb); -#ifdef ERTS_SMP - db_lock_take_over_ref(tb, kind); -#endif -} - static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) { + /* + * In NON-SMP case tb may refer to an already deallocated + * DbTable structure. That is, ONLY the SMP case is allowed + * to follow the tb pointer! + */ #ifdef ERTS_SMP ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); @@ -331,7 +360,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) } } #endif - (void) db_unref(tb); /* May delete table... */ } @@ -349,56 +377,85 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind) } static ERTS_INLINE -DbTable* db_get_table(Process *p, - Eterm id, - int what, - db_lock_kind_t kind) +DbTable* db_get_table_aux(Process *p, + Eterm id, + int what, + db_lock_kind_t kind, + int meta_already_locked) { DbTable *tb = NULL; + erts_smp_rwmtx_t *mtl = NULL; + + /* + * IMPORTANT: Only scheduler threads are allowed + * to access tables. Memory management + * depend on it. + */ + ASSERT(erts_get_scheduler_data()); if (is_small(id)) { Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; - meta_main_tab_lock(slot); - if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) { - /* SMP: inc to prevent race, between unlock of meta_main_tab_lock - * and the table locking outside the meta_main_tab_lock - */ - tb = db_ref(meta_main_tab[slot].u.tb); + if (!meta_already_locked) { + mtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rlock(mtl); } - meta_main_tab_unlock(slot); +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + else { + erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl) + || erts_lc_rwmtx_is_rwlocked(test_mtl)); + } +#endif + if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) + tb = meta_main_tab[slot].u.tb; } else if (is_atom(id)) { - erts_smp_rwmtx_t* rwlock; - struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock); - erts_smp_rwmtx_rlock(rwlock); + struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); + if (!meta_already_locked) + erts_smp_rwmtx_rlock(mtl); + else{ + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) + || erts_lc_rwmtx_is_rwlocked(mtl)); + mtl = NULL; + } + if (bucket->pu.tb != NULL) { if (is_atom(bucket->u.name_atom)) { /* single */ - if (bucket->u.name_atom == id) { - tb = db_ref(bucket->pu.tb); - } + if (bucket->u.name_atom == id) + tb = bucket->pu.tb; } else { /* multi */ Uint cnt = unsigned_val(bucket->u.mcnt); Uint i; for (i=0; i<cnt; i++) { if (bucket->pu.mvec[i].u.name_atom == id) { - tb = db_ref(bucket->pu.mvec[i].pu.tb); + tb = bucket->pu.mvec[i].pu.tb; break; } } } } - erts_smp_rwmtx_runlock(rwlock); } if (tb) { - db_lock_take_over_ref(tb, kind); - if (tb->common.id == id && ((tb->common.status & what) != 0 || - p->id == tb->common.owner)) { - return tb; + db_lock(tb, kind); + if (tb->common.id != id + || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) { + db_unlock(tb, kind); + tb = NULL; } - db_unlock(tb, kind); } - return NULL; + if (mtl) + erts_smp_rwmtx_runlock(mtl); + return tb; +} + +static ERTS_INLINE +DbTable* db_get_table(Process *p, + Eterm id, + int what, + db_lock_kind_t kind) +{ + return db_get_table_aux(p, id, what, kind, 0); } /* Requires meta_main_tab_locks[slot] locked. @@ -413,15 +470,15 @@ static ERTS_INLINE void free_slot(int slot) erts_smp_spin_unlock(&meta_main_tab_main_lock); } -static int insert_named_tab(Eterm name_atom, DbTable* tb) +static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; struct meta_name_tab_entry* new_entry; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - - erts_smp_rwmtx_rwlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwlock(rwlock); if (bucket->pu.tb == NULL) { /* empty */ new_entry = bucket; @@ -468,17 +525,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } -static int remove_named_tab(Eterm name_atom) +static int remove_named_tab(DbTable *tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; + Eterm name_atom = tb->common.id; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - erts_smp_rwmtx_rwlock(rwlock); +#ifdef ERTS_SMP + if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(rwlock); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock)); + if (bucket->pu.tb == NULL) { goto done; } @@ -529,7 +601,8 @@ static int remove_named_tab(Eterm name_atom) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } @@ -538,11 +611,11 @@ done: */ static ERTS_INLINE void local_fix_table(DbTable* tb) { - erts_refc_inc(&tb->common.fixref, 1); + erts_refc_inc(&tb->common.ref, 1); } static ERTS_INLINE void local_unfix_table(DbTable* tb) { - if (erts_refc_dectest(&tb->common.fixref, 0) == 0) { + if (erts_refc_dectest(&tb->common.ref, 0) == 0) { ASSERT(IS_HASH_TABLE(tb->common.status)); db_unfix_table_hash(&(tb->hash)); } @@ -704,12 +777,13 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3) int cret = DB_ERROR_BADITEM; Eterm list; Eterm iter; - Eterm cell[2]; + DeclareTmpHeap(cell,2,BIF_P); DbUpdateHandle handle; if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { BIF_ERROR(BIF_P, BADARG); } + UseTmpHeap(2,BIF_P); if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) { goto bail_out; } @@ -762,6 +836,7 @@ finalize: tb->common.meth->db_finalize_dbterm(&handle); bail_out: + UnUseTmpHeap(2,BIF_P); db_unlock(tb, LCK_WRITE_REC); switch (cret) { @@ -794,8 +869,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* ret_list_currp = NULL; Eterm* ret_list_prevp = NULL; Eterm iter; - Eterm cell[2]; - Eterm tuple[3]; + DeclareTmpHeap(cell,5,BIF_P); + Eterm *tuple = cell+2; DbUpdateHandle handle; Uint halloc_size = 0; /* overestimated heap usage */ Eterm* htop; /* actual heap usage */ @@ -805,6 +880,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { BIF_ERROR(BIF_P, BADARG); } + + UseTmpHeap(5,BIF_P); + if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) { goto bail_out; } @@ -832,7 +910,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm upop; Eterm* tpl; Sint position; - Eterm incr, warp, oldcnt; + Eterm incr, warp; + Wterm oldcnt; if (is_not_list(iter)) { goto finalize; @@ -871,7 +950,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) position > arityval(handle.dbterm->tpl[0])) { goto finalize; } - oldcnt = handle.dbterm->tpl[position]; + oldcnt = db_do_read_element(&handle, position); if (is_big(oldcnt)) { halloc_size += BIG_NEED_SIZE(big_arity(oldcnt)); } @@ -907,7 +986,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* tpl = tuple_val(CAR(list_val(iter))); Sint position = signed_val(tpl[1]); Eterm incr = tpl[2]; - Eterm oldcnt = handle.dbterm->tpl[position]; + Wterm oldcnt = db_do_read_element(&handle,position); Eterm newcnt = db_add_counter(&htop, oldcnt, incr); if (newcnt == NIL) { @@ -920,9 +999,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) if (arityval(*tpl) == 4) { /* Maybe warp it */ Eterm threshold = tpl[3]; - if ((cmp(incr,make_small(0)) < 0) ? /* negative increment? */ - (cmp(newcnt,threshold) < 0) : /* if negative, check if below */ - (cmp(newcnt,threshold) > 0)) { /* else check if above threshold */ + if ((CMP(incr,make_small(0)) < 0) ? /* negative increment? */ + (CMP(newcnt,threshold) < 0) : /* if negative, check if below */ + (CMP(newcnt,threshold) > 0)) { /* else check if above threshold */ newcnt = tpl[4]; } @@ -951,6 +1030,7 @@ finalize: tb->common.meth->db_finalize_dbterm(&handle); bail_out: + UnUseTmpHeap(5,BIF_P); db_unlock(tb, LCK_WRITE_REC); switch (cret) { @@ -1127,6 +1207,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) { DbTable* tb; Eterm ret; + erts_smp_rwmtx_t *lck1, *lck2; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1135,34 +1216,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { + + if (is_not_atom(BIF_ARG_2)) { BIF_ERROR(BIF_P, BADARG); } - if (is_not_atom(BIF_ARG_2)) { - goto badarg; + (void) meta_name_tab_bucket(BIF_ARG_2, &lck1); + + if (is_small(BIF_ARG_1)) { + Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask; + lck2 = get_meta_main_tab_lock(slot); + } + else if (is_atom(BIF_ARG_1)) { + (void) meta_name_tab_bucket(BIF_ARG_1, &lck2); + if (lck1 == lck2) + lck2 = NULL; + else if (lck1 > lck2) { + erts_smp_rwmtx_t *tmp = lck1; + lck1 = lck2; + lck2 = tmp; + } } + else { + BIF_ERROR(BIF_P, BADARG); + } + + erts_smp_rwmtx_rwlock(lck1); + if (lck2) + erts_smp_rwmtx_rwlock(lck2); + + tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1); + if (!tb) + goto badarg; if (is_not_atom(tb->common.id)) { /* Not a named table */ tb->common.the_name = BIF_ARG_2; goto done; } - if (!insert_named_tab(BIF_ARG_2,tb)) { + if (!insert_named_tab(BIF_ARG_2, tb, 1)) goto badarg; - } - if (!remove_named_tab(tb->common.id)) { + + if (!remove_named_tab(tb, 1)) erl_exit(1,"Could not find named tab %s", tb->common.id); - } tb->common.id = tb->common.the_name = BIF_ARG_2; done: ret = tb->common.id; db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_RET(ret); badarg: - db_unlock(tb, LCK_WRITE); + if (tb) + db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_ERROR(BIF_P, BADARG); } @@ -1180,13 +1292,14 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) Eterm val; Eterm ret; Eterm heir; - Eterm heir_data; + UWord heir_data; Uint32 status; Sint keypos; - int is_named, is_fine_locked; + int is_named, is_fine_locked, frequent_read, is_compressed; int cret; - Eterm meta_tuple[3]; + DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; + erts_smp_rwmtx_t *mmtl; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1199,8 +1312,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) keypos = 1; is_named = 0; is_fine_locked = 0; + frequent_read = 0; heir = am_none; - heir_data = am_undefined; + heir_data = (UWord) am_undefined; + is_compressed = erts_ets_always_compress; list = BIF_ARG_2; while(is_list(list)) { @@ -1232,6 +1347,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) is_fine_locked = 0; } else break; } + else if (tp[1] == am_read_concurrency) { + if (tp[2] == am_true) { + frequent_read = 1; + } else if (tp[2] == am_false) { + frequent_read = 0; + } else break; + } else if (tp[1] == am_heir && tp[2] == am_none) { heir = am_none; heir_data = am_undefined; @@ -1256,6 +1378,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) else if (val == am_named_table) { is_named = 1; } + else if (val == am_compressed) { + is_compressed = 1; + } else if (val == am_set || val == am_protected) ; else break; @@ -1280,6 +1405,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } +#ifdef ERTS_SMP + if (frequent_read && !(status & DB_PRIVATE)) + status |= DB_FREQ_READ; +#endif + /* we create table outside any table lock * and take the unusal cost of destroy table if it * fails to find a slot @@ -1302,7 +1432,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif - db_init_lock(tb, "db_tab", "db_tab_fix"); + erts_refc_init(&tb->common.ref, 0); + db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), + "db_tab", "db_tab_fix"); tb->common.keypos = keypos; tb->common.owner = BIF_P->id; set_heir(BIF_P, tb, heir, heir_data); @@ -1310,6 +1442,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_smp_atomic_init(&tb->common.nitems, 0); tb->common.fixations = NULL; + tb->common.compress = is_compressed; cret = meth->db_create(BIF_P, tb); ASSERT(cret == DB_ERROR_NONE); @@ -1322,8 +1455,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) "** Too many db tables **\n"); free_heir_data(tb); tb->common.meth->db_free_table(tb); - erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); - ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); + free_dbtable(tb); BIF_ERROR(BIF_P, SYSTEM_LIMIT); } @@ -1345,19 +1477,22 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.id = ret; tb->common.slot = slot; /* store slot for erase */ - meta_main_tab_lock(slot); + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); meta_main_tab[slot].u.tb = tb; ASSERT(IS_SLOT_ALIVE(slot)); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); - if (is_named && !insert_named_tab(BIF_ARG_1, tb)) { - meta_main_tab_lock(slot); + if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) { + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); free_slot(slot); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); - db_lock_take_over_ref(tb,LCK_WRITE); + db_lock(tb,LCK_WRITE); free_heir_data(tb); tb->common.meth->db_free_table(tb); + schedule_free_dbtable(tb); db_unlock(tb,LCK_WRITE); BIF_ERROR(BIF_P, BADARG); } @@ -1375,6 +1510,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); #endif + UseTmpHeap(3,BIF_P); + db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); if (db_put_hash(meta_pid_to_tab, TUPLE2(meta_tuple, BIF_P->id, make_small(slot)), @@ -1383,6 +1520,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) } db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); + UnUseTmpHeap(3,BIF_P); + BIF_RET(ret); } @@ -1489,6 +1628,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { int trap; DbTable* tb; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1510,16 +1650,26 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif /* We must keep the slot, to be found by db_proc_dead() if process dies */ MARK_SLOT_DEAD(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); - if (is_atom(tb->common.id)) { - remove_named_tab(tb->common.id); - } + erts_smp_rwmtx_rwunlock(mmtl); + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); if (tb->common.owner != BIF_P->id) { - Eterm meta_tuple[3]; + DeclareTmpHeap(meta_tuple,3,BIF_P); /* * The table is being deleted by a process other than its owner. @@ -1527,6 +1677,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * current process will be killed (e.g. by an EXIT signal), we will * now transfer the ownership to the current process. */ + UseTmpHeap(3,BIF_P); db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, make_small(tb->common.slot)); @@ -1538,6 +1689,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) TUPLE2(meta_tuple,BIF_P->id,make_small(tb->common.slot)), 0); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); + UnUseTmpHeap(3,BIF_P); } /* disable inheritance */ free_heir_data(tb); @@ -1554,9 +1706,15 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * (it looks like an continuation pointer), but that is will crash the * emulator if this BIF is call traced. */ +#if HALFWORD_HEAP + Eterm *hp = HAlloc(BIF_P, 3); + hp[0] = make_pos_bignum_header(2); + *((UWord *) (UWord) (hp+1)) = (UWord) tb; +#else Eterm *hp = HAlloc(BIF_P, 2); hp[0] = make_pos_bignum_header(1); hp[1] = (Eterm) tb; +#endif BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp)); } else { @@ -1571,7 +1729,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) { Process* to_proc = NULL; ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN; - Eterm buf[5]; + DeclareTmpHeap(buf,5,BIF_P); Eterm to_pid = BIF_ARG_2; Eterm from_pid; DbTable* tb = NULL; @@ -1593,6 +1751,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) goto badarg; /* or should we be idempotent? return false maybe */ } + UseTmpHeap(5,BIF_P); db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, make_small(tb->common.slot)); @@ -1610,6 +1769,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, from_pid, BIF_ARG_3), 0); erts_smp_proc_unlock(to_proc, to_locks); + UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); badarg: @@ -1624,11 +1784,12 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2) Eterm* tp; Eterm opt; Eterm heir = THE_NON_VALUE; - Eterm heir_data = THE_NON_VALUE; + UWord heir_data = (UWord) THE_NON_VALUE; Uint32 protection = 0; - Eterm fakelist[2]; + DeclareTmpHeap(fakelist,2,BIF_P); Eterm tail; + UseTmpHeap(2,BIF_P); for (tail = is_tuple(BIF_ARG_2) ? CONS(fakelist, BIF_ARG_2, NIL) : BIF_ARG_2; is_list(tail); tail = CDR(list_val(tail))) { @@ -1681,9 +1842,11 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2) } db_unlock (tb,LCK_WRITE); + UnUseTmpHeap(2,BIF_P); BIF_RET(am_true); badarg: + UnUseTmpHeap(2,BIF_P); if (tb != NULL) { db_unlock(tb,LCK_WRITE); } @@ -1896,14 +2059,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0) previous = NIL; j = 0; for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) { - meta_main_tab_lock(i); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(i)) { j++; tb = meta_main_tab[i].u.tb; previous = CONS(hp, tb->common.id, previous); hp += 2; } - meta_main_tab_unlock(i); + erts_smp_rwmtx_runlock(mmtl); } HRelease(BIF_P, hendp, hp); BIF_RET(previous); @@ -1949,29 +2113,37 @@ BIF_RETTYPE ets_match_1(BIF_ALIST_1) BIF_RETTYPE ets_match_2(BIF_ALIST_2) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarDollar, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select_2(BIF_P, BIF_ARG_1, ms); + UnUseTmpHeap(8,BIF_P); + return res; } BIF_RETTYPE ets_match_3(BIF_ALIST_3) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarDollar, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + UnUseTmpHeap(8,BIF_P); + return res; } @@ -2385,29 +2557,37 @@ BIF_RETTYPE ets_match_object_1(BIF_ALIST_1) BIF_RETTYPE ets_match_object_2(BIF_ALIST_2) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarUnderscore, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select_2(BIF_P, BIF_ARG_1, ms); + UnUseTmpHeap(8,BIF_P); + return res; } BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) { Eterm ms; - Eterm buff[8]; + DeclareTmpHeap(buff,8,BIF_P); Eterm *hp = buff; - /*hp = HAlloc(BIF_P, 8);*/ + Eterm res; + + UseTmpHeap(8,BIF_P); ms = CONS(hp, am_DollarUnderscore, NIL); hp += 2; ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + UnUseTmpHeap(8,BIF_P); + return res; } /* @@ -2417,7 +2597,7 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) BIF_RETTYPE ets_info_1(BIF_ALIST_1) { static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, - am_node, am_size, am_name, am_heir, am_owner, am_memory}; + am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -2533,7 +2713,6 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) Binary *mp; Eterm res; Uint32 dummy; - Uint sz; if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) || !is_binary(BIF_ARG_2)) { error: @@ -2558,11 +2737,10 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), 0, &dummy); + res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0, + ERTS_PAM_COPY_RESULT, &dummy); if (is_value(res)) { - sz = size_object(res); - hp = HAlloc(BIF_P, sz + 2); - res = copy_struct(res, sz, &hp, &MSO(BIF_P)); + hp = HAlloc(BIF_P, 2); ret = CONS(hp,res,ret); /*hp += 2;*/ } @@ -2585,18 +2763,29 @@ void init_db(void) { DbTable init_tb; int i; - extern Eterm* em_apply_bif; + extern BeamInstr* em_apply_bif; Eterm *hp; unsigned bits; size_t size; #ifdef ERTS_SMP - for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) { - erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i)); + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + meta_main_tab_locks = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES, + sizeof(erts_meta_main_tab_lock_t) + * ERTS_META_MAIN_TAB_LOCK_TAB_SIZE); + + for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { + erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, + "meta_main_tab_slot", make_small(i)); } erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main"); for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) { - erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i)); + erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt, + "meta_name_tab", make_small(i)); } #endif @@ -2664,9 +2853,9 @@ void init_db(void) erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0); meta_pid_to_tab->common.slot = -1; meta_pid_to_tab->common.meth = &db_hash; + meta_pid_to_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_tab->common.ref, 1); - erts_refc_init(&meta_pid_to_tab->common.fixref, 0); + erts_refc_init(&meta_pid_to_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ @@ -2696,9 +2885,9 @@ void init_db(void) erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0); meta_pid_to_fixed_tab->common.slot = -1; meta_pid_to_fixed_tab->common.meth = &db_hash; + meta_pid_to_fixed_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 1); - erts_refc_init(&meta_pid_to_fixed_tab->common.fixref, 0); + erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ @@ -2714,9 +2903,9 @@ void init_db(void) ets_select_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); ets_select_delete_continue_exp.code[2] = 1; ets_select_delete_continue_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_delete_continue_exp.code[4] = - (Eterm) &ets_select_delete_1; + (BeamInstr) &ets_select_delete_1; /* Non visual BIF to trap to. */ memset(&ets_select_count_continue_exp, 0, sizeof(Export)); @@ -2726,9 +2915,9 @@ void init_db(void) ets_select_count_continue_exp.code[1] = am_atom_put("count_trap",11); ets_select_count_continue_exp.code[2] = 1; ets_select_count_continue_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_count_continue_exp.code[4] = - (Eterm) &ets_select_count_1; + (BeamInstr) &ets_select_count_1; /* Non visual BIF to trap to. */ memset(&ets_select_continue_exp, 0, sizeof(Export)); @@ -2738,9 +2927,9 @@ void init_db(void) ets_select_continue_exp.code[1] = am_atom_put("select_trap",11); ets_select_continue_exp.code[2] = 1; ets_select_continue_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_continue_exp.code[4] = - (Eterm) &ets_select_trap_1; + (BeamInstr) &ets_select_trap_1; /* Non visual BIF to trap to. */ memset(&ets_delete_continue_exp, 0, sizeof(Export)); @@ -2748,8 +2937,8 @@ void init_db(void) ets_delete_continue_exp.code[0] = am_ets; ets_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); ets_delete_continue_exp.code[2] = 1; - ets_delete_continue_exp.code[3] = (Eterm) em_apply_bif; - ets_delete_continue_exp.code[4] = (Eterm) &ets_delete_trap; + ets_delete_continue_exp.code[3] = (BeamInstr) em_apply_bif; + ets_delete_continue_exp.code[4] = (BeamInstr) &ets_delete_trap; hp = ms_delete_all_buff; ms_delete_all = CONS(hp, am_true, NIL); @@ -2843,9 +3032,9 @@ static int give_away_to_heir(Process* p, DbTable* tb) { Process* to_proc; ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN; - Eterm buf[5]; + DeclareTmpHeap(buf,5,p); Eterm to_pid; - Eterm heir_data; + UWord heir_data; ASSERT(tb->common.owner == p->id); ASSERT(is_internal_pid(tb->common.heir)); @@ -2856,12 +3045,10 @@ retry: to_pid, to_locks, ERTS_P2P_FLG_TRY_LOCK); if (to_proc == ERTS_PROC_LOCK_BUSY) { - db_ref(tb); /* while unlocked */ db_unlock(tb,LCK_WRITE); to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to_pid, to_locks); db_lock(tb,LCK_WRITE); - tb = db_unref(tb); ASSERT(tb != NULL); if (tb->common.owner != p->id) { @@ -2888,6 +3075,7 @@ retry: erts_smp_proc_unlock(to_proc, to_locks); return 0; /* heir dead and pid reused, table still mine */ } + UseTmpHeap(5,p); db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner, make_small(tb->common.slot)); @@ -2899,11 +3087,11 @@ retry: TUPLE2(buf,to_pid,make_small(tb->common.slot)), 0); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); - + UnUseTmpHeap(5,p); db_unlock(tb,LCK_WRITE); heir_data = tb->common.heir_data; if (!is_immed(heir_data)) { - Eterm* tpv = DBTERM_BUF((DbTerm*)heir_data); /* tuple_val */ + Eterm* tpv = ((DbTerm*)heir_data)->tpl; /* tuple_val */ ASSERT(arityval(*tpv) == 1); heir_data = tpv[1]; } @@ -2968,15 +3156,16 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (!IS_SLOT_FREE(ix)) { - tb = db_ref(GET_ANY_SLOT_TAB(ix)); + tb = GET_ANY_SLOT_TAB(ix); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int do_yield; - db_lock_take_over_ref(tb, LCK_WRITE); + db_lock(tb, LCK_WRITE); /* Ownership may have changed since we looked up the table. */ if (tb->common.owner != pid) { @@ -3005,7 +3194,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) tb->common.status |= DB_DELETE; if (is_atom(tb->common.id)) - remove_named_tab(tb->common.id); + remove_named_tab(tb, 0); free_heir_data(tb); free_fixations_locked(tb); @@ -3055,17 +3244,18 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(ix)) { - tb = db_ref(meta_main_tab[ix].u.tb); + tb = meta_main_tab[ix].u.tb; ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int reds; DbFixation** pp; - db_lock_take_over_ref(tb, LCK_WRITE_REC); + db_lock(tb, LCK_WRITE_REC); #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif @@ -3075,7 +3265,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) pp = &(*pp)->next) { if ((*pp)->pid == pid) { DbFixation* fix = *pp; - erts_refc_add(&tb->common.fixref,-fix->counter,0); + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); *pp = fix->next; erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, fix, sizeof(DbFixation)); @@ -3145,12 +3336,12 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) static void fix_table_locked(Process* p, DbTable* tb) { DbFixation *fix; - Eterm meta_tuple[3]; + DeclareTmpHeap(meta_tuple,3,p); #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif - erts_refc_inc(&tb->common.fixref,1); + erts_refc_inc(&tb->common.ref,1); fix = tb->common.fixations; if (fix == NULL) { get_now(&(tb->common.megasec), @@ -3179,12 +3370,15 @@ static void fix_table_locked(Process* p, DbTable* tb) erts_smp_mtx_unlock(&tb->common.fixlock); #endif p->flags |= F_USING_DB; + UseTmpHeap(3,p); db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); if (db_put_hash(meta_pid_to_fixed_tab, TUPLE2(meta_tuple, p->id, make_small(tb->common.slot)), 0) != DB_ERROR_NONE) { + UnUseTmpHeap(3,p); erl_exit(1,"Could not insert ets metadata in safe_fixtable."); } + UnUseTmpHeap(3,p); db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); } @@ -3201,7 +3395,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) { if ((*pp)->pid == p->id) { DbFixation* fix = *pp; - erts_refc_dec(&tb->common.fixref,0); + erts_refc_dec(&tb->common.ref,0); --(fix->counter); ASSERT(fix->counter >= 0); if (fix->counter > 0) { @@ -3227,7 +3421,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, unlocked: if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status) - && erts_smp_atomic_read(&tb->hash.fixdel) != (long)NULL) { + && erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) { #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ @@ -3249,6 +3443,8 @@ static void free_fixations_locked(DbTable *tb) fix = tb->common.fixations; while (fix != NULL) { + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); next_fix = fix->next; db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_fixed_tab, @@ -3264,7 +3460,7 @@ static void free_fixations_locked(DbTable *tb) tb->common.fixations = NULL; } -static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data) +static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) { tb->common.heir = heir; if (heir == am_none) { @@ -3285,10 +3481,26 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data) } if (!is_immed(heir_data)) { - Eterm tmp[2]; - /* Make a dummy 1-tuple around data to use db_get_term() */ - heir_data = (Eterm) db_get_term(&tb->common, NULL, 0, - TUPLE1(tmp,heir_data)); + DeclareTmpHeap(tmp,2,me); + Eterm wrap_tpl; + int size; + DbTerm* dbterm; + Eterm* top; + ErlOffHeap tmp_offheap; + + UseTmpHeap(2,me); + /* Make a dummy 1-tuple around data to use DbTerm */ + wrap_tpl = TUPLE1(tmp,heir_data); + size = size_object(wrap_tpl); + dbterm = erts_db_alloc(ERTS_ALC_T_DB_HEIR_DATA, (DbTable *)tb, + (sizeof(DbTerm) + sizeof(Eterm)*(size-1))); + dbterm->size = size; + top = dbterm->tpl; + tmp_offheap.first = NULL; + copy_struct(wrap_tpl, size, &top, &tmp_offheap); + dbterm->first_oh = tmp_offheap.first; + heir_data = (UWord)dbterm; + UnUseTmpHeap(2,me); ASSERT(!is_immed(heir_data)); } tb->common.heir_data = heir_data; @@ -3298,8 +3510,8 @@ static void free_heir_data(DbTable* tb) { if (tb->common.heir != am_none && !is_immed(tb->common.heir_data)) { DbTerm* p = (DbTerm*) tb->common.heir_data; - db_free_term_data(p); - erts_db_free(ERTS_ALC_T_DB_TERM, tb, (void *)p, + db_cleanup_offheap_comp(p); + erts_db_free(ERTS_ALC_T_DB_HEIR_DATA, tb, (void *)p, sizeof(DbTerm) + (p->size-1)*sizeof(Eterm)); } #ifdef DEBUG @@ -3311,10 +3523,13 @@ static BIF_RETTYPE ets_delete_trap(Process *p, Eterm cont) { int trap; Eterm* ptr = big_val(cont); - DbTable *tb = (DbTable *) ptr[1]; + DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); +#if HALFWORD_HEAP + ASSERT(*ptr == make_pos_bignum_header(2)); +#else ASSERT(*ptr == make_pos_bignum_header(1)); - +#endif db_lock(tb, LCK_WRITE); trap = free_table_cont(p, tb, 0, 1); db_unlock(tb, LCK_WRITE); @@ -3337,6 +3552,7 @@ static int free_table_cont(Process *p, int clean_meta_tab) { Eterm result; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG if (!first) { @@ -3362,9 +3578,16 @@ static int free_table_cont(Process *p, tb->common.id); #endif /* Completely done - we will not get called again. */ - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif free_slot(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); + erts_smp_rwmtx_rwunlock(mmtl); if (clean_meta_tab) { db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); @@ -3372,7 +3595,7 @@ static int free_table_cont(Process *p, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); } - db_unref(tb); + schedule_free_dbtable(tb); BUMP_REDS(p, 100); return 0; } @@ -3420,10 +3643,13 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = erts_this_dist_entry->sysname; } else if (What == am_named_table) { ret = is_atom(tb->common.id) ? am_true : am_false; + } else if (What == am_compressed) { + ret = tb->common.compress ? am_true : am_false; + } /* * For debugging purposes */ - } else if (What == am_data) { + else if (What == am_data) { print_table(ERTS_PRINT_STDOUT, NULL, 1, tb); ret = am_true; } else if (What == am_atom_put("fixed",5)) { @@ -3511,7 +3737,7 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb) erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read(&tb->common.nitems)); erts_print(to, to_arg, "Words: %bpu\n", - (Uint) ((erts_smp_atomic_read(&tb->common.memory_size) + (UWord) ((erts_smp_atomic_read(&tb->common.memory_size) + sizeof(Uint) - 1) / sizeof(Uint))); diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 7da28fad29..e0bdebcb01 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -61,6 +61,7 @@ void erts_db_foreach_offheap(DbTable *, extern int user_requested_db_max_tabs; /* set in erl_init */ extern int erts_ets_realloc_always_moves; /* set in erl_init */ +extern int erts_ets_always_compress; /* set in erl_init */ extern Export ets_select_delete_continue_exp; extern Export ets_select_count_continue_exp; extern Export ets_select_continue_exp; @@ -82,7 +83,8 @@ Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); #define ERTS_DB_ALC_MEM_UPDATE_(TAB, FREE_SZ, ALLOC_SZ) \ do { \ - long sz__ = ((long) (ALLOC_SZ)) - ((long) (FREE_SZ)); \ + erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ + - ((erts_aint_t) (FREE_SZ))); \ ASSERT((TAB)); \ erts_smp_atomic_add(&(TAB)->common.memory_size, sz__); \ } while (0) diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 4141f9766b..fdc82c8b88 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -105,7 +105,7 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab)) +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab)) #define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) @@ -122,8 +122,8 @@ */ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) { - Uint mask = erts_smp_atomic_read(&tb->szm); - Uint ix = hval & mask; + Uint mask = erts_smp_atomic_read_acqb(&tb->szm); + Uint ix = hval & mask; if (ix >= erts_smp_atomic_read(&tb->nactive)) { ix &= mask>>1; ASSERT(ix < erts_smp_atomic_read(&tb->nactive)); @@ -135,8 +135,8 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) */ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) { - long was_next; - long exp_next; + erts_aint_t was_next; + erts_aint_t exp_next; FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL, (DbTable *) tb, sizeof(FixedDeletion)); @@ -146,7 +146,9 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) do { /* Lockless atomic insertion in linked list: */ exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixd, exp_next); + was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, + (erts_aint_t) fixd, + exp_next); }while (was_next != exp_next); } @@ -256,22 +258,16 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix, } -/* - * tplp is an untagged pointer to a tuple we know is large enough - * and dth is a pointer to a DbTableHash. - */ -#define GETKEY(dth, tplp) (*((tplp) + (dth)->common.keypos)) - /* * Some special binary flags */ #define BIN_FLAG_ALL_OBJECTS BIN_FLAG_USR1 -/* - * Size calculations - */ -#define SIZ_OVERHEAD ((sizeof(HashDbTerm)/sizeof(Eterm)) - 1) -#define SIZ_DBTERM(HDT) (SIZ_OVERHEAD + (HDT)->dbterm.size) + +static ERTS_INLINE void free_term(DbTableHash *tb, HashDbTerm* p) +{ + db_free_term((DbTable*)tb, p, offsetof(HashDbTerm, dbterm)); +} /* * Local types @@ -358,10 +354,8 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); static void shrink(DbTableHash* tb, int nactive); static void grow(DbTableHash* tb, int nactive); -static void free_term(DbTableHash *tb, HashDbTerm* p); -static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2); -static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, - Eterm obj, HashValue hval); +static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, + DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, struct mp_info *mpi); @@ -434,6 +428,9 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb) } } +#define EQ_REL(x,y,y_base) \ + (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base))) + /* Is this a live object (not pseodo-deleted) with the specified key? */ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, @@ -442,7 +439,8 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, if (b->hvalue != hval) return 0; else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); - return EQ(key,itemKey); + ASSERT(!is_header(itemKey)); + return EQ_REL(key, itemKey, b->dbterm.tpl); } } @@ -454,10 +452,38 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b, if (b->hvalue != hval && b->hvalue != INVALID_HASH) return 0; else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); - return EQ(key,itemKey); + ASSERT(!is_header(itemKey)); + return EQ_REL(key, itemKey, b->dbterm.tpl); + } +} + +static ERTS_INLINE HashDbTerm* new_dbterm(DbTableHash* tb, Eterm obj) +{ + HashDbTerm* p; + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); } + return p; } +static ERTS_INLINE HashDbTerm* replace_dbterm(DbTableHash* tb, HashDbTerm* old, + Eterm obj) +{ + HashDbTerm* ret; + ASSERT(old != NULL); + if (tb->common.compress) { + ret = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + } + else { + ret = db_store_term(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + } + return ret; +} + + /* ** External interface @@ -514,12 +540,12 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) { /*int tries = 0;*/ DEBUG_WAIT(); - if (erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel, - (long)NULL) != (long)NULL) { + if (erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, + (erts_aint_t)NULL) != (erts_aint_t)NULL) { /* Oboy, must join lists */ FixedDeletion* last = fixdel; - long was_tail; - long exp_tail; + erts_aint_t was_tail; + erts_aint_t exp_tail; while (last->next != NULL) last = last->next; was_tail = erts_smp_atomic_read(&tb->fixdel); @@ -528,7 +554,7 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) last->next = (FixedDeletion*) exp_tail; /*++tries;*/ DEBUG_WAIT(); - was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel, + was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, exp_tail); }while (was_tail != exp_tail); } @@ -546,7 +572,7 @@ void db_unfix_table_hash(DbTableHash *tb) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (long)NULL); + fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (erts_aint_t)NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; @@ -615,20 +641,24 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init(&tb->szm, SEGSZ_MASK); erts_smp_atomic_init(&tb->nactive, SEGSZ); - erts_smp_atomic_init(&tb->fixdel, (long)NULL); - erts_smp_atomic_init(&tb->segtab, (long) alloc_ext_seg(tb,0,NULL)->segtab); + erts_smp_atomic_init(&tb->fixdel, (erts_aint_t)NULL); + erts_smp_atomic_init(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); tb->nsegs = NSEG_1; tb->nslots = SEGSZ; erts_smp_atomic_init(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; + if (tb->common.type & DB_FREQ_READ) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { - erts_rwmtx_init_x(&tb->locks->lck_vec[i].lck, "db_hash_slot", make_small(i)); + erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, + "db_hash_slot", make_small(i)); } /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: @@ -638,6 +668,7 @@ int db_create_hash(Process *p, DbTable *tbl) else { /* coarse locking */ tb->locks = NULL; } + ERTS_THR_MEMORY_BARRIER; #endif /* ERST_SMP */ return DB_ERROR_NONE; } @@ -663,9 +694,7 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret) } } if (list != NULL) { - Eterm key = GETKEY(tb, list->dbterm.tpl); - - COPY_OBJECT(key, p, ret); + *ret = db_copy_key(p, tbl, &list->dbterm); RUNLOCK_HASH(lck); } else { @@ -713,7 +742,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) *ret = am_EOT; } else { - COPY_OBJECT(GETKEY(tb, b->dbterm.tpl), p, ret); + *ret = db_copy_key(p, tbl, &b->dbterm); RUNLOCK_HASH(lck); } return DB_ERROR_NONE; @@ -760,7 +789,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) ret = DB_ERROR_BADKEY; goto Ldone; } - q = get_term(tb, b, obj, hval); + q = replace_dbterm(tb, b, obj); q->next = bnext; q->hvalue = hval; /* In case of INVALID_HASH */ *bp = q; @@ -780,7 +809,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) HashDbTerm** qp = bp; q = b; do { - if (eq(make_tuple(q->dbterm.tpl), obj)) { + if (db_eq(&tb->common,obj,&q->dbterm)) { if (q->hvalue == INVALID_HASH) { erts_smp_atomic_inc(&tb->common.nitems); q->hvalue = hval; @@ -799,7 +828,8 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) /*else DB_DUPLICATE_BAG */ Lnew: - q = get_term(tb, NULL, obj, hval); + q = new_dbterm(tb, obj); + q->hvalue = hval; q->next = b; *bp = q; nitems = erts_smp_atomic_inctest(&tb->common.nitems); @@ -840,7 +870,7 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) while(b2 != NULL && has_key(tb,b2,key,hval)) b2 = b2->next; } - copy = put_term_list(p, b1, b2); + copy = build_term_list(p, b1, b2, tb); CHECK_TABLES(); *ret = copy; goto done; @@ -963,13 +993,10 @@ static int db_get_element_hash(Process *p, DbTable *tbl, while(b1 != 0) { if (has_live_key(tb,b1,key,hval)) { - Eterm copy; - if (ndex > arityval(b1->dbterm.tpl[0])) { retval = DB_ERROR_BADITEM; goto done; } - if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { HashDbTerm* b; HashDbTerm* b2 = b1->next; @@ -983,15 +1010,12 @@ static int db_get_element_hash(Process *p, DbTable *tbl, } b2 = b2->next; } - b = b1; while(b != b2) { if (b->hvalue != INVALID_HASH) { Eterm *hp; - Uint sz = size_object(b->dbterm.tpl[ndex])+2; - - hp = HAlloc(p, sz); - copy = copy_struct(b->dbterm.tpl[ndex], sz-2, &hp, &MSO(p)); + Eterm copy = db_copy_element_from_ets(&tb->common, p, + &b->dbterm, ndex, &hp, 2); elem_list = CONS(hp, copy, elem_list); hp += 2; } @@ -1000,8 +1024,8 @@ static int db_get_element_hash(Process *p, DbTable *tbl, *ret = elem_list; } else { - COPY_OBJECT(b1->dbterm.tpl[ndex], p, ©); - *ret = copy; + Eterm* hp; + *ret = db_copy_element_from_ets(&tb->common, p, &b1->dbterm, ndex, &hp, 0); } retval = DB_ERROR_NONE; goto done; @@ -1036,6 +1060,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value) ASSERT(!IS_FIXED(tb)); ASSERT((tb->common.status & DB_BAG)); + ASSERT(!tb->common.compress); while(b != 0) { if (has_live_key(tb,b,key,hval)) { @@ -1135,7 +1160,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) while(b != 0) { if (has_live_key(tb,b,key,hval)) { ++nkeys; - if (eq(object, make_tuple(b->dbterm.tpl))) { + if (db_eq(&tb->common,object, &b->dbterm)) { --nitems_diff; if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */ add_fixed_deletion(tb,ix); @@ -1184,7 +1209,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) lck = RLOCK_HASH(tb, slot); nactive = NACTIVE(tb); if (slot < nactive) { - *ret = put_term_list(p, BUCKET(tb, slot), 0); + *ret = build_term_list(p, BUCKET(tb, slot), 0, tb); retval = DB_ERROR_NONE; } else if (slot == nactive) { @@ -1228,8 +1253,6 @@ static int db_select_continue_hash(Process *p, int num_left = 1000; HashDbTerm *current = 0; Eterm match_list; - Uint32 dummy; - unsigned sz; Eterm *hp; Eterm match_res; Sint got; @@ -1281,26 +1304,14 @@ static int db_select_continue_hash(Process *p, } for(;;) { if (current->hvalue != INVALID_HASH && - (match_res = - db_prog_match(p,mp, - make_tuple(current->dbterm.tpl), - 0,&dummy), + (match_res = db_match_dbterm(&tb->common, p, mp, all_objects, + ¤t->dbterm, &hp, 2), is_value(match_res))) { - if (all_objects) { - hp = HAlloc(p, current->dbterm.size + 2); - match_res = copy_shallow(DBTERM_BUF(¤t->dbterm), - current->dbterm.size, - &hp, - &MSO(p)); - } else { - sz = size_object(match_res); - - hp = HAlloc(p, sz + 2); - match_res = copy_struct(match_res, sz, &hp, &MSO(p)); - } - match_list = CONS(hp, match_res, match_list); + + match_list = CONS(hp, match_res, match_list); ++got; } + --num_left; save_slot_ix = slot_ix; if ((current = next(tb, (Uint*)&slot_ix, &lck, current)) == NULL) { @@ -1391,9 +1402,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, HashDbTerm *current = 0; unsigned current_list_pos = 0; Eterm match_list; - Uint32 dummy; Eterm match_res; - unsigned sz; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1460,22 +1469,9 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, for(;;) { if (current != NULL) { if (current->hvalue != INVALID_HASH) { - match_res = db_prog_match(p,mpi.mp, - make_tuple(current->dbterm.tpl), - 0,&dummy); + match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, + ¤t->dbterm, &hp, 2); if (is_value(match_res)) { - if (mpi.all_objects) { - hp = HAlloc(p, current->dbterm.size + 2); - match_res = copy_shallow(DBTERM_BUF(¤t->dbterm), - current->dbterm.size, - &hp, - &MSO(p)); - } else { - sz = size_object(match_res); - - hp = HAlloc(p, sz + 2); - match_res = copy_struct(match_res, sz, &hp, &MSO(p)); - } match_list = CONS(hp, match_res, match_list); ++got; } @@ -1590,7 +1586,6 @@ static int db_select_count_hash(Process *p, Uint slot_ix = 0; HashDbTerm* current = NULL; unsigned current_list_pos = 0; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1640,8 +1635,8 @@ static int db_select_count_hash(Process *p, for(;;) { if (current != NULL) { if (current->hvalue != INVALID_HASH) { - if (db_prog_match(p, mpi.mp, make_tuple(current->dbterm.tpl), - 0, &dummy) == am_true) { + if (db_match_dbterm(&tb->common, p, mpi.mp, 0, + ¤t->dbterm, NULL,0) == am_true) { ++got; } --num_left; @@ -1709,7 +1704,6 @@ static int db_select_delete_hash(Process *p, Uint slot_ix = 0; HashDbTerm **current = NULL; unsigned current_list_pos = 0; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1719,9 +1713,9 @@ static int db_select_delete_hash(Process *p, Eterm mpb; Eterm egot; #ifdef ERTS_SMP - int fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */ + erts_aint_t fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */ #else - int fixated_by_me = 0; + erts_aint_t fixated_by_me = 0; #endif erts_smp_rwmtx_t* lck; @@ -1790,9 +1784,8 @@ static int db_select_delete_hash(Process *p, } else { int did_erase = 0; - if ((db_prog_match(p,mpi.mp, - make_tuple((*current)->dbterm.tpl), - 0,&dummy)) == am_true) { + if (db_match_dbterm(&tb->common, p, mpi.mp, 0, + &(*current)->dbterm, NULL, 0) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -1855,7 +1848,6 @@ static int db_select_delete_continue_hash(Process *p, Uint slot_ix; Uint last_pseudo_delete = (Uint)-1; HashDbTerm **current = NULL; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got; @@ -1903,8 +1895,8 @@ static int db_select_delete_continue_hash(Process *p, } else { int did_erase = 0; - if ((db_prog_match(p,mp,make_tuple((*current)->dbterm.tpl), - 0,&dummy)) == am_true) { + if (db_match_dbterm(&tb->common, p, mp, 0, + &(*current)->dbterm, NULL, 0) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -1966,7 +1958,6 @@ static int db_select_count_continue_hash(Process *p, DbTableHash *tb = &tbl->hash; Uint slot_ix; HashDbTerm* current; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got; @@ -2004,8 +1995,8 @@ static int db_select_count_continue_hash(Process *p, current = current->next; continue; } - if (db_prog_match(p, mp, make_tuple(current->dbterm.tpl), - 0,&dummy) == am_true) { + if (db_match_dbterm(&tb->common, p, mp, 0, ¤t->dbterm, + NULL, 0) == am_true) { ++got; } --num_left; @@ -2095,7 +2086,14 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) while(list != 0) { if (list->hvalue == INVALID_HASH) erts_print(to, to_arg, "*"); - erts_print(to, to_arg, "%T", make_tuple(list->dbterm.tpl)); + if (tb->common.compress) { + Eterm key = GETKEY(tb, list->dbterm.tpl); + erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl); + } + else { + Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl); + erts_print(to, to_arg, "%R", obj, list->dbterm.tpl); + } if (list->next != 0) erts_print(to, to_arg, ","); list = list->next; @@ -2131,11 +2129,11 @@ static int db_free_table_continue_hash(DbTable *tbl) sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); if (++done >= 2*DELETE_RECORD_LIMIT) { - erts_smp_atomic_set(&tb->fixdel, (long)fixdel); + erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)fixdel); return 0; /* Not done */ } } - erts_smp_atomic_set(&tb->fixdel, (long)NULL); + erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)NULL); done /= 2; while(tb->nslots != 0) { @@ -2184,7 +2182,7 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, HashValue hval = NIL; int num_heads = 0; int i; - + mpi->lists = mpi->dlists; mpi->num_lists = 0; mpi->key_given = 1; @@ -2352,7 +2350,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set(&tb->segtab, (long) eseg->segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2424,7 +2422,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set(&tb->segtab, (long)newtop->prev_segtab); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2441,7 +2439,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set(&tb->segtab, (long)NULL); + erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2450,31 +2448,19 @@ static int free_seg(DbTableHash *tb, int free_records) } -static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, - Eterm obj, HashValue hval) -{ - HashDbTerm* p = db_get_term((DbTableCommon *) tb, - (old != NULL) ? &(old->dbterm) : NULL, - ((char *) &(old->dbterm)) - ((char *) old), - obj); - p->hvalue = hval; - /*p->next = NULL;*/ /*No Need */ - return p; -} - - /* ** Copy terms from ptr1 until ptr2 ** works for ptr1 == ptr2 == 0 => [] ** or ptr2 == 0 */ -static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2) +static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, + DbTableHash* tb) { int sz = 0; HashDbTerm* ptr; Eterm list = NIL; Eterm copy; - Eterm *hp; + Eterm *hp, *hend; ptr = ptr1; while(ptr != ptr2) { @@ -2486,26 +2472,20 @@ static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2) } hp = HAlloc(p, sz); + hend = hp + sz; ptr = ptr1; while(ptr != ptr2) { if (ptr->hvalue != INVALID_HASH) { - copy = copy_shallow(DBTERM_BUF(&ptr->dbterm), ptr->dbterm.size, &hp, &MSO(p)); + copy = db_copy_object_from_ets(&tb->common, &ptr->dbterm, &hp, &MSO(p)); list = CONS(hp, copy, list); hp += 2; } ptr = ptr->next; } - return list; -} + HRelease(p,hend,hp); -static void free_term(DbTableHash *tb, HashDbTerm* p) -{ - db_free_term_data(&(p->dbterm)); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (void *) p, - SIZ_DBTERM(p)*sizeof(Eterm)); + return list; } /* Grow table with one new bucket. @@ -2554,9 +2534,9 @@ static void grow(DbTableHash* tb, int nactive) } erts_smp_atomic_inc(&tb->nactive); if (from_ix == 0) { - erts_smp_atomic_set(&tb->szm, szm); + erts_smp_atomic_set_relb(&tb->szm, szm); } - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); /* Finally, let's split the bucket. We try to do it in a smart way to keep link order and avoid unnecessary updates of next-pointers */ @@ -2588,7 +2568,7 @@ static void grow(DbTableHash* tb, int nactive) return; abort: - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } @@ -2632,7 +2612,7 @@ static void shrink(DbTableHash* tb, int nactive) erts_smp_atomic_set(&tb->nactive, src_ix); if (dst_ix == 0) { - erts_smp_atomic_set(&tb->szm, low_szm); + erts_smp_atomic_set_relb(&tb->szm, low_szm); } WUNLOCK_HASH(lck); @@ -2646,7 +2626,7 @@ static void shrink(DbTableHash* tb, int nactive) } /*else already done */ - erts_smp_atomic_set(&tb->is_resizing, 0); + erts_smp_atomic_set_relb(&tb->is_resizing, 0); } @@ -2716,8 +2696,11 @@ static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle handle->tb = tbl; handle->bp = (void**) prevp; handle->dbterm = &b->dbterm; - handle->new_size = b->dbterm.size; handle->mustResize = 0; + handle->new_size = b->dbterm.size; + #if HALFWORD_HEAP + handle->abs_vec = NULL; + #endif handle->lck = lck; /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */ return 1; @@ -2738,39 +2721,14 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck)); /* locked by db_lookup_dbterm_hash */ - ASSERT(&oldp->dbterm == handle->dbterm); - if (handle->mustResize) { - Eterm* top; - Eterm copy; - DbTerm* newDbTerm; - HashDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, - sizeof(HashDbTerm)+sizeof(Eterm)*(handle->new_size-1)); - sys_memcpy(newp, oldp, sizeof(HashDbTerm)-sizeof(DbTerm)); /* copy only hashtab header */ - *(handle->bp) = newp; - newDbTerm = &newp->dbterm; - - newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif - newDbTerm->off_heap.overhead = 0; - - /* make a flat copy */ - top = DBTERM_BUF(newDbTerm); - copy = copy_struct(make_tuple(handle->dbterm->tpl), - handle->new_size, - &top, &newDbTerm->off_heap); - DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); + ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize)); + if (handle->mustResize) { + db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); WUNLOCK_HASH(lck); - - db_free_term_data(handle->dbterm); - erts_db_free(ERTS_ALC_T_DB_TERM, tbl, - (void *) (((char *) handle->dbterm) - (sizeof(HashDbTerm) - sizeof(DbTerm))), - sizeof(HashDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1)); + + free_term(&tbl->hash, oldp); } else { WUNLOCK_HASH(lck); @@ -2779,7 +2737,7 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) handle->dbterm = 0; #endif return; -} +} static int db_delete_all_objects_hash(Process* p, DbTable* tbl) { @@ -2805,7 +2763,11 @@ void db_foreach_offheap_hash(DbTable *tbl, for (i = 0; i < nactive; i++) { list = BUCKET(tb,i); while(list != 0) { - (*func)(&(list->dbterm.off_heap), arg); + ErlOffHeap tmp_offheap; + tmp_offheap.first = list->dbterm.first_oh; + tmp_offheap.overhead = 0; + (*func)(&tmp_offheap, arg); + list->dbterm.first_oh = tmp_offheap.first; list = list->next; } } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index b421da591b..9a0ba3a418 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -48,9 +48,6 @@ #include "erl_db_tree.h" - - -#define GETKEY(dtt, tplp) (*((tplp) + (dtt)->common.keypos)) #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos)) #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) @@ -114,7 +111,7 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack) { if (stack == &tb->static_stack) { ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1); - erts_smp_atomic_set(&tb->is_stack_busy, 0); + erts_smp_atomic_set_relb(&tb->is_stack_busy, 0); } else { erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb, @@ -122,12 +119,41 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack) } } -static void reset_static_stack(DbTableTree* tb) +static ERTS_INLINE void reset_static_stack(DbTableTree* tb) { tb->static_stack.pos = 0; tb->static_stack.slot = 0; } +static ERTS_INLINE void free_term(DbTableTree *tb, TreeDbTerm* p) +{ + db_free_term((DbTable*)tb, p, offsetof(TreeDbTerm, dbterm)); +} + +static ERTS_INLINE TreeDbTerm* new_dbterm(DbTableTree *tb, Eterm obj) +{ + TreeDbTerm* p; + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + return p; +} +static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old, + Eterm obj) +{ + TreeDbTerm* p; + ASSERT(old != NULL); + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); + } + return p; +} /* ** Some macros for "direction stacks" @@ -153,7 +179,7 @@ static void reset_static_stack(DbTableTree* tb) static TreeDbTerm *traverse_until(TreeDbTerm *t, int *current, int to); static void check_slot_pos(DbTableTree *tb); static void check_saved_stack(DbTableTree *tb); -static int check_table_tree(TreeDbTerm *t); +static int check_table_tree(DbTableTree* tb, TreeDbTerm *t); #define TREE_DEBUG #endif @@ -168,8 +194,8 @@ static int check_table_tree(TreeDbTerm *t); ** Debugging dump */ -static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, - int offset); +static void do_dump_tree2(DbTableTree*, int to, void *to_arg, int show, + TreeDbTerm *t, int offset); #else @@ -178,12 +204,6 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, #endif /* - * Size calculations - */ -#define SIZ_OVERHEAD ((sizeof(TreeDbTerm)/sizeof(Eterm)) - 1) -#define SIZ_DBTERM(TDT) (SIZ_OVERHEAD + (TDT)->dbterm.size) - -/* ** Datatypes */ @@ -259,13 +279,10 @@ struct select_delete_context { /* ** Forward declarations */ -static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key); +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base); static TreeDbTerm *linkout_object_tree(DbTableTree *tb, Eterm object); static int do_free_tree_cont(DbTableTree *tb, int num_left); -static TreeDbTerm* get_term(DbTableTree *tb, - TreeDbTerm* old, - Eterm obj); static void free_term(DbTableTree *tb, TreeDbTerm* p); static int balance_left(TreeDbTerm **this); static int balance_right(TreeDbTerm **this); @@ -273,15 +290,15 @@ static int delsub(TreeDbTerm **this); static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot); static TreeDbTerm *find_node(DbTableTree *tb, Eterm key); static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key); -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key); -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key); +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static void traverse_backwards(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -289,7 +306,7 @@ static void traverse_backwards(DbTableTree *tb, void *context); static void traverse_forward(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -297,8 +314,8 @@ static void traverse_forward(DbTableTree *tb, void *context); static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, Eterm *partly_bound_key); -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); -static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done); +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base); +static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done); static int analyze_pattern(DbTableTree *tb, Eterm pattern, struct mp_info *mpi); @@ -318,7 +335,6 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, int forward); -static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t); static int partly_bound_can_match_lesser(Eterm partly_bound_1, Eterm partly_bound_2); @@ -443,9 +459,9 @@ void db_initialize_tree(void) ets_select_reverse_exp.code[1] = am_reverse; ets_select_reverse_exp.code[2] = 3; ets_select_reverse_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; ets_select_reverse_exp.code[4] = - (Eterm) &ets_select_reverse; + (BeamInstr) &ets_select_reverse; return; }; @@ -472,9 +488,6 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret) DbTableTree *tb = &tbl->tree; DbTreeStack* stack; TreeDbTerm *this; - Eterm e; - Eterm *hp; - Uint sz; if (( this = tb->root ) == NULL) { *ret = am_EOT; @@ -493,13 +506,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret) stack->slot = 1; release_stack(tb,stack); } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); - - hp = HAlloc(p, sz); - - *ret = copy_struct(e,sz,&hp,&MSO(p)); - + *ret = db_copy_key(p, tbl, &this->dbterm); return DB_ERROR_NONE; } @@ -508,26 +515,17 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) DbTableTree *tb = &tbl->tree; DbTreeStack* stack; TreeDbTerm *this; - Eterm e; - Eterm *hp; - Uint sz; if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_next(tb, stack, key); + this = find_next(tb, stack, key, NULL); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; return DB_ERROR_NONE; } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); - - hp = HAlloc(p, sz); - - *ret = copy_struct(e,sz,&hp,&MSO(p)); - + *ret = db_copy_key(p, tbl, &this->dbterm); return DB_ERROR_NONE; } @@ -536,9 +534,6 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret) DbTableTree *tb = &tbl->tree; TreeDbTerm *this; DbTreeStack* stack; - Eterm e; - Eterm *hp; - Uint sz; if (( this = tb->root ) == NULL) { *ret = am_EOT; @@ -557,13 +552,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret) stack->slot = NITEMS(tb); release_stack(tb,stack); } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); - - hp = HAlloc(p, sz); - - *ret = copy_struct(e,sz,&hp,&MSO(p)); - + *ret = db_copy_key(p, tbl, &this->dbterm); return DB_ERROR_NONE; } @@ -572,27 +561,33 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) DbTableTree *tb = &tbl->tree; TreeDbTerm *this; DbTreeStack* stack; - Eterm e; - Eterm *hp; - Uint sz; if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_prev(tb, stack, key); + this = find_prev(tb, stack, key, NULL); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; return DB_ERROR_NONE; } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); + *ret = db_copy_key(p, tbl, &this->dbterm); + return DB_ERROR_NONE; +} - hp = HAlloc(p, sz); +static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base, + TreeDbTerm* obj) +{ + return cmp_rel(key, key_base, + GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl); +} - *ret = copy_struct(e,sz,&hp,&MSO(p)); - - return DB_ERROR_NONE; +static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base, + TreeDbTerm* obj) +{ + Eterm obj_key = GETKEY(tb,obj->dbterm.tpl); + return is_same(key, key_base, obj_key, obj->dbterm.tpl) + || cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0; } static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) @@ -622,12 +617,12 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) erts_smp_atomic_dec(&tb->common.nitems); return DB_ERROR_SYSRES; } - *this = get_term(tb, NULL, obj); + *this = new_dbterm(tb, obj); (*this)->balance = 0; (*this)->left = (*this)->right = NULL; break; - } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { - /* go left */ + } else if ((c = cmp_key(tb, key, NULL, *this)) < 0) { + /* go lefts */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -636,7 +631,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) tstack[tpos++] = this; this = &((*this)->right); } else if (!key_clash_fail) { /* Equal key and this is a set, replace. */ - *this = get_term(tb, *this, obj); + *this = replace_dbterm(tb, *this, obj); break; } else { return DB_ERROR_BADKEY; /* key already exists */ @@ -714,7 +709,7 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableTree *tb = &tbl->tree; Eterm copy; - Eterm *hp; + Eterm *hp, *hend; TreeDbTerm *this; /* @@ -728,11 +723,11 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) *ret = NIL; } else { hp = HAlloc(p, this->dbterm.size + 2); - copy = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(p)); + hend = hp + this->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, &this->dbterm, &hp, &MSO(p)); *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p,hend,hp); } return DB_ERROR_NONE; } @@ -766,18 +761,10 @@ static int db_get_element_tree(Process *p, DbTable *tbl, if (this == NULL) { return DB_ERROR_BADKEY; } else { - Eterm element; - Uint sz; if (ndex > arityval(this->dbterm.tpl[0])) { return DB_ERROR_BADPARAM; } - element = this->dbterm.tpl[ndex]; - sz = size_object(element); - hp = HAlloc(p, sz); - *ret = copy_struct(element, - sz, - &hp, - &MSO(p)); + *ret = db_copy_element_from_ets(&tb->common, p, &this->dbterm, ndex, &hp, 0); } return DB_ERROR_NONE; } @@ -789,7 +776,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret) *ret = am_true; - if ((res = linkout_tree(tb, key)) != NULL) { + if ((res = linkout_tree(tb, key, NULL)) != NULL) { free_term(tb, res); } return DB_ERROR_NONE; @@ -815,7 +802,7 @@ static int db_slot_tree(Process *p, DbTable *tbl, DbTableTree *tb = &tbl->tree; Sint slot; TreeDbTerm *st; - Eterm *hp; + Eterm *hp, *hend; Eterm copy; /* @@ -847,11 +834,11 @@ static int db_slot_tree(Process *p, DbTable *tbl, return DB_ERROR_UNSPEC; } hp = HAlloc(p, st->dbterm.size + 2); - copy = copy_shallow(DBTERM_BUF(&st->dbterm), - st->dbterm.size, - &hp, - &MSO(p)); + hend = hp + st->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, &st->dbterm, &hp, &MSO(p)); *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p,hend,hp); return DB_ERROR_NONE; } @@ -981,15 +968,15 @@ static int db_select_continue_tree(Process *p, stack = get_any_stack(tb); if (chunk_size) { if (reverse) { - traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); } else { - traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); } } else { if (reverse) { - traverse_forward(tb, stack, lastkey, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc); } else { - traverse_backwards(tb, stack, lastkey, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc); } } release_stack(tb,stack); @@ -1014,10 +1001,9 @@ static int db_select_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); - - sz = size_object(key); + sz = size_object_rel(key,sc.lastobj); hp = HAlloc(p, 9 + sz); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); continuation = TUPLE8 (hp, tptr[1], @@ -1038,8 +1024,8 @@ static int db_select_continue_tree(Process *p, key = GETKEY(tb, sc.lastobj); if (chunk_size) { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key) < 0) || - (reverse && cmp_partly_bound(end_condition,key) > 0))) { + ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) || + (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) { /* done anyway */ if (!sc.got) { RET_TO_BIF(am_EOT, DB_ERROR_NONE); @@ -1051,16 +1037,16 @@ static int db_select_continue_tree(Process *p, } } else { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key) > 0) || - (reverse && cmp_partly_bound(end_condition,key) < 0))) { + ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) || + (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) { /* done anyway */ RET_TO_BIF(sc.accum,DB_ERROR_NONE); } } /* Not done yet, let's trap. */ - sz = size_object(key); + sz = size_object_rel(key,sc.lastobj); hp = HAlloc(p, 9 + sz); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); continuation = TUPLE8 (hp, tptr[1], @@ -1081,11 +1067,13 @@ static int db_select_continue_tree(Process *p, static int db_select_tree(Process *p, DbTable *tbl, Eterm pattern, int reverse, Eterm *ret) { + /* Strategy: Traverse backwards to build resulting list from tail to head */ DbTableTree *tb = &tbl->tree; DbTreeStack* stack; struct select_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1127,7 +1115,7 @@ static int db_select_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */); RET_TO_BIF(sc.accum,DB_ERROR_NONE); } @@ -1137,20 +1125,20 @@ static int db_select_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - - traverse_forward(tb, stack, lastkey, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc); } else { if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - - traverse_backwards(tb, stack, lastkey, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc); } release_stack(tb,stack); #ifdef HARDDEBUG @@ -1163,9 +1151,9 @@ static int db_select_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb=db_make_mp_binary(p,mpi.mp,&hp); @@ -1246,7 +1234,7 @@ static int db_select_count_continue_tree(Process *p, } stack = get_any_stack(tb); - traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); @@ -1256,12 +1244,12 @@ static int db_select_count_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); if (end_condition != NIL && - (cmp_partly_bound(end_condition,key) > 0)) { + (cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) { /* done anyway */ RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + 6); egot = make_small(sc.got); @@ -1271,7 +1259,7 @@ static int db_select_count_continue_tree(Process *p, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); continuation = TUPLE5 (hp, tptr[1], @@ -1293,7 +1281,8 @@ static int db_select_count_tree(Process *p, DbTable *tbl, DbTreeStack* stack; struct select_count_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1334,7 +1323,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */); RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE); } @@ -1343,11 +1332,12 @@ static int db_select_count_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1355,7 +1345,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); egot = make_small(sc.got); @@ -1365,7 +1355,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1395,7 +1385,8 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, DbTreeStack* stack; struct select_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1437,7 +1428,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); if (sc.accum != NIL) { hp=HAlloc(p, 3); @@ -1452,20 +1443,20 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - - traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); } else { if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - - traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); } release_stack(tb,stack); @@ -1490,9 +1481,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1515,9 +1506,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; @@ -1593,7 +1584,7 @@ static int db_select_delete_continue_tree(Process *p, sc.keypos = tb->common.keypos; ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy)); - traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -1602,11 +1593,11 @@ static int db_select_delete_continue_tree(Process *p, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); if (end_condition != NIL && - cmp_partly_bound(end_condition,key) > 0) { /* done anyway */ + cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */ RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object(key); + sz = size_object_rel(key, sc.lastterm->dbterm.tpl); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + 6); eaccsum = make_small(sc.accum); @@ -1616,7 +1607,7 @@ static int db_select_delete_continue_tree(Process *p, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); continuation = TUPLE5 (hp, tptr[1], @@ -1636,7 +1627,8 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, DbTableTree *tb = &tbl->tree; struct select_delete_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1680,7 +1672,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, sc.mp = mpi.mp; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); @@ -1689,11 +1681,12 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1701,7 +1694,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); - sz = size_object(key); + sz = size_object_rel(key, sc.lastterm->dbterm.tpl); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); eaccsum = make_small(sc.accum); @@ -1711,7 +1704,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); mpb = db_make_mp_binary(p,mpi.mp,&hp); continuation = TUPLE5 @@ -1737,7 +1730,8 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, ** Other interface routines (not directly coupled to one bif) */ -/* Display hash table contents (for dump) */ + +/* Display tree contents (for dump) */ static void db_print_tree(int to, void *to_arg, int show, DbTable *tbl) @@ -1747,13 +1741,12 @@ static void db_print_tree(int to, void *to_arg, if (show) erts_print(to, to_arg, "\nTree data dump:\n" "------------------------------------------------\n"); - do_dump_tree2(to, to_arg, show, tb->root, 0); + do_dump_tree2(&tbl->tree, to, to_arg, show, tb->root, 0); if (show) erts_print(to, to_arg, "\n" "------------------------------------------------\n"); #else erts_print(to, to_arg, "Ordered set (AVL tree), Elements: %d\n", NITEMS(tb)); - do_dump_tree(to, to_arg, tb->root); #endif } @@ -1817,15 +1810,19 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt, void (*func)(ErlOffHeap *, void *), void * arg) { + ErlOffHeap tmp_offheap; if(!tdbt) return; do_db_tree_foreach_offheap(tdbt->left, func, arg); - (*func)(&(tdbt->dbterm.off_heap), arg); + tmp_offheap.first = tdbt->dbterm.first_oh; + tmp_offheap.overhead = 0; + (*func)(&tmp_offheap, arg); + tdbt->dbterm.first_oh = tmp_offheap.first; do_db_tree_foreach_offheap(tdbt->right, func, arg); } static TreeDbTerm *linkout_tree(DbTableTree *tb, - Eterm key) + Eterm key, Eterm* key_base) { TreeDbTerm **tstack[STACK_NEED]; int tpos = 0; @@ -1848,7 +1845,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { + } else if ((c = cmp_key(tb, key, key_base, *this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -1912,7 +1909,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { + } else if ((c = cmp_key(tb,key,NULL,*this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -1921,7 +1918,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, tstack[tpos++] = this; this = &((*this)->right); } else { /* Equal key, found the only possible matching object*/ - if (!eq(object,make_tuple((*this)->dbterm.tpl))) { + if (!db_eq(&tb->common,object,&(*this)->dbterm)) { return NULL; } q = (*this); @@ -2065,24 +2062,6 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern, return DB_ERROR_NONE; } -static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t) -{ - if (t != NULL) { - do_dump_tree(to, to_arg, t->left); - erts_print(to, to_arg, "%T\n", make_tuple(t->dbterm.tpl)); - do_dump_tree(to, to_arg, t->right); - } -} - -static void free_term(DbTableTree *tb, TreeDbTerm* p) -{ - db_free_term_data(&(p->dbterm)); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (void *) p, - SIZ_DBTERM(p)*sizeof(Uint)); -} - static int do_free_tree_cont(DbTableTree *tb, int num_left) { TreeDbTerm *root; @@ -2113,17 +2092,6 @@ static int do_free_tree_cont(DbTableTree *tb, int num_left) return 1; } -static TreeDbTerm* get_term(DbTableTree *tb, - TreeDbTerm* old, - Eterm obj) -{ - TreeDbTerm* p = db_get_term((DbTableCommon *) tb, - (old != NULL) ? &(old->dbterm) : NULL, - ((char *) &(old->dbterm)) - ((char *) old), - obj); - return p; -} - /* * Deletion helpers */ @@ -2327,14 +2295,15 @@ done: * Find next and previous in sort order */ -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, + Eterm key, Eterm* key_base) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) { + if (!cmp_key_eq(tb,key,key_base,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2344,14 +2313,14 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) < 0) { + if (( c = cmp_key(tb,key,key_base,this) ) > 0) { if (this->right == NULL) /* We are at the previos and the element does not exist */ break; else this = this->right; - } else if (c > 0) { + } else if (c < 0) { if (this->left == NULL) /* Done */ return this; else @@ -2384,14 +2353,15 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) return this; } -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, + Eterm key, Eterm* key_base) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) { + if (!cmp_key_eq(tb,key,key_base,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2401,14 +2371,14 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) > 0) { + if (( c = cmp_key(tb,key,key_base,this) ) < 0) { if (this->left == NULL) /* We are at the next and the element does not exist */ break; else this = this->left; - } else if (c < 0) { + } else if (c > 0) { if (this->right == NULL) /* Done */ return this; else @@ -2454,7 +2424,8 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) >= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), + this->dbterm.tpl) ) >= 0) { if (this->right == NULL) { do { tmp = POP_NODE(stack); @@ -2487,7 +2458,8 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) <= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), + this->dbterm.tpl) ) <= 0) { if (this->left == NULL) { do { tmp = POP_NODE(stack); @@ -2517,12 +2489,11 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key) Sint res; DbTreeStack* stack = get_static_stack(tb); - if(!stack || EMPTY_NODE(stack) - || !CMP_EQ(GETKEY(tb, ( this = TOP_NODE(stack) )->dbterm.tpl), key)) { + if(!stack || EMPTY_NODE(stack) + || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) { this = tb->root; - while (this != NULL && - ( res = cmp(key, GETKEY(tb, this->dbterm.tpl)) ) != 0) { + while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) { if (res < 0) this = this->left; else @@ -2544,8 +2515,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) Sint res; this = &tb->root; - while ((*this) != NULL && - ( res = cmp(key, GETKEY(tb, (*this)->dbterm.tpl)) ) != 0) { + while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) { if (res < 0) this = &((*this)->left); else @@ -2565,48 +2535,24 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle handle->tb = tbl; handle->dbterm = &(*pp)->dbterm; + handle->mustResize = 0; handle->bp = (void**) pp; handle->new_size = (*pp)->dbterm.size; - handle->mustResize = 0; +#if HALFWORD_HEAP + handle->abs_vec = NULL; +#endif return 1; } static void db_finalize_dbterm_tree(DbUpdateHandle* handle) { if (handle->mustResize) { - Eterm* top; - Eterm copy; - DbTerm* newDbTerm; - DbTableTree *tb = &handle->tb->tree; TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp; - TreeDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - handle->tb, - sizeof(TreeDbTerm)+sizeof(Eterm)*(handle->new_size-1)); - memcpy(newp, oldp, sizeof(TreeDbTerm)-sizeof(DbTerm)); /* copy only tree header */ - *(handle->bp) = newp; - reset_static_stack(tb); - newDbTerm = &newp->dbterm; - - newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif - newDbTerm->off_heap.overhead = 0; - - /* make a flat copy */ - top = DBTERM_BUF(newDbTerm); - copy = copy_struct(make_tuple(handle->dbterm->tpl), - handle->new_size, - &top, &newDbTerm->off_heap); - DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); - - db_free_term_data(handle->dbterm); - erts_db_free(ERTS_ALC_T_DB_TERM, - handle->tb, - (void *) (((char *) handle->dbterm) - (sizeof(TreeDbTerm) - sizeof(DbTerm))), - sizeof(TreeDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1)); + + db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm)); + reset_static_stack(&handle->tb->tree); + + free_term(&handle->tb->tree, oldp); } #ifdef DEBUG handle->dbterm = 0; @@ -2619,7 +2565,7 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle) */ static void traverse_backwards(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2628,7 +2574,7 @@ static void traverse_backwards(DbTableTree *tb, { TreeDbTerm *this, *next; - if (lastkey == NIL) { + if (lastkey == THE_NON_VALUE) { stack->pos = stack->slot = 0; if (( this = tb->root ) == NULL) { return; @@ -2638,15 +2584,16 @@ static void traverse_backwards(DbTableTree *tb, this = this->right; } this = TOP_NODE(stack); - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), + this->dbterm.tpl); if (!((*doit)(tb, this, context, 0))) return; } else { - next = find_prev(tb, stack, lastkey); + next = find_prev(tb, stack, lastkey, lk_base); } while ((this = next) != NULL) { - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); if (!((*doit)(tb, this, context, 0))) return; } @@ -2657,7 +2604,7 @@ static void traverse_backwards(DbTableTree *tb, */ static void traverse_forward(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2666,7 +2613,7 @@ static void traverse_forward(DbTableTree *tb, { TreeDbTerm *this, *next; - if (lastkey == NIL) { + if (lastkey == THE_NON_VALUE) { stack->pos = stack->slot = 0; if (( this = tb->root ) == NULL) { return; @@ -2676,15 +2623,15 @@ static void traverse_forward(DbTableTree *tb, this = this->left; } this = TOP_NODE(stack); - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); if (!((*doit)(tb, this, context, 1))) return; } else { - next = find_next(tb, stack, lastkey); + next = find_next(tb, stack, lastkey, lk_base); } while ((this = next) != NULL) { - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); if (!((*doit)(tb, this, context, 1))) return; } @@ -2710,7 +2657,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, if (( this = find_node(tb, key) ) == NULL) { return -1; } - *ret = this; + *ret = this; return 1; } else if (partly_bound != NULL && key != am_Underscore && db_is_variable(key) < 0) @@ -2721,7 +2668,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, -static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) +static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) { Eterm* aa; Eterm* bb; @@ -2735,44 +2682,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) *done = 1; return 0; } - if (a == b) + if (is_same(a,NULL,b,b_base)) return 0; switch (a & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_LIST: if (!is_list(b)) { - return cmp(a,b); + return cmp_rel(a,NULL,b,b_base); } aa = list_val(a); - bb = list_val(b); + bb = list_val_rel(b,b_base); while (1) { - if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done) + if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done) return j; - if (*aa==*bb) + if (is_same(*aa, NULL, *bb, b_base)) return 0; if (is_not_list(*aa) || is_not_list(*bb)) - return do_cmp_partly_bound(*aa, *bb, done); + return do_cmp_partly_bound(*aa, *bb, b_base, done); aa = list_val(*aa); - bb = list_val(*bb); + bb = list_val_rel(*bb,b_base); } case TAG_PRIMARY_BOXED: if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) { - return cmp(a,b); + return cmp_rel(a,NULL,b,b_base); } a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; - b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; + b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; if (a_hdr != b_hdr) { - return cmp(a, b); + return cmp_rel(a, NULL, b, b_base); } if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) { aa = tuple_val(a); - bb = tuple_val(b); + bb = tuple_val_rel(b, b_base); /* compare the arities */ i = arityval(*aa); /* get the arity*/ if (i < arityval(*bb)) return(-1); if (i > arityval(*bb)) return(1); while (i--) { - if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0 + if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0 || *done) return j; } @@ -2780,14 +2727,14 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) } /* Drop through */ default: - return cmp(a, b); + return cmp_rel(a, NULL, b, b_base); } } -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base) { int done = 0; - Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done); + Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done); #ifdef HARDDEBUG erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key); if (ret < 0) @@ -2796,7 +2743,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) erts_fprintf(stderr," > "); else erts_fprintf(stderr," == "); - erts_fprintf(stderr,"%T\n",bound_key); + erts_fprintf(stderr,"%R\n", bound_key, bk_base); #endif return ret; } @@ -2883,7 +2830,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b, if (not_eq_tags(a,b)) { *done = 1; - return (cmp(a, b) < 0) ? 1 : 0; + return (CMP(a, b) < 0) ? 1 : 0; } /* we now know that tags are the same */ @@ -2919,7 +2866,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b, bb = list_val(*bb); } default: - if((i = cmp(a, b)) != 0) { + if((i = CMP(a, b)) != 0) { *done = 1; } return (i < 0) ? 1 : 0; @@ -2954,7 +2901,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b, if (not_eq_tags(a,b)) { *done = 1; - return (cmp(a, b) > 0) ? 1 : 0; + return (CMP(a, b) > 0) ? 1 : 0; } /* we now know that tags are the same */ @@ -2990,7 +2937,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b, bb = list_val(*bb); } default: - if((i = cmp(a, b)) != 0) { + if((i = CMP(a, b)) != 0) { *done = 1; } return (i > 0) ? 1 : 0; @@ -3006,39 +2953,24 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_context *sc = (struct select_context *) ptr; Eterm ret; - Uint32 dummy; + Eterm* hp; sc->lastobj = this->dbterm.tpl; if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0))) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - 0, &dummy); + ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects, + &this->dbterm, &hp, 2); if (is_value(ret)) { - Uint sz; - Eterm *hp; - if (sc->all_objects) { - hp = HAlloc(sc->p, this->dbterm.size + 2); - ret = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(sc->p)); - } else { - sz = size_object(ret); - hp = HAlloc(sc->p, sz + 2); - ret = copy_struct(ret, sz, - &hp, &MSO(sc->p)); - } sc->accum = CONS(hp, ret, sc->accum); } if (MBUF(sc->p)) { @@ -3059,20 +2991,18 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_count_context *sc = (struct select_count_context *) ptr; Eterm ret; - Uint32 dummy; sc->lastobj = this->dbterm.tpl; /* Always backwards traversing */ if (sc->end_condition != NIL && (cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0)) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0)) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - 0, &dummy); + ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, + &this->dbterm, NULL, 0); if (ret == am_true) { ++(sc->got); } @@ -3087,41 +3017,26 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_context *sc = (struct select_context *) ptr; Eterm ret; - Uint32 dummy; + Eterm* hp; sc->lastobj = this->dbterm.tpl; if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0))) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - 0, &dummy); + ret = db_match_dbterm(&tb->common, sc->p, sc->mp, sc->all_objects, + &this->dbterm, &hp, 2); if (is_value(ret)) { - Uint sz; - Eterm *hp; - ++(sc->got); - if (sc->all_objects) { - hp = HAlloc(sc->p, this->dbterm.size + 2); - ret = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(sc->p)); - } else { - sz = size_object(ret); - hp = HAlloc(sc->p, sz + 2); - ret = copy_struct(ret, sz, &hp, &MSO(sc->p)); - } sc->accum = CONS(hp, ret, sc->accum); } if (MBUF(sc->p)) { @@ -3143,7 +3058,6 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_delete_context *sc = (struct select_delete_context *) ptr; Eterm ret; - Uint32 dummy; Eterm key; if (sc->erase_lastterm) @@ -3153,15 +3067,14 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0) + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0) return 0; - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - 0, &dummy); + ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, + &this->dbterm, NULL, 0); if (ret == am_true) { key = GETKEY(sc->tb, this->dbterm.tpl); - linkout_tree(sc->tb, key); + linkout_tree(sc->tb, key, this->dbterm.tpl); sc->erase_lastterm = 1; ++sc->accum; } @@ -3172,19 +3085,28 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, } #ifdef TREE_DEBUG -static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, - int offset) +static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, + TreeDbTerm *t, int offset) { if (t == NULL) - return 0; - do_dump_tree2(to, to_arg, show, t->right, offset + 4); + return; + do_dump_tree2(tb, to, to_arg, show, t->right, offset + 4); if (show) { - erts_print(to, to_arg, "%*s%T (addr = %p, bal = %d)\n" - offset, "", make_tuple(t->dbterm.tpl), + const char* prefix; + Eterm term; + if (tb->common.compress) { + prefix = "key="; + term = GETKEY(tb, t->dbterm.tpl); + } + else { + prefix = ""; + term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl); + } + erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n", + offset, "", prefix, term, t->dbterm.tpl, t, t->balance); } - do_dump_tree2(to, to_arg, show, t->left, offset + 4); - return sum; + do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); } #endif @@ -3194,7 +3116,7 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, void db_check_table_tree(DbTable *tbl) { DbTableTree *tb = &tbl->tree; - check_table_tree(tb->root); + check_table_tree(tb, tb->root); check_saved_stack(tb); check_slot_pos(tb); } @@ -3225,7 +3147,7 @@ static void check_slot_pos(DbTableTree *tb) "element position %d is really 0x%08X, when stack says " "it's 0x%08X\n", tb->stack.slot, t, tb->stack.array[tb->stack.pos - 1]); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); } } @@ -3240,14 +3162,14 @@ static void check_saved_stack(DbTableTree *tb) if (t != stack->array[0]) { erts_fprintf(stderr,"tb->stack[0] is 0x%08X, should be 0x%08X\n", stack->array[0], t); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } while (n < stack->pos) { if (t == NULL) { erts_fprintf(stderr, "NULL pointer in tree when stack not empty," " stack depth is %d\n", n); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } n++; @@ -3261,28 +3183,26 @@ static void check_saved_stack(DbTableTree *tb) "represent child pointer in tree!" "(left == 0x%08X, right == 0x%08X\n", n, tb->stack[n], t->left, t->right); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0); return; } } } } -static int check_table_tree(TreeDbTerm *t) +static int check_table_tree(DbTableTree* tb, TreeDbTerm *t) { int lh, rh; if (t == NULL) return 0; - lh = check_table_tree(t->left); - rh = check_table_tree(t->right); + lh = check_table_tree(tb, t->left); + rh = check_table_tree(tb, t->right); if ((rh - lh) != t->balance) { erts_fprintf(stderr, "Invalid tree balance for this node:\n"); - erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n" - "data = %T", - t->balance, t->left, t->right, - make_tuple(t->dbterm.tpl)); + erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n", + t->balance, t->left, t->right); erts_fprintf(stderr,"\nDump:\n---------------------------------\n"); - do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, t, 0); + do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, t, 0); erts_fprintf(stderr,"\n---------------------------------\n"); } return ((rh > lh) ? rh : lh) + 1; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 8c373451fd..c3b074f782 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -25,7 +25,6 @@ #ifdef HAVE_CONFIG_H # include "config.h" #endif - #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -58,6 +57,7 @@ DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY +#define HEAP_XTRA 100 /* ** Some convenience macros for stacks (DMC == db_match_compile) @@ -117,6 +117,10 @@ do { \ erts_free(ERTS_ALC_T_DB_MC_STK, (Name).data); \ } while (0) + +#define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm)))) + + static ERTS_INLINE Process * get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks) { @@ -226,6 +230,11 @@ typedef enum { matchCall2, matchCall3, matchPushV, +#if HALFWORD_HEAP + matchPushVGuard, /* First guard-only variable reference */ +#endif + matchPushVResult, /* First variable reference in result, or (if HALFWORD) + in guard if also referenced in result */ matchPushExpr, /* Push the whole expression we're matching ('$_') */ matchPushArrayAsList, /* Only when parameter is an Array and not an erlang term (DCOMP_TRACE) */ @@ -281,7 +290,7 @@ typedef struct dmc_guard_bif { */ DMC_DECLARE_STACK_TYPE(Eterm); -DMC_DECLARE_STACK_TYPE(Uint); +DMC_DECLARE_STACK_TYPE(UWord); DMC_DECLARE_STACK_TYPE(unsigned); @@ -289,11 +298,19 @@ DMC_DECLARE_STACK_TYPE(unsigned); ** Data about the heap during compilation */ +typedef struct DMCVariable { + int is_bound; + int is_in_body; +#if HALFWORD_HEAP + int first_guard_label; /* to maybe change from PushVGuard to PushVResult */ +#endif +} DMCVariable; + typedef struct DMCHeap { int size; - unsigned def[DMC_DEFAULT_SIZE]; - unsigned *data; - int used; + DMCVariable vars_def[DMC_DEFAULT_SIZE]; + DMCVariable* vars; + int vars_used; } DMCHeap; /* @@ -320,7 +337,6 @@ typedef struct dmc_context { Eterm *bodyexpr; int num_match; int current_match; - int eheap_need; Uint cflags; int is_guard; /* 1 if in guard, 0 if in body */ int special; /* 1 if the head in the match was a single expression */ @@ -343,9 +359,22 @@ typedef struct dmc_context { #define ERTS_DEFAULT_MS_HEAP_SIZE 128 +/* Runtime info about a $-variable +*/ +typedef struct MatchVariable { + Eterm term; +#ifdef DEBUG + Process* proc; + Eterm* base; +#endif +} MatchVariable; + typedef struct { Process process; - Eterm *heap; + union { + Eterm* heap; + MatchVariable* variables; /* first on "heap" */ + }u; Eterm default_heap[ERTS_DEFAULT_MS_HEAP_SIZE]; } ErtsMatchPseudoProcess; @@ -359,12 +388,7 @@ static ErtsMatchPseudoProcess *match_pseudo_process; static ERTS_INLINE void cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) { - if (mpsp->process.mbuf - || mpsp->process.off_heap.mso -#ifndef HYBRID /* FIND ME! */ - || mpsp->process.off_heap.funs -#endif - || mpsp->process.off_heap.externals) { + if (mpsp->process.mbuf || mpsp->process.off_heap.first) { erts_cleanup_empty_process(&mpsp->process); } #ifdef DEBUG @@ -373,16 +397,16 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) } #endif if (!keep_heap) { - if (mpsp->heap != &mpsp->default_heap[0]) { + if (mpsp->u.heap != mpsp->default_heap) { /* Have to be done *after* call to erts_cleanup_empty_process() */ - erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->heap); - mpsp->heap = &mpsp->default_heap[0]; + erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->u.heap); + mpsp->u.heap = mpsp->default_heap; } #ifdef DEBUG else { int i; for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef; #else mpsp->default_heap[i] = (Eterm) 0xdeadbeef; @@ -400,7 +424,7 @@ create_match_pseudo_process(void) mpsp = (ErtsMatchPseudoProcess *)erts_alloc(ERTS_ALC_T_DB_MS_PSDO_PROC, sizeof(ErtsMatchPseudoProcess)); erts_init_empty_process(&mpsp->process); - mpsp->heap = &mpsp->default_heap[0]; + mpsp->u.heap = mpsp->default_heap; return mpsp; } @@ -424,11 +448,11 @@ get_match_pseudo_process(Process *c_p, Uint heap_size) mpsp = match_pseudo_process; cleanup_match_pseudo_process(mpsp, 0); #endif - if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE) - mpsp->heap = (Eterm *) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, - heap_size*sizeof(Uint)); + if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) { + mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size); + } else { - ASSERT(mpsp->heap == &mpsp->default_heap[0]); + ASSERT(mpsp->u.heap == mpsp->default_heap); } return mpsp; } @@ -469,23 +493,6 @@ erts_match_set_release_result(Process* c_p) static erts_smp_atomic_t trace_control_word; - -Eterm -erts_ets_copy_object(Eterm obj, Process* to) -{ - Uint size = size_object(obj); - Eterm* hp = HAlloc(to, size); - Eterm res; - - res = copy_struct(obj, size, &hp, &MSO(to)); -#ifdef DEBUG - if (eq(obj, res) == 0) { - erl_exit(1, "copy not equal to source\n"); - } -#endif - return res; -} - /* This needs to be here, before the bif table... */ static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val); @@ -830,50 +837,50 @@ static Uint my_size_object(Eterm t); static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap); /* Guard compilation */ -static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text, +static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t); static DMCRet dmc_list(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_tuple(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_fun(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet dmc_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); static DMCRet compile_guard_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t); /* match expression subroutine */ static DMCRet dmc_one_term(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(Eterm) *stack, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm c); #ifdef DMC_DEBUG static int test_disassemble_next = 0; -static void db_match_dis(Binary *prog); +void db_match_dis(Binary *prog); #define TRACE erts_fprintf(stderr,"Trace: %s:%d\n",__FILE__,__LINE__) -#define FENCE_PATTERN_SIZE 1 +#define FENCE_PATTERN_SIZE (1*sizeof(Uint)) #define FENCE_PATTERN 0xDEADBEEFUL #else #define TRACE /* Nothing */ @@ -891,6 +898,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace); static Eterm seq_trace_fake(Process *p, Eterm arg1); +static void db_free_tmp_uncompressed(DbTerm* obj); + /* ** Interface routines. @@ -915,7 +924,7 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (long) val); + old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (erts_aint_t) val); BIF_RET(erts_make_integer((Uint) old_tcw, p)); } @@ -1179,14 +1188,14 @@ done: } Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, Uint32 *return_flags) { Eterm ret; - ret = db_prog_match(p, mpsp, - (Eterm) args, - num_args, return_flags); + ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args, + in_flags, return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { erts_fprintf(stderr, "Failed\n"); @@ -1204,6 +1213,32 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp, */ } +static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, + Eterm args, int num_args, + Uint32 *return_flags) +{ + Eterm ret; + + ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args, + ERTS_PAM_CONTIGUOUS_TUPLE | ERTS_PAM_COPY_RESULT, + return_flags); +#if defined(HARDDEBUG) + if (is_non_value(ret)) { + erts_fprintf(stderr, "Failed\n"); + } else { + erts_fprintf(stderr, "Returning : %T\n", ret); + } +#endif + return ret; + /* Returns + * THE_NON_VALUE if no match + * am_false if {message,false} has been called, + * am_true if {message,_} has not been called or + * if {message,true} has been called, + * Msg if {message,Msg} has been called. + */ +} + /* ** API Used by other erl_db modules. */ @@ -1245,7 +1280,7 @@ Binary *db_match_compile(Eterm *matchexpr, { DMCHeap heap; DMC_STACK_TYPE(Eterm) stack; - DMC_STACK_TYPE(Uint) text; + DMC_STACK_TYPE(UWord) text; DMCContext context; MatchProg *ret = NULL; Eterm t; @@ -1254,7 +1289,6 @@ Binary *db_match_compile(Eterm *matchexpr, int structure_checked; DMCRet res; int current_try_label; - Uint max_eheap_need; Binary *bp = NULL; unsigned clause_start; @@ -1267,27 +1301,24 @@ Binary *db_match_compile(Eterm *matchexpr, context.matchexpr = matchexpr; context.guardexpr = guards; context.bodyexpr = body; - context.eheap_need = 0; context.err_info = err_info; context.cflags = flags; heap.size = DMC_DEFAULT_SIZE; - heap.data = heap.def; + heap.vars = heap.vars_def; /* ** Compile the match expression */ restart: - heap.used = 0; - max_eheap_need = 0; + heap.vars_used = 0; for (context.current_match = 0; context.current_match < num_progs; ++context.current_match) { /* This loop is long, too long */ - memset(heap.data, 0, heap.size * sizeof(*heap.data)); + memset(heap.vars, 0, heap.size * sizeof(*heap.vars)); t = context.matchexpr[context.current_match]; context.stack_used = 0; - context.eheap_need = 0; structure_checked = 0; if (context.current_match < num_progs - 1) { DMC_PUSH(text,matchTryMeElse); @@ -1380,7 +1411,7 @@ restart: /* ** There is one single top variable in the match expression - ** iff the text is tho Uint's and the single instruction + ** iff the text is two Uint's and the single instruction ** is 'matchBind' or it is only a skip. */ context.special = @@ -1459,10 +1490,6 @@ restart: if (current_try_label >= 0) { DMC_POKE(text, current_try_label, DMC_STACK_NUM(text)); } - /* So, how much eheap did this part of the match program need? */ - if (context.eheap_need > max_eheap_need) { - max_eheap_need = context.eheap_need; - } } /* for (context.current_match = 0 ...) */ @@ -1491,23 +1518,20 @@ restart: ** A special case is when the match expression is a single binding ** (i.e '$1'), then the field single_variable is set to 1. */ - bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(Uint)) + - (DMC_STACK_NUM(text) * sizeof(Uint))), + bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(UWord)) + + (DMC_STACK_NUM(text) * sizeof(UWord))), erts_db_match_prog_destructor); ret = Binary2MatchProg(bp); ret->saved_program_buf = NULL; ret->saved_program = NIL; ret->term_save = context.save; - ret->num_bindings = heap.used; + ret->num_bindings = heap.vars_used; ret->single_variable = context.special; sys_memcpy(ret->text, DMC_STACK_DATA(text), - DMC_STACK_NUM(text) * sizeof(Uint)); - ret->heap_size = ((heap.used * sizeof(Eterm)) + - (max_eheap_need * sizeof(Eterm)) + - (context.stack_need * sizeof(Eterm *)) + - (3 * (FENCE_PATTERN_SIZE * sizeof(Eterm *)))); - ret->eheap_offset = heap.used + FENCE_PATTERN_SIZE; - ret->stack_offset = ret->eheap_offset + max_eheap_need + FENCE_PATTERN_SIZE; + DMC_STACK_NUM(text) * sizeof(UWord)); + ret->stack_offset = heap.vars_used*sizeof(MatchVariable) + FENCE_PATTERN_SIZE; + ret->heap_size = ret->stack_offset + context.stack_need * sizeof(Eterm*) + FENCE_PATTERN_SIZE; + #ifdef DMC_DEBUG ret->prog_end = ret->text + DMC_STACK_NUM(text); #endif @@ -1517,17 +1541,16 @@ restart: */ context.save = NULL; error: /* Here is were we land when compilation failed. */ - while (context.save != NULL) { - ErlHeapFragment *ll = context.save->next; + if (context.save != NULL) { free_message_buffer(context.save); - context.save = ll; + context.save = NULL; } DMC_FREE(stack); DMC_FREE(text); if (context.copy != NULL) free_message_buffer(context.copy); - if (heap.data != heap.def) - erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.data); + if (heap.vars != heap.vars_def) + erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.vars); return bp; } @@ -1537,15 +1560,11 @@ error: /* Here is were we land when compilation failed. */ void erts_db_match_prog_destructor(Binary *bprog) { MatchProg *prog; - ErlHeapFragment *tmp, *ll; if (bprog == NULL) return; prog = Binary2MatchProg(bprog); - tmp = prog->term_save; - while (tmp != NULL) { - ll = tmp->next; - free_message_buffer(tmp); - tmp = ll; + if (prog->term_save != NULL) { + free_message_buffer(prog->term_save); } if (prog->saved_program_buf != NULL) free_message_buffer(prog->saved_program_buf); @@ -1576,7 +1595,7 @@ erts_match_prog_foreach_offheap(Binary *bprog, */ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) { - Eterm *hp = HAlloc(psp, arity * 2); + Eterm *hp = HAllocX(psp, arity * 2, HEAP_XTRA); Eterm ret = NIL; while (--arity >= 0) { ret = CONS(hp, arr[arity], ret); @@ -1584,14 +1603,83 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) } return ret; } + + +#if HALFWORD_HEAP +struct heap_checkpoint_t +{ + Process *p; + Eterm* htop; + ErlHeapFragment* mbuf; + unsigned used_size; + ErlOffHeap off_heap; +}; + +static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp) +{ + hcp->p = p; + hcp->htop = HEAP_TOP(p); + hcp->mbuf = MBUF(p); + hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0; + hcp->off_heap = MSO(p); +} + +static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp) +{ + struct erl_off_heap_header* oh = MSO(hcp->p).first; + + if (oh != hcp->off_heap.first) { + ASSERT(oh != NULL); + if (hcp->off_heap.first) { + while (oh->next != hcp->off_heap.first) { + oh = oh->next; + } + oh->next = NULL; + } + erts_cleanup_offheap(&MSO(hcp->p)); + MSO(hcp->p) = hcp->off_heap; + } + if (MBUF(hcp->p) != hcp->mbuf) { + ErlHeapFragment* hf = MBUF(hcp->p); + ASSERT(hf != NULL); + if (hcp->mbuf) { + while (hf->next != hcp->mbuf) { + hf = hf->next; + } + hf->next = NULL; + } + free_message_buffer(MBUF(hcp->p)); + MBUF(hcp->p) = hcp->mbuf; + } + if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) { + hcp->mbuf->used_size = hcp->used_size; + } + HEAP_TOP(hcp->p) = hcp->htop; +} +#endif /* HALFWORD_HEAP */ + +static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) +{ + if (!is_immed(term)) { + Uint sz = size_object_rel(term, base); + Eterm* top = HAllocX(p, sz, HEAP_XTRA); + return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL); + } + return term; +} + + /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. -** the para meter 'arity' is only used if 'term' is actually an array, +** the parameter 'arity' is only used if 'term' is actually an array, ** i.e. 'DCOMP_TRACE' was specified */ -Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, +Eterm db_prog_match(Process *c_p, Binary *bprog, + Eterm term, Eterm* base, + Eterm *termp, int arity, + enum erts_pam_run_flags in_flags, Uint32 *return_flags) { MatchProg *prog = Binary2MatchProg(bprog); @@ -1600,8 +1688,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, Eterm t; Eterm **sp; Eterm *esp; - Eterm *hp; - Uint *pc = prog->text; + MatchVariable* variables; + BeamInstr *cp; + UWord *pc = prog->text; Eterm *ehp; Eterm ret; Uint n = 0; /* To avoid warning. */ @@ -1609,19 +1698,24 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, unsigned do_catch; ErtsMatchPseudoProcess *mpsp; Process *psp; + Process* build_proc; Process *tmpp; Process *current_scheduled; ErtsSchedulerData *esdp; Eterm (*bif)(Process*, ...); int fail_label; int atomic_trace; +#if HALFWORD_HEAP + struct heap_checkpoint_t c_p_checkpoint = {}; +#endif #ifdef DMC_DEBUG - unsigned long *heap_fence; - unsigned long *eheap_fence; - unsigned long *stack_fence; + Uint *heap_fence; + Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ + ASSERT(base==NULL || HALFWORD_HEAP); + mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; @@ -1631,7 +1725,6 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p); ASSERT(esdp != NULL); current_scheduled = esdp->current_process; - esdp->current_process = psp; /* SMP: psp->scheduler_data is set by get_match_pseudo_process */ atomic_trace = 0; @@ -1654,11 +1747,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, #ifdef DMC_DEBUG save_op = 0; - heap_fence = (unsigned long *) mpsp->heap + prog->eheap_offset - 1; - eheap_fence = (unsigned long *) mpsp->heap + prog->stack_offset - 1; - stack_fence = (unsigned long *) mpsp->heap + prog->heap_size - 1; + heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1; + stack_fence = (Eterm*)((char*) mpsp->u.heap + prog->heap_size) - 1; *heap_fence = FENCE_PATTERN; - *eheap_fence = FENCE_PATTERN; *stack_fence = FENCE_PATTERN; #endif /* DMC_DEBUG */ @@ -1672,36 +1763,48 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, *return_flags = 0U; + variables = mpsp->u.variables; +#if HALFWORD_HEAP + c_p_checkpoint.p = NULL; +#endif + restart: ep = &term; - esp = mpsp->heap + prog->stack_offset; + esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset); sp = (Eterm **) esp; - hp = mpsp->heap; - ehp = mpsp->heap + prog->eheap_offset; ret = am_true; do_catch = 0; fail_label = -1; + build_proc = psp; + esdp->current_process = psp; + ASSERT_HALFWORD(!c_p_checkpoint.p); + +#ifdef DEBUG + ASSERT(variables == mpsp->u.variables); + for (i=0; i<prog->num_bindings; i++) { + variables[i].term = THE_NON_VALUE; + variables[i].proc = NULL; + variables[i].base = base; + } +#endif for (;;) { -#ifdef DMC_DEBUG + + #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { erl_exit(1, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } - if (*eheap_fence != FENCE_PATTERN) { - erl_exit(1, "Eheap fence overwritten in db_prog_match after op " - "0x%08x, overwritten with 0x%08x.", save_op, - *eheap_fence); - } if (*stack_fence != FENCE_PATTERN) { erl_exit(1, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } save_op = *pc; -#endif + #endif switch (*pc++) { case matchTryMeElse: + ASSERT(fail_label == -1); fail_label = *pc++; break; case matchArray: /* only when DCOMP_TRACE, is always first @@ -1709,16 +1812,17 @@ restart: n = *pc++; if ((int) n != arity) FAIL(); - ep = (Eterm *) *ep; + ep = termp; break; case matchArrayBind: /* When the array size is unknown. */ + ASSERT(termp); n = *pc++; - hp[n] = dpm_array_to_list(psp, (Eterm *) term, arity); + variables[n].term = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ - if (!is_tuple(*ep)) + if (!is_tuple_rel(*ep,base)) FAIL(); - ep = tuple_val(*ep); + ep = tuple_val_rel(*ep,base); n = *pc++; if (arityval(*ep) != n) FAIL(); @@ -1726,9 +1830,9 @@ restart: break; case matchPushT: /* *ep is a tuple of arity n, push ptr to first element */ - if (!is_tuple(*ep)) + if (!is_tuple_rel(*ep,base)) FAIL(); - tp = tuple_val(*ep); + tp = tuple_val_rel(*ep,base); n = *pc++; if (arityval(*tp) != n) FAIL(); @@ -1738,12 +1842,12 @@ restart: case matchList: if (!is_list(*ep)) FAIL(); - ep = list_val(*ep); + ep = list_val_rel(*ep,base); break; case matchPushL: if (!is_list(*ep)) FAIL(); - *sp++ = list_val(*ep); + *sp++ = list_val_rel(*ep,base); ++ep; break; case matchPop: @@ -1751,52 +1855,61 @@ restart: break; case matchBind: n = *pc++; - hp[n] = *ep++; + variables[n].term = *ep++; break; case matchCmp: n = *pc++; - if (!eq(hp[n],*ep)) + if (!eq_rel(variables[n].term, base, *ep, base)) FAIL(); ++ep; break; case matchEqBin: t = (Eterm) *pc++; - if (!eq(*ep,t)) + if (!eq_rel(t,NULL,*ep,base)) FAIL(); ++ep; break; case matchEqFloat: - if (!is_float(*ep)) + if (!is_float_rel(*ep,base)) FAIL(); - if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) + if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double))) FAIL(); - pc += 2; + pc += TermWords(2); ++ep; break; - case matchEqRef: - if (!is_ref(*ep)) + case matchEqRef: { + Eterm* epc = (Eterm*)pc; + if (!is_ref_rel(*ep,base)) FAIL(); - if (!eq(*ep, make_internal_ref(pc))) + if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) { FAIL(); - i = thing_arityval(*pc); - pc += i+1; + } + i = thing_arityval(*epc); + pc += TermWords(i+1); ++ep; break; + } case matchEqBig: - if (!is_big(*ep)) + if (!is_big_rel(*ep,base)) FAIL(); - tp = big_val(*ep); - if (*tp != *pc) - FAIL(); - i = BIG_ARITY(pc); - while(i--) - if (*++tp != *++pc) + tp = big_val_rel(*ep,base); + { + Eterm *epc = (Eterm *) pc; + if (*tp != *epc) FAIL(); - ++pc; + i = BIG_ARITY(epc); + pc += TermWords(i+1); + while(i--) { + if (*++tp != *++epc) { + FAIL(); + } + } + } ++ep; break; case matchEq: - t = (Eterm) *pc++; + t = (Eterm) *pc++; + ASSERT(is_immed(t)); if (t != *ep++) FAIL(); break; @@ -1804,25 +1917,32 @@ restart: ++ep; break; /* - * Here comes guard instructions + * Here comes guard & body instructions */ case matchPushC: /* Push constant */ - *esp++ = *pc++; + if ((in_flags & ERTS_PAM_COPY_RESULT) + && do_catch && !is_immed(*pc)) { + *esp++ = copy_object(*pc++, c_p); + } + else { + *esp++ = *pc++; + } break; case matchConsA: - ehp[1] = *--esp; - ehp[0] = esp[-1]; + ehp = HAllocX(build_proc, 2, HEAP_XTRA); + CDR(ehp) = *--esp; + CAR(ehp) = esp[-1]; esp[-1] = make_list(ehp); - ehp += 2; break; case matchConsB: - ehp[0] = *--esp; - ehp[1] = esp[-1]; + ehp = HAllocX(build_proc, 2, HEAP_XTRA); + CAR(ehp) = *--esp; + CDR(ehp) = esp[-1]; esp[-1] = make_list(ehp); - ehp += 2; break; case matchMkTuple: n = *pc++; + ehp = HAllocX(build_proc, n+1, HEAP_XTRA); t = make_tuple(ehp); *ehp++ = make_arityval(n); while (n--) { @@ -1832,7 +1952,7 @@ restart: break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp); + t = (*bif)(build_proc); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1843,7 +1963,7 @@ restart: break; case matchCall1: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1]); + t = (*bif)(build_proc, esp[-1]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1854,7 +1974,7 @@ restart: break; case matchCall2: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1], esp[-2]); + t = (*bif)(build_proc, esp[-1], esp[-2]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1866,7 +1986,7 @@ restart: break; case matchCall3: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1], esp[-2], esp[-3]); + t = (*bif)(build_proc, esp[-1], esp[-2], esp[-3]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1876,15 +1996,73 @@ restart: esp -= 2; esp[-1] = t; break; + + #if HALFWORD_HEAP + case matchPushVGuard: + if (!base) goto case_matchPushV; + /* Build NULL-based copy on pseudo heap for easy disposal */ + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].proc); + variables[n].term = copy_object_rel(psp, variables[n].term, base); + *esp++ = variables[n].term; + #ifdef DEBUG + variables[n].proc = psp; + variables[n].base = NULL; + #endif + break; + #endif + case matchPushVResult: + if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV; + + /* Build (NULL-based) copy on callers heap */ + #if HALFWORD_HEAP + if (!do_catch && !c_p_checkpoint.p) { + heap_checkpoint_init(c_p, &c_p_checkpoint); + } + #endif + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].proc); + variables[n].term = copy_object_rel(c_p, variables[n].term, base); + *esp++ = variables[n].term; + #ifdef DEBUG + variables[n].proc = c_p; + variables[n].base = NULL; + #endif + break; case matchPushV: - *esp++ = hp[*pc++]; + case_matchPushV: + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].base); + *esp++ = variables[n].term; break; case matchPushExpr: - *esp++ = term; + if (in_flags & ERTS_PAM_COPY_RESULT) { + Uint sz; + Eterm* top; + sz = size_object_rel(term, base); + top = HAllocX(build_proc, sz, HEAP_XTRA); + if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) { + ASSERT(is_tuple_rel(term,base)); + *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz, + &top, &MSO(build_proc), base); + } + else { + *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc), + base, NULL); + } + } + else { + *esp = term; + } break; case matchPushArrayAsList: + ASSERT_HALFWORD(base == NULL); n = arity; /* Only happens when 'term' is an array */ - tp = (Eterm *) term; + tp = termp; + ehp = HAllocX(build_proc, n*2, HEAP_XTRA); *esp++ = make_list(ehp); while (n--) { *ehp++ = *tp++; @@ -1897,7 +2075,8 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - *esp++ = dpm_array_to_list(psp, (Eterm *) term, arity); + ASSERT_HALFWORD(base == NULL); + *esp++ = dpm_array_to_list(build_proc, termp, arity); break; case matchTrue: if (*--esp != am_true) @@ -1983,7 +2162,8 @@ restart: case matchProcessDump: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p); - *esp++ = new_binary(psp, (byte *)dsbufp->str, (int)dsbufp->str_len); + *esp++ = new_binary(build_proc, (byte *)dsbufp->str, + dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); break; } @@ -2027,29 +2207,24 @@ restart: if (SEQ_TRACE_TOKEN(c_p) == NIL) *esp++ = NIL; else { + Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); + Uint sender_sz = is_immed(sender) ? 0 : size_object(sender); + ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA); + if (sender_sz) { + sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc)); + } *esp++ = make_tuple(ehp); ehp[0] = make_arityval(5); ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p); ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p); ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p); - ehp[4] = SEQ_TRACE_TOKEN_SENDER(c_p); + ehp[4] = sender; ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p); ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); ASSERT(is_immed(ehp[1])); ASSERT(is_immed(ehp[2])); ASSERT(is_immed(ehp[3])); ASSERT(is_immed(ehp[5])); - if(!is_immed(ehp[4])) { - Eterm *sender = &ehp[4]; - ehp += 6; - *sender = copy_struct(*sender, - size_object(*sender), - &ehp, - &MSO(psp)); - } - else - ehp += 6; - } break; case matchEnableTrace: @@ -2095,17 +2270,17 @@ restart: } break; case matchCaller: - if (!(c_p->cp) || !(hp = find_function_from_pc(c_p->cp))) { + if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { *esp++ = am_undefined; } else { + ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); - ehp[0] = make_arityval(3); - ehp[1] = hp[0]; - ehp[2] = hp[1]; - ehp[3] = make_small(hp[2]); - ehp += 4; - } - break; + ehp[0] = make_arityval(3); + ehp[1] = cp[0]; + ehp[2] = cp[1]; + ehp[3] = make_small((Uint) cp[2]); + } + break; case matchSilent: --esp; if (*esp == am_true) { @@ -2180,8 +2355,12 @@ restart: } } break; - case matchCatch: + case matchCatch: /* Match success, now build result */ do_catch = 1; + if (in_flags & ERTS_PAM_COPY_RESULT) { + build_proc = c_p; + esdp->current_process = c_p; + } break; case matchHalt: goto success; @@ -2190,9 +2369,16 @@ restart: } } fail: +#if HALFWORD_HEAP + if (c_p_checkpoint.p) { + /* Dispose garbage built by guards on caller heap */ + heap_checkpoint_revert(&c_p_checkpoint); + c_p_checkpoint.p = NULL; + } +#endif *return_flags = 0U; - if (fail_label >= 0) { /* We failed during a "TryMeElse", - lets restart, with the next match + if (fail_label >= 0) { /* We failed during a "TryMeElse", + lets restart, with the next match program */ pc = (prog->text) + fail_label; cleanup_match_pseudo_process(mpsp, 1); @@ -2206,11 +2392,6 @@ success: erl_exit(1, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } - if (*eheap_fence != FENCE_PATTERN) { - erl_exit(1, "Eheap fence overwritten in db_prog_match after op " - "0x%08x, overwritten with 0x%08x.", save_op, - *eheap_fence); - } if (*stack_fence != FENCE_PATTERN) { erl_exit(1, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, @@ -2221,6 +2402,7 @@ success: esdp->current_process = current_scheduled; END_ATOMIC_TRACE(c_p); + return ret; #undef FAIL #undef FAIL_TERM @@ -2232,7 +2414,8 @@ success: /* * Convert a match program to a "magic" binary to return up to erlang */ -Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) { +Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) +{ return erts_mk_magic_binary_term(hpp, &MSO(p), mp); } @@ -2298,13 +2481,13 @@ void db_free_dmc_err_info(DMCErrInfo *ei){ ** Store bignum in *hpp and increase *hpp accordingly. ** *hpp is assumed to be large enough to hold the result. */ -Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) +Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr) { - Eterm big_tmp[2]; + DeclareTmpHeapNoproc(big_tmp,2); Eterm res; Sint ires; - Eterm arg1; - Eterm arg2; + Wterm arg1; + Wterm arg2; if (is_both_small(counter,incr)) { ires = signed_val(counter) + signed_val(incr); @@ -2318,6 +2501,7 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) } } else { + UseTmpHeapNoproc(2); switch(NUMBER_CODE(counter, incr)) { case SMALL_BIG: arg1 = small_to_big(signed_val(counter), big_tmp); @@ -2332,16 +2516,46 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) arg2 = counter; break; default: + UnUseTmpHeapNoproc(2); return THE_NON_VALUE; } res = big_plus(arg1, arg2, *hpp); if (is_big(res)) { *hpp += BIG_NEED_SIZE(big_size(res)); } + UnUseTmpHeapNoproc(2); return res; } } +/* Must be called to read elements after db_lookup_dbterm. +** Will decompress if needed. +** HEALFWORD_HEAP: +** Will convert from relative to Wterm format if needed. +** (but only on top level, tuples and lists will still contain rterms) +*/ +Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) +{ + Eterm elem = handle->dbterm->tpl[position]; + if (!is_header(elem)) { +#if HALFWORD_HEAP + if (!is_immed(elem) + && !handle->tb->common.compress + && !(handle->abs_vec && handle->abs_vec[position])) { + return rterm2wterm(elem, handle->dbterm->tpl); + } +#endif + return elem; + } + + ASSERT(((DbTableCommon*)handle->tb)->compress); + ASSERT(!handle->mustResize); + handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, + handle->dbterm); + handle->mustResize = 1; + return handle->dbterm->tpl[position]; +} + /* ** Update one element: ** handle: Initialized by db_lookup_dbterm() @@ -2358,127 +2572,488 @@ void db_do_update_element(DbUpdateHandle* handle, Eterm* oldp; Uint newval_sz; Uint oldval_sz; +#if HALFWORD_HEAP + Eterm* old_base; +#endif if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; + #ifdef DEBUG_CLONE + if (handle->dbterm->debug_clone) { + handle->dbterm->debug_clone[position] = newval; + } + #endif return; } - else if (!handle->mustResize && is_boxed(newval)) { - newp = boxed_val(newval); - switch (*newp & _TAG_HEADER_MASK) { - case _TAG_HEADER_POS_BIG: - case _TAG_HEADER_NEG_BIG: - case _TAG_HEADER_FLOAT: - case _TAG_HEADER_HEAP_BIN: - newval_sz = header_arity(*newp) + 1; - if (is_boxed(oldval)) { - oldp = boxed_val(oldval); - switch (*oldp & _TAG_HEADER_MASK) { + if (!handle->mustResize) { + if (handle->tb->common.compress) { + handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, + handle->dbterm); + handle->mustResize = 1; + oldval = handle->dbterm->tpl[position]; + #if HALFWORD_HEAP + old_base = NULL; + #endif + } + else { + #if HALFWORD_HEAP + ASSERT(!handle->abs_vec); + old_base = handle->dbterm->tpl; + #endif + if (is_boxed(newval)) { + newp = boxed_val(newval); + switch (*newp & _TAG_HEADER_MASK) { case _TAG_HEADER_POS_BIG: case _TAG_HEADER_NEG_BIG: case _TAG_HEADER_FLOAT: case _TAG_HEADER_HEAP_BIN: - oldval_sz = header_arity(*oldp) + 1; - if (oldval_sz == newval_sz) { - /* "self contained" terms of same size, do memcpy */ - sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); - return; + newval_sz = header_arity(*newp) + 1; + if (is_boxed(oldval)) { + oldp = boxed_val_rel(oldval,old_base); + switch (*oldp & _TAG_HEADER_MASK) { + case _TAG_HEADER_POS_BIG: + case _TAG_HEADER_NEG_BIG: + case _TAG_HEADER_FLOAT: + case _TAG_HEADER_HEAP_BIN: + oldval_sz = header_arity(*oldp) + 1; + if (oldval_sz == newval_sz) { + /* "self contained" terms of same size, do memcpy */ + sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); + return; + } + goto both_size_set; + } } - goto both_size_set; + goto new_size_set; } } - goto new_size_set; } } +#if HALFWORD_HEAP + else { + old_base = (handle->tb->common.compress + || (handle->abs_vec && handle->abs_vec[position])) ? + NULL : handle->dbterm->tpl; + } +#endif /* Not possible for simple memcpy or dbterm is already non-contiguous, */ /* need to realloc... */ newval_sz = is_immed(newval) ? 0 : size_object(newval); new_size_set: - - oldval_sz = is_immed(oldval) ? 0 : size_object(oldval); + + oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base); both_size_set: handle->new_size = handle->new_size - oldval_sz + newval_sz; - /* write new value in old dbterm, finalize will make a flat copy */ + /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; handle->mustResize = 1; + +#if HALFWORD_HEAP + if (old_base && newval_sz > 0) { + ASSERT(!handle->tb->common.compress); + if (!handle->abs_vec) { + int i = header_arity(handle->dbterm->tpl[0]); + handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char)); + sys_memset(handle->abs_vec, 0, i+1); + /* abs_vec[0] not used */ + } + handle->abs_vec[position] = 1; + } +#endif +} + +static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, + Uint old_sz, Uint new_sz, Uint offset) +{ + byte* ret; + if (erts_ets_realloc_always_moves) { + ret = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz); + sys_memcpy(ret, old, offset); + erts_db_free(ERTS_ALC_T_DB_TERM, (DbTable*)tb, old, old_sz); + } else { + ret = erts_db_realloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, + old, old_sz, new_sz); + } + return ret; } +/* Allocated size of a compressed dbterm +*/ +static ERTS_INLINE Uint db_alloced_size_comp(DbTerm* obj) +{ + return obj->tpl[arityval(*obj->tpl) + 1]; +} + +void db_free_term(DbTable *tb, void* basep, Uint offset) +{ + DbTerm* db = (DbTerm*) ((byte*)basep + offset); + Uint size; + if (tb->common.compress) { + db_cleanup_offheap_comp(db); + size = db_alloced_size_comp(db); + } + else { + ErlOffHeap tmp_oh; + tmp_oh.first = db->first_oh; + erts_cleanup_offheap(&tmp_oh); + size = offset + offsetof(DbTerm,tpl) + db->size*sizeof(Eterm); + } + erts_db_free(ERTS_ALC_T_DB_TERM, tb, basep, size); +} + +static ERTS_INLINE Uint align_up(Uint value, Uint pow2) +{ + ASSERT((pow2 & (pow2-1)) == 0); + return (value + (pow2-1)) & ~(pow2-1); +} + +/* Compressed size of an uncompressed term +*/ +static Uint db_size_dbterm_comp(DbTableCommon* tb, Eterm obj) +{ + Eterm* tpl = tuple_val(obj); + int i; + Uint size = sizeof(DbTerm) + + arityval(*tpl) * sizeof(Eterm) + + sizeof(Uint); /* "alloc_size" */ + + for (i = arityval(*tpl); i>0; i--) { + if (i != tb->keypos && is_not_immed(tpl[i])) { + size += erts_encode_ext_size_ets(tpl[i]); + } + } + size += size_object(tpl[tb->keypos]) * sizeof(Eterm); + return align_up(size, sizeof(Uint)); +} + +/* Conversion between top tuple element and pointer to compressed data +*/ +static ERTS_INLINE Eterm ext2elem(Eterm* tpl, byte* ext) +{ + return (((Uint)(ext - (byte*)tpl)) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; +} +static ERTS_INLINE byte* elem2ext(Eterm* tpl, Uint ix) +{ + ASSERT(is_header(tpl[ix])); + return (byte*)tpl + (tpl[ix] >> _TAG_PRIMARY_SIZE); +} + +static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, + Uint alloc_size) +{ + ErlOffHeap tmp_offheap; + Eterm* src = tuple_val(obj); + Eterm* tpl = dest->tpl; + Eterm key = src[tb->keypos]; + int arity = arityval(src[0]); + union { + Eterm* ep; + byte* cp; + UWord ui; + }top; + int i; + + top.ep = tpl+ 1 + arity + 1; + tpl[0] = src[0]; + tpl[arity + 1] = alloc_size; + + tmp_offheap.first = NULL; + tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl); + dest->first_oh = tmp_offheap.first; + for (i=1; i<=arity; i++) { + if (i != tb->keypos) { + if (is_immed(src[i])) { + tpl[i] = src[i]; + } + else { + tpl[i] = ext2elem(tpl, top.cp); + top.cp = erts_encode_ext_ets(src[i], top.cp, &dest->first_oh); + } + } + } + +#ifdef DEBUG_CLONE + { + Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); + dest->debug_clone = dbg_top; + tmp_offheap.first = dest->first_oh; + copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top); + dest->first_oh = tmp_offheap.first; + ASSERT(dbg_top == dest->debug_clone + dest->size); + } +#endif + return top.cp; +} /* ** Copy the object into a possibly new DbTerm, ** offset is the offset of the DbTerm from the start -** of the sysAllocaed structure, The possibly realloced and copied +** of the allocated structure, The possibly realloced and copied ** structure is returned. Make sure (((char *) old) - offset) is a ** pointer to a ERTS_ALC_T_DB_TERM allocated data area. */ -void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) +void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) { + byte* basep; + DbTerm* newp; + Eterm* top; int size = size_object(obj); - void *structp = ((char*) old) - offset; - DbTerm* p; - Eterm copy; - Eterm *top; + ErlOffHeap tmp_offheap; if (old != 0) { - erts_cleanup_offheap(&old->off_heap); + basep = ((byte*) old) - offset; + tmp_offheap.first = old->first_oh; + erts_cleanup_offheap(&tmp_offheap); + old->first_oh = tmp_offheap.first; if (size == old->size) { - p = old; - } else { + newp = old; + } + else { Uint new_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1); Uint old_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(old->size-1); - if (erts_ets_realloc_always_moves) { - void *nstructp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - new_sz); - memcpy(nstructp,structp,offset); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - structp, - old_sz); - structp = nstructp; - } else { - structp = erts_db_realloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - structp, - old_sz, - new_sz); - } - p = (DbTerm*) ((void *)(((char *) structp) + offset)); + basep = db_realloc_term(tb, basep, old_sz, new_sz, offset); + newp = (DbTerm*) (basep + offset); } } else { - structp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (offset - + sizeof(DbTerm) - + sizeof(Eterm)*(size-1))); - p = (DbTerm*) ((void *)(((char *) structp) + offset)); - } - p->size = size; - p->off_heap.mso = NULL; - p->off_heap.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; + basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable *)tb, + (offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1))); + newp = (DbTerm*) (basep + offset); + } + newp->size = size; + top = newp->tpl; + tmp_offheap.first = NULL; + copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top); + newp->first_oh = tmp_offheap.first; +#ifdef DEBUG_CLONE + newp->debug_clone = NULL; #endif - p->off_heap.overhead = 0; + return basep; +} + - top = DBTERM_BUF(p); - copy = copy_struct(obj, size, &top, &p->off_heap); - DBTERM_SET_TPL(p,tuple_val(copy)); +void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) +{ + Uint new_sz = offset + db_size_dbterm_comp(tb, obj); + byte* basep; + DbTerm* newp; + byte* top; + + ASSERT(tb->compress); + if (old != 0) { + Uint old_sz = db_alloced_size_comp(old); + db_cleanup_offheap_comp(old); - return structp; + basep = ((byte*) old) - offset; + if (new_sz == old_sz) { + newp = old; + } + else { + basep = db_realloc_term(tb, basep, old_sz, new_sz, offset); + newp = (DbTerm*) (basep + offset); + } + } + else { + basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz); + newp = (DbTerm*) (basep + offset); + } + + newp->size = size_object(obj); + top = copy_to_comp(tb, obj, newp, new_sz); + ASSERT(top <= basep + new_sz); + + /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */ + + return basep; } -void db_free_term_data(DbTerm* p) +void db_finalize_resize(DbUpdateHandle* handle, Uint offset) { - erts_cleanup_offheap(&p->off_heap); + DbTable* tbl = handle->tb; + DbTerm* newDbTerm; + Uint alloc_sz = offset + + (tbl->common.compress ? + db_size_dbterm_comp(&tbl->common, make_tuple(handle->dbterm->tpl)) : + sizeof(DbTerm)+sizeof(Eterm)*(handle->new_size-1)); + byte* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, alloc_sz); + byte* oldp = *(handle->bp); + + sys_memcpy(newp, oldp, offset); /* copy only hash/tree header */ + *(handle->bp) = newp; + newDbTerm = (DbTerm*) (newp + offset); + newDbTerm->size = handle->new_size; +#ifdef DEBUG_CLONE + newDbTerm->debug_clone = NULL; +#endif + + /* make a flat copy */ + + if (tbl->common.compress) { + copy_to_comp(&tbl->common, make_tuple(handle->dbterm->tpl), + newDbTerm, alloc_sz); + db_free_tmp_uncompressed(handle->dbterm); + } + else { + ErlOffHeap tmp_offheap; + Eterm* tpl = handle->dbterm->tpl; + Eterm* top = newDbTerm->tpl; + + tmp_offheap.first = NULL; + + #if HALFWORD_HEAP + if (handle->abs_vec) { + int i, arity = header_arity(handle->dbterm->tpl[0]); + + top[0] = tpl[0]; + top += arity + 1; + for (i=1; i<=arity; i++) { + Eterm* src_base = handle->abs_vec[i] ? NULL : tpl; + + newDbTerm->tpl[i] = copy_struct_rel(tpl[i], + size_object_rel(tpl[i],src_base), + &top, &tmp_offheap, src_base, + newDbTerm->tpl); + } + newDbTerm->first_oh = tmp_offheap.first; + ASSERT((byte*)top <= (newp + alloc_sz)); + erts_free(ERTS_ALC_T_TMP, handle->abs_vec); + } + else + #endif /* HALFWORD_HEAP */ + { + copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top, + &tmp_offheap, tpl, top); + newDbTerm->first_oh = tmp_offheap.first; + ASSERT((byte*)top == (newp + alloc_sz)); + } + } +} + +Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, + ErlOffHeap* off_heap) +{ + Eterm* hp = *hpp; + int i, arity = arityval(bp->tpl[0]); + + hp[0] = bp->tpl[0]; + *hpp += arity + 1; + + hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], + size_object_rel(bp->tpl[tb->keypos], bp->tpl), + hpp, off_heap, bp->tpl, NULL); + for (i=arity; i>0; i--) { + if (i != tb->keypos) { + if (is_immed(bp->tpl[i])) { + hp[i] = bp->tpl[i]; + } + else { + hp[i] = erts_decode_ext_ets(hpp, off_heap, + elem2ext(bp->tpl, i)); + } + } + } + ASSERT((*hpp - hp) <= bp->size); +#ifdef DEBUG_CLONE + ASSERT(eq_rel(make_tuple(hp),make_tuple(bp->debug_clone),bp->debug_clone)); +#endif + return make_tuple(hp); +} + +Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, + DbTerm* obj, Uint pos, + Eterm** hpp, Uint extra) +{ + if (is_immed(obj->tpl[pos])) { + *hpp = HAlloc(p, extra); + return obj->tpl[pos]; + } + if (tb->compress && pos != tb->keypos) { + byte* ext = elem2ext(obj->tpl, pos); + Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; + Eterm* hp = HAlloc(p, sz); + Eterm* endp = hp + sz; + Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); + *hpp = hp; + hp += extra; + HRelease(p, endp, hp); +#ifdef DEBUG_CLONE + ASSERT(eq_rel(copy, obj->debug_clone[pos], obj->debug_clone)); +#endif + return copy; + } + else { + Uint sz = size_object_rel(obj->tpl[pos], obj->tpl); + *hpp = HAlloc(p, sz + extra); + return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL); + } +} + + +/* Our own "cleanup_offheap" + * as refc-binaries may be unaligned in compressed terms +*/ +void db_cleanup_offheap_comp(DbTerm* obj) +{ + union erl_off_heap_ptr u; + ProcBin tmp; + + for (u.hdr = obj->first_oh; u.hdr; u.hdr = u.hdr->next) { + if ((UWord)u.voidp % sizeof(Uint) != 0) { /* unaligned ptr */ + sys_memcpy(&tmp, u.voidp, sizeof(tmp)); + /* Warning, must pass (void*)-variable to memcpy. Otherwise it will + cause Bus error on Sparc due to false compile time assumptions + about word aligned memory (type cast is not enough) */ + u.pb = &tmp; + } + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { + erts_bin_free(u.pb->val); + } + break; + case FUN_SUBTAG: + ASSERT(u.pb != &tmp); + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + erts_erase_fun_entry(u.fun->fe); + } + break; + default: + ASSERT(is_external_header(u.hdr->thing_word)); + ASSERT(u.pb != &tmp); + erts_deref_node_entry(u.ext->node); + break; + } + } +#ifdef DEBUG_CLONE + if (obj->debug_clone != NULL) { + erts_free(ERTS_ALC_T_DB_TERM, obj->debug_clone); + obj->debug_clone = NULL; + } +#endif } +int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b) +{ + ErlOffHeap tmp_offheap; + Eterm* allocp; + Eterm* hp; + Eterm tmp_b; + int is_eq; + + ASSERT(tb->compress); + hp = allocp = erts_alloc(ERTS_ALC_T_TMP, b->size*sizeof(Eterm)); + tmp_offheap.first = NULL; + tmp_b = db_copy_from_comp(tb, b, &hp, &tmp_offheap); + is_eq = eq(a,tmp_b); + erts_cleanup_offheap(&tmp_offheap); + erts_free(ERTS_ALC_T_TMP, allocp); + return is_eq; +} /* ** Check if object represents a "match" variable @@ -2606,7 +3181,7 @@ static void add_dmc_err(DMCErrInfo *err_info, static DMCRet dmc_one_term(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(Eterm) *stack, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm c) { Sint n; @@ -2624,7 +3199,7 @@ static DMCRet dmc_one_term(DMCContext *context, ** Ouch, big integer in match variable. */ Eterm *save_hp; - ASSERT(heap->data == heap->def); + ASSERT(heap->vars == heap->vars_def); sz = sz2 = sz3 = 0; for (j = 0; j < context->num_match; ++j) { sz += size_object(context->matchexpr[j]); @@ -2662,24 +3237,23 @@ static DMCRet dmc_one_term(DMCContext *context, may be atoms that changed */ context->matchexpr[j] = context->copy->mem[j]; } - heap->data = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP, - heap->size*sizeof(unsigned)); - sys_memset(heap->data, 0, - heap->size * sizeof(unsigned)); + heap->vars = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP, + heap->size*sizeof(DMCVariable)); + sys_memset(heap->vars, 0, heap->size * sizeof(DMCVariable)); DMC_CLEAR(*stack); /*DMC_PUSH(*stack,NIL);*/ DMC_CLEAR(*text); return retRestart; } - if (heap->data[n]) { /* already bound ? */ + if (heap->vars[n].is_bound) { DMC_PUSH(*text,matchCmp); DMC_PUSH(*text,n); } else { /* Not bound, bind! */ - if (n >= heap->used) - heap->used = n + 1; + if (n >= heap->vars_used) + heap->vars_used = n + 1; DMC_PUSH(*text,matchBind); DMC_PUSH(*text,n); - heap->data[n] = 1; + heap->vars[n].is_bound = 1; } } else if (c == am_Underscore) { DMC_PUSH(*text, matchSkip); @@ -2704,27 +3278,84 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*stack, c); break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): - n = thing_arityval(*internal_ref_val(c)); + { + Eterm* ref_val = internal_ref_val(c); DMC_PUSH(*text, matchEqRef); - DMC_PUSH(*text, *internal_ref_val(c)); - for (i = 1; i <= n; ++i) { - DMC_PUSH(*text, (Uint) internal_ref_val(c)[i]); +#if HALFWORD_HEAP + { + union { + UWord u; + Uint t[2]; + } fiddle; + ASSERT(thing_arityval(ref_val[0]) == 3); + fiddle.t[0] = ref_val[0]; + fiddle.t[1] = ref_val[1]; + DMC_PUSH(*text, fiddle.u); + fiddle.t[0] = ref_val[2]; + fiddle.t[1] = ref_val[3]; + DMC_PUSH(*text, fiddle.u); } +#else + n = thing_arityval(ref_val[0]); + for (i = 0; i <= n; ++i) { + DMC_PUSH(*text, ref_val[i]); + } +#endif break; + } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - n = thing_arityval(*big_val(c)); + { + Eterm* bval = big_val(c); + n = thing_arityval(bval[0]); DMC_PUSH(*text, matchEqBig); - DMC_PUSH(*text, *big_val(c)); - for (i = 1; i <= n; ++i) { - DMC_PUSH(*text, (Uint) big_val(c)[i]); +#if HALFWORD_HEAP + { + union { + UWord u; + Uint t[2]; + } fiddle; + ASSERT(n >= 1); + fiddle.t[0] = bval[0]; + fiddle.t[1] = bval[1]; + DMC_PUSH(*text, fiddle.u); + for (i = 2; i <= n; ++i) { + fiddle.t[0] = bval[i]; + if (++i <= n) { + fiddle.t[1] = bval[i]; + } else { + fiddle.t[1] = (Uint) 0; + } + DMC_PUSH(*text, fiddle.u); + } } +#else + for (i = 0; i <= n; ++i) { + DMC_PUSH(*text, (Uint) bval[i]); + } +#endif break; + } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); +#if HALFWORD_HEAP + { + union { + UWord u; + Uint t[2]; + } fiddle; + fiddle.t[0] = float_val(c)[1]; + fiddle.t[1] = float_val(c)[2]; + DMC_PUSH(*text, fiddle.u); + } +#else DMC_PUSH(*text, (Uint) float_val(c)[1]); - /* XXX: this reads and pushes random junk on ARCH_64 */ +#ifdef ARCH_64 + DMC_PUSH(*text, (Uint) 0); +#else DMC_PUSH(*text, (Uint) float_val(c)[2]); +#endif +#endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ /* @@ -2753,7 +3384,7 @@ static DMCRet dmc_one_term(DMCContext *context, ** Match guard compilation */ -static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text, +static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t) { int sz; @@ -2807,7 +3438,7 @@ add_dmc_err((ContextP)->err_info, String, -1, T, dmcWarning) static DMCRet dmc_list(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2837,17 +3468,16 @@ static DMCRet dmc_list(DMCContext *context, DMC_PUSH(*text, matchConsB); } --context->stack_used; /* Two objects on stack becomes one */ - context->eheap_need += 2; return retOk; } static DMCRet dmc_tuple(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { - DMC_STACK_TYPE(Uint) instr_save; + DMC_STACK_TYPE(UWord) instr_save; int all_constant = 1; int textpos = DMC_STACK_NUM(*text); Eterm *p = tuple_val(t); @@ -2896,14 +3526,13 @@ static DMCRet dmc_tuple(DMCContext *context, DMC_PUSH(*text, matchMkTuple); DMC_PUSH(*text, nelems); context->stack_used -= (nelems - 1); - context->eheap_need += (nelems + 1); *constant = 0; return retOk; } static DMCRet dmc_whole_expression(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2914,9 +3543,6 @@ static DMCRet dmc_whole_expression(DMCContext *context, } else { ASSERT(is_tuple(context->matchexpr [context->current_match])); - context->eheap_need += - arityval(*(tuple_val(context->matchexpr - [context->current_match]))) * 2; DMC_PUSH(*text, matchPushArrayAsList); } } else { @@ -2929,20 +3555,55 @@ static DMCRet dmc_whole_expression(DMCContext *context, return retOk; } +/* Figure out which PushV instruction to use. +*/ +static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap, + DMC_STACK_TYPE(UWord) *text, Uint n) +{ + DMCVariable* v = &heap->vars[n]; + MatchOps instr = matchPushV; + + ASSERT(n < heap->vars_used && v->is_bound); + if (context->is_guard) { + #if HALFWORD_HEAP + if (!v->first_guard_label) { + v->first_guard_label = DMC_STACK_NUM(*text); + ASSERT(v->first_guard_label); + instr = matchPushVGuard; /* may be changed to PushVResult below */ + } + #endif + } + else { /* body */ + #if HALFWORD_HEAP + if (v->first_guard_label) { + /* Avoid double-copy, copy to result heap at first encounter in guard */ + DMC_POKE(*text, v->first_guard_label, matchPushVResult); + v->is_in_body = 1; + } + #endif + if (!v->is_in_body) { + instr = matchPushVResult; + v->is_in_body = 1; + } + } + DMC_PUSH(*text, instr); + DMC_PUSH(*text, n); +} + static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { Uint n = db_is_variable(t); - ASSERT(n >= 0); - if (n >= heap->used) - RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); - if (heap->data[n] == 0U) + + if (n >= heap->vars_used || !heap->vars[n].is_bound) { RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); - DMC_PUSH(*text, matchPushV); - DMC_PUSH(*text, n); + } + + dmc_add_pushv_variant(context, heap, text, n); + ++context->stack_used; if (context->stack_used > context->stack_need) context->stack_need = context->stack_used; @@ -2952,7 +3613,7 @@ static DMCRet dmc_variable(DMCContext *context, static DMCRet dmc_all_bindings(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2961,10 +3622,9 @@ static DMCRet dmc_all_bindings(DMCContext *context, DMC_PUSH(*text, matchPushC); DMC_PUSH(*text, NIL); - for (i = heap->used - 1; i >= 0; --i) { - if (heap->data[i]) { - DMC_PUSH(*text, matchPushV); - DMC_PUSH(*text, i); + for (i = heap->vars_used - 1; i >= 0; --i) { + if (heap->vars[i].is_bound) { + dmc_add_pushv_variant(context, heap, text, i); DMC_PUSH(*text, matchConsB); heap_used += 2; } @@ -2972,14 +3632,13 @@ static DMCRet dmc_all_bindings(DMCContext *context, ++context->stack_used; if ((context->stack_used + 1) > context->stack_need) context->stack_need = (context->stack_used + 1); - context->eheap_need += heap_used; *constant = 0; return retOk; } static DMCRet dmc_const(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -2996,7 +3655,7 @@ static DMCRet dmc_const(DMCContext *context, static DMCRet dmc_and(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3025,7 +3684,7 @@ static DMCRet dmc_and(DMCContext *context, static DMCRet dmc_or(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3055,7 +3714,7 @@ static DMCRet dmc_or(DMCContext *context, static DMCRet dmc_andalso(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3104,7 +3763,7 @@ static DMCRet dmc_andalso(DMCContext *context, static DMCRet dmc_orelse(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3152,7 +3811,7 @@ static DMCRet dmc_orelse(DMCContext *context, static DMCRet dmc_message(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3194,7 +3853,7 @@ static DMCRet dmc_message(DMCContext *context, static DMCRet dmc_self(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3214,7 +3873,7 @@ static DMCRet dmc_self(DMCContext *context, static DMCRet dmc_return_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3244,7 +3903,7 @@ static DMCRet dmc_return_trace(DMCContext *context, static DMCRet dmc_exception_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3276,7 +3935,7 @@ static DMCRet dmc_exception_trace(DMCContext *context, static DMCRet dmc_is_seq_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3302,7 +3961,7 @@ static DMCRet dmc_is_seq_trace(DMCContext *context, static DMCRet dmc_set_seq_token(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3351,7 +4010,7 @@ static DMCRet dmc_set_seq_token(DMCContext *context, static DMCRet dmc_get_seq_token(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3375,10 +4034,6 @@ static DMCRet dmc_get_seq_token(DMCContext *context, *constant = 0; DMC_PUSH(*text, matchGetSeqToken); - context->eheap_need += (6 /* A 5-tuple is built */ - + EXTERNAL_THING_HEAD_SIZE + 2 /* Sender can - be an external - pid */); if (++context->stack_used > context->stack_need) context->stack_need = context->stack_used; return retOk; @@ -3388,7 +4043,7 @@ static DMCRet dmc_get_seq_token(DMCContext *context, static DMCRet dmc_display(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3428,7 +4083,7 @@ static DMCRet dmc_display(DMCContext *context, static DMCRet dmc_process_dump(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3458,7 +4113,7 @@ static DMCRet dmc_process_dump(DMCContext *context, static DMCRet dmc_enable_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3518,7 +4173,7 @@ static DMCRet dmc_enable_trace(DMCContext *context, static DMCRet dmc_disable_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3578,7 +4233,7 @@ static DMCRet dmc_disable_trace(DMCContext *context, static DMCRet dmc_trace(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3652,7 +4307,7 @@ static DMCRet dmc_trace(DMCContext *context, static DMCRet dmc_caller(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3675,7 +4330,6 @@ static DMCRet dmc_caller(DMCContext *context, } *constant = 0; DMC_PUSH(*text, matchCaller); /* Creates binary */ - context->eheap_need += 4; /* A 3-tuple is built */ if (++context->stack_used > context->stack_need) context->stack_need = context->stack_used; return retOk; @@ -3685,7 +4339,7 @@ static DMCRet dmc_caller(DMCContext *context, static DMCRet dmc_silent(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3727,7 +4381,7 @@ static DMCRet dmc_silent(DMCContext *context, static DMCRet dmc_fun(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3844,7 +4498,7 @@ static DMCRet dmc_fun(DMCContext *context, erl_exit(1,"ets:match() internal error, " "guard with more than 3 arguments."); } - DMC_PUSH(*text, (Uint) b->biff); + DMC_PUSH(*text, (UWord) b->biff); context->stack_used -= (((int) a) - 2); if (context->stack_used > context->stack_need) context->stack_need = context->stack_used; @@ -3853,7 +4507,7 @@ static DMCRet dmc_fun(DMCContext *context, static DMCRet dmc_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant) { @@ -3916,7 +4570,7 @@ static DMCRet dmc_expr(DMCContext *context, static DMCRet compile_guard_expr(DMCContext *context, DMCHeap *heap, - DMC_STACK_TYPE(Uint) *text, + DMC_STACK_TYPE(UWord) *text, Eterm l) { DMCRet ret; @@ -4031,7 +4685,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) DMC_INIT_STACK(heap); p = expr->mem; - i = expr->size; + i = expr->used_size; while (i--) { if (is_thing(*p)) { a = thing_arityval(*p); @@ -4060,7 +4714,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) } p = expr->mem; - i = expr->size; + i = expr->used_size; while (i--) { if (is_thing(*p)) { a = thing_arityval(*p); @@ -4230,7 +4884,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) Eterm l; Uint32 ret_flags; Uint sz; - Eterm *save_cp; + BeamInstr *save_cp; if (trace && !(is_list(against) || against == NIL)) { return THE_NON_VALUE; @@ -4271,25 +4925,26 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) ++n; l = CDR(list_val(l)); } + save_cp = p->cp; + p->cp = NULL; + res = erts_match_set_run(p, mps, arr, n, + ERTS_PAM_COPY_RESULT, &ret_flags); + p->cp = save_cp; } else { n = 0; - arr = (Eterm *) against; + arr = NULL; + res = erts_match_set_run_ets(p, mps, against, n, &ret_flags); } /* We are in the context of a BIF, {caller} should return 'undefined' */ - save_cp = p->cp; - p->cp = NULL; - res = erts_match_set_run(p, mps, arr, n, &ret_flags); - p->cp = save_cp; if (is_non_value(res)) { res = am_false; } - sz = size_object(res); + sz = 0; if (ret_flags & MATCH_SET_EXCEPTION_TRACE) sz += 2; if (ret_flags & MATCH_SET_RETURN_TRACE) sz += 2; hp = HAlloc(p, 5 + sz); - res = copy_struct(res, sz, &hp, &MSO(p)); flg = NIL; if (ret_flags & MATCH_SET_EXCEPTION_TRACE) { flg = CONS(hp, am_exception_trace, flg); @@ -4316,15 +4971,70 @@ static Eterm seq_trace_fake(Process *p, Eterm arg1) } return result; } - + +DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org) +{ + ErlOffHeap tmp_offheap; + DbTerm* res = erts_alloc(ERTS_ALC_T_TMP, + sizeof(DbTerm) + org->size*sizeof(Eterm)); + Eterm* hp = res->tpl; + tmp_offheap.first = NULL; + db_copy_from_comp(tb, org, &hp, &tmp_offheap); + res->first_oh = tmp_offheap.first; + res->size = org->size; +#ifdef DEBUG_CLONE + res->debug_clone = NULL; +#endif + return res; +} + +void db_free_tmp_uncompressed(DbTerm* obj) +{ + ErlOffHeap off_heap; + off_heap.first = obj->first_oh; + erts_cleanup_offheap(&off_heap); +#ifdef DEBUG_CLONE + ASSERT(obj->debug_clone == NULL); +#endif + erts_free(ERTS_ALC_T_TMP, obj); +} + +Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, + int all, DbTerm* obj, Eterm** hpp, Uint extra) +{ + Uint32 dummy; + Eterm* base; + Eterm res; + + if (tb->compress) { + obj = db_alloc_tmp_uncompressed(tb, obj); + base = NULL; + } + else base = HALFWORD_HEAP ? obj->tpl : NULL; + + res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0, + ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); + + if (is_value(res) && hpp!=NULL) { + *hpp = HAlloc(c_p, extra); + } + + if (tb->compress) { + db_free_tmp_uncompressed(obj); + } + return res; +} + + #ifdef DMC_DEBUG + /* ** Disassemble match program */ -static void db_match_dis(Binary *bp) +void db_match_dis(Binary *bp) { MatchProg *prog = Binary2MatchProg(bp); - Uint *t = prog->text; + UWord *t = prog->text; Uint n; Eterm p; int first; @@ -4336,31 +5046,31 @@ static void db_match_dis(Binary *bp) ++t; n = *t; ++t; - erts_printf("TryMeElse\t%bpu\n", n); + erts_printf("TryMeElse\t%beu\n", n); break; case matchArray: ++t; n = *t; ++t; - erts_printf("Array\t%bpu\n", n); + erts_printf("Array\t%beu\n", n); break; case matchArrayBind: ++t; n = *t; ++t; - erts_printf("ArrayBind\t%bpu\n", n); + erts_printf("ArrayBind\t%beu\n", n); break; case matchTuple: ++t; n = *t; ++t; - erts_printf("Tuple\t%bpu\n", n); + erts_printf("Tuple\t%beu\n", n); break; case matchPushT: ++t; n = *t; ++t; - erts_printf("PushT\t%bpu\n", n); + erts_printf("PushT\t%beu\n", n); break; case matchPushL: ++t; @@ -4374,13 +5084,13 @@ static void db_match_dis(Binary *bp) ++t; n = *t; ++t; - erts_printf("Bind\t%bpu\n", n); + erts_printf("Bind\t%beu\n", n); break; case matchCmp: ++t; n = *t; ++t; - erts_printf("Cmp\t%bpu\n", n); + erts_printf("Cmp\t%beu\n", n); break; case matchEqBin: ++t; @@ -4390,41 +5100,48 @@ static void db_match_dis(Binary *bp) break; case matchEqRef: ++t; - n = thing_arityval(*t); - ++t; - erts_printf("EqRef\t(%d) {", (int) n); - first = 1; - while (n--) { - if (first) - first = 0; - else - erts_printf(", "); -#ifdef ARCH_64 - erts_printf("0x%016bpx", *t); + { + RefThing *rt = (RefThing *) t; + int ri; + n = thing_arityval(rt->header); + erts_printf("EqRef\t(%d) {", (int) n); + first = 1; + for (ri = 0; ri < n; ++ri) { + if (first) + first = 0; + else + erts_printf(", "); +#if defined(ARCH_64) && !HALFWORD_HEAP + erts_printf("0x%016bex", rt->data.ui[ri]); #else - erts_printf("0x%08bpx", *t); + erts_printf("0x%08bex", rt->data.ui[ri]); #endif - ++t; + } } + t += TermWords(REF_THING_SIZE); erts_printf("}\n"); break; case matchEqBig: ++t; n = thing_arityval(*t); - ++t; - erts_printf("EqBig\t(%d) {", (int) n); - first = 1; - while (n--) { - if (first) - first = 0; - else - erts_printf(", "); -#ifdef ARCH_64 - erts_printf("0x%016bpx", *t); + { + Eterm *et = (Eterm *) t; + t += TermWords(n+1); + erts_printf("EqBig\t(%d) {", (int) n); + first = 1; + ++n; + while (n--) { + if (first) + first = 0; + else + erts_printf(", "); +#if defined(ARCH_64) && !HALFWORD_HEAP + erts_printf("0x%016bex", *et); #else - erts_printf("0x%08bpx", *t); + erts_printf("0x%08bex", *et); #endif - ++t; + ++et; + } } erts_printf("}\n"); break; @@ -4432,8 +5149,8 @@ static void db_match_dis(Binary *bp) ++t; { double num; - memcpy(&num,t, 2 * sizeof(*t)); - t += 2; + memcpy(&num,t,sizeof(double)); + t += TermWords(2); erts_printf("EqFloat\t%f\n", num); } break; @@ -4473,31 +5190,31 @@ static void db_match_dis(Binary *bp) ++t; n = *t; ++t; - erts_printf("MkTuple\t%bpu\n", n); + erts_printf("MkTuple\t%beu\n", n); break; case matchOr: ++t; n = *t; ++t; - erts_printf("Or\t%bpu\n", n); + erts_printf("Or\t%beu\n", n); break; case matchAnd: ++t; n = *t; ++t; - erts_printf("And\t%bpu\n", n); + erts_printf("And\t%beu\n", n); break; case matchOrElse: ++t; n = *t; ++t; - erts_printf("OrElse\t%bpu\n", n); + erts_printf("OrElse\t%beu\n", n); break; case matchAndAlso: ++t; n = *t; ++t; - erts_printf("AndAlso\t%bpu\n", n); + erts_printf("AndAlso\t%beu\n", n); break; case matchCall0: ++t; @@ -4527,7 +5244,19 @@ static void db_match_dis(Binary *bp) ++t; n = (Uint) *t; ++t; - erts_printf("PushV\t%bpu\n", n); + erts_printf("PushV\t%beu\n", n); + break; + #if HALFWORD_HEAP + case matchPushVGuard: + n = (Uint) *++t; + ++t; + erts_printf("PushVGuard\t%beu\n", n); + break; + #endif + case matchPushVResult: + n = (Uint) *++t; + ++t; + erts_printf("PushVResult\t%beu\n", n); break; case matchTrue: ++t; @@ -4638,9 +5367,8 @@ static void db_match_dis(Binary *bp) } erts_printf("}\n"); erts_printf("num_bindings: %d\n", prog->num_bindings); - erts_printf("heap_size: %bpu\n", prog->heap_size); - erts_printf("eheap_offset: %bpu\n", prog->eheap_offset); - erts_printf("stack_offset: %bpu\n", prog->stack_offset); + erts_printf("heap_size: %beu\n", prog->heap_size); + erts_printf("stack_offset: %beu\n", prog->stack_offset); erts_printf("text: 0x%08x\n", (unsigned long) prog->text); erts_printf("stack_size: %d (words)\n", prog->heap_size-prog->stack_offset); diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 4fc7b4f52e..bb1751d309 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -52,22 +52,27 @@ is broken.*/ #define DB_ERROR_UNSPEC -10 /* Unspecified error */ +/*#define DEBUG_CLONE*/ /* * A datatype for a database entry stored out of a process heap */ typedef struct db_term { - ErlOffHeap off_heap; /* Off heap data for term. */ - Uint size; /* Size of term in "words" */ - Eterm tpl[1]; /* Untagged "constant pointer" to top tuple */ - /* (assumed to be first in buffer) */ + struct erl_off_heap_header* first_oh; /* Off heap data for term. */ + Uint size; /* Heap size of term in "words" */ +#ifdef DEBUG_CLONE + Eterm* debug_clone; /* An uncompressed copy */ +#endif + Eterm tpl[1]; /* Term data. Top tuple always first */ + + /* Compression: is_immed and key element are uncompressed. + Compressed elements are stored in external format after each other + last in dbterm. The top tuple elements contains byte offsets, to + the start of the data, tagged as headers. + The allocated size of the dbterm in bytes is stored at tpl[arity+1]. + */ } DbTerm; -/* "Assign" a value to DbTerm.tpl */ -#define DBTERM_SET_TPL(dbtermPtr,tplPtr) ASSERT((tplPtr)==(dbtermPtr->tpl)) -/* Get start of term buffer */ -#define DBTERM_BUF(dbtermPtr) ((dbtermPtr)->tpl) - union db_table; typedef union db_table DbTable; @@ -81,6 +86,9 @@ typedef struct { Uint new_size; int mustResize; void* lck; +#if HALFWORD_HEAP + unsigned char* abs_vec; /* [i] true if dbterm->tpl[i] is absolute Eterm */ +#endif } DbUpdateHandle; @@ -186,6 +194,12 @@ typedef struct db_table_method } DbTableMethod; +typedef struct db_fixation { + Eterm pid; + Uint counter; + struct db_fixation *next; +} DbFixation; + /* * This structure contains data for all different types of database * tables. Note that these fields must match the same fields @@ -194,16 +208,8 @@ typedef struct db_table_method * operations may be the same on different types of tables. */ -typedef struct db_fixation { - Eterm pid; - Uint counter; - struct db_fixation *next; -} DbFixation; - - typedef struct db_table_common { - erts_refc_t ref; - erts_refc_t fixref; /* fixation counter */ + erts_refc_t ref; /* fixation counter and delete counter */ #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ @@ -212,7 +218,7 @@ typedef struct db_table_common { #endif Eterm owner; /* Pid of the creator */ Eterm heir; /* Pid of the heir */ - Eterm heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ + UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ SysTimeval heir_started; /* To further identify the heir */ Eterm the_name; /* an atom */ Eterm id; /* atom | integer */ @@ -226,6 +232,7 @@ typedef struct db_table_common { Uint32 status; /* bit masks defined below */ int slot; /* slot index in meta_main_tab */ int keypos; /* defaults to 1 */ + int compress; } DbTableCommon; /* These are status bit patterns */ @@ -240,30 +247,78 @@ typedef struct db_table_common { #define DB_DUPLICATE_BAG (1 << 8) #define DB_ORDERED_SET (1 << 9) #define DB_DELETE (1 << 10) /* table is being deleted */ +#define DB_FREQ_READ (1 << 11) -#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED) +#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ) #define IS_HASH_TABLE(Status) (!!((Status) & \ (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) #define IS_TREE_TABLE(Status) (!!((Status) & \ DB_ORDERED_SET)) -#define NFIXED(T) (erts_refc_read(&(T)->common.fixref,0)) +#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0)) #define IS_FIXED(T) (NFIXED(T) != 0) -Eterm erts_ets_copy_object(Eterm, Process*); +/* + * tplp is an untagged pointer to a tuple we know is large enough + * and dth is a pointer to a DbTableHash. + */ +#define GETKEY(dth, tplp) (*((tplp) + ((DbTableCommon*)(dth))->keypos)) + + +ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj); +Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, + ErlOffHeap* off_heap); +int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b); +DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org); + +ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, + Eterm** hpp, ErlOffHeap* off_heap); +ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b); +Wterm db_do_read_element(DbUpdateHandle* handle, Sint position); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj) +{ + Eterm key = GETKEY(tb, obj->tpl); + if IS_CONST(key) return key; + else { + Uint size = size_object_rel(key, obj->tpl); + Eterm* hp = HAlloc(p, size); + Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL); + ASSERT(eq_rel(res,NULL,key,obj->tpl)); + return res; + } +} + +ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, + Eterm** hpp, ErlOffHeap* off_heap) +{ + if (tb->compress) { + return db_copy_from_comp(tb, bp, hpp, off_heap); + } + else { + return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl); + } +} + +ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) +{ + if (!tb->compress) { + return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl); + } + else { + return db_eq_comp(tb, a, b); + } +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ -/* optimised version of copy_object (normal case? atomic object) */ -#define COPY_OBJECT(obj, p, objp) \ - if (IS_CONST(obj)) { *(objp) = (obj); } \ - else { *objp = erts_ets_copy_object(obj, p); } #define DB_READ (DB_PROTECTED|DB_PUBLIC) #define DB_WRITE DB_PUBLIC #define DB_INFO (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE) -/* tb is an DbTableCommon and obj is an Eterm (tagged) */ -#define TERM_GETKEY(tb, obj) db_getkey((tb)->common.keypos, (obj)) - #define ONLY_WRITER(P,T) (((T)->common.status & (DB_PRIVATE|DB_PROTECTED)) \ && (T)->common.owner == (P)->id) @@ -276,15 +331,19 @@ Eterm db_set_trace_control_word_1(Process *p, Eterm val); void db_initialize_util(void); Eterm db_getkey(int keypos, Eterm obj); -void db_free_term_data(DbTerm* p); -void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +void db_cleanup_offheap_comp(DbTerm* p); +void db_free_term(DbTable *tb, void* basep, Uint offset); +void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj, + Uint pos, Eterm** hpp, Uint extra); int db_has_variable(Eterm obj); int db_is_variable(Eterm obj); void db_do_update_element(DbUpdateHandle* handle, Sint position, Eterm newval); -void db_finalize_update_element(DbUpdateHandle* handle); -Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr); +void db_finalize_resize(DbUpdateHandle* handle, Uint offset); +Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr); Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags); Binary *db_match_set_compile(Process *p, Eterm matchexpr, Uint flags); @@ -301,12 +360,11 @@ typedef struct match_prog { struct erl_heap_fragment *saved_program_buf; Eterm saved_program; Uint heap_size; /* size of: heap + eheap + stack */ - Uint eheap_offset; Uint stack_offset; #ifdef DMC_DEBUG - Uint* prog_end; /* End of program */ + UWord* prog_end; /* End of program */ #endif - Uint text[1]; /* Beginning of program */ + UWord text[1]; /* Beginning of program */ } MatchProg; /* @@ -366,8 +424,15 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Uint flags, DMCErrInfo *err_info); /* Returns newly allocated MatchProg binary with refc == 0*/ -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, int arity, + +Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, + int all, DbTerm* obj, Eterm** hpp, Uint extra); + +Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base, + Eterm *termp, int arity, + enum erts_pam_run_flags in_flags, Uint32 *return_flags /* Zeroed on enter */); + /* returns DB_ERROR_NONE if matches, 1 if not matches and some db error on error. */ DMCErrInfo *db_new_dmc_err_info(void); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index e5c3c76fdd..d7d6fcf0a2 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -207,11 +207,7 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj) case FLOAT_DEF: { FloatDef ff; GET_DOUBLE(obj, ff); -#ifdef _OSE_ - erts_print(to, to_arg, "%e", ff.fd); -#else erts_print(to, to_arg, "%.20e", ff.fd); -#endif } break; case BINARY_DEF: @@ -235,9 +231,9 @@ pps(Process* p, Eterm* stop) } while(sp >= stop) { - erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (Eterm) sp); + erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (UWord) sp); if (is_catch(*sp)) { - erts_print(to, to_arg, "catch %d", (Uint)catch_pc(*sp)); + erts_print(to, to_arg, "catch %ld", (UWord)catch_pc(*sp)); } else { paranoid_display(to, to_arg, p, *sp); } @@ -265,7 +261,7 @@ static int verify_eterm(Process *p,Eterm element) return 1; for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) { - if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) { + if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) { return 1; } } @@ -312,7 +308,7 @@ void erts_check_stack(Process *p) if (IN_HEAP(p, ptr)) continue; for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) - if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) { + if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) { in_mbuf = 1; break; } @@ -750,7 +746,7 @@ static void print_process_memory(Process *p) PTR_SIZE, "heap fragments", dashes, dashes, dashes, dashes); while (bp) { - print_untagged_memory(bp->mem,bp->mem + bp->size); + print_untagged_memory(bp->mem,bp->mem + bp->used_size); bp = bp->next; } } @@ -895,5 +891,29 @@ void print_memory_info(Process *p) #endif erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes); } +#if !HEAP_ON_C_STACK && defined(DEBUG) +Eterm *erts_debug_allocate_tmp_heap(int size, Process *p) +{ + ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); + int offset = sd->num_tmp_heap_used; + + ASSERT(offset+size <= TMP_HEAP_SIZE); + return (sd->tmp_heap)+offset; +} +void erts_debug_use_tmp_heap(int size, Process *p) +{ + ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); + + sd->num_tmp_heap_used += size; + ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE); +} +void erts_debug_unuse_tmp_heap(int size, Process *p) +{ + ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); + + sd->num_tmp_heap_used -= size; + ASSERT(sd->num_tmp_heap_used >= 0); +} +#endif #endif diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h index 74f4a00b63..bdfbaddbbf 100644 --- a/erts/emulator/beam/erl_debug.h +++ b/erts/emulator/beam/erl_debug.h @@ -1,26 +1,27 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifndef _ERL_DEBUG_H_ #define _ERL_DEBUG_H_ - #ifdef DEBUG +#include "erl_term.h" + #ifdef HIPE #include "hipe_debug.h" #endif @@ -92,6 +93,11 @@ extern void print_tagged_memory(Eterm *start, Eterm *end); extern void print_untagged_memory(Eterm *start, Eterm *end); extern void print_memory(Process *p); extern void print_memory_info(Process *p); +#if defined(DEBUG) && !HEAP_ON_C_STACK +extern Eterm *erts_debug_allocate_tmp_heap(int, Process *); +extern void erts_debug_use_tmp_heap(int, Process *); +extern void erts_debug_unuse_tmp_heap(int, Process *); +#endif #ifdef HYBRID extern void print_ma_info(void); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 489e74d960..401967a8de 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -48,6 +48,10 @@ # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif #include "erl_int_sizes_config.h" #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR # error SIZEOF_CHAR mismatch @@ -65,6 +69,11 @@ # error SIZEOF_LONG_LONG mismatch #endif +/* This is OK to override by the NIF/driver implementor */ +#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR) +#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__ +#endif + #include "erl_drv_nif.h" #include <stdlib.h> @@ -141,6 +150,27 @@ typedef struct { #define ERL_DRV_FLAG_SOFT_BUSY (1 << 1) /* + * Integer types + */ + +typedef unsigned long ErlDrvTermData; +typedef unsigned long ErlDrvUInt; +typedef signed long ErlDrvSInt; + +#if defined(__WIN32__) +typedef unsigned __int64 ErlDrvUInt64; +typedef __int64 ErlDrvSInt64; +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlDrvUInt64; +typedef long ErlDrvSInt64; +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlDrvUInt64; +typedef long long ErlDrvSInt64; +#else +#error No 64-bit integer type +#endif + +/* * A binary as seen in a driver. Note that a binary should never be * altered by the driver when it has been sent to Erlang. */ @@ -170,26 +200,6 @@ struct erl_drv_event_data { #endif typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */ -/* - * Used in monitors... - */ -typedef unsigned long ErlDrvTermData; -typedef unsigned long ErlDrvUInt; -typedef signed long ErlDrvSInt; - -#if defined(__WIN32__) -typedef unsigned __int64 ErlDrvUInt64; -typedef __int64 ErlDrvSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlDrvUInt64; -typedef long ErlDrvSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlDrvUInt64; -typedef long long ErlDrvSInt64; -#else -#error No 64-bit integer type -#endif - /* * A driver monitor */ @@ -272,7 +282,7 @@ typedef struct erl_drv_entry { the port */ void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); /* called when we have input from one of - the driver's handles) */ + the driver's handles */ void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event); /* called when output is possible to one of the driver's handles */ @@ -284,7 +294,7 @@ typedef struct erl_drv_entry { int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, int len, char **rbuf, int rlen); /* "ioctl" for drivers - invoked by - port_control/3) */ + port_control/3 */ void (*timeout)(ErlDrvData drv_data); /* Handling of timeout in driver */ void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev); /* called when we have output from erlang @@ -297,7 +307,7 @@ typedef struct erl_drv_entry { before 'stop' can be called */ int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, int len, char **rbuf, int rlen, unsigned int *flags); - /* Works mostly like 'control', a syncronous + /* Works mostly like 'control', a synchronous call into the driver. */ void (*event)(ErlDrvData drv_data, ErlDrvEvent event, ErlDrvEventData event_data); @@ -385,9 +395,9 @@ EXTERN int driver_exit (ErlDrvPort port, int err); EXTERN ErlDrvPDL driver_pdl_create(ErlDrvPort); EXTERN void driver_pdl_lock(ErlDrvPDL); EXTERN void driver_pdl_unlock(ErlDrvPDL); -EXTERN long driver_pdl_get_refc(ErlDrvPDL); -EXTERN long driver_pdl_inc_refc(ErlDrvPDL); -EXTERN long driver_pdl_dec_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL); /* * Process monitors @@ -423,9 +433,9 @@ EXTERN ErlDrvBinary* driver_realloc_binary(ErlDrvBinary *bin, int size); EXTERN void driver_free_binary(ErlDrvBinary *bin); /* Referenc count on driver binaries */ -EXTERN long driver_binary_get_refc(ErlDrvBinary *dbp); -EXTERN long driver_binary_inc_refc(ErlDrvBinary *dbp); -EXTERN long driver_binary_dec_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp); /* Allocation interface */ EXTERN void *driver_alloc(size_t size); diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 50d8c25c46..dc578f6d2a 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -24,6 +24,10 @@ #include "global.h" #include <string.h> +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) +#define __DARWIN__ 1 +#endif + #define ERL_DRV_THR_OPTS_SIZE(LAST_FIELD) \ (((size_t) &((ErlDrvThreadOpts *) 0)->LAST_FIELD) \ + sizeof(((ErlDrvThreadOpts *) 0)->LAST_FIELD)) @@ -186,10 +190,9 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_trylock(&dmtx->mtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_mutex_trylock()"); - return res; + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_trylock()"); + return ethr_mutex_trylock(&dmtx->mtx); #else return 0; #endif @@ -199,9 +202,9 @@ void erl_drv_mutex_lock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_lock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_lock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_lock()"); + ethr_mutex_lock(&dmtx->mtx); #endif } @@ -209,9 +212,9 @@ void erl_drv_mutex_unlock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_unlock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_unlock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_unlock()"); + ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -256,9 +259,9 @@ void erl_drv_cond_signal(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_signal(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_signal()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_signal()"); + ethr_cond_signal(&dcnd->cnd); #endif } @@ -266,9 +269,9 @@ void erl_drv_cond_broadcast(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_broadcast(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_broadcast()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_broadcast()"); + ethr_cond_broadcast(&dcnd->cnd); #endif } @@ -277,18 +280,13 @@ void erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res; if (!dcnd || !dmtx) { - res = EINVAL; - error: - fatal_error(res, "erl_drv_cond_wait()"); + fatal_error(EINVAL, "erl_drv_cond_wait()"); } while (1) { - res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); + int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); if (res == 0) break; - if (res != EINTR) - goto error; } #endif } @@ -333,10 +331,9 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); + return ethr_rwmutex_tryrlock(&drwlck->rwmtx); #else return 0; #endif @@ -346,9 +343,9 @@ void erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); + ethr_rwmutex_rlock(&drwlck->rwmtx); #endif } @@ -356,9 +353,9 @@ void erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_runlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_runlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); + ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -366,10 +363,9 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrwlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); + return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); #else return 0; #endif @@ -379,9 +375,9 @@ void erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); + ethr_rwmutex_rwlock(&drwlck->rwmtx); #endif } @@ -389,9 +385,9 @@ void erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwunlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwunlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); + ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } @@ -536,7 +532,7 @@ erl_drv_tsd_get(ErlDrvTSDKey key) if (!dtid) return NULL; #endif - if (ERL_DRV_TSD_LEN__ < key) + if (ERL_DRV_TSD_LEN__ <= key) return NULL; return ERL_DRV_TSD__[key]; } @@ -603,11 +599,7 @@ erl_drv_thread_create(char *name, dtid->name = ((char *) dtid) + sizeof(struct ErlDrvTid_); sys_strcpy(dtid->name, name); } -#ifdef ERTS_ENABLE_LOCK_COUNT - res = erts_lcnt_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts); -#else res = ethr_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts); -#endif if (res != 0) { erts_free(ERTS_ALC_T_DRV_TID, dtid); @@ -704,3 +696,64 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp) #endif } +#if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP) +extern int erts_darwin_main_thread_pipe[2]; +extern int erts_darwin_main_thread_result_pipe[2]; + +int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp); +int erl_drv_steal_main_thread(char *name, + ErlDrvTid *dtid, + void* (*func)(void*), + void* arg, + ErlDrvThreadOpts *opts); + + +int +erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp) +{ + void *dummy; + void **x; + if (respp == NULL) + x = &dummy; + else + x = respp; + read(erts_darwin_main_thread_result_pipe[0],x,sizeof(void *)); + return 0; +} + +int +erl_drv_steal_main_thread(char *name, + ErlDrvTid *tid, + void* (*func)(void*), + void* arg, + ErlDrvThreadOpts *opts) +{ + char buff[sizeof(void* (*)(void*)) + sizeof(void *)]; + int buff_sz = sizeof(void* (*)(void*)) + sizeof(void *); + /*struct ErlDrvTid_ *dtid; + + dtid = erts_alloc_fnf(ERTS_ALC_T_DRV_TID, + (sizeof(struct ErlDrvTid_) + + (name ? sys_strlen(name) + 1 : 0))); + if (!dtid) + return ENOMEM; + memset(dtid,0,sizeof(ErlDrvTid_)); + dtid->tid = (void * ) -1; + dtid->drv_thr = 1; + dtid->func = func; + dtid->arg = arg; + dtid->tsd = NULL; + dtid->tsd_len = 0; + dtid->name = no_name; + *tid = (ErlDrvTid) dtid; + */ + *tid = NULL; + /* Ignore options and name... */ + + memcpy(buff,&func,sizeof(void* (*)(void*))); + memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *)); + write(erts_darwin_main_thread_pipe[1],buff,buff_sz); + return 0; +} + +#endif diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 79e844b315..88947b5536 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -37,8 +37,6 @@ static erts_smp_rwmtx_t erts_fun_table_lock; #define erts_fun_read_unlock() erts_smp_rwmtx_runlock(&erts_fun_table_lock) #define erts_fun_write_lock() erts_smp_rwmtx_rwlock(&erts_fun_table_lock) #define erts_fun_write_unlock() erts_smp_rwmtx_rwunlock(&erts_fun_table_lock) -#define erts_fun_init_lock() erts_smp_rwmtx_init(&erts_fun_table_lock, \ - "fun_tab") static HashValue fun_hash(ErlFunEntry* obj); static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2); @@ -50,15 +48,19 @@ static void fun_free(ErlFunEntry* obj); * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted * as an illegal arity when attempting to call a fun. */ -static Eterm unloaded_fun_code[3] = {NIL, -1, 0}; -static Eterm* unloaded_fun = unloaded_fun_code + 2; +static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0}; +static BeamInstr* unloaded_fun = unloaded_fun_code + 2; void erts_init_fun_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab"); - erts_fun_init_lock(); f.hash = (H_FUN) fun_hash; f.cmp = (HCMP_FUN) fun_cmp; f.alloc = (HALLOC_FUN) fun_alloc; @@ -95,7 +97,7 @@ erts_put_fun_entry(Eterm mod, int uniq, int index) { ErlFunEntry template; ErlFunEntry* fe; - long refc; + erts_aint_t refc; ASSERT(is_atom(mod)); template.old_uniq = uniq; template.old_index = index; @@ -117,7 +119,7 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, { ErlFunEntry template; ErlFunEntry* fe; - long refc; + erts_aint_t refc; ASSERT(is_atom(mod)); template.old_uniq = old_uniq; @@ -155,7 +157,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index) erts_fun_read_lock(); ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template); if (ret) { - long refc = erts_refc_inctest(&ret->refc, 1); + erts_aint_t refc = erts_refc_inctest(&ret->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&ret->refc, 1); } @@ -192,22 +194,8 @@ erts_erase_fun_entry(ErlFunEntry* fe) erts_fun_write_unlock(); } -#ifndef HYBRID /* FIND ME! */ -void -erts_cleanup_funs(ErlFunThing* funp) -{ - while (funp) { - ErlFunEntry* fe = funp->fe; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - funp = funp->next; - } -} -#endif - void -erts_cleanup_funs_on_purge(Eterm* start, Eterm* end) +erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) { int limit; HashBucket** bucket; @@ -222,7 +210,7 @@ erts_cleanup_funs_on_purge(Eterm* start, Eterm* end) while (b) { ErlFunEntry* fe = (ErlFunEntry *) b; - Eterm* addr = fe->address; + BeamInstr* addr = fe->address; if (start <= addr && addr < end) { fe->address = unloaded_fun; @@ -269,7 +257,7 @@ erts_dump_fun_entries(int to, void *to_arg) #ifdef HIPE erts_print(to, to_arg, "Native_address: %p\n", fe->native_address); #endif - erts_print(to, to_arg, "Refc: %d\n", erts_refc_read(&fe->refc, 1)); + erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); b = b->next; } } diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index fb5e75649b..2f165afa06 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -33,10 +33,10 @@ typedef struct erl_fun_entry { int index; /* New style index. */ int old_uniq; /* Unique number (old_style) */ int old_index; /* Old style index */ - Eterm* address; /* Pointer to code for fun */ + BeamInstr* address; /* Pointer to code for fun */ #ifdef HIPE - Eterm* native_address; /* Native entry code for fun. */ + UWord* native_address; /* Native entry code for fun. */ #endif Uint arity; /* The arity of the fun. */ @@ -53,12 +53,12 @@ typedef struct erl_fun_entry { typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ + ErlFunEntry* fe; /* Pointer to fun entry. */ #ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* next; /* Next fun in mso list. */ + struct erl_off_heap_header* next; #endif - ErlFunEntry* fe; /* Pointer to fun entry. */ #ifdef HIPE - Eterm* native_address; /* Native code for the fun. */ + UWord* native_address; /* Native code for the fun. */ #endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ @@ -86,7 +86,7 @@ void erts_erase_fun_entry(ErlFunEntry* fe); #ifndef HYBRID /* FIND ME! */ void erts_cleanup_funs(ErlFunThing* funp); #endif -void erts_cleanup_funs_on_purge(Eterm* start, Eterm* end); +void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end); void erts_dump_fun_entries(int, void *); #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index e9bf37a173..5edcd667e7 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,6 +33,7 @@ #include "erl_gc.h" #if HIPE #include "hipe_stack.h" +#include "hipe_mode_switch.h" #endif #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 @@ -110,9 +111,7 @@ static Uint adjust_after_fullsweep(Process *p, int size_before, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); -static void sweep_proc_bins(Process *p, int fullsweep); -static void sweep_proc_funs(Process *p, int fullsweep); -static void sweep_proc_externals(Process *p, int fullsweep); +static void sweep_off_heap(Process *p, int fullsweep); static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, @@ -126,7 +125,7 @@ static void disallow_heap_frag_ref_in_old_heap(Process* p); static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj); #endif -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP # define MAX_HEAP_SIZES 154 #else # define MAX_HEAP_SIZES 55 @@ -145,6 +144,16 @@ erts_init_gc(void) { int i = 0; + ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); + ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); + ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); + ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); + ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); + erts_smp_spinlock_init(&info_lck, "gc_info"); garbage_cols = 0; reclaimed = 0; @@ -206,7 +215,7 @@ erts_next_heap_size(Uint size, Uint offset) low = mid + 1; } } - erl_exit(1, "no next heap size found: %d, offset %d\n", size, offset); + erl_exit(1, "no next heap size found: %lu, offset %lu\n", (unsigned long)size, (unsigned long)offset); } return 0; } @@ -286,25 +295,14 @@ erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low)); } + #define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low)) void erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high) { - if (ohp->mso && ptr_within((Eterm *)ohp->mso, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->mso; - *uptr += offs; - } - -#ifndef HYBRID /* FIND ME! */ - if (ohp->funs && ptr_within((Eterm *)ohp->funs, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->funs; - *uptr += offs; - } -#endif - - if (ohp->externals && ptr_within((Eterm *)ohp->externals, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->externals; + if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) { + Eterm** uptr = (Eterm**) (void *) &ohp->first; *uptr += offs; } } @@ -457,7 +455,6 @@ erts_garbage_collect_hibernate(Process* p) Eterm* heap; Eterm* htop; Rootset rootset; - int n; char* src; Uint src_size; Uint actual_size; @@ -488,7 +485,10 @@ erts_garbage_collect_hibernate(Process* p) sizeof(Eterm)*heap_size); htop = heap; - n = setup_rootset(p, p->arg_reg, p->arity, &rootset); + (void) setup_rootset(p, p->arg_reg, p->arity, &rootset); +#if HIPE + hipe_empty_nstack(p); +#endif src = (char *) p->heap; src_size = (char *) p->htop - src; @@ -504,14 +504,8 @@ erts_garbage_collect_hibernate(Process* p) cleanup_rootset(&rootset); - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } /* @@ -667,7 +661,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) case TAG_PRIMARY_BOXED: ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, area, area_size)) { @@ -679,7 +673,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) case TAG_PRIMARY_LIST: ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (in_area(ptr, area, area_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); @@ -752,7 +746,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * is large enough. */ - if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) { + if (OLD_HEAP(p) && + ((mature <= OLD_HEND(p) - OLD_HTOP(p)) && + ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) && + ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { ErlMessage *msgp; Uint size_after; Uint need_after; @@ -913,7 +910,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, heap, mature_size)) { @@ -929,7 +926,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); @@ -972,7 +969,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, heap, mature_size)) { @@ -987,7 +984,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,n_hp++); @@ -1008,7 +1005,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) Eterm* origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(val); } else if (in_area(ptr, heap, mature_size)) { @@ -1041,15 +1038,8 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) OLD_HTOP(p) = old_htop; HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop; - if (MSO(p).mso) { - sweep_proc_bins(p, 0); - } - - if (MSO(p).funs) { - sweep_proc_funs(p, 0); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 0); + if (MSO(p).first) { + sweep_off_heap(p, 0); } #ifdef HARDDEBUG @@ -1161,7 +1151,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -1175,7 +1165,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,g_ptr++); @@ -1216,7 +1206,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -1229,7 +1219,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); @@ -1249,7 +1239,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); } else if (in_area(ptr, src, src_size) || @@ -1271,17 +1261,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } - if (OLD_HEAP(p) != NULL) { + if (OLD_HEAP(p) != NULL) { ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP, OLD_HEAP(p), (OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm)); @@ -1305,6 +1289,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) HIGH_WATER(p) = HEAP_TOP(p); ErtsGcQuickSanityCheck(p); + /* * Copy newly received message onto the end of the new heap. */ @@ -1392,17 +1377,12 @@ combined_message_size(Process* p) static void remove_message_buffers(Process* p) { - ErlHeapFragment* bp = MBUF(p); - - MBUF(p) = NULL; - MBUF_SIZE(p) = 0; - while (bp != NULL) { - ErlHeapFragment* next_bp = bp->next; - free_message_buffer(bp); - bp = next_bp; - } + if (MBUF(p) != NULL) { + free_message_buffer(MBUF(p)); + MBUF(p) = NULL; + } + MBUF_SIZE(p) = 0; } - #ifdef HARDDEBUG /* @@ -1433,12 +1413,12 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = _unchecked_boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); objv++; } else { for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1450,11 +1430,11 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = _unchecked_list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { objv++; } else { for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1499,7 +1479,7 @@ disallow_heap_frag_ref_in_heap(Process* p) ptr = _unchecked_boxed_val(val); if (!in_area(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1509,7 +1489,7 @@ disallow_heap_frag_ref_in_heap(Process* p) ptr = _unchecked_list_val(val); if (!in_area(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1551,26 +1531,26 @@ disallow_heap_frag_ref_in_old_heap(Process* p) val = *hp++; switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: - ptr = (Eterm *) val; + ptr = (Eterm *) EXPAND_POINTER(val); if (!in_area(ptr, old_heap, old_heap_size)) { if (in_area(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } } break; case TAG_PRIMARY_LIST: - ptr = (Eterm *) val; + ptr = (Eterm *) EXPAND_POINTER(val); if (!in_area(ptr, old_heap, old_heap_size)) { if (in_area(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1610,7 +1590,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1623,7 +1603,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,g_ptr++); @@ -1657,7 +1637,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1670,7 +1650,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); @@ -1690,7 +1670,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); } else if (in_area(ptr, src, src_size)) { @@ -1722,7 +1702,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1735,7 +1715,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,heap_ptr++); @@ -1830,28 +1810,6 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, return n_htop; } -#ifdef DEBUG -static Eterm follow_moved(Eterm term) -{ - Eterm* ptr; - switch (primary_tag(term)) { - case TAG_PRIMARY_IMMED1: - break; - case TAG_PRIMARY_BOXED: - ptr = boxed_val(term); - if (IS_MOVED(*ptr)) term = *ptr; - break; - case TAG_PRIMARY_LIST: - ptr = list_val(term); - if (is_non_value(ptr[0])) term = ptr[1]; - break; - default: - abort(); - } - return term; -} -#endif - static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { @@ -2030,8 +1988,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj) HEAP_SIZE(p) = new_sz; } -static Uint -do_next_vheap_size(Uint vheap, Uint vheap_sz) { +static Uint64 +do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) { /* grow * @@ -2048,131 +2006,53 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) { * ---------------------- */ - if (vheap > (Uint) (vheap_sz*3/4)) { + if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) { + Uint64 new_vheap_sz = vheap_sz; - while(vheap > (Uint) (vheap_sz*3/4)) { - vheap_sz = vheap_sz*2; + while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) { + /* the golden ratio = 1.618 */ + new_vheap_sz = (Uint64) vheap_sz * 1.618; + if (new_vheap_sz < vheap_sz ) { + return vheap_sz; + } + vheap_sz = new_vheap_sz; } - return erts_next_heap_size(vheap_sz, 0); + return vheap_sz; } - if (vheap < (Uint) (vheap_sz/4)) { - return erts_next_heap_size((Uint) (vheap_sz / 2), 0); + if (vheap < (Uint64) (vheap_sz/4)) { + return (vheap_sz >> 1); } return vheap_sz; } -static Uint -next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) { - vheap_sz = do_next_vheap_size(vheap, vheap_sz); - return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz; -} - -static void -sweep_proc_externals(Process *p, int fullsweep) -{ - ExternalThing** prev; - ExternalThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).externals; - ptr = MSO(p).externals; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - ExternalThing* ro = external_thing_ptr(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - erts_deref_node_entry(ptr->node); - *prev = ptr = ptr->next; - } - } - ASSERT(*prev == NULL); -} - -static void -sweep_proc_funs(Process *p, int fullsweep) -{ - ErlFunThing** prev; - ErlFunThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).funs; - ptr = MSO(p).funs; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - ErlFunThing* ro = (ErlFunThing *) fun_val(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - ErlFunEntry* fe = ptr->fe; - - *prev = ptr = ptr->next; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - } - } - ASSERT(*prev == NULL); +static Uint64 +next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) { + Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz); + return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz; } struct shrink_cand_data { - ProcBin* new_candidates; - ProcBin* new_candidates_end; - ProcBin* old_candidates; + struct erl_off_heap_header* new_candidates; + struct erl_off_heap_header* new_candidates_end; + struct erl_off_heap_header* old_candidates; Uint no_of_candidates; Uint no_of_active; }; static ERTS_INLINE void link_live_proc_bin(struct shrink_cand_data *shrink, - ProcBin ***prevppp, - ProcBin **pbpp, + struct erl_off_heap_header*** prevppp, + struct erl_off_heap_header** currpp, int new_heap) { - ProcBin *pbp = *pbpp; - - *pbpp = pbp->next; + ProcBin *pbp = (ProcBin*) *currpp; + ASSERT(**prevppp == *currpp); + *currpp = pbp->next; if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) { ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) == (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) @@ -2189,15 +2069,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink, /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ if (unused >= 8) { /* A shrink candidate; save in candidate list */ + **prevppp = pbp->next; if (new_heap) { if (!shrink->new_candidates) - shrink->new_candidates_end = pbp; + shrink->new_candidates_end = (struct erl_off_heap_header*)pbp; pbp->next = shrink->new_candidates; - shrink->new_candidates = pbp; + shrink->new_candidates = (struct erl_off_heap_header*)pbp; } else { pbp->next = shrink->old_candidates; - shrink->old_candidates = pbp; + shrink->old_candidates = (struct erl_off_heap_header*)pbp; } shrink->no_of_candidates++; return; @@ -2205,83 +2086,117 @@ link_live_proc_bin(struct shrink_cand_data *shrink, } } - /* Not a shrink candidate; keep in original mso list */ - **prevppp = pbp; + /* Not a shrink candidate; keep in original mso list */ *prevppp = &pbp->next; - } -static void -sweep_proc_bins(Process *p, int fullsweep) +static void +sweep_off_heap(Process *p, int fullsweep) { struct shrink_cand_data shrink = {0}; - ProcBin** prev; - ProcBin* ptr; - Binary* bptr; - char* oh = NULL; - Uint oh_size = 0; - Uint bin_vheap = 0; + struct erl_off_heap_header* ptr; + struct erl_off_heap_header** prev; + char* oheap = NULL; + Uint oheap_sz = 0; + Uint64 bin_vheap = 0; +#ifdef DEBUG + int seen_mature = 0; +#endif if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; + oheap = (char *) OLD_HEAP(p); + oheap_sz = (char *) OLD_HEND(p) - oheap; } BIN_OLD_VHEAP(p) = 0; - prev = &MSO(p).mso; - ptr = MSO(p).mso; + prev = &MSO(p).first; + ptr = MSO(p).first; - /* - * Note: In R7 we no longer force a fullsweep when we find binaries - * on the old heap. The reason is that with the introduction of the - * bit syntax we can expect binaries to be used a lot more. Note that - * in earlier releases a brand new binary (or any other term) could - * be put on the old heap during a gen-gc fullsweep, but this is - * no longer the case in R7. + /* Firts part of the list will reside on the (old) new-heap. + * Keep if moved, otherwise deref. */ while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - bin_vheap += ptr->size / sizeof(Eterm); - ptr = (ProcBin*) binary_val(*ppt); - link_live_proc_bin(&shrink, - &prev, - &ptr, - !in_area(ptr, oh, oh_size)); - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ - link_live_proc_bin(&shrink, &prev, &ptr, 0); - } else { /* Object has not been moved - deref it */ - - *prev = ptr->next; - bptr = ptr->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) - erts_bin_free(bptr); - ptr = *prev; - } + if (IS_MOVED_BOXED(ptr->thing_word)) { + ASSERT(!in_area(ptr, oheap, oheap_sz)); + *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + int to_new_heap = !in_area(ptr, oheap, oheap_sz); + ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); + if (to_new_heap) { + bin_vheap += ptr->size / sizeof(Eterm); + } else { + BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ + } + link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap); + } + else { + prev = &ptr->next; + ptr = ptr->next; + } + } + else if (!in_area(ptr, oheap, oheap_sz)) { + /* garbage */ + switch (thing_subtag(ptr->thing_word)) { + case REFC_BINARY_SUBTAG: + { + Binary* bptr = ((ProcBin*)ptr)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + break; + } + case FUN_SUBTAG: + { + ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe; + if (erts_refc_dectest(&fe->refc, 0) == 0) { + erts_erase_fun_entry(fe); + } + break; + } + default: + ASSERT(is_external_header(ptr->thing_word)); + erts_deref_node_entry(((ExternalThing*)ptr)->node); + } + *prev = ptr = ptr->next; + } + else break; /* and let old-heap loop continue */ } - if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) { - FLAGS(p) |= F_NEED_FULLSWEEP; + /* The rest of the list resides on old-heap, and we just did a + * generational collection - keep objects in list. + */ + while (ptr) { + ASSERT(in_area(ptr, oheap, oheap_sz)); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ + link_live_proc_bin(&shrink, &prev, &ptr, 0); + } + else { + ASSERT(is_fun_header(ptr->thing_word) || + is_external_header(ptr->thing_word)); + prev = &ptr->next; + ptr = ptr->next; + } } - BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); - BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p)); - MSO(p).overhead = bin_vheap; + if (fullsweep) { + BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p)); + } + BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); + MSO(p).overhead = bin_vheap; + BIN_VHEAP_MATURE(p) = bin_vheap; /* * If we got any shrink candidates, check them out. */ if (shrink.no_of_candidates) { - ProcBin *candlist[] = {shrink.new_candidates, shrink.old_candidates}; + ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates, + (ProcBin*)shrink.old_candidates }; Uint leave_unused = 0; int i; @@ -2293,21 +2208,21 @@ sweep_proc_bins(Process *p, int fullsweep) } for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) { - - for (ptr = candlist[i]; ptr; ptr = ptr->next) { - Uint new_size = ptr->size; + ProcBin* pb; + for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) { + Uint new_size = pb->size; if (leave_unused) { new_size += (new_size * 100) / leave_unused; /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ - if (new_size + 8 >= ptr->val->orig_size) + if (new_size + 8 >= pb->val->orig_size) continue; } - ptr->val = erts_bin_realloc(ptr->val, new_size); - ptr->val->orig_size = new_size; - ptr->bytes = (byte *) ptr->val->orig_bytes; + pb->val = erts_bin_realloc(pb->val, new_size); + pb->val->orig_size = new_size; + pb->bytes = (byte *) pb->val->orig_bytes; } } @@ -2316,21 +2231,20 @@ sweep_proc_bins(Process *p, int fullsweep) * We now potentially have the mso list divided into three lists: * - shrink candidates on new heap (inactive writable with unused data) * - shrink candidates on old heap (inactive writable with unused data) - * - other binaries (read only + active writable ...) + * - other binaries (read only + active writable ...) + funs and externals * * Put them back together: new candidates -> other -> old candidates * This order will ensure that the list only refers from new * generation to old and never from old to new *which is important*. */ if (shrink.new_candidates) { - if (prev == &MSO(p).mso) /* empty other binaries list */ + if (prev == &MSO(p).first) /* empty other binaries list */ prev = &shrink.new_candidates_end->next; else - shrink.new_candidates_end->next = MSO(p).mso; - MSO(p).mso = shrink.new_candidates; + shrink.new_candidates_end->next = MSO(p).first; + MSO(p).first = shrink.new_candidates; } } - *prev = shrink.old_candidates; } @@ -2361,15 +2275,17 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) tari = thing_arityval(val); switch (thing_subtag(val)) { case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { - ProcBin* pb = (ProcBin*) hp; - Eterm** uptr = (Eterm **) (void *) &pb->next; + struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp; - if (*uptr && in_area((Eterm *)pb->next, area, area_size)) { + if (in_area(oh->next, area, area_size)) { + Eterm** uptr = (Eterm **) (void *) &oh->next; *uptr += offs; /* Patch the mso chain */ } - sz -= tari; - hp += tari + 1; } break; case BIN_MATCHSTATE_SUBTAG: @@ -2380,40 +2296,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) mb->orig = offset_ptr(mb->orig, offs); mb->base = binary_bytes(mb->orig); } - sz -= tari; - hp += tari + 1; } break; - case FUN_SUBTAG: - { - ErlFunThing* funp = (ErlFunThing *) hp; - Eterm** uptr = (Eterm **) (void *) &funp->next; - - if (*uptr && in_area((Eterm *)funp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - { - ExternalThing* etp = (ExternalThing *) hp; - Eterm** uptr = (Eterm **) (void *) &etp->next; - - if (*uptr && in_area((Eterm *)etp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - default: - sz -= tari; - hp += tari + 1; } + sz -= tari; + hp += tari + 1; break; } default: @@ -2450,18 +2337,8 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) { - if (MSO(p).mso && in_area((Eterm *)MSO(p).mso, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).mso; - *uptr += offs; - } - - if (MSO(p).funs && in_area((Eterm *)MSO(p).funs, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).funs; - *uptr += offs; - } - - if (MSO(p).externals && in_area((Eterm *)MSO(p).externals, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).externals; + if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) { + Eterm** uptr = (Eterm**) (void *) &MSO(p).first; *uptr += offs; } } @@ -2542,7 +2419,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) return 1; } while (bp != NULL) { - if (bp->mem <= ptr && ptr < bp->mem + bp->size) { + if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { return 1; } bp = bp->next; @@ -2556,7 +2433,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) hfp = erts_dist_ext_trailer(mp->data.dist_ext); else hfp = NULL; - if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->size) + if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size) return 1; } mp = mp->next; @@ -2582,8 +2459,8 @@ do { \ __FILE__, __LINE__, #EXP); \ } while (0) -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST -# define ERTS_EXTERNAL_VISITED_BIT ((Eterm) 1 << 31) +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST +# define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31) #endif @@ -2593,62 +2470,45 @@ erts_check_off_heap2(Process *p, Eterm *htop) Eterm *oheap = (Eterm *) OLD_HEAP(p); Eterm *ohtop = (Eterm *) OLD_HTOP(p); int old; - ProcBin *pb; - ErlFunThing *eft; - ExternalThing *et; + union erl_off_heap_ptr u; old = 0; - for (pb = MSO(p).mso; pb; pb = pb->next) { - Eterm *ptr = (Eterm *) pb; - long refc = erts_refc_read(&pb->val->refc, 1); + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) { + erts_aint_t refc; + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + refc = erts_refc_read(&u.pb->val->refc, 1); + break; + case FUN_SUBTAG: + refc = erts_refc_read(&u.fun->fe->refc, 1); + break; + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + refc = erts_refc_read(&u.ext->node->refc, 1); + break; + default: + ASSERT(!!"erts_check_off_heap2: Invalid thing_word"); + } ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST + ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT)); + u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT; +#endif if (old) { - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); + ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop); } - else if (oheap <= ptr && ptr < ohtop) + else if (oheap <= u.ep && u.ep < ohtop) old = 1; else { - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); + ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop)); } } - old = 0; - for (eft = MSO(p).funs; eft; eft = eft->next) { - Eterm *ptr = (Eterm *) eft; - long refc = erts_refc_read(&eft->fe->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); - } - - old = 0; - for (et = MSO(p).externals; et; et = et->next) { - Eterm *ptr = (Eterm *) et; - long refc = erts_refc_read(&et->node->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - ERTS_CHK_OFFHEAP_ASSERT(!(et->header & ERTS_EXTERNAL_VISITED_BIT)); -#endif - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - et->header |= ERTS_EXTERNAL_VISITED_BIT; -#endif - } - #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - for (et = MSO(p).externals; et; et = et->next) - et->header &= ~ERTS_EXTERNAL_VISITED_BIT; + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) + u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT; #endif - } void diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index af55b6363f..807ef8ae8d 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -22,11 +22,12 @@ /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */ -#ifdef DEBUG +#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF # define HARDDEBUG 1 #endif -#define IS_MOVED(x) (!is_header((x))) +#define IS_MOVED_BOXED(x) (!is_header((x))) +#define IS_MOVED_CONS(x) (is_non_value((x))) #define MOVE_CONS(PTR,CAR,HTOP,ORIG) \ do { \ @@ -69,4 +70,28 @@ extern Uint erts_test_long_gc_sleep; int within(Eterm *ptr, Process *p); #endif +ERTS_GLB_INLINE Eterm follow_moved(Eterm term); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Eterm follow_moved(Eterm term) +{ + Eterm* ptr; + switch (primary_tag(term)) { + case TAG_PRIMARY_IMMED1: + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(term); + if (IS_MOVED_BOXED(*ptr)) term = *ptr; + break; + case TAG_PRIMARY_LIST: + ptr = list_val(term); + if (IS_MOVED_CONS(ptr[0])) term = ptr[1]; + break; + default: + ASSERT(!"strange tag in follow_moved"); + } + return term; +} +#endif + #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index ea2ba4d55c..76b206d76f 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -49,30 +49,30 @@ #define MIN_MBC_FIRST_FREE_SZ (4*1024) #define MAX_SUB_MASK_IX \ - ((((Uint)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1) -#define MAX_SUB_BKT_IX ((((Uint)1) << SUB_MASK_IX_SHIFT) - 1) + ((((UWord)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1) +#define MAX_SUB_BKT_IX ((((UWord)1) << SUB_MASK_IX_SHIFT) - 1) #define MAX_BKT_IX (NO_OF_BKTS - 1) -#define MIN_BLK_SZ UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(Uint)) +#define MIN_BLK_SZ UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(UWord)) -#define IX2SBIX(IX) ((IX) & (~(~((Uint)0) << SUB_MASK_IX_SHIFT))) +#define IX2SBIX(IX) ((IX) & (~(~((UWord)0) << SUB_MASK_IX_SHIFT))) #define IX2SMIX(IX) ((IX) >> SUB_MASK_IX_SHIFT) #define MAKE_BKT_IX(SMIX, SBIX) \ - ((((Uint)(SMIX)) << SUB_MASK_IX_SHIFT) | ((Uint)(SBIX))) + ((((UWord)(SMIX)) << SUB_MASK_IX_SHIFT) | ((UWord)(SBIX))) #define SET_BKT_MASK_IX(BM, IX) \ do { \ int sub_mask_ix__ = IX2SMIX((IX)); \ - (BM).main |= (((Uint) 1) << sub_mask_ix__); \ - (BM).sub[sub_mask_ix__] |= (((Uint)1) << IX2SBIX((IX))); \ + (BM).main |= (((UWord) 1) << sub_mask_ix__); \ + (BM).sub[sub_mask_ix__] |= (((UWord)1) << IX2SBIX((IX))); \ } while (0) #define UNSET_BKT_MASK_IX(BM, IX) \ do { \ int sub_mask_ix__ = IX2SMIX((IX)); \ - (BM).sub[sub_mask_ix__] &= ~(((Uint)1) << IX2SBIX((IX))); \ + (BM).sub[sub_mask_ix__] &= ~(((UWord)1) << IX2SBIX((IX))); \ if (!(BM).sub[sub_mask_ix__]) \ - (BM).main &= ~(((Uint)1) << sub_mask_ix__); \ + (BM).main &= ~(((UWord)1) << sub_mask_ix__); \ } while (0) /* Buckets ... */ @@ -263,8 +263,8 @@ find_bucket(BucketMask_t *bmask, int min_index) while(max != min) { \ mid = ((max - min) >> 1) + min; \ if((BitMask) \ - & (~(~((Uint) 0) << (mid + 1))) \ - & (~((Uint) 0) << min)) \ + & (~(~((UWord) 0) << (mid + 1))) \ + & (~((UWord) 0) << min)) \ max = mid; \ else \ min = mid + 1; \ @@ -272,21 +272,21 @@ find_bucket(BucketMask_t *bmask, int min_index) (MinBit) = min - ASSERT(bmask->main < (((Uint) 1) << (MAX_SUB_MASK_IX+1))); + ASSERT(bmask->main < (((UWord) 1) << (MAX_SUB_MASK_IX+1))); sub_mask_ix = IX2SMIX(min_index); - if ((bmask->main & (~((Uint) 0) << sub_mask_ix)) == 0) + if ((bmask->main & (~((UWord) 0) << sub_mask_ix)) == 0) return -1; /* There exists a non empty bucket; find it... */ - if (bmask->main & (((Uint) 1) << sub_mask_ix)) { + if (bmask->main & (((UWord) 1) << sub_mask_ix)) { sub_bkt_ix = IX2SBIX(min_index); - if ((bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix)) == 0) { + if ((bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix)) == 0) { sub_mask_ix++; sub_bkt_ix = 0; - if ((bmask->main & (~((Uint) 0)<< sub_mask_ix)) == 0) + if ((bmask->main & (~((UWord) 0)<< sub_mask_ix)) == 0) return -1; } else @@ -299,17 +299,17 @@ find_bucket(BucketMask_t *bmask, int min_index) ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX); /* Has to be a bit > sub_mask_ix */ - ASSERT(bmask->main & (~((Uint) 0) << (sub_mask_ix))); + ASSERT(bmask->main & (~((UWord) 0) << (sub_mask_ix))); GET_MIN_BIT(sub_mask_ix, bmask->main, sub_mask_ix, MAX_SUB_MASK_IX); find_sub_bkt_ix: ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX); ASSERT(sub_bkt_ix <= MAX_SUB_BKT_IX); - if ((bmask->sub[sub_mask_ix] & (((Uint) 1) << sub_bkt_ix)) == 0) { + if ((bmask->sub[sub_mask_ix] & (((UWord) 1) << sub_bkt_ix)) == 0) { ASSERT(sub_mask_ix + 1 <= MAX_SUB_BKT_IX); /* Has to be a bit > sub_bkt_ix */ - ASSERT(bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix)); + ASSERT(bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix)); GET_MIN_BIT(sub_bkt_ix, bmask->sub[sub_mask_ix], @@ -336,7 +336,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) Uint min_sz; Uint blk_sz; Uint cand_sz = 0; - Uint max_blk_search; + UWord max_blk_search; GFFreeBlock_t *blk; GFFreeBlock_t *cand = NULL; int blk_on_lambc; @@ -615,9 +615,9 @@ check_block(Allctr_t *allctr, Block_t * blk, int free_block) Uint blk_sz = BLK_SZ(blk); bi = BKT_IX(gfallctr, blk_sz); - ASSERT(gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi))); + ASSERT(gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi))); ASSERT(gfallctr->bucket_mask.sub[IX2SMIX(bi)] - & (((Uint) 1) << IX2SBIX(bi))); + & (((UWord) 1) << IX2SBIX(bi))); found = 0; for (fblk = gfallctr->buckets[bi]; fblk; fblk = fblk->next) @@ -648,9 +648,9 @@ check_mbc(Allctr_t *allctr, Carrier_t *mbc) int bi; for(bi = 0; bi < NO_OF_BKTS; bi++) { - if ((gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi))) + if ((gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi))) && (gfallctr->bucket_mask.sub[IX2SMIX(bi)] - & (((Uint) 1) << IX2SBIX(bi)))) { + & (((UWord) 1) << IX2SBIX(bi)))) { ASSERT(gfallctr->buckets[bi] != NULL); } else { diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h index 3d1b8c01f6..a554a6f466 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.h +++ b/erts/emulator/beam/erl_goodfit_alloc.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -28,7 +28,7 @@ typedef struct GFAllctr_t_ GFAllctr_t; typedef struct { - Uint mbsd; + UWord mbsd; } GFAllctrInit_t; #define ERTS_DEFAULT_GF_ALLCTR_INIT { \ @@ -49,18 +49,18 @@ Allctr_t *erts_gfalc_start(GFAllctr_t *, GFAllctrInit_t *, AllctrInit_t *); #include "erl_alloc_util.h" #define NO_OF_BKT_IX_BITS (8) -#ifdef ARCH_64 +#if defined(ARCH_64) # define SUB_MASK_IX_SHIFT (6) #else # define SUB_MASK_IX_SHIFT (5) #endif -#define NO_OF_BKTS (((Uint) 1) << NO_OF_BKT_IX_BITS) -#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((Uint) 1) << SUB_MASK_IX_SHIFT)) +#define NO_OF_BKTS (((UWord) 1) << NO_OF_BKT_IX_BITS) +#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((UWord) 1) << SUB_MASK_IX_SHIFT)) typedef struct { - Uint main; - Uint sub[NO_OF_SUB_MASKS]; -} BucketMask_t; + UWord main; + UWord sub[NO_OF_SUB_MASKS]; +} BucketMask_t; typedef struct GFFreeBlock_t_ GFFreeBlock_t; struct GFFreeBlock_t_ { @@ -74,11 +74,11 @@ struct GFAllctr_t_ { char * last_aux_mbc_start; char * last_aux_mbc_end; - Uint bkt_max_size_d; - Uint bkt_intrvl_d; + UWord bkt_max_size_d; + UWord bkt_intrvl_d; BucketMask_t bucket_mask; GFFreeBlock_t * buckets[NO_OF_BKTS]; - Uint max_blk_search; + UWord max_blk_search; }; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e97ab328cd..0a57eb6d88 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -41,6 +41,7 @@ #include "erl_printf_term.h" #include "erl_misc_utils.h" #include "packet_parser.h" +#include "erl_cpu_topology.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -63,6 +64,8 @@ extern void ConNormalExit(void); extern void ConWaitForExit(void); #endif +static void erl_init(int ncpu); + #define ERTS_MIN_COMPAT_REL 7 #ifdef ERTS_SMP @@ -76,8 +79,6 @@ int erts_initialized = 0; static erts_tid_t main_thread; #endif -erts_cpu_info_t *erts_cpuinfo; - int erts_use_sender_punish; /* @@ -99,7 +100,7 @@ int erts_backtrace_depth; /* How many functions to show in a backtrace int erts_async_max_threads; /* number of threads for async support */ int erts_async_thread_suggested_stack_size; -erts_smp_atomic_t erts_max_gen_gcs; +erts_smp_atomic32_t erts_max_gen_gcs; Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, am_info or am_warning, am_error is @@ -228,18 +229,18 @@ void erl_error(char *fmt, va_list args) erts_vfprintf(stderr, fmt, args); } -static void early_init(int *argc, char **argv); +static int early_init(int *argc, char **argv); void erts_short_init(void) { - early_init(NULL, NULL); - erl_init(); + int ncpu = early_init(NULL, NULL); + erl_init(ncpu); erts_initialized = 1; } -void -erl_init(void) +static void +erl_init(int ncpu) { init_benchmarking(); @@ -249,12 +250,13 @@ erl_init(void) erts_init_monitors(); erts_init_gc(); - init_time(); - erts_init_process(); + erts_init_time(); + erts_init_sys_common_misc(); + erts_init_process(ncpu); erts_init_scheduling(use_multi_run_queue, no_schedulers, no_schedulers_online); - + erts_init_cpu_topology(); /* Must be after init_scheduling */ H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); @@ -281,20 +283,18 @@ erl_init(void) init_load(); erts_init_bif(); erts_init_bif_chksum(); + erts_init_bif_binary(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG - erts_mseg_late_init(); /* Must be after timer (init_time()) and thread + erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread initializations */ #endif #ifdef HIPE hipe_mode_switch_init(); /* Must be after init_load/beam_catches/init */ #endif -#ifdef _OSE_ - erl_sys_init_final(); -#endif packet_parser_init(); erl_nif_init(); } @@ -323,7 +323,7 @@ init_shared_memory(int argc, char **argv) #endif global_gen_gcs = 0; - global_max_gen_gcs = erts_smp_atomic_read(&erts_max_gen_gcs); + global_max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); global_gc_flags = erts_default_process_flags; erts_global_offheap.mso = NULL; @@ -338,59 +338,6 @@ init_shared_memory(int argc, char **argv) #endif } - -/* - * Create the very first process. - */ - -void -erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv) -{ - int i; - Eterm args; - Eterm pid; - Eterm* hp; - Process parent; - Process* p; - ErlSpawnOpts so; - - if (erts_find_function(modname, am_start, 1) == NULL) { - char sbuf[256]; - Atom* ap; - - ap = atom_tab(atom_val(modname)); - memcpy(sbuf, ap->name, ap->len); - sbuf[ap->len] = '\0'; - erl_exit(5, "No function %s:start/1\n", sbuf); - } - - /* - * We need a dummy parent process to be able to call erl_create_process(). - */ - erts_init_empty_process(&parent); - hp = HAlloc(&parent, argc*2 + 4); - args = NIL; - for (i = argc-1; i >= 0; i--) { - int len = sys_strlen(argv[i]); - args = CONS(hp, new_binary(&parent, (byte*)argv[i], len), args); - hp += 2; - } - args = CONS(hp, new_binary(&parent, code, size), args); - hp += 2; - args = CONS(hp, args, NIL); - - so.flags = 0; - pid = erl_create_process(&parent, modname, am_start, args, &so); - p = process_tab[internal_pid_index(pid)]; - p->group_leader = pid; - - erts_cleanup_empty_process(&parent); -} - -/* - * XXX Old way of starting. Hopefully soon obsolete. - */ - static void erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) { @@ -560,10 +507,13 @@ void erts_usage(void) ERTS_MIN_COMPAT_REL, this_rel_num()); erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); + erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); + erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, @@ -585,7 +535,8 @@ void erts_usage(void) erts_fprintf(stderr, "-W<i|w> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); - + erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); + erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); erts_fprintf(stderr, "\n"); erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n"); @@ -593,7 +544,51 @@ void erts_usage(void) erl_exit(-1, ""); } -static void +#ifdef USE_THREADS +/* + * allocators for thread lib + */ + +static void *ethr_std_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_STD, (Uint) size); +} +static void *ethr_std_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_STD, ptr, (Uint) size); +} +static void ethr_std_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_STD, ptr); +} +static void *ethr_sl_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_SL, (Uint) size); +} +static void *ethr_sl_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_SL, ptr, (Uint) size); +} +static void ethr_sl_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_SL, ptr); +} +static void *ethr_ll_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_LL, (Uint) size); +} +static void *ethr_ll_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_LL, ptr, (Uint) size); +} +static void ethr_ll_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_LL, ptr); +} + +#endif + +static int early_init(int *argc, char **argv) /* * Only put things here which are * really important initialize @@ -606,6 +601,10 @@ early_init(int *argc, char **argv) /* int ncpuavail; int schdlrs; int schdlrs_onln; + int max_main_threads; + int max_reader_groups; + int reader_groups; + use_multi_run_queue = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; @@ -621,13 +620,11 @@ early_init(int *argc, char **argv) /* erts_use_sender_punish = 1; - erts_cpuinfo = erts_cpu_info_create(); - -#ifdef ERTS_SMP - ncpu = erts_get_cpu_configured(erts_cpuinfo); - ncpuonln = erts_get_cpu_online(erts_cpuinfo); - ncpuavail = erts_get_cpu_available(erts_cpuinfo); -#else + erts_pre_early_init_cpu_topology(&max_reader_groups, + &ncpu, + &ncpuonln, + &ncpuavail); +#ifndef ERTS_SMP ncpu = 1; ncpuonln = 1; ncpuavail = 1; @@ -654,7 +651,7 @@ early_init(int *argc, char **argv) /* erts_writing_erl_crash_dump = 0; #endif - erts_smp_atomic_init(&erts_max_gen_gcs, (long)((Uint16) -1)); + erts_smp_atomic32_init(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); #if defined(USE_THREADS) && !defined(ERTS_SMP) @@ -682,6 +679,24 @@ early_init(int *argc, char **argv) /* } if (argv[i][0] == '-') { switch (argv[i][1]) { + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + char *arg = get_arg(sub_param+1, argv[i+1], &i); + if (sscanf(arg, "%d", &max_reader_groups) != 1) { + erts_fprintf(stderr, + "bad reader groups limit: %s\n", arg); + erts_usage(); + } + if (max_reader_groups < 0) { + erts_fprintf(stderr, + "bad reader groups limit: %d\n", + max_reader_groups); + erts_usage(); + } + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -750,13 +765,40 @@ early_init(int *argc, char **argv) /* erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ + /* Require allocators */ + erts_early_init_scheduling(); + erts_init_utils(); + erts_early_init_cpu_topology(no_schedulers, + &max_main_threads, + max_reader_groups, + &reader_groups); - erts_early_init_scheduling(); /* Require allocators */ - erts_init_utils(); /* Require allocators */ +#ifdef USE_THREADS + { + erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER; + elid.mem.std.alloc = ethr_std_alloc; + elid.mem.std.realloc = ethr_std_realloc; + elid.mem.std.free = ethr_std_free; + elid.mem.sl.alloc = ethr_sl_alloc; + elid.mem.sl.realloc = ethr_sl_realloc; + elid.mem.sl.free = ethr_sl_free; + elid.mem.ll.alloc = ethr_ll_alloc; + elid.mem.ll.realloc = ethr_ll_realloc; + elid.mem.ll.free = ethr_ll_free; + elid.main_threads = max_main_threads; + elid.reader_groups = reader_groups; + + erts_thr_late_init(&elid); + } +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); #endif + +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_late_init(); +#endif #if defined(HIPE) hipe_signal_init(); /* must be done very early */ @@ -766,7 +808,10 @@ early_init(int *argc, char **argv) /* erl_sys_args(argc, argv); erts_ets_realloc_always_moves = 0; + erts_ets_always_compress = 0; + erts_dist_buf_busy_limit = ERTS_DE_BUSY_LIMIT; + return ncpu; } #ifndef ERTS_SMP @@ -800,8 +845,7 @@ erl_start(int argc, char **argv) char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; int async_max_threads = erts_async_max_threads; - - early_init(&argc, argv); + int ncpu = early_init(&argc, argv); envbufsz = sizeof(envbuf); if (erts_sys_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -812,14 +856,20 @@ erl_start(int argc, char **argv) envbufsz = sizeof(envbuf); if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { Uint16 max_gen_gcs = atoi(envbuf); - erts_smp_atomic_set(&erts_max_gen_gcs, (long) max_gen_gcs); + erts_smp_atomic32_set(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); } envbufsz = sizeof(envbuf); if (erts_sys_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) { async_max_threads = atoi(envbuf); } - + +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) + /* + * The default stack size on MacOS X is too small for pcre. + */ + erts_sched_thread_suggested_stack_size = 256; +#endif #ifdef DEBUG verbose = DEBUG_DEFAULT; @@ -858,7 +908,27 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("using display items %d\n",display_items)); break; - + case 'f': + if (!strncmp(argv[i],"-fn",3)) { + arg = get_arg(argv[i]+3, argv[i+1], &i); + switch (*arg) { + case 'u': + erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8); + break; + case 'l': + erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1); + break; + case 'a': + erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN); + default: + erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg); + erts_usage(); + } + break; + } else { + erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); + erts_usage(); + } case 'l': display_loads++; break; @@ -980,15 +1050,20 @@ erl_start(int argc, char **argv) break; case 'e': - /* set maximum number of ets tables */ - arg = get_arg(argv[i]+2, argv[i+1], &i); - if (( user_requested_db_max_tabs = atoi(arg) ) < 0) { - erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg); - erts_usage(); + if (sys_strcmp("c", argv[i]+2) == 0) { + erts_ets_always_compress = 1; + } + else { + /* set maximum number of ets tables */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (( user_requested_db_max_tabs = atoi(arg) ) < 0) { + erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("using maximum number of ets tables %d\n", + user_requested_db_max_tabs)); } - VERBOSE(DEBUG_SYSTEM, - ("using maximum number of ets tables %d\n", - user_requested_db_max_tabs)); break; case 'i': @@ -1052,7 +1127,7 @@ erl_start(int argc, char **argv) char *sub_param = argv[i]+2; if (has_prefix("bt", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - res = erts_init_scheduler_bind_type(arg); + res = erts_init_scheduler_bind_type_string(arg); if (res != ERTS_INIT_SCHED_BIND_TYPE_SUCCESS) { switch (res) { case ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED: @@ -1077,7 +1152,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ct", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - res = erts_init_cpu_topology(arg); + res = erts_init_cpu_topology_string(arg); if (res != ERTS_INIT_CPU_TOPOLOGY_OK) { switch (res) { case ERTS_INIT_CPU_TOPOLOGY_INVALID_ID: @@ -1120,10 +1195,20 @@ erl_start(int argc, char **argv) } else if (sys_strcmp("mrq", sub_param) == 0) use_multi_run_queue = 1; - else if (sys_strcmp("srq", sub_param) == 0) - use_multi_run_queue = 0; else if (sys_strcmp("nsp", sub_param) == 0) erts_use_sender_punish = 0; + else if (sys_strcmp("srq", sub_param) == 0) + use_multi_run_queue = 0; + else if (sys_strcmp("wt", sub_param) == 0) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if (erts_sched_set_wakeup_limit(arg) != 0) { + erts_fprintf(stderr, "scheduler wakeup threshold: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("scheduler wakup threshold: %s\n", arg)); + } else if (has_prefix("ss", sub_param)) { /* suggested stack size (Kilo Words) for scheduler threads */ arg = get_arg(sub_param+2, argv[i+1], &i); @@ -1238,9 +1323,17 @@ erl_start(int argc, char **argv) erts_async_thread_suggested_stack_size)); break; - case 'r': - erts_ets_realloc_always_moves = 1; + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + get_arg(sub_param+1, argv[i+1], &i); + /* already handled */ + } + else { + erts_ets_realloc_always_moves = 1; + } break; + } case 'n': /* XXX obsolete */ break; case 'c': @@ -1270,6 +1363,26 @@ erl_start(int argc, char **argv) } break; + case 'z': { + char *sub_param = argv[i]+2; + int new_limit; + + if (has_prefix("dbbl", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); + new_limit = atoi(arg); + if (new_limit < 1 || INT_MAX/1024 < new_limit) { + erts_fprintf(stderr, "Invalid dbbl limit: %d\n", new_limit); + erts_usage(); + } else { + erts_dist_buf_busy_limit = new_limit*1024; + } + } else { + erts_fprintf(stderr, "bad -z option %s\n", argv[i]); + erts_usage(); + } + break; + } + default: erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); erts_usage(); @@ -1310,7 +1423,7 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; - erl_init(); + erl_init(ncpu); init_shared_memory(boot_argc, boot_argv); load_preloaded(); @@ -1325,6 +1438,7 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else + erts_thr_set_main_status(1, 1); set_main_stack_size(); process_main(); #endif @@ -1398,7 +1512,7 @@ system_cleanup(int exit_code) erts_cleanup_incgc(); #endif -#if defined(USE_THREADS) && !defined(ERTS_SMP) +#if defined(USE_THREADS) exit_async(); #endif #if HAVE_ERTS_MSEG @@ -1447,13 +1561,7 @@ __decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...) if (fmt != NULL && *fmt != '\0') erl_error(fmt, args); /* Print error message. */ va_end(args); -#ifdef __WIN32__ - if(n > 0) ConWaitForExit(); - else ConNormalExit(); -#endif -#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_) - sys_tty_reset(); -#endif + sys_tty_reset(n); if (n == ERTS_INTR_EXIT) exit(0); @@ -1493,13 +1601,7 @@ __decl_noreturn void erl_exit(int n, char *fmt,...) if (fmt != NULL && *fmt != '\0') erl_error(fmt, args); /* Print error message. */ va_end(args); -#ifdef __WIN32__ - if(n > 0) ConWaitForExit(); - else ConNormalExit(); -#endif -#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_) - sys_tty_reset(); -#endif + sys_tty_reset(n); if (n == ERTS_INTR_EXIT) exit(0); diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index 3f022f92b8..f3f3c22933 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -540,18 +540,18 @@ static void dump_memory_map_to_stream(FILE *fp) if (is_internal_pid(bp->pid)) fprintf(fp, "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n", - (Uint) bp->type_no, - (Uint) bp->mem, - (Uint) bp->size, - (Uint) pid_channel_no(bp->pid), - (Uint) pid_number(bp->pid), - (Uint) pid_serial(bp->pid)); + (UWord) bp->type_no, + (UWord) bp->mem, + (UWord) bp->size, + (UWord) pid_channel_no(bp->pid), + (UWord) pid_number(bp->pid), + (UWord) pid_serial(bp->pid)); else fprintf(fp, "{%lu, %lu, %lu, undefined}.\n", - (Uint) bp->type_no, - (Uint) bp->mem, - (Uint) bp->size); + (UWord) bp->type_no, + (UWord) bp->mem, + (UWord) bp->size); } if (lock) @@ -638,7 +638,7 @@ Eterm erts_instr_get_memory_map(Process *proc) hsz += 4; } - if ((Uint) bp->mem > MAX_SMALL) + if ((UWord) bp->mem > MAX_SMALL) hsz += BIG_UINT_HEAP_SIZE; if (bp->size > MAX_SMALL) hsz += BIG_UINT_HEAP_SIZE; @@ -749,12 +749,12 @@ Eterm erts_instr_get_memory_map(Process *proc) #endif type = make_small(bp->type_no); - if ((Uint) bp->mem > MAX_SMALL) { - ptr = uint_to_big((Uint) bp->mem, hp); + if ((UWord) bp->mem > MAX_SMALL) { + ptr = uint_to_big((UWord) bp->mem, hp); hp += BIG_UINT_HEAP_SIZE; } else - ptr = make_small((Uint) bp->mem); + ptr = make_small((UWord) bp->mem); if (bp->size > MAX_SMALL) { size = uint_to_big(bp->size, hp); @@ -962,12 +962,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) fprintf(fp, "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n", - stats->tot.size, - stats->tot.max_size, - stats->tot.max_size_ever, - stats->tot.blocks, - stats->tot.max_blocks, - stats->tot.max_blocks_ever); + (UWord) stats->tot.size, + (UWord) stats->tot.max_size, + (UWord) stats->tot.max_size_ever, + (UWord) stats->tot.blocks, + (UWord) stats->tot.max_blocks, + (UWord) stats->tot.max_blocks_ever); a_max = 0; a_min = ~0; @@ -992,12 +992,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == a_min ? "{allocators,\n [" : " ", ERTS_ALC_A2AD(i), - stats->a[i].size, - stats->a[i].max_size, - stats->a[i].max_size_ever, - stats->a[i].blocks, - stats->a[i].max_blocks, - stats->a[i].max_blocks_ever, + (UWord) stats->a[i].size, + (UWord) stats->a[i].max_size, + (UWord) stats->a[i].max_size_ever, + (UWord) stats->a[i].blocks, + (UWord) stats->a[i].max_blocks, + (UWord) stats->a[i].max_blocks_ever, i == a_max ? "]}.\n" : ",\n"); } } @@ -1009,12 +1009,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_C_MIN ? "{classes,\n [" : " ", ERTS_ALC_C2CD(i), - stats->c[i].size, - stats->c[i].max_size, - stats->c[i].max_size_ever, - stats->c[i].blocks, - stats->c[i].max_blocks, - stats->c[i].max_blocks_ever, + (UWord) stats->c[i].size, + (UWord) stats->c[i].max_size, + (UWord) stats->c[i].max_size_ever, + (UWord) stats->c[i].blocks, + (UWord) stats->c[i].max_blocks, + (UWord) stats->c[i].max_blocks_ever, i == ERTS_ALC_C_MAX ? "]}.\n" : ",\n" ); } @@ -1025,12 +1025,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_N_MIN ? "{types,\n [" : " ", ERTS_ALC_N2TD(i), - stats->n[i].size, - stats->n[i].max_size, - stats->n[i].max_size_ever, - stats->n[i].blocks, - stats->n[i].max_blocks, - stats->n[i].max_blocks_ever, + (UWord) stats->n[i].size, + (UWord) stats->n[i].max_size, + (UWord) stats->n[i].max_size_ever, + (UWord) stats->n[i].blocks, + (UWord) stats->n[i].max_blocks, + (UWord) stats->n[i].max_blocks_ever, i == ERTS_ALC_N_MAX ? "]}.\n" : ",\n" ); } diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 074b08ea57..9180508a49 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -96,16 +96,15 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_status", "pid" }, { "proc_tab", NULL }, { "ports_snapshot", NULL }, - { "db_tab", "address" }, - { "db_tab_fix", "address" }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, + { "db_tab", "address" }, + { "db_tab_fix", "address" }, { "meta_main_tab_main", NULL }, { "db_hash_slot", "address" }, { "node_table", NULL }, { "dist_table", NULL }, { "sys_tracers", NULL }, - { "trace_pattern", NULL }, { "module_tab", NULL }, { "export_tab", NULL }, { "fun_tab", NULL }, @@ -120,17 +119,17 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "child_status", NULL }, #endif #ifdef __WIN32__ - { "sys_driver_data_lock", NULL }, + { "sys_driver_data_lock", NULL }, #endif - { "drv_ev_state_grow", NULL, }, + { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, { "pollset_rm_list", NULL }, { "removed_fd_pre_alloc_lock", NULL }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, - { "cpu_bind", NULL }, { "run_queue", "address" }, + { "cpu_info", NULL }, { "pollset", "address" }, #ifdef __WIN32__ { "pollwaiter", "address" }, @@ -154,7 +153,11 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "fix_alloc", "index" }, { "alcu_allocator", "index" }, + { "alcu_delayed_free", "index" }, { "mseg", NULL }, +#if HALFWORD_HEAP + { "pmmap", NULL }, +#endif #ifdef ERTS_SMP { "port_task_pre_alloc_lock", "address" }, { "port_taskq_pre_alloc_lock", "address" }, @@ -174,18 +177,22 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "async_id", NULL }, { "pix_lock", "address" }, { "run_queues_lists", NULL }, + { "misc_aux_work_queue", "index" }, + { "misc_aux_work_pre_alloc_lock", "address" }, { "sched_stat", NULL }, + { "run_queue_sleep_list", "address" }, #endif { "alloc_thr_ix_lock", NULL }, #ifdef ERTS_SMP - { "proc_lck_wtr_alloc", NULL }, + { "proc_lck_qs_alloc", NULL }, #endif #ifdef __WIN32__ #ifdef DEBUG { "save_ops_lock", NULL }, #endif #endif - { "mtrace_buf", NULL } + { "mtrace_buf", NULL }, + { "erts_alloc_hard_debug", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ @@ -197,6 +204,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { & ERTS_LC_FLG_LT_ALL \ & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK))) +static __decl_noreturn void __noreturn lc_abort(void); + static char * lock_type(Uint16 flags) { @@ -220,7 +229,7 @@ rw_op_str(Uint16 flags) return " (r)"; case ERTS_LC_FLG_LO_WRITE: erts_fprintf(stderr, "\nInternal error\n"); - abort(); + lc_abort(); default: break; } @@ -231,7 +240,7 @@ typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t; struct erts_lc_locked_lock_t_ { erts_lc_locked_lock_t *next; erts_lc_locked_lock_t *prev; - Eterm extra; + UWord extra; Sint16 id; Uint16 flags; }; @@ -269,28 +278,18 @@ static erts_lc_free_block_t *free_blocks; #define ERTS_LC_FB_CHUNK_SIZE 10 #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS static ethr_spinlock_t free_blocks_lock; -#define ERTS_LC_LOCK ethr_spin_lock -#define ERTS_LC_UNLOCK ethr_spin_unlock -#else -static ethr_mutex free_blocks_lock; -#define ERTS_LC_LOCK ethr_mutex_lock -#define ERTS_LC_UNLOCK ethr_mutex_unlock -#endif static ERTS_INLINE void lc_lock(void) { - if (ERTS_LC_LOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_lock(&free_blocks_lock); } static ERTS_INLINE void lc_unlock(void) { - if (ERTS_LC_UNLOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_unlock(&free_blocks_lock); } static ERTS_INLINE void lc_free(void *p) @@ -311,7 +310,7 @@ static void *lc_core_alloc(void) { lc_unlock(); erts_fprintf(stderr, "Lock checker out of memory!\n"); - abort(); + lc_abort(); } #else @@ -325,7 +324,7 @@ static void *lc_core_alloc(void) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { erts_fprintf(stderr, "Lock checker failed to allocate memory!\n"); - abort(); + lc_abort(); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG @@ -365,11 +364,11 @@ create_locked_locks(char *thread_name) { erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); if (!l_lcks) - abort(); + lc_abort(); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); l_lcks->tid = erts_thr_self(); l_lcks->required.first = NULL; @@ -442,12 +441,12 @@ new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags) } static void -print_lock2(char *prefix, Sint16 id, Eterm extra, Uint16 flags, char *suffix) +print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix) { char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE ? erts_lock_order[id].name : "unknown"); - if (is_boxed(extra)) + if (is_not_immed(extra)) erts_fprintf(stderr, "%s'%s:%p%s'%s%s", prefix, @@ -511,7 +510,7 @@ uninitialized_lock(void) { erts_fprintf(stderr, "Performing operations on uninitialized lock!\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } static void @@ -521,7 +520,7 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags)); print_lock(" ", lck, " lock which is already locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -531,7 +530,7 @@ unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags)); print_lock("", lck, " lock which mismatch previous lock operation!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -539,7 +538,7 @@ unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking ", lck, " lock which is not locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -548,7 +547,7 @@ lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) print_lock("Lock order violation occured when locking ", lck, "!\n"); print_curr_locks(l_lcks); print_lock_order(); - abort(); + lc_abort(); } static void @@ -559,7 +558,7 @@ type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks, print_lock(op, lck, "!\n"); ASSERT(l_lcks); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -611,7 +610,7 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact, } } print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -619,7 +618,7 @@ unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking required ", lck, " lock!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -627,7 +626,7 @@ unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *l { print_lock("Unrequire on ", lck, " lock not required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -635,7 +634,7 @@ require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Require on ", lck, " lock already required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -643,7 +642,7 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Required ", lck, " lock not locked!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } @@ -656,13 +655,23 @@ thread_exit_handler(void) erts_fprintf(stderr, "Thread exiting while having locked locks!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } destroy_locked_locks(l_lcks); /* erts_tsd_set(locks_key, NULL); */ } } +static __decl_noreturn void +lc_abort(void) +{ +#ifdef __WIN32__ + DebugBreak(); +#else + abort(); +#endif +} + void erts_lc_set_thread_name(char *thread_name) { @@ -674,7 +683,7 @@ erts_lc_set_thread_name(char *thread_name) free((void *) l_lcks->thread_name); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); } } @@ -684,7 +693,7 @@ erts_lc_assert_failed(char *file, int line, char *assertion) erts_fprintf(stderr, "%s:%d: Lock check assertion \"%s\" failed!\n", file, line, assertion); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); return 0; } @@ -697,7 +706,7 @@ void erts_lc_fail(char *fmt, ...) va_end(args); erts_fprintf(stderr, "\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } @@ -717,7 +726,7 @@ erts_lc_get_lock_order_id(char *name) "(update erl_lock_check.c)\n", name); } - abort(); + lc_abort(); return (Sint16) -1; } @@ -893,6 +902,25 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) } } +void +erts_lc_check_no_locked_of_type(Uint16 flags) +{ + erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + if (l_lcks) { + erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) { + if (l_lck->flags & flags) { + erts_fprintf(stderr, + "Locked lock of type %s found which isn't " + "allowed here!\n", + lock_type(l_lck->flags)); + print_curr_locks(l_lcks); + lc_abort(); + } + } + } +} + int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) { @@ -952,10 +980,10 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) /* We only force busy if a lock order violation would occur and when on an even millisecond. */ { - erts_thr_timeval_t time; - erts_thr_time_now(&time); + SysTimeval tv; + sys_gettimeofday(&tv); - if ((time.tv_nsec / 1000000) & 1) + if ((tv.tv_usec / 1000) & 1) return 0; } #endif @@ -1231,7 +1259,9 @@ void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags) { lck->id = erts_lc_get_lock_order_id(name); - lck->extra = make_boxed(&lck->extra); + + lck->extra = &lck->extra; + ASSERT(is_not_immed(lck->extra)); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; } @@ -1241,6 +1271,7 @@ erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra) { lck->id = erts_lc_get_lock_order_id(name); lck->extra = extra; + ASSERT(is_immed(lck->extra)); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; } @@ -1279,13 +1310,8 @@ erts_lc_init(void) free_blocks = NULL; #endif /* #ifdef ERTS_LC_STATIC_ALLOC */ -#ifdef ETHR_HAVE_NATIVE_LOCKS if (ethr_spinlock_init(&free_blocks_lock) != 0) - abort(); -#else - if (ethr_mutex_init(&free_blocks_lock) != 0) - abort(); -#endif + lc_abort(); erts_tsd_key_create(&locks_key); } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index d5e2ede9ac..b67f36fa06 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -39,7 +39,7 @@ typedef struct { int inited; Sint16 id; Uint16 flags; - Eterm extra; + UWord extra; } erts_lc_lock_t; #define ERTS_LC_INITITALIZED 0x7f7f7f7f @@ -77,6 +77,7 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len, void erts_lc_check_exact(erts_lc_lock_t *have, int have_len); void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len); void erts_lc_have_lock_ids(int *resv, int *ids, int len); +void erts_lc_check_no_locked_of_type(Uint16 flags); int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 0d7e1335c1..a36c53560e 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -159,15 +159,15 @@ static char* lock_opt(Uint16 flag) { } static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) { - long int colls, tries, w_state, r_state; + erts_aint_t colls, tries, w_state, r_state; erts_lcnt_lock_stats_t *stats = NULL; char *type; int i; type = lcnt_lock_type(lock->flag); - ethr_atomic_read(&lock->r_state, &r_state); - ethr_atomic_read(&lock->w_state, &w_state); + r_state = ethr_atomic_read(&lock->r_state); + w_state = ethr_atomic_read(&lock->w_state); if (lock->flag & flag) { @@ -257,6 +257,10 @@ void erts_lcnt_init() { erts_lcnt_clear_counters(); } +void erts_lcnt_late_init() { + erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); +} + /* list operations */ /* BEGIN ASSUMPTION: lcnt_data_lock taken */ @@ -381,7 +385,7 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { /* lock */ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { - long r_state = 0, w_state = 0; + erts_aint_t r_state = 0, w_state = 0; erts_lcnt_thread_data_t *eltd; if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; @@ -390,10 +394,10 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { ASSERT(eltd); - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); if (option & ERTS_LCNT_LO_WRITE) { - ethr_atomic_read(&lock->r_state, &r_state); + r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); } if (option & ERTS_LCNT_LO_READ) { @@ -414,12 +418,12 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { } void erts_lcnt_lock(erts_lcnt_lock_t *lock) { - long w_state; + erts_aint_t w_state; erts_lcnt_thread_data_t *eltd; if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ethr_atomic_inc( &lock->w_state); eltd = lcnt_get_thread_data(); @@ -467,14 +471,14 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line erts_lcnt_time_t time_wait; erts_lcnt_lock_stats_t *stats; #ifdef DEBUG - long flowstate; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); } @@ -512,18 +516,18 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { #ifdef DEBUG - long w_state; - long flowstate; + erts_aint_t w_state; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG /* flowstate */ - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 1); ethr_atomic_dec( &lock->flowstate); /* write state */ - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ASSERT(w_state > 0) #endif ethr_atomic_dec(&lock->w_state); @@ -548,13 +552,13 @@ void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* Determine lock_state via res instead of state */ #ifdef DEBUG - long flowstate; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; if (res != EBUSY) { #ifdef DEBUG - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); #endif @@ -570,36 +574,26 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* thread operations */ -static void *lcnt_thr_init(erts_lcnt_thread_data_t *eltd) { - void *(*function)(void *); - void *argument; - void *res; - function = eltd->function; - argument = eltd->argument; - - ethr_tsd_set(lcnt_thr_data_key, eltd); - - res = (void *)function(argument); - free(eltd); - return (void *)res; -} - - - -int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts) { +void erts_lcnt_thread_setup(void) { erts_lcnt_thread_data_t *eltd; - + lcnt_lock(); /* lock for thread id global update */ eltd = lcnt_thread_data_alloc(); lcnt_unlock(); - - eltd->function = function; - eltd->argument = arg; - - return ethr_thr_create(tid, (void *)lcnt_thr_init, (void *)eltd, opts); + ASSERT(eltd); + ethr_tsd_set(lcnt_thr_data_key, eltd); } +void erts_lcnt_thread_exit_handler() { + erts_lcnt_thread_data_t *eltd; + + eltd = ethr_tsd_get(lcnt_thr_data_key); + + if (eltd) { + free(eltd); + } +} /* bindings for bifs */ diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index e3044c371f..6306580ae4 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -155,11 +155,6 @@ typedef struct { erts_lcnt_time_t timer; /* timer */ int timer_set; /* bool */ int lock_in_conflict; /* bool */ - - /* function pointer */ - void *(*function)(void *); - void *argument; - } erts_lcnt_thread_data_t; /* globals */ @@ -169,6 +164,11 @@ extern Uint16 erts_lcnt_rt_options; /* function declerations */ void erts_lcnt_init(void); +void erts_lcnt_late_init(void); + +/* thread operations */ +void erts_lcnt_thread_setup(void); +void erts_lcnt_thread_exit_handler(void); /* list operations (local) */ erts_lcnt_lock_list_t *erts_lcnt_list_init(void); @@ -194,12 +194,7 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option); void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option); void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res); -/* thread operations */ - -int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts); - /* bif interface */ - Uint16 erts_lcnt_set_rt_opt(Uint16 opt); Uint16 erts_lcnt_clear_rt_opt(Uint16 opt); void erts_lcnt_clear_counters(void); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index a056fce0c5..82f272d28a 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -30,6 +30,7 @@ #include "erl_message.h" #include "erl_process.h" #include "erl_nmgc.h" +#include "erl_binary.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, ErlMessage, @@ -42,6 +43,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, #undef HARD_DEBUG #endif + + + +static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp) +{ + return ((unsigned)(ptr - bp->mem) < bp->used_size); +} + + void init_message(void) { @@ -81,9 +91,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif ErlHeapFragment* nbp; + /* ToDo: Make use of 'used_size' to avoid realloc + when shrinking just a few words */ + #ifdef DEBUG { - Uint off_sz = size < bp->size ? size : bp->size; + Uint off_sz = size < bp->used_size ? size : bp->used_size; for (i = 0; i < brefs_size; i++) { Eterm *ptr; if (is_immed(brefs[i])) @@ -95,12 +108,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif - if (size == bp->size) + if (size == bp->used_size) return bp; #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); - dbg_bp = new_message_buffer(bp->size); + dbg_bp = new_message_buffer(bp->used_size); dbg_hp = dbg_bp->mem; dbg_tot_size = 0; for (i = 0; i < brefs_size; i++) { @@ -109,15 +122,15 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp, &dbg_bp->off_heap); } - ASSERT(dbg_tot_size == (size < bp->size ? size : bp->size)); + ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size)); #endif nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size), + ERTS_HEAP_FRAG_SIZE(bp->alloc_size), ERTS_HEAP_FRAG_SIZE(size)); if (bp != nbp) { - Uint off_sz = size < nbp->size ? size : nbp->size; + Uint off_sz = size < nbp->used_size ? size : nbp->used_size; Eterm *sp = &bp->mem[0]; Eterm *ep = sp + off_sz; Sint offs = &nbp->mem[0] - sp; @@ -135,7 +148,7 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif } - nbp->size = size; + nbp->alloc_size = size; nbp->used_size = size; #ifdef HARD_DEBUG @@ -152,26 +165,40 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, void erts_cleanup_offheap(ErlOffHeap *offheap) { - if (offheap->mso) { - erts_cleanup_mso(offheap->mso); - } -#ifndef HYBRID /* FIND ME! */ - if (offheap->funs) { - erts_cleanup_funs(offheap->funs); - } -#endif - if (offheap->externals) { - erts_cleanup_externals(offheap->externals); + union erl_off_heap_ptr u; + + for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { + erts_bin_free(u.pb->val); + } + break; + case FUN_SUBTAG: + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + erts_erase_fun_entry(u.fun->fe); + } + break; + default: + ASSERT(is_external_header(u.hdr->thing_word)); + erts_deref_node_entry(u.ext->node); + break; + } } } void free_message_buffer(ErlHeapFragment* bp) { - erts_cleanup_offheap(&bp->off_heap); - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, - (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size)); + ASSERT(bp != NULL); + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); } static ERTS_INLINE void @@ -181,43 +208,19 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) /* Link the message buffer */ bp->next = MBUF(proc); MBUF(proc) = bp; - MBUF_SIZE(proc) += bp->size; + MBUF_SIZE(proc) += bp->used_size; FLAGS(proc) |= F_FORCE_GC; - /* Move any binaries into the process */ - if (bp->off_heap.mso != NULL) { - ProcBin** next_p = &bp->off_heap.mso; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).mso; - MSO(proc).mso = bp->off_heap.mso; - bp->off_heap.mso = NULL; - MSO(proc).overhead += bp->off_heap.overhead; - } - - /* Move any funs into the process */ -#ifndef HYBRID - if (bp->off_heap.funs != NULL) { - ErlFunThing** next_p = &bp->off_heap.funs; + /* Move any off_heap's into the process */ + if (bp->off_heap.first != NULL) { + struct erl_off_heap_header** next_p = &bp->off_heap.first; while (*next_p != NULL) { next_p = &((*next_p)->next); } - *next_p = MSO(proc).funs; - MSO(proc).funs = bp->off_heap.funs; - bp->off_heap.funs = NULL; - } -#endif - - /* Move any external things into the process */ - if (bp->off_heap.externals != NULL) { - ExternalThing** next_p = &bp->off_heap.externals; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).externals; - MSO(proc).externals = bp->off_heap.externals; - bp->off_heap.externals = NULL; + *next_p = MSO(proc).first; + MSO(proc).first = bp->off_heap.first; + bp->off_heap.first = NULL; + OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); } } } @@ -242,7 +245,7 @@ erts_msg_distext2heap(Process *pp, goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - tok_sz = heap_frag->size; + tok_sz = heap_frag->used_size; sz += tok_sz; } if (pp) @@ -283,12 +286,13 @@ erts_msg_distext2heap(Process *pp, erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (*bpp) + if (*bpp) { free_message_buffer(*bpp); + *bpp = NULL; + } else if (hp) { HRelease(pp, hp_end, hp); } - *bpp = NULL; return THE_NON_VALUE; } @@ -436,11 +440,10 @@ erts_queue_message(Process* receiver, ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; mp->next = NULL; + mp->data.heap_frag = bp; #ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { - mp->data.heap_frag = bp; - /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order @@ -453,11 +456,9 @@ erts_queue_message(Process* receiver, LINK_MESSAGE_PRIVQ(receiver, mp); } else { - mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); } #else - mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); #endif @@ -491,19 +492,7 @@ erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp) void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { - /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */ - union { - Uint** upp; - ProcBin **pbpp; - ErlFunThing **efpp; - ExternalThing **etpp; - } oh_list_pp, oh_el_next_pp; - union { - Uint *up; - ProcBin *pbp; - ErlFunThing *efp; - ExternalThing *etp; - } oh_el_p; + struct erl_off_heap_header* oh; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; @@ -530,40 +519,33 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); - ASSERT(bp->size == dbg_term_sz + dbg_token_sz); - - dbg_bp = new_message_buffer(bp->size); + /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); + Copied size may be smaller due to removed SubBins's or garbage. + Copied size may be larger due to duplicated shared terms. + */ + dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); dbg_thp_start = *hpp; #endif - ASSERT(bp); - msg->data.attached = NULL; - - off_heap->overhead += bp->off_heap.overhead; - sz = bp->size; - -#ifdef DEBUG - if (is_not_immed(term)) { - ASSERT(bp->mem <= ptr_val(term)); - ASSERT(bp->mem + bp->size > ptr_val(term)); + if (bp->next != NULL) { + move_multi_frags(hpp, off_heap, bp, msg->m, 2); + goto copy_done; } - if (is_not_immed(token)) { - ASSERT(bp->mem <= ptr_val(token)); - ASSERT(bp->mem + bp->size > ptr_val(token)); - } -#endif + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + sz = bp->used_size; + + ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp)); + ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp)); fhp = bp->mem; hp = *hpp; offs = hp - fhp; - oh_list_pp.upp = NULL; - oh_el_next_pp.upp = NULL; /* Shut up compiler warning */ - oh_el_p.up = NULL; /* Shut up compiler warning */ + oh = NULL; while (sz--) { Uint cpy_sz; Eterm val = *fhp++; @@ -574,8 +556,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - ASSERT(bp->mem <= ptr_val(val)); - ASSERT(bp->mem + bp->size > ptr_val(val)); + ASSERT(in_heapfrag(ptr_val(val), bp)); *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: @@ -584,31 +565,18 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: - oh_list_pp.pbpp = &off_heap->mso; - oh_el_p.up = (hp-1); - oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next; - cpy_sz = thing_arityval(val); - goto cpy_words; case FUN_SUBTAG: -#ifndef HYBRID - oh_list_pp.efpp = &off_heap->funs; - oh_el_p.up = (hp-1); - oh_el_next_pp.efpp = &(oh_el_p.efp)->next; -#endif - cpy_sz = thing_arityval(val); - goto cpy_words; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: - oh_list_pp.etpp = &off_heap->externals; - oh_el_p.up = (hp-1); - oh_el_next_pp.etpp = &(oh_el_p.etp)->next; + oh = (struct erl_off_heap_header*) (hp-1); cpy_sz = thing_arityval(val); goto cpy_words; default: cpy_sz = header_arity(val); cpy_words: + ASSERT(sz >= cpy_sz); sz -= cpy_sz; while (cpy_sz >= 8) { cpy_sz -= 8; @@ -631,44 +599,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case 1: *hp++ = *fhp++; default: break; } - if (oh_list_pp.upp) { -#ifdef HARD_DEBUG - Uint *dbg_old_oh_list_p = *oh_list_pp.upp; -#endif + if (oh) { /* Add to offheap list */ - *oh_el_next_pp.upp = *oh_list_pp.upp; - *oh_list_pp.upp = oh_el_p.up; - ASSERT(*hpp <= oh_el_p.up); - ASSERT(hp > oh_el_p.up); -#ifdef HARD_DEBUG - switch (val & _HEADER_SUBTAG_MASK) { - case REFC_BINARY_SUBTAG: - ASSERT(off_heap->mso == *oh_list_pp.pbpp); - ASSERT(off_heap->mso->next - == (ProcBin *) dbg_old_oh_list_p); - break; -#ifndef HYBRID - case FUN_SUBTAG: - ASSERT(off_heap->funs == *oh_list_pp.efpp); - ASSERT(off_heap->funs->next - == (ErlFunThing *) dbg_old_oh_list_p); - break; -#endif - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - ASSERT(off_heap->externals - == *oh_list_pp.etpp); - ASSERT(off_heap->externals->next - == (ExternalThing *) dbg_old_oh_list_p); - break; - default: - ASSERT(0); - } -#endif - oh_list_pp.upp = NULL; - - + oh->next = off_heap->first; + off_heap->first = oh; + ASSERT(*hpp <= (Eterm*)oh); + ASSERT(hp > (Eterm*)oh); + oh = NULL; } break; } @@ -676,12 +613,11 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } } - ASSERT(bp->size == hp - *hpp); + ASSERT(bp->used_size == hp - *hpp); *hpp = hp; if (is_not_immed(token)) { - ASSERT(bp->mem <= ptr_val(token)); - ASSERT(bp->mem + bp->size > ptr_val(token)); + ASSERT(in_heapfrag(ptr_val(token), bp)); ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); @@ -690,8 +626,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } if (is_not_immed(term)) { - ASSERT(bp->mem <= ptr_val(term)); - ASSERT(bp->mem + bp->size > ptr_val(term)); + ASSERT(in_heapfrag(ptr_val(term),bp)); ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); @@ -699,10 +634,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif } +copy_done: #ifdef HARD_DEBUG { int i, j; + ErlHeapFragment* frag; { ProcBin *mso = off_heap->mso; i = j = 0; @@ -710,10 +647,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) mso = mso->next; i++; } - mso = bp->off_heap.mso; - while (mso) { - mso = mso->next; - j++; + for (frag=bp; frag; frag=frag->next) { + mso = frag->off_heap.mso; + while (mso) { + mso = mso->next; + j++; + } } ASSERT(i == j); } @@ -724,10 +663,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) fun = fun->next; i++; } - fun = bp->off_heap.funs; - while (fun) { - fun = fun->next; - j++; + for (frag=bp; frag; frag=frag->next) { + fun = frag->off_heap.funs; + while (fun) { + fun = fun->next; + j++; + } } ASSERT(i == j); } @@ -738,10 +679,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) external = external->next; i++; } - external = bp->off_heap.externals; - while (external) { - external = external->next; - j++; + for (frag=bp; frag; frag=frag->next) { + external = frag->off_heap.externals; + while (external) { + external = external->next; + j++; + } } ASSERT(i == j); } @@ -749,12 +692,9 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif - bp->off_heap.mso = NULL; -#ifndef HYBRID - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; free_message_buffer(bp); + msg->data.heap_frag = NULL; #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); @@ -764,6 +704,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } + Uint erts_msg_attached_data_size_aux(ErlMessage *msg) { @@ -789,7 +730,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) if (is_not_nil(msg->m[1])) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->size; + sz += heap_frag->used_size; } return sz; } @@ -805,7 +746,7 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), - heap_frag->size, + heap_frag->used_size, hpp, ohp); erts_cleanup_offheap(&heap_frag->off_heap); @@ -1062,3 +1003,4 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, erts_queue_message(to, to_locksp, bp, save, NIL); } } + diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 5cf7c209bd..5aca0db6fe 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -28,13 +28,22 @@ struct external_thing_; * but is stored outside of any heap. */ -typedef struct erl_off_heap { - struct proc_bin* mso; /* List of associated binaries. */ -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; /* List of funs. */ +struct erl_off_heap_header { + Eterm thing_word; + Uint size; +#if HALFWORD_HEAP + void* dummy_ptr_padding__; #endif - struct external_thing_* externals; /* List of external things. */ - int overhead; /* Administrative overhead (used to force GC). */ + struct erl_off_heap_header* next; +}; + +#define OH_OVERHEAD(oh, size) do { \ + (oh)->overhead += size; \ +} while(0) + +typedef struct erl_off_heap { + struct erl_off_heap_header* first; + Uint64 overhead; /* Administrative overhead (used to force GC). */ } ErlOffHeap; #include "external.h" @@ -49,7 +58,7 @@ typedef struct erl_heap_fragment ErlHeapFragment; struct erl_heap_fragment { ErlHeapFragment* next; /* Next heap fragment */ ErlOffHeap off_heap; /* Offset heap data. */ - unsigned size; /* Size in words of mem */ + unsigned alloc_size; /* Size in (half)words of mem */ unsigned used_size; /* With terms to be moved to heap by GC */ Eterm mem[1]; /* Data */ }; @@ -75,6 +84,13 @@ typedef struct { ErlMessage** last; /* point to the last next pointer */ ErlMessage** save; int len; /* queue length */ + + /* + * The following two fields are used by the recv_mark/1 and + * recv_set/1 instructions. + */ + BeamInstr* mark; /* address to rec_loop/2 instruction */ + ErlMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; #ifdef ERTS_SMP @@ -137,6 +153,7 @@ do { \ (p)->msg.len--; \ if (__mp == NULL) \ (p)->msg.last = (p)->msg.save; \ + (p)->msg.mark = 0; \ } while(0) /* Reset message save point (after receive match) */ @@ -191,11 +208,9 @@ do { \ #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS) \ do { \ (HEAP_FRAG_P)->next = NULL; \ - (HEAP_FRAG_P)->size = (DATA_WORDS); \ + (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->off_heap.mso = NULL; \ - (HEAP_FRAG_P)->off_heap.funs = NULL; \ - (HEAP_FRAG_P)->off_heap.externals = NULL; \ + (HEAP_FRAG_P)->off_heap.first = NULL; \ (HEAP_FRAG_P)->off_heap.overhead = 0; \ } while (0) @@ -219,14 +234,25 @@ void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); +ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) +{ + const ErlHeapFragment *bp; + Uint sz = 0; + for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + sz += bp->used_size; + } + return sz; +} + ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) { ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) - return msg->data.heap_frag->size; + return erts_msg_used_frag_sz(msg); else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { @@ -234,7 +260,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->size; + sz += heap_frag->used_size; } return sz; } diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index d873c7a701..9751b5d77c 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -85,7 +85,7 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2) if (is_ref_thing_header(*b2)) { return 1; } - return cmp(ref1,ref2); + return CMP(ref1,ref2); } #define CP_LINK_VAL(To, Hp, From) \ @@ -380,7 +380,7 @@ int erts_add_link(ErtsLink **root, Uint type, Eterm pid) state = 1; *this = create_link(type,pid); break; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { /* go left */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -415,7 +415,7 @@ erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid) state = 1; res = *this = create_suspend_monitor(pid); break; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { /* go left */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -453,7 +453,7 @@ ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid) *this = create_link(type,pid); ret = *this; break; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { /* go left */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -663,7 +663,7 @@ ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid) for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -715,7 +715,7 @@ erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid) for (;;) { if (!*this) { /* Nothing found */ return; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -771,7 +771,7 @@ ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid) Sint c; for (;;) { - if (root == NULL || (c = cmp(pid,root->pid)) == 0) { + if (root == NULL || (c = CMP(pid,root->pid)) == 0) { return root; } else if (c < 0) { root = root->left; @@ -787,7 +787,7 @@ erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid) Sint c; for (;;) { - if (root == NULL || (c = cmp(pid,root->pid)) == 0) { + if (root == NULL || (c = CMP(pid,root->pid)) == 0) { return root; } else if (c < 0) { root = root->left; diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index 8b8ac2ec80..b1478758a1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -79,7 +79,7 @@ static erts_mtx_t mtrace_buf_mutex; #define UI16_SZ (2) #define UI32_SZ (4) #define UI64_SZ (8) -#ifdef ARCH_64 +#ifdef ARCH_64 /* XXX:PaN Halfword? (whole file...) */ # define UI_SZ UI64_SZ #else # define UI_SZ UI32_SZ @@ -188,7 +188,7 @@ check_alloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 type, int type_n, - Uint res, int res_n, + UWord res, int res_n, Uint size, int size_n, Uint32 ti,int ti_n); void @@ -196,8 +196,8 @@ check_realloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 type, int type_n, - Uint res, int res_n, - Uint ptr, int ptr_n, + UWord res, int res_n, + UWord ptr, int ptr_n, Uint size, int size_n, Uint32 ti,int ti_n); void @@ -205,7 +205,7 @@ check_free_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint ptr, int ptr_n, + UWord ptr, int ptr_n, Uint32 ti,int ti_n); void check_time_inc_entry(byte *sp, byte *ep, @@ -585,9 +585,7 @@ void erts_mtrace_init(char *receiver, char *nodename) Uint16 port; erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf"); - erts_mtx_set_forksafe(&mtrace_buf_mutex); erts_mtx_init(&mtrace_op_mutex, "mtrace_op"); - erts_mtx_set_forksafe(&mtrace_op_mutex); socket_desc = erts_sock_open(); if (socket_desc == ERTS_SOCK_INVALID_SOCKET) { @@ -785,7 +783,7 @@ write_alloc_entry(byte tag, tag, ct_no, ct_no_n, t_no, t_no_n, - (Uint) res, res_n, + (UWord) res, res_n, size, size_n, ti, ti_n); #endif @@ -865,8 +863,8 @@ write_realloc_entry(byte tag, tag, ct_no, ct_no_n, t_no, t_no_n, - (Uint) res, res_n, - (Uint) ptr, ptr_n, + (UWord) res, res_n, + (UWord) ptr, ptr_n, size, size_n, ti, ti_n); #endif @@ -934,7 +932,7 @@ write_free_entry(byte tag, tag, ct_no, ct_no_n, t_no, t_no_n, - (Uint) ptr, ptr_n, + (UWord) ptr, ptr_n, ti, ti_n); #endif } @@ -1135,7 +1133,7 @@ check_alloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint res, int res_n, + UWord res, int res_n, Uint size, int size_n, Uint32 ti,int ti_n) { @@ -1163,8 +1161,8 @@ check_realloc_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint res, int res_n, - Uint ptr, int ptr_n, + UWord res, int res_n, + UWord ptr, int ptr_n, Uint size, int size_n, Uint32 ti,int ti_n) { @@ -1193,7 +1191,7 @@ check_free_entry(byte *sp, byte *ep, byte tag, Uint16 ct_no, int ct_no_n, Uint16 t_no, int t_no_n, - Uint ptr, int ptr_n, + UWord ptr, int ptr_n, Uint32 ti,int ti_n) { byte *p = sp; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 585a6c1fdf..68421b4387 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -48,9 +48,23 @@ struct erl_module_nif { struct enif_entry_t* entry; erts_refc_t rt_cnt; /* number of resource types */ erts_refc_t rt_dtor_cnt; /* number of resource types with destructors */ - int is_orphan; /* if erlang module has been purged */ + Module* mod; /* Can be NULL if orphan with dtor-resources left */ }; +#ifdef DEBUG +# define READONLY_CHECK +#endif +#ifdef READONLY_CHECK +# define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE) +static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz); +#else +# define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0) +#endif + +#ifdef DEBUG +static int is_offheap(const ErlOffHeap* off_heap); +#endif + #define MIN_HEAP_FRAG_SZ 200 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp); @@ -67,23 +81,33 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need) static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp) { - unsigned frag_sz; env->hp = hp; if (env->heap_frag == NULL) { ASSERT(HEAP_LIMIT(env->proc) == env->hp_end); HEAP_TOP(env->proc) = env->hp; } else { - HRelease(env->proc, env->hp_end, env->hp); + env->heap_frag->used_size = hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } - frag_sz = need + MIN_HEAP_FRAG_SZ; - hp = erts_heap_alloc(env->proc, frag_sz); - env->hp = hp + need; - env->hp_end = hp + frag_sz; + hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ); env->heap_frag = MBUF(env->proc); + env->hp = hp + need; + env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; + return hp; } +#if SIZEOF_LONG != ERTS_SIZEOF_ETERM +static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need) +{ + if (env->hp + may_need > env->hp_end) { + alloc_heap_heavy(env, may_need, env->hp); + env->hp -= may_need; + } +} +#endif + void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) { env->mod_nif = mod_nif; @@ -106,6 +130,13 @@ static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) env->tmp_obj_list = NULL; } +/* Temporary object header, auto-deallocated when NIF returns. */ +struct enif_tmp_obj_t { + struct enif_tmp_obj_t* next; + void (*dtor)(struct enif_tmp_obj_t*); + /*char data[];*/ +}; + static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) { while (env->tmp_obj_list != NULL) { @@ -126,8 +157,9 @@ void erts_post_nif(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); - HRelease(env->proc, env->hp_end, env->hp); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } free_tmp_objs(env); } @@ -141,7 +173,7 @@ static void post_nif_noproc(ErlNifEnv* env) /* Flush out our cached heap pointers to allow an ordinary HAlloc */ -static void enable_halloc(ErlNifEnv* env) +static void flush_env(ErlNifEnv* env) { if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); @@ -151,14 +183,15 @@ static void enable_halloc(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); - HRelease(env->proc, env->hp_end, env->hp); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } } -/* Restore cached heap pointers +/* Restore cached heap pointers to allow alloc_heap again. */ -static void disable_halloc(ErlNifEnv* env) +static void cache_env(ErlNifEnv* env) { if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); @@ -168,35 +201,190 @@ static void disable_halloc(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); env->heap_frag = MBUF(env->proc); ASSERT(env->heap_frag != NULL); env->hp = env->heap_frag->mem + env->heap_frag->used_size; - env->hp_end = env->heap_frag->mem + env->heap_frag->size; + env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } - - void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; } -void* enif_alloc(ErlNifEnv* env, size_t size) +void* enif_alloc(size_t size) { return erts_alloc_fnf(ERTS_ALC_T_NIF, (Uint) size); } -void* enif_realloc(ErlNifEnv* env, void* ptr, size_t size) +void* enif_realloc(void* ptr, size_t size) { return erts_realloc_fnf(ERTS_ALC_T_NIF, ptr, size); } -void enif_free(ErlNifEnv* env, void* ptr) +void enif_free(void* ptr) { erts_free(ERTS_ALC_T_NIF, ptr); } +struct enif_msg_environment_t +{ + ErlNifEnv env; + Process phony_proc; +}; + +ErlNifEnv* enif_alloc_env(void) +{ + struct enif_msg_environment_t* msg_env = + erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t)); + Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */ + + msg_env->env.hp = phony_heap; + msg_env->env.hp_end = phony_heap; + msg_env->env.heap_frag = NULL; + msg_env->env.mod_nif = NULL; + msg_env->env.tmp_obj_list = (struct enif_tmp_obj_t*) 1; /* invalid non-NULL */ + msg_env->env.proc = &msg_env->phony_proc; + memset(&msg_env->phony_proc, 0, sizeof(Process)); + HEAP_START(&msg_env->phony_proc) = phony_heap; + HEAP_TOP(&msg_env->phony_proc) = phony_heap; + HEAP_LIMIT(&msg_env->phony_proc) = phony_heap; + HEAP_END(&msg_env->phony_proc) = phony_heap; + MBUF(&msg_env->phony_proc) = NULL; + msg_env->phony_proc.id = ERTS_INVALID_PID; +#ifdef FORCE_HEAP_FRAGS + msg_env->phony_proc.space_verified = 0; + msg_env->phony_proc.space_verified_from = NULL; +#endif + return &msg_env->env; +} +void enif_free_env(ErlNifEnv* env) +{ + enif_clear_env(env); + erts_free(ERTS_ALC_T_NIF, env); +} + +static ERTS_INLINE void clear_offheap(ErlOffHeap* oh) +{ + oh->first = NULL; + oh->overhead = 0; +} + +void enif_clear_env(ErlNifEnv* env) +{ + struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env; + Process* p = &menv->phony_proc; + ASSERT(p == menv->env.proc); + ASSERT(p->id == ERTS_INVALID_PID); + ASSERT(MBUF(p) == menv->env.heap_frag); + if (MBUF(p) != NULL) { + erts_cleanup_offheap(&MSO(p)); + clear_offheap(&MSO(p)); + free_message_buffer(MBUF(p)); + MBUF(p) = NULL; + menv->env.heap_frag = NULL; + } + ASSERT(HEAP_TOP(p) == HEAP_END(p)); + menv->env.hp = menv->env.hp_end = HEAP_TOP(p); + + ASSERT(!is_offheap(&MSO(p))); +} +int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, + ErlNifEnv* msg_env, ERL_NIF_TERM msg) +{ + struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; + ErtsProcLocks rp_locks = 0; + Process* rp; + Process* c_p; + ErlHeapFragment* frags; +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + ErtsProcLocks rp_had_locks; +#endif + Eterm receiver = to_pid->pid; + int flush_me = 0; + + if (env != NULL) { + c_p = env->proc; + if (receiver == c_p->id) { + rp_locks = ERTS_PROC_LOCK_MAIN; + flush_me = 1; + } + } + else { +#ifdef ERTS_SMP + c_p = NULL; +#else + erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); +#endif + } + +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + rp_had_locks = rp_locks; +#endif + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + if (rp == NULL) { + ASSERT(env == NULL || receiver != c_p->id); + return 0; + } + flush_env(msg_env); + frags = menv->env.heap_frag; + ASSERT(frags == MBUF(&menv->phony_proc)); + if (frags != NULL) { + /* Move all offheap's from phony proc to the first fragment. + Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */ + ASSERT(!is_offheap(&frags->off_heap)); + frags->off_heap = MSO(&menv->phony_proc); + clear_offheap(&MSO(&menv->phony_proc)); + menv->env.heap_frag = NULL; + MBUF(&menv->phony_proc) = NULL; + } + ASSERT(!is_offheap(&MSO(&menv->phony_proc))); + + if (flush_me) { + flush_env(env); /* Needed for ERTS_HOLE_CHECK */ + } + erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); + if (rp_locks) { + ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | + ERTS_PROC_LOCK_STATUS))); + erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS)); + } + erts_smp_proc_dec_refc(rp); + if (flush_me) { + cache_env(env); + } + return 1; +} + +ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) +{ + Uint sz; + Eterm* hp; + sz = size_object(src_term); + hp = alloc_heap(dst_env, sz); + return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); +} + + +#ifdef DEBUG +static int is_offheap(const ErlOffHeap* oh) +{ + return oh->first != NULL; +} +#endif + +ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid) +{ + pid->pid = caller_env->proc->id; + return pid; +} +int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid) +{ + return is_internal_pid(term) ? (pid->pid=term, 1) : 0; +} + int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term) { return is_atom(term); @@ -232,9 +420,24 @@ int enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term) return is_ref(term); } +int enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_tuple(term); +} + +int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_list(term) || is_nil(term); +} + +int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return term == THE_NON_VALUE; +} + static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { - erts_free_aligned_binary_bytes((byte*)obj); + erts_free_aligned_binary_bytes_extra((byte*)obj,ERTS_ALC_T_TMP); } int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) @@ -244,7 +447,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) byte* raw_ptr; }u; u.tmp = NULL; - bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, + bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, ERTS_ALC_T_TMP, sizeof(struct enif_tmp_obj_t)); if (bin->data == NULL) { return 0; @@ -257,6 +460,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; + ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } @@ -268,7 +472,7 @@ static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { struct enif_tmp_obj_t* tobj; - int sz; + Uint sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); } @@ -279,7 +483,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) bin->ref_bin = NULL; return 1; } - if ((sz = io_list_len(term)) < 0) { + if (erts_iolist_size(term, &sz)) { return 0; } @@ -293,10 +497,11 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; io_list_to_buf(term, (char*) bin->data, sz); + ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin) +int enif_alloc_binary(size_t size, ErlNifBinary* bin) { Binary* refbin; @@ -315,7 +520,7 @@ int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin) return 1; } -int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size) +int enif_realloc_binary(ErlNifBinary* bin, size_t size) { if (bin->ref_bin != NULL) { Binary* oldbin; @@ -333,15 +538,15 @@ int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size) } else { unsigned char* old_data = bin->data; - unsigned cpy_sz = (size < bin->size ? size : bin->size); - enif_alloc_binary(env, size, bin); + size_t cpy_sz = (size < bin->size ? size : bin->size); + enif_alloc_binary(size, bin); sys_memcpy(bin->data, old_data, cpy_sz); } return 1; } -void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) +void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; @@ -357,14 +562,23 @@ void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) #endif } -int enif_is_identical(ErlNifEnv* env, Eterm lhs, Eterm rhs) +unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size, + ERL_NIF_TERM* termp) +{ + flush_env(env); + *termp = new_binary(env->proc, NULL, size); + cache_env(env); + return binary_bytes(*termp); +} + +int enif_is_identical(Eterm lhs, Eterm rhs) { return EQ(lhs,rhs); } -int enif_compare(ErlNifEnv* env, Eterm lhs, Eterm rhs) +int enif_compare(Eterm lhs, Eterm rhs) { - return cmp(lhs,rhs); + return CMP(lhs,rhs); } int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array) @@ -425,13 +639,13 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = bptr->orig_size; - pb->next = MSO(env->proc).mso; - MSO(env->proc).mso = pb; + pb->next = MSO(env->proc).first; + MSO(env->proc).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(env->proc).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); bin_term = make_binary(pb); if (erts_refc_read(&bptr->refc, 1) == 1) { /* Total ownership transfer */ @@ -441,15 +655,15 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) return bin_term; } else { - enable_halloc(env); + flush_env(env); bin->bin_term = new_binary(env->proc, bin->data, bin->size); - disable_halloc(env); + cache_env(env); return bin->bin_term; } } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, - unsigned pos, unsigned size) + size_t pos, size_t size) { ErlSubBin* sb; Eterm orig; @@ -479,9 +693,11 @@ Eterm enif_make_badarg(ErlNifEnv* env) BIF_ERROR(env->proc, BADARG); } -int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len) +int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len, + ErlNifCharEncoding encoding) { Atom* ap; + ASSERT(encoding == ERL_NIF_LATIN1); if (is_not_atom(atom)) { return 0; } @@ -496,9 +712,9 @@ int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len) int enif_get_int(ErlNifEnv* env, Eterm term, int* ip) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return term_to_Sint(term, (Sint*)ip); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM Sint i; if (!term_to_Sint(term, &i) || i < INT_MIN || i > INT_MAX) { return 0; @@ -512,9 +728,9 @@ int enif_get_int(ErlNifEnv* env, Eterm term, int* ip) int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return term_to_Uint(term, (Uint*)ip); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM Uint i; if (!term_to_Uint(term, &i) || i > UINT_MAX) { return 0; @@ -526,8 +742,10 @@ int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip) int enif_get_long(ErlNifEnv* env, Eterm term, long* ip) { -#if SIZEOF_LONG == SIZEOF_VOID_P +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Sint(term, ip); +#elif SIZEOF_LONG == 8 + return term_to_Sint64(term, ip); #else # error Unknown long word size #endif @@ -535,14 +753,28 @@ int enif_get_long(ErlNifEnv* env, Eterm term, long* ip) int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip) { -#if SIZEOF_LONG == SIZEOF_VOID_P +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Uint(term, ip); +#elif SIZEOF_LONG == 8 + return term_to_Uint64(term, ip); #else # error Unknown long word size #endif } -int enif_get_double(ErlNifEnv* env, Eterm term, double* dp) +#if HAVE_INT64 && SIZEOF_LONG != 8 +int enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip) +{ + return term_to_Sint64(term, ip); +} + +int enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip) +{ + return term_to_Uint64(term, ip); +} +#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */ + +int enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp) { FloatDef f; if (is_not_float(term)) { @@ -553,6 +785,17 @@ int enif_get_double(ErlNifEnv* env, Eterm term, double* dp) return 1; } +int enif_get_atom_length(ErlNifEnv* env, Eterm atom, unsigned* len, + ErlNifCharEncoding enc) +{ + Atom* ap; + ASSERT(enc == ERL_NIF_LATIN1); + if (is_not_atom(atom)) return 0; + ap = atom_tab(atom_val(atom)); + *len = ap->len; + return 1; +} + int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) { Eterm* val; @@ -563,37 +806,71 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) return 1; } +int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len) +{ + if (is_not_list(term) && is_not_nil(term)) return 0; + *len = list_length(term); + return 1; +} + ERL_NIF_TERM enif_make_int(ErlNifEnv* env, int i) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return IS_SSMALL(i) ? make_small(i) : small_to_big(i,alloc_heap(env,2)); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM return make_small(i); #endif } ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i) { -#if SIZEOF_INT == SIZEOF_VOID_P +#if SIZEOF_INT == ERTS_SIZEOF_ETERM return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); -#elif SIZEOF_LONG == SIZEOF_VOID_P +#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM return make_small(i); #endif } ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) { -#if SIZEOF_LONG != SIZEOF_VOID_P -# error Unknown long word size -#endif +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2)); +#elif SIZEOF_LONG == 8 + ensure_heap(env,3); + return erts_sint64_to_big(i, &env->hp); +#endif } ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) { +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); +#elif SIZEOF_LONG == 8 + ensure_heap(env,3); + return erts_uint64_to_big(i, &env->hp); +#endif +} + +#if HAVE_INT64 && SIZEOF_LONG != 8 +ERL_NIF_TERM enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i) +{ + Uint* hp; + Uint need = 0; + erts_bld_sint64(NULL, &need, i); + hp = alloc_heap(env, need); + return erts_bld_sint64(&hp, NULL, i); } +ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i) +{ + Uint* hp; + Uint need = 0; + erts_bld_uint64(NULL, &need, i); + hp = alloc_heap(env, need); + return erts_bld_uint64(&hp, NULL, i); +} +#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */ + ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) { Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT); @@ -605,12 +882,25 @@ ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) { - return am_atom_put(name, sys_strlen(name)); + return enif_make_atom_len(env, name, sys_strlen(name)); } -int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom) +ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len) { - return erts_atom_get(name, sys_strlen(name), atom); + return am_atom_put(name, len); +} + +int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, + ErlNifCharEncoding enc) +{ + return enif_make_existing_atom_len(env, name, sys_strlen(name), atom, enc); +} + +int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, + ERL_NIF_TERM* atom, ErlNifCharEncoding encoding) +{ + ASSERT(encoding == ERL_NIF_LATIN1); + return erts_atom_get(name, len, atom); } ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) @@ -653,21 +943,26 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr) ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) { - Eterm* hp = alloc_heap(env,cnt*2); - Eterm ret = make_list(hp); - Eterm* last = &ret; - va_list ap; - - va_start(ap,cnt); - while (cnt--) { - *last = make_list(hp); - *hp = va_arg(ap,Eterm); - last = ++hp; - ++hp; + if (cnt == 0) { + return NIL; + } + else { + Eterm* hp = alloc_heap(env,cnt*2); + Eterm ret = make_list(hp); + Eterm* last = &ret; + va_list ap; + + va_start(ap,cnt); + while (cnt--) { + *last = make_list(hp); + *hp = va_arg(ap,Eterm); + last = ++hp; + ++hp; + } + va_end(ap); + *last = NIL; + return ret; } - va_end(ap); - *last = NIL; - return ret; } ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt) @@ -689,11 +984,16 @@ ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], ERL_NIF_TERM enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding) -{ - Sint n = sys_strlen(string); - Eterm* hp = alloc_heap(env,n*2); +{ + return enif_make_string_len(env, string, sys_strlen(string), encoding); +} + +ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string, + size_t len, ErlNifCharEncoding encoding) +{ + Eterm* hp = alloc_heap(env,len*2); ASSERT(encoding == ERL_NIF_LATIN1); - return erts_bld_string_n(&hp,NULL,string,n); + return erts_bld_string_n(&hp,NULL,string,len); } ERL_NIF_TERM enif_make_ref(ErlNifEnv* env) @@ -764,7 +1064,8 @@ struct enif_resource_type_t ErlNifResourceDtor* dtor; /* user destructor function */ erts_refc_t refc; /* num of resources of this type (HOTSPOT warning) +1 for active erl_module_nif */ - char name[1]; + Eterm module; + Eterm name; }; /* dummy node in circular list */ @@ -782,14 +1083,14 @@ typedef struct enif_resource_t #define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE)) #define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data))) -static ErlNifResourceType* find_resource_type(const char* name) +static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) { ErlNifResourceType* type; for (type = resource_type_list.next; type != &resource_type_list; type = type->next) { - if (sys_strcmp(type->name, name) == 0) { + if (type->module == module && type->name == name) { return type; } } @@ -822,33 +1123,42 @@ static void steal_resource_type(ErlNifResourceType* type) if (type->dtor != NULL && erts_refc_dectest(&lib->rt_dtor_cnt, 0) == 0 - && lib->is_orphan) { + && lib->mod == NULL) { /* last type with destructor gone, close orphan lib */ close_lib(lib); } if (erts_refc_dectest(&lib->rt_cnt, 0) == 0 - && lib->is_orphan) { + && lib->mod == NULL) { erts_free(ERTS_ALC_T_NIF, lib); } } ErlNifResourceType* -enif_open_resource_type(ErlNifEnv* env, const char* type_name, - ErlNifResourceDtor* dtor, - enum ErlNifResourceFlags flags, - enum ErlNifResourceFlags* tried) +enif_open_resource_type(ErlNifEnv* env, + const char* module_str, + const char* name_str, + ErlNifResourceDtor* dtor, + ErlNifResourceFlags flags, + ErlNifResourceFlags* tried) { - ErlNifResourceType* type = find_resource_type(type_name); - enum ErlNifResourceFlags op = flags; + ErlNifResourceType* type = NULL; + ErlNifResourceFlags op = flags; + Eterm module_am, name_am; + ASSERT(erts_smp_is_system_blocked(0)); + ASSERT(module_str == NULL); /* for now... */ + module_am = make_atom(env->mod_nif->mod->module); + name_am = enif_make_atom(env, name_str); + + type = find_resource_type(module_am, name_am); if (type == NULL) { if (flags & ERL_NIF_RT_CREATE) { type = erts_alloc(ERTS_ALC_T_NIF, - sizeof(struct enif_resource_type_t) - + sys_strlen(type_name)); + sizeof(struct enif_resource_type_t)); type->dtor = dtor; - sys_strcpy(type->name, type_name); + type->module = module_am; + type->name = name_am; erts_refc_init(&type->refc, 1); type->owner = env->mod_nif; type->prev = &resource_type_list; @@ -896,13 +1206,13 @@ static void nif_resource_dtor(Binary* bin) if (erts_refc_dectest(&type->refc, 0) == 0) { ASSERT(type->next == NULL); ASSERT(type->owner != NULL); - ASSERT(type->owner->is_orphan); + ASSERT(type->owner->mod == NULL); steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } } -void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size) +void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); @@ -915,7 +1225,7 @@ void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned siz return resource->data; } -void enif_release_resource(ErlNifEnv* env, void* obj) +void enif_release_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); @@ -929,6 +1239,18 @@ void enif_release_resource(ErlNifEnv* env, void* obj) } } +void enif_keep_resource(void* obj) +{ + ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); +#ifdef DEBUG + erts_refc_inc(&resource->nif_refc, 1); +#endif + erts_refc_inc(&bin->binary.refc, 2); +} + ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); @@ -937,15 +1259,30 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } +ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, + const void* data, size_t size) +{ + Eterm bin = enif_make_resource(env, obj); + ProcBin* pb = (ProcBin*) binary_val(bin); + pb->bytes = (byte*) data; + pb->size = size; + return bin; +} + int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp) { + ProcBin* pb; Binary* mbin; ErlNifResource* resource; if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { return 0; } - mbin = ((ProcBin*) binary_val(term))->val; + pb = (ProcBin*) binary_val(term); + /*if (pb->size != 0) { + return 0; / * Or should we allow "resource binaries" as handles? * / + }*/ + mbin = pb->val; resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { @@ -955,7 +1292,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 1; } -unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj) +size_t enif_sizeof_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary; @@ -967,37 +1304,22 @@ unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj) ***************************************************************************/ -static Uint** get_func_pp(Eterm* mod_code, Eterm f_atom, unsigned arity) +static BeamInstr** get_func_pp(BeamInstr* mod_code, Eterm f_atom, unsigned arity) { int n = (int) mod_code[MI_NUM_FUNCTIONS]; int j; for (j = 0; j < n; ++j) { - Uint* code_ptr = (Uint*) mod_code[MI_FUNCTIONS+j]; - ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI)); + BeamInstr* code_ptr = (BeamInstr*) mod_code[MI_FUNCTIONS+j]; + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (f_atom == ((Eterm) code_ptr[3]) && arity == ((unsigned) code_ptr[4])) { - return (Uint**) &mod_code[MI_FUNCTIONS+j]; + return (BeamInstr**) &mod_code[MI_FUNCTIONS+j]; } } return NULL; } -/*static void refresh_cached_nif_data(Eterm* mod_code, - struct erl_module_nif* mod_nif) -{ - int i; - for (i=0; i < mod_nif->entry->num_of_funcs; i++) { - Eterm f_atom; - ErlNifFunc* func = &mod_nif->entry->funcs[i]; - Uint* code_ptr; - - erts_atom_get(func->name, sys_strlen(func->name), &f_atom); - code_ptr = *get_func_pp(mod_code, f_atom, func->arity); - code_ptr[5+2] = (Uint) mod_nif->priv_data; - } -}*/ - static Eterm mkatom(const char *str) { return am_atom_put(str, sys_strlen(str)); @@ -1089,7 +1411,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) static const char bad_lib[] = "bad_lib"; static const char reload[] = "reload"; static const char upgrade[] = "upgrade"; - char lib_name[256]; /* BUGBUG: Max-length? */ + char* lib_name = NULL; void* handle = NULL; void* init_func; ErlNifEntry* entry = NULL; @@ -1098,15 +1420,20 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Module* mod; Eterm mod_atom; Eterm f_atom; - Eterm* caller; + BeamInstr* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; struct erl_module_nif* lib = NULL; - len = intlist_to_buf(BIF_ARG_1, lib_name, sizeof(lib_name)-1); - if (len < 1) { - /*erts_fprintf(stderr, "Invalid library path name '%T'\r\n", BIF_ARG_1);*/ + len = list_length(BIF_ARG_1); + if (len < 0) { + BIF_ERROR(BIF_P, BADARG); + } + lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1); + + if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) { + erts_free(ERTS_ALC_T_TMP, lib_name); BIF_ERROR(BIF_P, BADARG); } lib_name[len] = '\0'; @@ -1156,7 +1483,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); - } + } + else if (entry->minor >= 1 + && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { + ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for " + "this vm variant (%s).", + entry->vm_variant, ERL_NIF_VM_VARIANT); + } else if (!erts_is_atom_str((char*)entry->name, mod_atom)) { ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not" " match calling module '%T'", entry->name, mod_atom); @@ -1165,7 +1498,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/ for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { - Uint** code_pp; + BeamInstr** code_pp; ErlNifFunc* f = &entry->funcs[i]; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom) || (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { @@ -1195,7 +1528,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) lib->entry = entry; erts_refc_init(&lib->rt_cnt, 0); erts_refc_init(&lib->rt_dtor_cnt, 0); - lib->is_orphan = 0; + lib->mod = mod; env.mod_nif = lib; if (mod->nif != NULL) { /* Reload */ int k; @@ -1268,22 +1601,23 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) mod->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { - Uint* code_ptr; + BeamInstr* code_ptr; erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom); code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity); if (code_ptr[1] == 0) { - code_ptr[5+0] = (Uint) BeamOp(op_call_nif); + code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - BpData* bp = (BpData*) code_ptr[1]; - bp->orig_instr = (Uint) BeamOp(op_call_nif); + BpData** bps = (BpData**) code_ptr[1]; + BpData* bp = (BpData*) bps[bp_sched2ix()]; + bp->orig_instr = (BeamInstr) BeamOp(op_call_nif); } - code_ptr[5+1] = (Uint) entry->funcs[i].fptr; - code_ptr[5+2] = (Uint) lib; + code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr; + code_ptr[5+2] = (BeamInstr) lib; } } - else { + else { error: ASSERT(ret != am_ok); if (lib != NULL) { @@ -1297,6 +1631,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_smp_release_system(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_free(ERTS_ALC_T_TMP, lib_name); BIF_RET(ret); } @@ -1308,7 +1643,7 @@ erts_unload_nif(struct erl_module_nif* lib) ErlNifResourceType* next; ASSERT(erts_smp_is_system_blocked(0)); ASSERT(lib != NULL); - ASSERT(!lib->is_orphan); + ASSERT(lib->mod != NULL); for (rt = resource_type_list.next; rt != &resource_type_list; rt = next) { @@ -1338,7 +1673,7 @@ erts_unload_nif(struct erl_module_nif* lib) else { ASSERT(erts_refc_read(&lib->rt_cnt, 1) > 0); } - lib->is_orphan = 1; + lib->mod = NULL; /* orphan lib */ } void erl_nif_init() @@ -1347,6 +1682,53 @@ void erl_nif_init() resource_type_list.prev = &resource_type_list; resource_type_list.dtor = NULL; resource_type_list.owner = NULL; - resource_type_list.name[0] = '\0'; + resource_type_list.module = THE_NON_VALUE; + resource_type_list.name = THE_NON_VALUE; } +#ifdef READONLY_CHECK +/* Use checksums to assert that NIFs do not write into inspected binaries +*/ +static void readonly_check_dtor(struct enif_tmp_obj_t*); +static unsigned calc_checksum(unsigned char* ptr, unsigned size); + +struct readonly_check_t +{ + struct enif_tmp_obj_t hdr; + unsigned char* ptr; + unsigned size; + unsigned checksum; +}; +static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) +{ + struct readonly_check_t* obj = erts_alloc(ERTS_ALC_T_TMP, + sizeof(struct readonly_check_t)); + obj->hdr.next = env->tmp_obj_list; + env->tmp_obj_list = &obj->hdr; + obj->hdr.dtor = &readonly_check_dtor; + obj->ptr = ptr; + obj->size = sz; + obj->checksum = calc_checksum(ptr, sz); +} +static void readonly_check_dtor(struct enif_tmp_obj_t* o) +{ + struct readonly_check_t* obj = (struct readonly_check_t*) o; + unsigned chksum = calc_checksum(obj->ptr, obj->size); + if (chksum != obj->checksum) { + fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ" + " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); + abort(); + } + erts_free(ERTS_ALC_T_TMP, obj); +} +static unsigned calc_checksum(unsigned char* ptr, unsigned size) +{ + unsigned i, sum = 0; + for (i=0; i<size; i++) { + sum ^= ptr[i] << ((i % 4)*8); + } + return sum; +} + +#endif /* READONLY_CHECK */ + diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 1ccf00293e..d028567faf 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,18 +23,71 @@ #ifndef __ERL_NIF_H__ #define __ERL_NIF_H__ + #include "erl_drv_nif.h" /* Version history: ** 0.1: R13B03 ** 1.0: R13B04 +** 2.0: R14A +** 2.1: R14B02 "vm_variant" +** 2.2: R14B03 enif_is_exception */ -#define ERL_NIF_MAJOR_VERSION 1 -#define ERL_NIF_MINOR_VERSION 0 +#define ERL_NIF_MAJOR_VERSION 2 +#define ERL_NIF_MINOR_VERSION 2 #include <stdlib.h> +#ifdef SIZEOF_CHAR +# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR +# undef SIZEOF_CHAR +#endif +#ifdef SIZEOF_SHORT +# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT +# undef SIZEOF_SHORT +#endif +#ifdef SIZEOF_INT +# define SIZEOF_INT_SAVED__ SIZEOF_INT +# undef SIZEOF_INT +#endif +#ifdef SIZEOF_LONG +# define SIZEOF_LONG_SAVED__ SIZEOF_LONG +# undef SIZEOF_LONG +#endif +#ifdef SIZEOF_LONG_LONG +# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG +# undef SIZEOF_LONG_LONG +#endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif +#include "erl_int_sizes_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef unsigned __int64 ErlNifUInt64; +typedef __int64 ErlNifSInt64; +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlNifUInt64; +typedef long ErlNifSInt64; +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlNifUInt64; +typedef long long ErlNifSInt64; +#else +#error No 64-bit integer type +#endif + +#ifdef HALFWORD_HEAP_EMULATOR +# define ERL_NIF_VM_VARIANT "beam.halfword" +typedef unsigned int ERL_NIF_TERM; +#else +# define ERL_NIF_VM_VARIANT "beam.vanilla" typedef unsigned long ERL_NIF_TERM; +#endif struct enif_environment_t; typedef struct enif_environment_t ErlNifEnv; @@ -56,14 +109,15 @@ typedef struct enif_entry_t int (*load) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); - void (*unload) (ErlNifEnv*, void* priv_data); + void (*unload) (ErlNifEnv*, void* priv_data); + const char* vm_variant; }ErlNifEntry; typedef struct { - unsigned size; + size_t size; unsigned char* data; /* Internals (avert your eyes) */ @@ -73,17 +127,22 @@ typedef struct typedef struct enif_resource_type_t ErlNifResourceType; typedef void ErlNifResourceDtor(ErlNifEnv*, void*); -enum ErlNifResourceFlags +typedef enum { ERL_NIF_RT_CREATE = 1, ERL_NIF_RT_TAKEOVER = 2 -}; +}ErlNifResourceFlags; typedef enum { ERL_NIF_LATIN1 = 1 }ErlNifCharEncoding; +typedef struct +{ + ERL_NIF_TERM pid; /* internal, may change */ +}ErlNifPid; + typedef ErlDrvSysInfo ErlNifSysInfo; typedef struct ErlDrvTid_ *ErlNifTid; @@ -116,8 +175,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #endif - - #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; # define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) @@ -133,8 +190,20 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #endif +#ifdef __cplusplus +} +# define ERL_NIF_INIT_PROLOGUE extern "C" { +# define ERL_NIF_INIT_EPILOGUE } +#else +# define ERL_NIF_INIT_PROLOGUE +# define ERL_NIF_INIT_EPILOGUE +#endif + + #define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \ +ERL_NIF_INIT_PROLOGUE \ ERL_NIF_INIT_GLOB \ +ERL_NIF_INIT_DECL(NAME); \ ERL_NIF_INIT_DECL(NAME) \ { \ static ErlNifEntry entry = \ @@ -144,11 +213,14 @@ ERL_NIF_INIT_DECL(NAME) \ #NAME, \ sizeof(FUNCS) / sizeof(*FUNCS), \ FUNCS, \ - LOAD, RELOAD, UPGRADE, UNLOAD \ + LOAD, RELOAD, UPGRADE, UNLOAD, \ + ERL_NIF_VM_VARIANT \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ -} +} \ +ERL_NIF_INIT_EPILOGUE + #endif /* __ERL_NIF_H__ */ diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index ec07a976b2..c991b61abe 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,31 +21,38 @@ # error This file should not be included directly #endif +/* +** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list +** to keep compatibility on Windows!!! +** +** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h +** when adding functions to the API. +*/ #ifdef ERL_NIF_API_FUNC_DECL ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*)); -ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(ErlNifEnv*, size_t size)); -ERL_NIF_API_FUNC_DECL(void,enif_free,(ErlNifEnv*, void* ptr)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr)); ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(ErlNifEnv*, unsigned size, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifEnv*, ErlNifBinary* bin, unsigned size)); -ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifEnv*, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip)); ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip)); ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp)); ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)); ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); -ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); -ERL_NIF_API_FUNC_DECL(int,enif_compare,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name)); -ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)); +ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr)); @@ -82,13 +89,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2)); ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp)); ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp)); -ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(ErlNifEnv*, void* ptr, size_t size)); +ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size)); ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size)); ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, unsigned pos, unsigned size)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size)); ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding)); -ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding)); ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term)); @@ -99,17 +106,46 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term)); -ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* type_name, void (*dtor)(ErlNifEnv*,void *), enum ErlNifResourceFlags flags, enum ErlNifResourceFlags* tried)); -ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifEnv*, ErlNifResourceType* type, unsigned size)); -ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(ErlNifEnv*, void* obj)); +ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj)); ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)); -ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj)); +ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp)); +ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len)); +ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void)); +ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)); +ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size)); +#if SIZEOF_LONG != 8 +ERL_NIF_API_FUNC_DECL(int,enif_get_int64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifSInt64* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_uint64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifUInt64* ip)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); +#endif +ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); + /* -** Add last to keep compatibility on Windows!!! +** Add new entries here to keep compatibility on Windows!!! */ #endif +/* +** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order +** as the ERL_NIF_API_FUNC_DECL list above +*/ #ifdef ERL_NIF_API_FUNC_MACRO # define enif_priv_data ERL_NIF_API_FUNC_MACRO(enif_priv_data) # define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc) @@ -195,7 +231,35 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj)); # define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource) # define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource) # define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource) +# define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary) +# define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list) +# define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple) +# define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length) +# define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length) +# define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len) +# define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len) +# define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len) +# define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env) +# define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env) +# define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env) +# define enif_send ERL_NIF_API_FUNC_MACRO(enif_send) +# define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy) +# define enif_self ERL_NIF_API_FUNC_MACRO(enif_self) +# define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid) +# define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource) +# define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary) +#if SIZEOF_LONG != 8 +# define enif_get_int64 ERL_NIF_API_FUNC_MACRO(enif_get_int64) +# define enif_get_uint64 ERL_NIF_API_FUNC_MACRO(enif_get_uint64) +# define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64) +# define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64) +#endif + +# define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) +/* +** Add new entries here +*/ #endif #ifndef enif_make_list1 @@ -217,9 +281,15 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj)); # define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7) # define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) # define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) + +# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) + +#if SIZEOF_LONG == 8 +# define enif_get_int64 enif_get_long +# define enif_get_uint64 enif_get_ulong +# define enif_make_int64 enif_make_long +# define enif_make_uint64 enif_make_ulong #endif -#ifndef enif_get_data -# define enif_get_data enif_priv_data /* deprecated */ #endif diff --git a/erts/emulator/beam/erl_nmgc.c b/erts/emulator/beam/erl_nmgc.c index 626d4e295a..d7bfb2ab12 100644 --- a/erts/emulator/beam/erl_nmgc.c +++ b/erts/emulator/beam/erl_nmgc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,7 +26,6 @@ #include "erl_nmgc.h" #include "erl_debug.h" #if HIPE -#include "hipe_bif0.h" /* for hipe_constants_{start,next} */ #include "hipe_stack.h" #endif diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 87dbfc2a04..2c67e781e0 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -251,23 +251,36 @@ extern int erts_use_r9_pids_ports; * Refs * \* */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define internal_ref_no_of_numbers(x) \ (internal_ref_data((x))[0]) +#define internal_thing_ref_no_of_numbers(thing) \ + (internal_thing_ref_data(thing)[0]) #define internal_ref_numbers(x) \ (&internal_ref_data((x))[1]) +#define internal_thing_ref_numbers(thing) \ + (&internal_thing_ref_data(thing)[1]) #define external_ref_no_of_numbers(x) \ (external_ref_data((x))[0]) +#define external_thing_ref_no_of_numbers(thing) \ + (external_thing_ref_data(thing)[0]) #define external_ref_numbers(x) \ (&external_ref_data((x))[1]) +#define external_thing_ref_numbers(thing) \ + (&external_thing_ref_data(thing)[1]) + #else #define internal_ref_no_of_numbers(x) (internal_ref_data_words((x))) +#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t)) #define internal_ref_numbers(x) (internal_ref_data((x))) +#define internal_thing_ref_numbers(t) (internal_thing_ref_data(t)) #define external_ref_no_of_numbers(x) (external_ref_data_words((x))) +#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t))) #define external_ref_numbers(x) (external_ref_data((x))) +#define external_thing_ref_numbers(t) (external_thing_ref_data((t))) #endif @@ -311,6 +324,8 @@ extern int erts_use_r9_pids_ports; : external_ref_channel_no((x))) #define is_ref(x) (is_internal_ref((x)) \ || is_external_ref((x))) +#define is_ref_rel(x,Base) (is_internal_ref_rel((x),Base) \ + || is_external_ref_rel((x),Base)) #define is_not_ref(x) (!is_ref(x)) #endif diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 42b28d987c..6daa127d23 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -80,6 +80,8 @@ dist_table_alloc(void *dep_tmpl) Eterm chnl_nr; Eterm sysname; DistEntry *dep; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) return dep_tmpl; @@ -92,7 +94,7 @@ dist_table_alloc(void *dep_tmpl) dep->prev = NULL; erts_refc_init(&dep->refc, -1); - erts_smp_rwmtx_init_x(&dep->rwmtx, "dist_entry", chnl_nr); + erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr); dep->sysname = sysname; dep->cid = NIL; dep->connection_id = 0; @@ -105,7 +107,7 @@ dist_table_alloc(void *dep_tmpl) dep->nlinks = NULL; dep->monitors = NULL; - erts_smp_spinlock_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr); + erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr); dep->qflgs = 0; dep->qsize = 0; dep->out_queue.first = NULL; @@ -170,7 +172,7 @@ dist_table_free(void *vdep) ASSERT(!dep->cache); erts_smp_rwmtx_destroy(&dep->rwmtx); erts_smp_mtx_destroy(&dep->lnk_mtx); - erts_smp_spinlock_destroy(&dep->qlock); + erts_smp_mtx_destroy(&dep->qlock); #ifdef DEBUG sys_memset(vdep, 0x77, sizeof(DistEntry)); @@ -233,7 +235,7 @@ erts_sysname_to_connected_dist_entry(Eterm sysname) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de); if (res_dep) { - long refc = erts_refc_inctest(&res_dep->refc, 1); + erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&res_dep->refc, 1); } @@ -255,7 +257,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname) { DistEntry *res; DistEntry de; - long refc; + erts_aint_t refc; res = erts_find_dist_entry(sysname); if (res) return res; @@ -277,7 +279,7 @@ DistEntry *erts_find_dist_entry(Eterm sysname) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res = hash_get(&erts_dist_table, (void *) &de); if (res) { - long refc = erts_refc_inctest(&res->refc, 1); + erts_aint_t refc = erts_refc_inctest(&res->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&res->refc, 1); } @@ -580,11 +582,23 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) ErlNode ne; ne.sysname = sysname; ne.creation = creation; + + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + res = hash_get(&erts_node_table, (void *) &ne); + if (res && res != erts_this_node) { + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); + if (refc < 2) /* New or pending delete */ + erts_refc_inc(&res->refc, 1); + } + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + if (res) + return res; + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); if (res != erts_this_node) { - long refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ erts_refc_inc(&res->refc, 1); } @@ -696,8 +710,12 @@ erts_set_this_node(Eterm sysname, Uint creation) void erts_init_node_tables(void) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; @@ -719,9 +737,10 @@ void erts_init_node_tables(void) erts_this_dist_entry->prev = NULL; erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - erts_smp_rwmtx_init_x(&erts_this_dist_entry->rwmtx, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, + &rwmtx_opt, + "dist_entry", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->sysname = am_Noname; erts_this_dist_entry->cid = NIL; erts_this_dist_entry->connection_id = 0; @@ -736,9 +755,9 @@ void erts_init_node_tables(void) erts_this_dist_entry->nlinks = NULL; erts_this_dist_entry->monitors = NULL; - erts_smp_spinlock_init_x(&erts_this_dist_entry->qlock, - "dist_entry_out_queue", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_mtx_init_x(&erts_this_dist_entry->qlock, + "dist_entry_out_queue", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->qflgs = 0; erts_this_dist_entry->qsize = 0; erts_this_dist_entry->out_queue.first = NULL; @@ -772,8 +791,8 @@ void erts_init_node_tables(void) (void) hash_put(&erts_node_table, (void *) erts_this_node); - erts_smp_rwmtx_init(&erts_node_table_rwmtx, "node_table"); - erts_smp_rwmtx_init(&erts_dist_table_rwmtx, "dist_table"); + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); references_atoms_need_init = 1; } @@ -1087,49 +1106,62 @@ insert_offheap2(ErlOffHeap *oh, void *arg) static void insert_offheap(ErlOffHeap *oh, int type, Eterm id) { - if(oh->externals) { - ExternalThing *etp = oh->externals; - while (etp) { - insert_node(etp->node, type, id); - etp = etp->next; - } - } + union erl_off_heap_ptr u; + struct insert_offheap2_arg a; + a.type = BIN_REF; - if(oh->mso) { - ProcBin *pb; - struct insert_offheap2_arg a; - a.type = BIN_REF; - for(pb = oh->mso; pb; pb = pb->next) { - if(IsMatchProgBinary(pb->val)) { + for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if(IsMatchProgBinary(u.pb->val)) { InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) - if(ib->bin_val == pb->val) { + if(ib->bin_val == u.pb->val) { insert_bin = 0; break; } if (insert_bin) { - Uint id_heap[BIG_UINT_HEAP_SIZE]; +#if HALFWORD_HEAP + UWord val = (UWord) u.pb->val; + DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */ +#else + DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE); +#endif Uint *hp = &id_heap[0]; InsertedBin *nib; - a.id = erts_bld_uint(&hp, NULL, (Uint) pb->val); - erts_match_prog_foreach_offheap(pb->val, +#if HALFWORD_HEAP + int actual_need = BIG_UWORD_HEAP_SIZE(val); + ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2)); + UseTmpHeapNoproc(actual_need); + a.id = erts_bld_uword(&hp, NULL, (UWord) val); +#else + UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); + a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); +#endif + erts_match_prog_foreach_offheap(u.pb->val, insert_offheap2, (void *) &a); nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin)); - nib->bin_val = pb->val; + nib->bin_val = u.pb->val; nib->next = inserted_bins; inserted_bins = nib; +#if HALFWORD_HEAP + UnUseTmpHeapNoproc(actual_need); +#else + UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); +#endif } - } + } + break; + case FUN_SUBTAG: + break; /* No need to */ + default: + ASSERT(is_external_header(u.hdr->thing_word)); + insert_node(u.ext->node, type, id); + break; } } - -#if 0 - if(oh->funs) { - /* No need to */ - } -#endif } static void doit_insert_monitor(ErtsMonitor *monitor, void *p) @@ -1190,12 +1222,15 @@ static void insert_bif_timer(Eterm receiver, Eterm msg, ErlHeapFragment *bp, void *arg) { if (bp) { - Eterm heap[3]; + DeclareTmpHeapNoproc(heap,3); + + UseTmpHeapNoproc(3); insert_offheap(&bp->off_heap, TIMER_REF, (is_internal_pid(receiver) ? receiver : TUPLE2(&heap[0], AM_process, receiver))); + UnUseTmpHeapNoproc(3); } } @@ -1230,7 +1265,7 @@ setup_reference_table(void) DistEntry *dep; HashInfo hi; int i; - Eterm heap[3]; + DeclareTmpHeapNoproc(heap,3); inserted_bins = NULL; @@ -1251,6 +1286,7 @@ setup_reference_table(void) /* Go through the hole system, and build a table of all references to ErlNode and DistEntry structures */ + UseTmpHeapNoproc(3); insert_node(erts_this_node, SYSTEM_REF, TUPLE2(&heap[0], AM_system, am_undefined)); @@ -1261,11 +1297,13 @@ setup_reference_table(void) HEAP_REF, TUPLE2(&heap[0], AM_processes, am_undefined)); #endif + UnUseTmpHeapNoproc(3); /* Insert all processes */ for (i = 0; i < erts_max_processes; i++) if (process_tab[i]) { ErlMessage *msg; + /* Insert Heap */ insert_offheap(&(process_tab[i]->off_heap), HEAP_REF, @@ -1352,21 +1390,22 @@ setup_reference_table(void) { /* Add binaries stored elsewhere ... */ ErlOffHeap oh; - ProcBin pb[2] = {{0},{0}}; - ProcBin *mso = NULL; + ProcBin pb[2]; int i = 0; Binary *default_match_spec; Binary *default_meta_match_spec; - /* Only the ProcBin members val and next will be inspected + oh.first = NULL; + /* Only the ProcBin members thing_word, val and next will be inspected (by insert_offheap()) */ #undef ADD_BINARY -#define ADD_BINARY(Bin) \ - if ((Bin)) { \ - pb[i].val = (Bin); \ - pb[i].next = mso; \ - mso = &pb[i]; \ - i++; \ +#define ADD_BINARY(Bin) \ + if ((Bin)) { \ + pb[i].thing_word = REFC_BINARY_SUBTAG; \ + pb[i].val = (Bin); \ + pb[i].next = oh.first; \ + oh.first = (struct erl_off_heap_header*) &pb[i]; \ + i++; \ } erts_get_default_trace_pattern(NULL, @@ -1378,11 +1417,6 @@ setup_reference_table(void) ADD_BINARY(default_match_spec); ADD_BINARY(default_meta_match_spec); - oh.mso = mso; - oh.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - oh.funs = NULL; -#endif insert_offheap(&oh, BIN_REF, AM_match_spec); #undef ADD_BINARY } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index c48dac6219..b0a63ae035 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -61,7 +61,7 @@ #define ERTS_DE_QFLGS_ALL (ERTS_DE_QFLG_BUSY \ | ERTS_DE_QFLG_EXIT) -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL) #else #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713) @@ -131,7 +131,7 @@ typedef struct dist_entry_ { ErtsLink *nlinks; /* Link tree with subtrees */ ErtsMonitor *monitors; /* Monitor tree */ - erts_smp_spinlock_t qlock; /* Protects qflgs and out_queue */ + erts_smp_mtx_t qlock; /* Protects qflgs and out_queue */ Uint32 qflgs; Sint qsize; ErtsDistOutputQueue out_queue; diff --git a/erts/emulator/beam/erl_obsolete.c b/erts/emulator/beam/erl_obsolete.c deleted file mode 100644 index 9c5a7c7ff9..0000000000 --- a/erts/emulator/beam/erl_obsolete.c +++ /dev/null @@ -1,186 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_driver.h" - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * * - * ------------------------- OBSOLETE! DO NOT USE! ------------------------- * - * * -\* */ - -/* cut from ../obsolete/driver.h (since it doesn't mix well with other - * headers from the emulator). - */ -#ifdef __WIN32__ -#ifdef CONST -# undef CONST -#endif -#endif - -#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE) -# define _USING_PROTOTYPES_ 1 -# define _ANSI_ARGS_(x) x -# define CONST const -#else -# define _ANSI_ARGS_(x) () -# define CONST -#endif - -typedef void* erl_mutex_t; -typedef void* erl_cond_t; -typedef void* erl_thread_t; - -EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void)); -EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t)); - -EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void)); -EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t)); -EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long)); - -EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*, - void* (*func)(void*), - void* arg, - int detached)); -EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void)); -EXTERN void erts_thread_exit _ANSI_ARGS_((void*)); -EXTERN int erts_thread_join _ANSI_ARGS_((erl_thread_t, void**)); -EXTERN int erts_thread_kill _ANSI_ARGS_((erl_thread_t)); - -/* - * These functions implement the thread interface in ../obsolete/driver.h. - * Do *not* use this interface! Within the emulator, use the erl_threads.h, - * erl_smp.h, or ethread.h interface. From a driver use the thread interface - * in erl_driver.h. - */ - -erl_mutex_t -erts_mutex_create(void) -{ - return (erl_mutex_t) erl_drv_mutex_create(NULL); -} - -int -erts_mutex_destroy(erl_mutex_t mtx) -{ - erl_drv_mutex_destroy((ErlDrvMutex *) mtx); - return 0; -} - -int -erts_mutex_lock(erl_mutex_t mtx) -{ - erl_drv_mutex_lock((ErlDrvMutex *) mtx); - return 0; -} - -int -erts_mutex_unlock(erl_mutex_t mtx) -{ - erl_drv_mutex_unlock((ErlDrvMutex *) mtx); - return 0; -} - -erl_cond_t -erts_cond_create(void) -{ - return (erl_cond_t) erl_drv_cond_create(NULL); -} - -int -erts_cond_destroy(erl_cond_t cnd) -{ - erl_drv_cond_destroy((ErlDrvCond *) cnd); - return 0; -} - - -int -erts_cond_signal(erl_cond_t cnd) -{ - erl_drv_cond_signal((ErlDrvCond *) cnd); - return 0; -} - -int -erts_cond_broadcast(erl_cond_t cnd) -{ - erl_drv_cond_broadcast((ErlDrvCond *) cnd); - return 0; -} - - -int -erts_cond_wait(erl_cond_t cnd, erl_mutex_t mtx) -{ - erl_drv_cond_wait((ErlDrvCond *) cnd, (ErlDrvMutex *) mtx); - return 0; -} - -int -erts_cond_timedwait(erl_cond_t cnd, erl_mutex_t mtx, long ms) -{ - return ENOTSUP; -} - -int -erts_thread_create(erl_thread_t *tid, - void* (*func)(void*), - void* arg, - int detached) -{ - if (detached) - return ENOTSUP; - return erl_drv_thread_create(NULL, (ErlDrvTid *) tid, func, arg, NULL); -} - -erl_thread_t -erts_thread_self(void) -{ - return (erl_thread_t) erl_drv_thread_self(); -} - -void -erts_thread_exit(void *res) -{ - erl_drv_thread_exit(res); -} - -int -erts_thread_join(erl_thread_t tid, void **respp) -{ - return erl_drv_thread_join((ErlDrvTid) tid, respp); -} - -int -erts_thread_kill(erl_thread_t tid) -{ - return ENOTSUP; -} - diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 0b6bb0d8e9..e6b55c45e4 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -129,7 +129,7 @@ reset_handle(ErtsPortTask *ptp) { if (ptp->handle) { ASSERT(ptp == handle2task(ptp->handle)); - erts_smp_atomic_set(ptp->handle, (long) NULL); + erts_smp_atomic_set(ptp->handle, (erts_aint_t) NULL); } } @@ -138,7 +138,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->handle = pthp; if (pthp) { - erts_smp_atomic_set(pthp, (long) ptp); + erts_smp_atomic_set(pthp, (erts_aint_t) ptp); ASSERT(ptp == handle2task(ptp->handle)); } } @@ -568,7 +568,7 @@ erts_port_task_schedule(Eterm id, ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); if (xrunq) { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (long) xrunq); + erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); runq = xrunq; } @@ -625,6 +625,7 @@ erts_port_task_schedule(Eterm id, if (!enq_port) { ERTS_PT_CHK_PRES_PORTQ(runq, pp); + erts_smp_runq_unlock(runq); } else { enqueue_port(runq, pp); @@ -634,9 +635,10 @@ erts_port_task_schedule(Eterm id, profile_runnable_port(pp, am_active); } + erts_smp_runq_unlock(runq); + erts_smp_notify_inc_runq(runq); } - erts_smp_runq_unlock(runq); return 0; } @@ -656,8 +658,6 @@ erts_port_task_free_port(Port *pp) when scheduled out... */ ErtsPortTask *ptp = port_task_alloc(); erts_smp_port_state_lock(pp); - ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); @@ -679,7 +679,6 @@ erts_port_task_free_port(Port *pp) port_is_dequeued = 1; } erts_smp_port_state_lock(pp); - erts_smp_atomic_dec(&erts_ports_alive); pp->status &= ~ERTS_PORT_SFLG_CLOSING; pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; erts_may_save_closed_port(pp); @@ -725,7 +724,8 @@ resume_after_block(void *vd) ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; erts_smp_runq_lock(d->runq); if (d->resp) - *d->resp = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + *d->resp = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); } /* @@ -746,7 +746,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ErtsPortTask *ptp; int res = 0; int reds = ERTS_PORT_REDS_EXECUTE; - long io_tasks_executed = 0; + erts_aint_t io_tasks_executed = 0; int fpe_was_unmasked; ErtsPortTaskExeBlockData blk_data = {runq, NULL}; @@ -940,18 +940,19 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (long) xrunq); + erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; - erts_smp_notify_inc_runq(xrunq); erts_smp_runq_unlock(xrunq); + erts_smp_notify_inc_runq(xrunq); } #endif port_was_enqueued = 1; } - res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); ERTS_PT_CHK_PRES_PORTQ(runq, pp); @@ -969,15 +970,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_port_release(pp); #else { - long refc = erts_smp_atomic_dectest(&pp->refc); + erts_aint_t refc; + erts_smp_mtx_unlock(pp->lock); + refc = erts_smp_atomic_dectest(&pp->refc); ASSERT(refc >= 0); - if (refc > 0) - erts_smp_mtx_unlock(pp->lock); - else { + if (refc == 0) { erts_smp_runq_unlock(runq); erts_port_cleanup(pp); /* Might aquire runq lock */ erts_smp_runq_lock(runq); - res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); } } #endif @@ -1110,9 +1112,8 @@ erts_port_migrate(Port *prt, int *prt_locked, if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; dequeue_port(from_rq, prt); - erts_smp_atomic_set(&prt->run_queue, (long) to_rq); + erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) to_rq); enqueue_port(to_rq, prt); - erts_smp_notify_inc_runq(to_rq); return ERTS_MIGRATE_SUCCESS; } @@ -1124,7 +1125,7 @@ erts_port_migrate(Port *prt, int *prt_locked, void erts_port_task_init(void) { - erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (long) 0); + erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0); init_port_task_alloc(); init_port_taskq_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index f12d02da0c..3e2c5f07ab 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -79,7 +79,7 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp) { - erts_smp_atomic_init(pthp, (long) NULL); + erts_smp_atomic_init(pthp, (erts_aint_t) NULL); } ERTS_GLB_INLINE int @@ -102,6 +102,7 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp) ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void) { + ERTS_THR_MEMORY_BARRIER; return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0; } diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 7fe3f3bca5..34da9cab84 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -114,13 +114,13 @@ do { \ /* return 0 if list is not a non-empty flat list of printable characters */ static int -is_printable_string(Eterm list) +is_printable_string(Eterm list, Eterm* base) { int len = 0; int c; while(is_list(list)) { - Eterm* consp = list_val(list); + Eterm* consp = list_val_rel(list, base); Eterm hd = CAR(consp); if (!is_byte(hd)) @@ -216,240 +216,289 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) } +#define PRT_BAR ((Eterm) 0) +#define PRT_COMMA ((Eterm) 1) +#define PRT_CLOSE_LIST ((Eterm) 2) +#define PRT_CLOSE_TUPLE ((Eterm) 3) +#define PRT_TERM ((Eterm) 4) +#define PRT_ONE_CONS ((Eterm) 5) +#define PRT_PATCH_FUN_SIZE ((Eterm) 6) +#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */ static int -print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) +print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, + Eterm* obj_base) /* ignored if !HALFWORD_HEAP */ { + DECLARE_WSTACK(s); int res; int i; + Eterm val; Uint32 *ref_num; + union { + UWord word; + Eterm* ptr; + }popped; Eterm* nobj; + Wterm wobj; res = 0; - if ((*dcount)-- <= 0) - return res; + goto L_jump_start; + + L_outer_loop: + while (!WSTACK_ISEMPTY(s)) { + switch (val = WSTACK_POP(s)) { + case PRT_COMMA: + PRINT_CHAR(res, fn, arg, ','); + goto L_outer_loop; + case PRT_BAR: + PRINT_CHAR(res, fn, arg, '|'); + goto L_outer_loop; + case PRT_CLOSE_LIST: + PRINT_CHAR(res, fn, arg, ']'); + goto L_outer_loop; + case PRT_CLOSE_TUPLE: + PRINT_CHAR(res, fn, arg, '}'); + goto L_outer_loop; + default: + popped.word = WSTACK_POP(s); -#ifdef HYBRID___NOT_ACTIVE - /* Color coded output based on memory location */ - if(ptr_val(obj) >= global_heap && ptr_val(obj) < global_hend) - PRINT_STRING(res, fn, arg, "\033[32m"); -#ifdef INCREMENTAL - else if(ptr_val(obj) >= inc_fromspc && ptr_val(obj) < inc_fromend) - PRINT_STRING(res, fn, arg, "\033[33m"); -#endif - else if(IS_CONST(obj)) - PRINT_STRING(res, fn, arg, "\033[34m"); - else - PRINT_STRING(res, fn, arg, "\033[31m"); -#endif + switch (val) { + case PRT_TERM: + obj = (Eterm) popped.word; + break; + case PRT_ONE_CONS: + obj = (Eterm) popped.word; + L_print_one_cons: + { + Eterm* cons = list_val_rel(obj, obj_base); + Eterm tl; + + obj = CAR(cons); + tl = CDR(cons); + if (is_not_nil(tl)) { + if (is_list(tl)) { + WSTACK_PUSH(s, tl); + WSTACK_PUSH(s, PRT_ONE_CONS); + WSTACK_PUSH(s, PRT_COMMA); + } else { + WSTACK_PUSH(s, tl); + WSTACK_PUSH(s, PRT_TERM); + WSTACK_PUSH(s, PRT_BAR); + } + } + } + break; + case PRT_LAST_ARRAY_ELEMENT: + obj = *popped.ptr; + break; + default: /* PRT_LAST_ARRAY_ELEMENT+1 and upwards */ + obj = *popped.ptr; + WSTACK_PUSH(s, (UWord) (popped.ptr + 1)); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, PRT_COMMA); + break; + } + break; + } - if (is_CP(obj)) { - PRINT_STRING(res, fn, arg, "<cp/header:"); - PRINT_POINTER(res, fn, arg, obj); - PRINT_CHAR(res, fn, arg, '>'); - return res; - } + L_jump_start: - switch (tag_val_def(obj)) { - case NIL_DEF: - PRINT_STRING(res, fn, arg, "[]"); - break; - case ATOM_DEF: { - int tres = print_atom_name(fn, arg, obj, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; - break; - } - case SMALL_DEF: - PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj)); - break; - case BIG_DEF: { - int print_res; - char def_buf[64]; - char *buf, *big_str; - Uint sz = (Uint) big_decimal_estimate(obj); - sz++; - if (sz <= 64) - buf = &def_buf[0]; - else - buf = erts_alloc(ERTS_ALC_T_TMP, sz); - big_str = erts_big_to_string(obj, buf, sz); - print_res = erts_printf_string(fn, arg, big_str); - if (buf != &def_buf[0]) - erts_free(ERTS_ALC_T_TMP, (void *) buf); - if (print_res < 0) - return print_res; - res += print_res; - break; - } - case REF_DEF: - case EXTERNAL_REF_DEF: - PRINT_STRING(res, fn, arg, "#Ref<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) ref_channel_no(obj)); - ref_num = ref_numbers(obj); - for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) { - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); + if ((*dcount)-- <= 0) + goto L_done; + + if (is_CP(obj)) { + PRINT_STRING(res, fn, arg, "<cp/header:"); + PRINT_POINTER(res, fn, arg, cp_val(obj)); + PRINT_CHAR(res, fn, arg, '>'); + goto L_done; } - PRINT_CHAR(res, fn, arg, '>'); - break; - case PID_DEF: - case EXTERNAL_PID_DEF: - PRINT_CHAR(res, fn, arg, '<'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_channel_no(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_number(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_serial(obj)); - PRINT_CHAR(res, fn, arg, '>'); - break; - case PORT_DEF: - case EXTERNAL_PORT_DEF: - PRINT_STRING(res, fn, arg, "#Port<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_channel_no(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_number(obj)); - PRINT_CHAR(res, fn, arg, '>'); - break; - case LIST_DEF: - if (is_printable_string(obj)) { - int c; - PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val(obj); - while (1) { - if ((*dcount)-- <= 0) - return res; - c = signed_val(*nobj++); - if (c == '\n') - PRINT_STRING(res, fn, arg, "\\n"); - else { - if (c == '"') - PRINT_CHAR(res, fn, arg, '\\'); - PRINT_CHAR(res, fn, arg, (char) c); - } - if (is_not_list(*nobj)) - break; - nobj = list_val(*nobj); - } - PRINT_CHAR(res, fn, arg, '"'); - } else { - PRINT_CHAR(res, fn, arg, '['); - nobj = list_val(obj); - while (1) { - int tres = print_term(fn, arg, *nobj++, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; - if (is_not_list(*nobj)) - break; - PRINT_CHAR(res, fn, arg, ','); - nobj = list_val(*nobj); - } - if (is_not_nil(*nobj)) { - int tres; - PRINT_CHAR(res, fn, arg, '|'); - tres = print_term(fn, arg, *nobj, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; +#if HALFWORD_HEAP + wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base); +#else + wobj = (Wterm)obj; +#endif + switch (tag_val_def(wobj)) { + case NIL_DEF: + PRINT_STRING(res, fn, arg, "[]"); + break; + case ATOM_DEF: { + int tres = print_atom_name(fn, arg, obj, dcount); + if (tres < 0) { + res = tres; + goto L_done; } - PRINT_CHAR(res, fn, arg, ']'); - } - break; - case TUPLE_DEF: - nobj = tuple_val(obj); /* pointer to arity */ - i = arityval(*nobj); /* arity */ - PRINT_CHAR(res, fn, arg, '{'); - while (i--) { - int tres = print_term(fn, arg, *++nobj, dcount); - if (tres < 0) - return tres; res += tres; if (*dcount <= 0) - return res; - if (i >= 1) - PRINT_CHAR(res, fn, arg, ','); - } - PRINT_CHAR(res, fn, arg, '}'); - break; - case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(obj, ff); - PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); + goto L_done; + break; } - break; - case BINARY_DEF: - { - ProcBin* pb = (ProcBin *) binary_val(obj); - if (pb->size == 1) - PRINT_STRING(res, fn, arg, "<<1 byte>>"); - else { - PRINT_STRING(res, fn, arg, "<<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size); - PRINT_STRING(res, fn, arg, " bytes>>"); + case SMALL_DEF: + PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj)); + break; + case BIG_DEF: { + int print_res; + char def_buf[64]; + char *buf, *big_str; + Uint sz = (Uint) big_decimal_estimate(wobj); + sz++; + if (sz <= 64) + buf = &def_buf[0]; + else + buf = erts_alloc(ERTS_ALC_T_TMP, sz); + big_str = erts_big_to_string(wobj, buf, sz); + print_res = erts_printf_string(fn, arg, big_str); + if (buf != &def_buf[0]) + erts_free(ERTS_ALC_T_TMP, (void *) buf); + if (print_res < 0) { + res = print_res; + goto L_done; } + res += print_res; + break; } - break; - case EXPORT_DEF: - { - Export* ep = (Export *) (export_val(obj))[1]; - Atom* module = atom_tab(atom_val(ep->code[0])); - Atom* name = atom_tab(atom_val(ep->code[1])); - - PRINT_STRING(res, fn, arg, "#Fun<"); - PRINT_BUF(res, fn, arg, module->name, module->len); + case REF_DEF: + case EXTERNAL_REF_DEF: + PRINT_STRING(res, fn, arg, "#Ref<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) ref_channel_no(wobj)); + ref_num = ref_numbers(wobj); + for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) { + PRINT_CHAR(res, fn, arg, '.'); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); + } + PRINT_CHAR(res, fn, arg, '>'); + break; + case PID_DEF: + case EXTERNAL_PID_DEF: + PRINT_CHAR(res, fn, arg, '<'); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_BUF(res, fn, arg, name->name, name->len); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_number(wobj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) ep->code[2]); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_serial(wobj)); PRINT_CHAR(res, fn, arg, '>'); - } - break; - case FUN_DEF: - { - ErlFunThing *funp = (ErlFunThing *) fun_val(obj); - Atom *ap = atom_tab(atom_val(funp->fe->module)); - - PRINT_STRING(res, fn, arg, "#Fun<"); - PRINT_BUF(res, fn, arg, ap->name, ap->len); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_index); + break; + case PORT_DEF: + case EXTERNAL_PORT_DEF: + PRINT_STRING(res, fn, arg, "#Port<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) port_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_uniq); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) port_number(wobj)); + PRINT_CHAR(res, fn, arg, '>'); + break; + case LIST_DEF: + if (is_printable_string(obj, obj_base)) { + int c; + PRINT_CHAR(res, fn, arg, '"'); + nobj = list_val_rel(obj, obj_base); + while (1) { + if ((*dcount)-- <= 0) + goto L_done; + c = signed_val(*nobj++); + if (c == '\n') + PRINT_STRING(res, fn, arg, "\\n"); + else { + if (c == '"') + PRINT_CHAR(res, fn, arg, '\\'); + PRINT_CHAR(res, fn, arg, (char) c); + } + if (is_not_list(*nobj)) + break; + nobj = list_val_rel(*nobj, obj_base); + } + PRINT_CHAR(res, fn, arg, '"'); + } else { + PRINT_CHAR(res, fn, arg, '['); + WSTACK_PUSH(s,PRT_CLOSE_LIST); + goto L_print_one_cons; + } + break; + case TUPLE_DEF: + nobj = tuple_val(wobj); /* pointer to arity */ + i = arityval(*nobj); /* arity */ + PRINT_CHAR(res, fn, arg, '{'); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + ++nobj; + if (i > 0) { + WSTACK_PUSH(s, (UWord) nobj); + WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1); + } + break; + case FLOAT_DEF: { + FloatDef ff; + GET_DOUBLE(wobj, ff); + PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); + } + break; + case BINARY_DEF: + { + ProcBin* pb = (ProcBin *) binary_val(wobj); + if (pb->size == 1) + PRINT_STRING(res, fn, arg, "<<1 byte>>"); + else { + PRINT_STRING(res, fn, arg, "<<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size); + PRINT_STRING(res, fn, arg, " bytes>>"); + } + } + break; + case EXPORT_DEF: + { + Export* ep = *((Export **) (export_val(wobj) + 1)); + Atom* module = atom_tab(atom_val(ep->code[0])); + Atom* name = atom_tab(atom_val(ep->code[1])); + + PRINT_STRING(res, fn, arg, "#Fun<"); + PRINT_BUF(res, fn, arg, module->name, module->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_BUF(res, fn, arg, name->name, name->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) ep->code[2]); + PRINT_CHAR(res, fn, arg, '>'); + } + break; + case FUN_DEF: + { + ErlFunThing *funp = (ErlFunThing *) fun_val(wobj); + Atom *ap = atom_tab(atom_val(funp->fe->module)); + + PRINT_STRING(res, fn, arg, "#Fun<"); + PRINT_BUF(res, fn, arg, ap->name, ap->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) funp->fe->old_index); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) funp->fe->old_uniq); + PRINT_CHAR(res, fn, arg, '>'); + } + break; + default: + PRINT_STRING(res, fn, arg, "<unknown:"); + PRINT_POINTER(res, fn, arg, wobj); PRINT_CHAR(res, fn, arg, '>'); + break; } - break; - default: - PRINT_STRING(res, fn, arg, "<unknown:"); - PRINT_POINTER(res, fn, arg, obj); - PRINT_CHAR(res, fn, arg, '>'); - break; } + L_done: + + DESTROY_WSTACK(s); return res; } int -erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision) +erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, + unsigned long* term_base) { - int res = print_term(fn, arg, (Uint) term, &precision); + int res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); if (res < 0) return res; if (precision <= 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index 4f76028396..a48a3de34c 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,6 @@ #define ERL_PRINTF_TERM_H__ #include "erl_printf_format.h" -int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision); - +int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, + unsigned long* term_base); #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 996806fc75..2704359a8f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,7 +24,6 @@ #endif #include <stddef.h> /* offsetof() */ -#include <ctype.h> #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -38,6 +37,8 @@ #include "erl_instrument.h" #include "erl_threads.h" #include "erl_binary.h" +#include "beam_bp.h" +#include "erl_cpu_topology.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -45,14 +46,23 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#define ERTS_SCHED_SLEEP_SPINCOUNT 10000 +#define ERTS_SCHED_SPIN_UNTIL_YIELD 100 + +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \ + (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT) +#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0 + +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10) -#define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2) #define ERTS_WAKEUP_OTHER_DEC 10 #define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10) -#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) - #if 0 || defined(DEBUG) #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA #endif @@ -91,9 +101,9 @@ do { \ #define ERTS_EMPTY_RUNQ(RQ) \ ((RQ)->len == 0 && (RQ)->misc.start == NULL) -extern Eterm beam_apply[]; -extern Eterm beam_exit[]; -extern Eterm beam_continue_exit[]; +extern BeamInstr beam_apply[]; +extern BeamInstr beam_exit[]; +extern BeamInstr beam_continue_exit[]; static Sint p_last; static Sint p_next; @@ -105,6 +115,8 @@ Uint erts_no_schedulers; Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; Uint erts_process_tab_index_mask; +static int wakeup_other_limit; + int erts_sched_thread_suggested_stack_size = -1; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -115,19 +127,38 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; int erts_disable_proc_not_running_opt; -#define ERTS_SCHED_CHANGING_ONLINE 1 -#define ERTS_SCHED_CHANGING_MULTI_SCHED 2 +#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) +#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) + +#ifndef DEBUG + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ + erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL)) + +#else + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ +do { \ + erts_aint32_t old_val__; \ + old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing, \ + (VAL)); \ + ASSERT(old_val__ == (OLD_VAL)); \ +} while (0) + +#endif + static struct { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - int changing; int online; int curr_online; int wait_curr_online; - erts_smp_atomic_t active; + erts_smp_atomic32_t changing; + erts_smp_atomic32_t active; struct { - erts_smp_atomic_t ongoing; + erts_smp_atomic32_t ongoing; long wait_active; ErtsProcList *procs; } msb; /* Multi Scheduling Block */ @@ -135,11 +166,10 @@ static struct { static struct { erts_smp_mtx_t update_mtx; - erts_smp_atomic_t active_runqs; + erts_smp_atomic32_t no_runqs; int last_active_runqs; - erts_smp_atomic_t used_runqs; int forced_check_balance; - erts_smp_atomic_t checking_balance; + erts_smp_atomic32_t checking_balance; int halftime; int full_reds_history_index; struct { @@ -159,44 +189,6 @@ do { \ #endif -/* - * Cpu topology hierarchy. - */ -#define ERTS_TOPOLOGY_NODE 0 -#define ERTS_TOPOLOGY_PROCESSOR 1 -#define ERTS_TOPOLOGY_PROCESSOR_NODE 2 -#define ERTS_TOPOLOGY_CORE 3 -#define ERTS_TOPOLOGY_THREAD 4 -#define ERTS_TOPOLOGY_LOGICAL 5 - -#define ERTS_TOPOLOGY_MAX_DEPTH 6 - -typedef struct { - int bind_id; - int bound_id; -} ErtsCpuBindData; - -static ErtsCpuBindData *scheduler2cpu_map; -erts_smp_rwmtx_t erts_cpu_bind_rwmtx; - -typedef enum { - ERTS_CPU_BIND_SPREAD, - ERTS_CPU_BIND_PROCESSOR_SPREAD, - ERTS_CPU_BIND_THREAD_SPREAD, - ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD, - ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD, - ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD, - ERTS_CPU_BIND_NO_SPREAD, - ERTS_CPU_BIND_NONE -} ErtsCpuBindOrder; - -ErtsCpuBindOrder cpu_bind_order; - -static erts_cpu_topology_t *user_cpudata; -static int user_cpudata_size; -static erts_cpu_topology_t *system_cpudata; -static int system_cpudata_size; - erts_sched_stat_t erts_sched_stat; ErtsRunQueue *erts_common_run_queue; @@ -207,11 +199,11 @@ static erts_tsd_key_t sched_data_key; static erts_smp_mtx_t proc_tab_mtx; -static erts_smp_atomic_t function_calls; +static erts_smp_atomic32_t function_calls; #ifdef ERTS_SMP -static erts_smp_atomic_t doing_sys_schedule; -static erts_smp_atomic_t no_empty_run_queues; +static erts_smp_atomic32_t doing_sys_schedule; +static erts_smp_atomic32_t no_empty_run_queues; #else /* !ERTS_SMP */ ErtsSchedulerData *erts_scheduler_data; #endif @@ -219,12 +211,18 @@ ErtsSchedulerData *erts_scheduler_data; ErtsAlignedRunQueue *erts_aligned_run_queues; Uint erts_no_run_queues; +ErtsAlignedSchedulerData *erts_aligned_scheduler_data; + +#ifdef ERTS_SMP + typedef union { - ErtsSchedulerData esd; - char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; -} ErtsAlignedSchedulerData; + ErtsSchedulerSleepInfo ssi; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; +} ErtsAlignedSchedulerSleepInfo; -ErtsAlignedSchedulerData *erts_aligned_scheduler_data; +static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; + +#endif #ifndef BM_COUNTERS static int processes_busy; @@ -249,7 +247,10 @@ Uint erts_num_active_procs; Process** erts_active_procs; #endif -static erts_smp_atomic_t process_count; +#if ERTS_MAX_PROCESSES > 0x7fffffff +#error "Need to store process_count in another type" +#endif +static erts_smp_atomic32_t process_count; typedef struct ErtsTermProcElement_ ErtsTermProcElement; struct ErtsTermProcElement_ { @@ -283,8 +284,9 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) -#define ERTS_RUNQ_IX(IX) (&erts_aligned_run_queues[(IX)].runq) -#define ERTS_SCHEDULER_IX(IX) (&erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_SCHED_SLEEP_INFO_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -334,23 +336,14 @@ do { \ static void init_processes_bif(void); static void save_terminating_process(Process *p); static void exec_misc_ops(ErtsRunQueue *); -static void print_function_from_pc(int to, void *to_arg, Eterm* x); +static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg); #ifdef ERTS_SMP static void handle_pending_exiters(ErtsProcList *); -static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, - int size, - ErtsCpuBindOrder bind_order, - int mk_seq); -static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); - #endif -static void early_cpu_bind_init(void); -static void late_cpu_bind_init(void); - #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) @@ -388,7 +381,12 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks = ERTS_PSD_DIST_ENTRY_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; + = ERTS_PSD_DIST_ENTRY_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks + = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks + = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; /* Check that we have locks for all entries */ for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { @@ -401,18 +399,18 @@ erts_pre_init_process(void) /* initialize the scheduler */ void -erts_init_process(void) +erts_init_process(int ncpu) { Uint proc_bits = ERTS_PROC_BITS; #ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; - erts_init_proc_lock(); + erts_init_proc_lock(ncpu); #endif init_proclist_alloc(); - erts_smp_atomic_init(&process_count, 0); + erts_smp_atomic32_init(&process_count, 0); if (erts_use_r9_pids_ports) { proc_bits = ERTS_R9_PROC_BITS; @@ -572,6 +570,194 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP +void +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) +{ + switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_sys_schedule_interrupt(1); + break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_tse_set(ssi->event); + break; + case 0: + break; + default: + erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + __FILE__, __LINE__); + break; + } +} + +typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; +struct erts_misc_aux_work_t_ { + erts_misc_aux_work_t *next; + void (*func)(void *); + void *arg; +}; + +typedef struct { + erts_smp_mtx_t mtx; + erts_misc_aux_work_t *first; + erts_misc_aux_work_t *last; +} erts_misc_aux_work_q_t; + +typedef union { + erts_misc_aux_work_q_t data; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))]; +} erts_algnd_misc_aux_work_q_t; + +static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues; + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, + erts_misc_aux_work_t, + 200, + ERTS_ALC_T_MISC_AUX_WORK) + +static void +init_misc_aux_work(void) +{ + int ix; + + init_misc_aux_work_alloc(); + + misc_aux_work_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q, + erts_no_schedulers * + sizeof(erts_algnd_misc_aux_work_q_t)); + + for (ix = 0; ix < erts_no_schedulers; ix++) { + erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, + "misc_aux_work_queue", + make_small(ix + 1)); + misc_aux_work_queues[ix].data.first = NULL; + misc_aux_work_queues[ix].data.last = NULL; + } +} + +static void +handle_misc_aux_work(ErtsSchedulerData *esdp) +{ + int ix = (int) esdp->no - 1; + erts_misc_aux_work_t *mawp; + + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); + mawp = misc_aux_work_queues[ix].data.first; + misc_aux_work_queues[ix].data.first = NULL; + misc_aux_work_queues[ix].data.last = NULL; + erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + + while (mawp) { + erts_misc_aux_work_t *free_mawp; + mawp->func(mawp->arg); + free_mawp = mawp; + mawp = mawp->next; + misc_aux_work_free(free_mawp); + } +} + +void +erts_smp_schedule_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg) +{ + int ix, ignore_ix = -1; + + if (ignore_self) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + ignore_ix = (int) esdp->no - 1; + } + + ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers); + + for (ix = 0; ix < max_sched; ix++) { + erts_aint32_t aux_work; + erts_misc_aux_work_t *mawp; + ErtsSchedulerSleepInfo *ssi; + if (ix == ignore_ix) + continue; + + mawp = misc_aux_work_alloc(); + + mawp->func = func; + mawp->arg = arg; + mawp->next = NULL; + + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); + if (!misc_aux_work_queues[ix].data.last) + misc_aux_work_queues[ix].data.first = mawp; + else + misc_aux_work_queues[ix].data.last->next = mawp; + misc_aux_work_queues[ix].data.last = mawp; + erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + aux_work = erts_smp_atomic32_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_MISC); + if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0) + erts_sched_poke(ssi); + } +} + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN +void +erts_smp_notify_check_children_needed(void) +{ + int i; + + for (i = 0; i < erts_no_schedulers; i++) { + erts_aint32_t aux_work; + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(i); + aux_work = erts_smp_atomic32_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) + erts_sched_poke(ssi); + } +} +#endif + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +static ERTS_INLINE erts_aint32_t +blockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + erts_aint32_t aux_work) +{ + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + if (aux_work & ERTS_SSI_AUX_WORK_MISC) { + aux_work = erts_smp_atomic32_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_MISC); + aux_work &= ~ERTS_SSI_AUX_WORK_MISC; + handle_misc_aux_work(esdp); + } +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { + aux_work = erts_smp_atomic32_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; + erts_check_children(); + } +#endif + } + return aux_work; +} + +#endif + +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +static ERTS_INLINE erts_aint32_t +nonblockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + erts_aint32_t aux_work) +{ + if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + + } +} +#endif + static void prepare_for_block(void *vrq) { @@ -627,6 +813,46 @@ erts_active_schedulers(void) #ifdef ERTS_SMP static ERTS_INLINE void +clear_sys_scheduling(void) +{ + erts_smp_atomic32_set_relb(&doing_sys_schedule, 0); +} + +static ERTS_INLINE int +try_set_sys_scheduling(void) +{ + return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0); +} + +#endif + +static ERTS_INLINE int +prepare_for_sys_schedule(void) +{ +#ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && try_set_sys_scheduling()) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + clear_sys_scheduling(); + } + return 0; +#else + return !erts_port_task_have_outstanding_io_tasks(); +#endif +} + +#ifdef ERTS_SMP + +static ERTS_INLINE void +sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq->waiting < 0); + rq->waiting *= -1; +} + +static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -656,287 +882,544 @@ sched_active(Uint no, ErtsRunQueue *rq) static int ERTS_INLINE ongoing_multi_scheduling_block(void) { - return erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) != 0; + return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0; } static ERTS_INLINE void empty_runq(ErtsRunQueue *rq) { - long oifls = erts_smp_atomic_band(&rq->info_flags, ~ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags, + ~ERTS_RUNQ_IFLG_NONEMPTY); if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { #ifdef DEBUG - long empty = erts_smp_atomic_read(&no_empty_run_queues); - ASSERT(0 <= empty && empty < erts_no_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + /* + * For a short period of time no_empty_run_queues may have + * been increased twice for a specific run queue. + */ + ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif - erts_smp_atomic_inc(&no_empty_run_queues); + erts_smp_atomic32_inc(&no_empty_run_queues); } } static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { - long oifls = erts_smp_atomic_bor(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags, + ERTS_RUNQ_IFLG_NONEMPTY); if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { #ifdef DEBUG - long empty = erts_smp_atomic_read(&no_empty_run_queues); - ASSERT(0 < empty && empty <= erts_no_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + /* + * For a short period of time no_empty_run_queues may have + * been increased twice for a specific run queue. + */ + ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif - erts_smp_atomic_dec(&no_empty_run_queues); + erts_smp_atomic32_dec(&no_empty_run_queues); } } -static ERTS_INLINE int -sched_spin_wake(ErtsRunQueue *rq) +static erts_aint32_t +sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + erts_aint32_t xflgs = 0; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_atomic_inc(&rq->spin_wake); - return 1; - } - return 0; -#endif + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED)); + return oflgs; } -static ERTS_INLINE int -sched_spin_wake_all(ErtsRunQueue *rq) +static erts_aint32_t +sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) - erts_smp_atomic_add(&rq->spin_wake, val); - return val; -#endif + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } while (oflgs & ERTS_SSI_FLG_WAITING); + return oflgs; +} + +static erts_aint32_t +sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + erts_aint32_t flgs; + + do { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; } +static erts_aint32_t +sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) +{ + erts_aint32_t oflgs; + erts_aint32_t nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + + if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + return oflgs; + } + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } +} + +#define ERTS_SCHED_WAIT_WOKEN(FLGS) \ + (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ + != ERTS_SSI_FLG_WAITING) + static void -sched_sys_wait(Uint no, ErtsRunQueue *rq) +scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { - long dt; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + int spincount; + erts_aint32_t flgs; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + erts_aint32_t aux_work; +#endif + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_smp_spin_lock(&rq->sleepers.lock); + flgs = sched_prep_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { + /* Go suspend instead... */ + erts_smp_spin_unlock(&rq->sleepers.lock); + return; + } + + ssi->prev = NULL; + ssi->next = rq->sleepers.list; + if (rq->sleepers.list) + rq->sleepers.list->prev = ssi; + rq->sleepers.list = ssi; + erts_smp_spin_unlock(&rq->sleepers.lock); + + /* + * If all schedulers are waiting, one of them *should* + * be waiting in erl_sys_schedule() + */ + + if (!prepare_for_sys_schedule()) { + + sched_waiting(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + tse_wait: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + tse_blockable_aux_work: + aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - sched_waiting_sys(no, rq); + while (1) { -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif - erl_sys_schedule(1); /* Might give us something to do */ + flgs = sched_spin_wait(ssi, spincount); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + + flgs = sched_prep_cont_spin_wait(ssi); + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto tse_blockable_aux_work; + } +#endif - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_runq_unlock(rq); } - } - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + erts_smp_runq_lock(rq); + sched_active(esdp->no, rq); + } else { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_aint_t dt; + + erts_smp_atomic32_set_relb(&function_calls, 0); + *fcalls = 0; + + sched_waiting_sys(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; + + while (spincount-- > 0) { + + sys_poll_aux_work: + + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + + erl_sys_schedule(1); /* Might give us something to do */ + + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); + + sys_aux_work: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + } + + /* + * If we got new I/O tasks we aren't allowed to + * call erl_sys_schedule() until it is handled. + */ + if (erts_port_task_have_outstanding_io_tasks()) { + clear_sys_scheduling(); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to continue checking for I/O... + */ + if (!prepare_for_sys_schedule()) { + spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; + goto tse_wait; + } + } + } + + erts_smp_runq_lock(rq); + /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). */ - if (!erts_port_task_have_outstanding_io_tasks()) { -#endif + if (erts_port_task_have_outstanding_io_tasks()) { + clear_sys_scheduling(); + + /* + * Got to check that we still got I/O tasks; otherwise + * we have to wait in erl_sys_schedule() after all... + */ + if (prepare_for_sys_schedule()) + goto do_sys_schedule; + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + else { + do_sys_schedule: erts_sys_schedule_interrupt(0); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) + goto sys_locked_woken; + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } + + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_smp_runq_unlock(rq); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + erl_sys_schedule(0); - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); - erts_smp_runq_lock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + sys_woken: + erts_smp_runq_lock(rq); + sys_locked_woken: + clear_sys_scheduling(); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + sched_active_sys(esdp->no, rq); } } -#endif - sched_active_sys(no, rq); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); +} + +static ERTS_INLINE erts_aint32_t +ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) +{ + /* reset all flags but suspended */ + erts_aint32_t oflgs; + erts_aint32_t nflgs = 0; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + while (1) { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return oflgs; + nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + xflgs = oflgs; + } } static void -sched_cnd_wait(Uint no, ErtsRunQueue *rq) +wake_scheduler(ErtsRunQueue *rq, int incq, int one) { -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif + ErtsSchedulerSleepInfo *ssi; + ErtsSchedulerSleepList *sl; - sched_waiting(no, rq); - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); + /* + * The unlocked run queue is not strictly necessary + * from a thread safety or deadlock prevention + * perspective. It will, however, cost us performance + * if it is locked during wakup of another scheduler, + * so all code *should* handle this without having + * the lock on the run queue. + */ + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + + sl = &rq->sleepers; + + erts_smp_spin_lock(&sl->lock); + ssi = sl->list; + if (!ssi) + erts_smp_spin_unlock(&sl->lock); + else if (one) { + erts_aint32_t flgs; + if (ssi->prev) + ssi->prev->next = ssi->next; + else { + ASSERT(sl->list == ssi); + sl->list = ssi->next; + } + if (ssi->next) + ssi->next->prev = ssi->prev; -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); -#else - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_mtx_unlock(&rq->mtx); - - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_mtx_unlock(&rq->mtx); - } - } - - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val == 0) { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); + erts_smp_spin_unlock(&sl->lock); + + ERTS_THR_MEMORY_BARRIER; + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); + + if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING)) + non_empty_runq(rq); } else { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - } -#endif - - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); + sl->list = NULL; + erts_smp_spin_unlock(&sl->lock); - sched_active(no, rq); + ERTS_THR_MEMORY_BARRIER; + do { + ErtsSchedulerSleepInfo *wake_ssi = ssi; + ssi = ssi->next; + erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi)); + } while (ssi); + } } static void -wake_one_scheduler(void) -{ - ASSERT(erts_common_run_queue); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue)); - if (erts_common_run_queue->waiting) { - if (!sched_spin_wake(erts_common_run_queue)) { - if (erts_common_run_queue->waiting == -1) /* One scheduler waiting - and doing so in - sys_schedule */ - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&erts_common_run_queue->cnd); +wake_all_schedulers(void) +{ + if (erts_common_run_queue) + wake_scheduler(erts_common_run_queue, 0, 0); + else { + int ix; + for (ix = 0; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq, 0, 1); } } } -static void -wake_scheduler(ErtsRunQueue *rq, int incq) +#define ERTS_NO_USED_RUNQS_SHIFT 16 +#define ERTS_NO_RUNQS_MASK 0xffff + +#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK +# error "Too large amount of schedulers allowed" +#endif + +static ERTS_INLINE void +init_no_runqs(int active, int used) { - ASSERT(!erts_common_run_queue); - ASSERT(-1 <= rq->waiting && rq->waiting <= 1); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - if (rq->waiting && !rq->woken) { - if (!sched_spin_wake(rq)) { - if (rq->waiting < 0) - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&rq->cnd); - } - rq->woken = 1; - if (incq) - non_empty_runq(rq); + erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK); + no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT); + erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs); +} + +static ERTS_INLINE void +get_no_runqs(int *active, int *used) +{ + erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs); + if (active) + *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK); + if (used) + *used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK); +} + +static ERTS_INLINE void +set_no_used_runqs(int used) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK); + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; } } -static void -wake_all_schedulers(void) +static ERTS_INLINE void +set_no_active_runqs(int active) { - if (erts_common_run_queue) { - erts_smp_runq_lock(erts_common_run_queue); - if (erts_common_run_queue->waiting) { - if (erts_common_run_queue->waiting < 0) - erts_sys_schedule_interrupt(1); - sched_spin_wake_all(erts_common_run_queue); - erts_smp_cnd_broadcast(&erts_common_run_queue->cnd); - } - erts_smp_runq_unlock(erts_common_run_queue); + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + while (1) { + erts_aint32_t act, new; + new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + break; + exp = act; } - else { - int ix; - for (ix = 0; ix < erts_no_run_queues; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - wake_scheduler(rq, 0); - erts_smp_runq_unlock(rq); - } +} + +static ERTS_INLINE int +try_inc_no_active_runqs(int active) +{ + erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs); + if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active) + return 0; + if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) { + erts_aint32_t new, act; + new = (exp & ~ERTS_NO_RUNQS_MASK) | active; + act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp); + if (act == exp) + return 1; } + return 0; } + static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { - long iflgs; + erts_aint32_t iflgs; ErtsRunQueue *wrq; if (crq->ix == ix) return 0; wrq = ERTS_RUNQ_IX(ix); - iflgs = erts_smp_atomic_read(&wrq->info_flags); + iflgs = erts_smp_atomic32_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { - erts_smp_xrunq_lock(crq, wrq); if (activate) { - if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) { + if (try_inc_no_active_runqs(ix+1)) { + erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; + erts_smp_xrunq_unlock(crq, wrq); } } - wake_scheduler(wrq, 0); - erts_smp_xrunq_unlock(crq, wrq); + wake_scheduler(wrq, 0, 1); return 1; } return 0; @@ -947,8 +1430,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq) { int ix = crq->ix; int stop_ix = ix; - int active_ix = erts_smp_atomic_read(&balance_info.active_runqs); - int balance_ix = erts_smp_atomic_read(&balance_info.used_runqs); + int active_ix, balance_ix; + + get_no_runqs(&active_ix, &balance_ix); if (active_ix > balance_ix) active_ix = balance_ix; @@ -982,19 +1466,42 @@ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP - if (erts_common_run_queue) - wake_one_scheduler(); - else - wake_scheduler(runq, 1); + if (runq) + wake_scheduler(runq, 1, 1); #endif } void -erts_smp_notify_inc_runq__(ErtsRunQueue *runq) +erts_smp_notify_inc_runq(ErtsRunQueue *runq) { smp_notify_inc_runq(runq); } +void +erts_sched_notify_check_cpu_bind(void) +{ +#ifdef ERTS_SMP + int ix; + if (erts_common_run_queue) { + for (ix = 0; ix < erts_no_schedulers; ix++) + erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); + wake_all_schedulers(); + } + else { + for (ix = 0; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); + }; + } +#else + erts_sched_check_cpu_bind(erts_get_scheduler_data()); +#endif +} + + #ifdef ERTS_SMP ErtsRunQueue * @@ -1136,20 +1643,24 @@ static void evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) { Port *prt; + int notify_to_rq = 0; int prio; int prt_locked = 0; int rq_locked = 0; int evac_rq_locked = 1; + ErtsMigrateResult mres; erts_smp_runq_lock(evac_rq); + erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); - erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); - + erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1177,9 +1688,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) /* Evacuate scheduled ports */ prt = evac_rq->ports.start; while (prt) { - (void) erts_port_migrate(prt, &prt_locked, + mres = erts_port_migrate(prt, &prt_locked, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (prt_locked) erts_smp_port_unlock(prt); if (!evac_rq_locked) { @@ -1208,9 +1721,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) goto end_of_proc; } - (void) erts_proc_migrate(proc, &proc_locks, + mres = erts_proc_migrate(proc, &proc_locks, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (proc_locks) erts_smp_proc_unlock(proc, proc_locks); if (!evac_rq_locked) { @@ -1242,10 +1757,13 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) if (rq_locked) erts_smp_runq_unlock(rq); - if (!evac_rq_locked) - erts_smp_runq_lock(evac_rq); - wake_scheduler(evac_rq, 0); - erts_smp_runq_unlock(evac_rq); + if (evac_rq_locked) + erts_smp_runq_unlock(evac_rq); + + if (notify_to_rq) + smp_notify_inc_runq(rq); + + wake_scheduler(evac_rq, 0, 1); } static int @@ -1391,7 +1909,7 @@ static ERTS_INLINE int check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); - long iflgs = erts_smp_atomic_read(&vrq->info_flags); + erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags); if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) return try_steal_task_from_victim(rq, rq_lockedp, vrq); else @@ -1421,8 +1939,7 @@ try_steal_task(ErtsRunQueue *rq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); - active_rqs = erts_smp_atomic_read(&balance_info.active_runqs); - blnc_rqs = erts_smp_atomic_read(&balance_info.used_runqs); + get_no_runqs(&active_rqs, &blnc_rqs); if (active_rqs > blnc_rqs) active_rqs = blnc_rqs; @@ -1433,7 +1950,7 @@ try_steal_task(ErtsRunQueue *rq) if (active_rqs < blnc_rqs) { int no = blnc_rqs - active_rqs; int stop_ix = vix = active_rqs + rq->ix % no; - while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { res = check_possible_steal_victim(rq, &rq_locked, vix); if (res) goto done; @@ -1448,7 +1965,7 @@ try_steal_task(ErtsRunQueue *rq) vix = rq->ix; /* ... then try to steal a job from another active queue... */ - while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { vix++; if (vix >= active_rqs) vix = 0; @@ -1473,31 +1990,6 @@ try_steal_task(ErtsRunQueue *rq) return res; } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) { - erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue); - ERTS_SCHEDULER_IX(i)->check_children = 1; - if (!erts_common_run_queue) - wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0); - erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue); - } - if (ongoing_multi_scheduling_block()) { - /* Also blocked schedulers need to check children */ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - for (i = 0; i < erts_no_schedulers; i++) - ERTS_SCHEDULER_IX(i)->blocked_check_children = 1; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } - if (erts_common_run_queue) - wake_all_schedulers(); -} -#endif - /* Run queue balancing */ typedef struct { @@ -1561,20 +2053,23 @@ do { \ static void check_balance(ErtsRunQueue *c_rq) { +#if ERTS_MAX_PROCESSES >= (1 << 27) +# error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27) +#endif ErtsRunQueueBalance avg = {0}; Sint64 scheds_reds, full_scheds_reds; int forced, active, current_active, oowc, half_full_scheds, full_scheds, mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix; - if (erts_smp_atomic_xchg(&balance_info.checking_balance, 1)) { + if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) { c_rq->check_balance_reds = INT_MAX; return; } - blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + get_no_runqs(NULL, &blnc_no_rqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); return; } @@ -1582,7 +2077,7 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.halftime) { balance_info.halftime = 0; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) @@ -1610,12 +2105,13 @@ check_balance(ErtsRunQueue *c_rq) forced = balance_info.forced_check_balance; balance_info.forced_check_balance = 0; - blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + get_no_runqs(¤t_active, &blnc_no_rqs); + if (blnc_no_rqs == 1) { erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); return; } @@ -1624,8 +2120,6 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) balance_info.full_reds_history_index = 0; - current_active = erts_smp_atomic_read(&balance_info.active_runqs); - /* Read balance information for all run queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); @@ -1684,12 +2178,14 @@ check_balance(ErtsRunQueue *c_rq) run_queue_info[qix].prio[pix].avail = 0; } else { - int xreds = 0; - int procreds = treds; - procreds -= run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds; + Sint64 xreds = 0; + Sint64 procreds = treds; + procreds -= + ((Sint64) + run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds); for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - int av; + Sint64 av; if (xreds == 0) av = 100; @@ -1700,9 +2196,10 @@ check_balance(ErtsRunQueue *c_rq) if (av == 0) av = 1; } - run_queue_info[qix].prio[pix].avail = av; + run_queue_info[qix].prio[pix].avail = (int) av; + ASSERT(run_queue_info[qix].prio[pix].avail >= 0); if (pix < PRIORITY_NORMAL) /* ie., max or high */ - xreds += run_queue_info[qix].prio[pix].reds; + xreds += (Sint64) run_queue_info[qix].prio[pix].reds; } run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].avail = 100; } @@ -1807,7 +2304,8 @@ check_balance(ErtsRunQueue *c_rq) if (max_len != 0) { int avail = avg.prio[pix].avail; if (avail != 0) { - max_len = ((100*max_len - 1) / avail) + 1; + max_len = (int) ((100*((Sint64) max_len) - 1) + / ((Sint64) avail)) + 1; avg.prio[pix].max_len = max_len; ASSERT(max_len >= 0); } @@ -1824,9 +2322,10 @@ check_balance(ErtsRunQueue *c_rq) || run_queue_info[qix].prio[pix].avail == 0) limit = 0; else - limit = (((avg.prio[pix].max_len - * run_queue_info[qix].prio[pix].avail) - 1) - / 100 + 1); + limit = (int) (((((Sint64) avg.prio[pix].max_len) + * ((Sint64) run_queue_info[qix].prio[pix].avail)) + - 1) + / 100 + 1); run_queue_info[qix].prio[pix].migration_limit = limit; } } @@ -1954,10 +2453,10 @@ erts_fprintf(stderr, "--------------------------------\n"); } balance_info.last_active_runqs = active; - erts_smp_atomic_set(&balance_info.active_runqs, active); + set_no_active_runqs(active); balance_info.halftime = 1; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); /* Write migration paths and reset balance statistics in all queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { @@ -2054,8 +2553,27 @@ erts_debug_nbalance(void) void erts_early_init_scheduling(void) { - early_cpu_bind_init(); + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; +} + +int +erts_sched_set_wakeup_limit(char *str) +{ + if (sys_strcmp(str, "very_high") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH; + else if (sys_strcmp(str, "high") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH; + else if (sys_strcmp(str, "medium") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; + else if (sys_strcmp(str, "low") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_LOW; + else if (sys_strcmp(str, "very_low") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW; + else + return EINVAL; + return 0; } + void erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) @@ -2076,24 +2594,21 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) n = (int) (mrq ? no_schedulers : 1); - erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS, - (sizeof(ErtsAlignedRunQueue)*(n+1))); - if ((((Uint) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0) - erts_aligned_run_queues = ((ErtsAlignedRunQueue *) - ((((Uint) erts_aligned_run_queues) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - + erts_aligned_run_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, + sizeof(ErtsAlignedRunQueue) * n); #ifdef ERTS_SMP - erts_smp_atomic_init(&no_empty_run_queues, 0); + erts_smp_atomic32_init(&no_empty_run_queues, 0); #endif + erts_no_run_queues = n; + for (ix = 0; ix < n; ix++) { int pix, rix; ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); rq->ix = ix; - erts_smp_atomic_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_smp_atomic32_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); /* make sure that the "extra" id correponds to the schedulers * id if the esdp->no <-> ix+1 mapping change. @@ -2102,8 +2617,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); - erts_smp_atomic_init(&rq->spin_waiter, 0); - erts_smp_atomic_init(&rq->spin_wake, 0); +#ifdef ERTS_SMP + erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list"); + rq->sleepers.list = NULL; +#endif rq->waiting = 0; rq->woken = 0; @@ -2154,7 +2671,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; - erts_no_run_queues = n; #ifdef ERTS_SMP @@ -2169,23 +2685,48 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #endif + n = (int) no_schedulers; + erts_no_schedulers = n; + +#ifdef ERTS_SMP + /* Create and initialize scheduler sleep info */ + + aligned_sched_sleep_info = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO, + n * sizeof(ErtsAlignedSchedulerSleepInfo)); + + for (ix = 0; ix < n; ix++) { + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); +#if 0 /* no need to initialize these... */ + ssi->next = NULL; + ssi->prev = NULL; +#endif + erts_smp_atomic32_init(&ssi->flags, 0); + ssi->event = NULL; /* initialized in sched_thread_func */ + erts_smp_atomic32_init(&ssi->aux_work, 0); + } +#endif + /* Create and initialize scheduler specific data */ - n = (int) no_schedulers; - erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, - (sizeof(ErtsAlignedSchedulerData) - *(n+1))); - if ((((Uint) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0) - erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *) - ((((Uint) erts_aligned_scheduler_data) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); + erts_aligned_scheduler_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + n*sizeof(ErtsAlignedSchedulerData)); + for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; +#if HALFWORD_HEAP + /* Registers need to be heap allocated (correct memory range) for tracing to work */ + esdp->save_reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); +#endif +#endif +#if !HEAP_ON_C_STACK + esdp->num_tmp_heap_used = 0; #endif esdp->no = (Uint) ix+1; esdp->current_process = NULL; @@ -2206,12 +2747,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->check_children = 0; - esdp->blocked_check_children = 0; -#endif - erts_smp_atomic_init(&esdp->suspended, 0); - erts_smp_atomic_init(&esdp->chk_cpu_bind, 0); + erts_smp_atomic32_init(&esdp->chk_cpu_bind, 0); #endif } @@ -2219,21 +2755,20 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - schdlr_sspnd.changing = 0; + erts_smp_atomic32_init(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; - erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0); - erts_smp_atomic_init(&schdlr_sspnd.active, no_schedulers); + erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); + erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - erts_smp_atomic_set(&balance_info.used_runqs, - erts_common_run_queue ? 1 : no_schedulers_online); - erts_smp_atomic_init(&balance_info.active_runqs, no_schedulers); + init_no_runqs(no_schedulers, + erts_common_run_queue ? 1 : no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; balance_info.halftime = 1; balance_info.full_reds_history_index = 0; - erts_smp_atomic_init(&balance_info.checking_balance, 0); + erts_smp_atomic32_init(&balance_info.checking_balance, 0); balance_info.prev_rise.active_runqs = 0; balance_info.prev_rise.max_len = 0; balance_info.prev_rise.reds = 0; @@ -2242,7 +2777,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1); + erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2253,9 +2789,12 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) schdlr_sspnd.wait_curr_online = no_schedulers_online; schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - erts_smp_atomic_init(&doing_sys_schedule, 0); + erts_smp_atomic32_init(&doing_sys_schedule, 0); + + init_misc_aux_work(); #else /* !ERTS_SMP */ { @@ -2269,12 +2808,19 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_no_schedulers = 1; #endif - erts_smp_atomic_init(&function_calls, 0); + erts_smp_atomic32_init(&function_calls, 0); /* init port tasks */ erts_port_task_init(); - late_cpu_bind_init(); +#ifndef ERTS_SMP +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC + erts_scheduler_data->verify_unused_temp_alloc + = erts_alloc_get_verify_unused_temp_alloc( + &erts_scheduler_data->verify_unused_temp_alloc_data); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); +#endif +#endif } ErtsRunQueue * @@ -2374,7 +2920,6 @@ resume_process(Process *p) return; switch(p->rstatus) { case P_RUNABLE: - *statusp = P_WAITING; /* make erts_add_to_runq work */ erts_add_to_runq(p); break; case P_WAITING: @@ -2386,6 +2931,19 @@ resume_process(Process *p) p->rstatus = P_FREE; } +int +erts_get_max_no_executing_schedulers(void) +{ +#ifdef ERTS_SMP + if (erts_smp_atomic32_read(&schdlr_sspnd.changing)) + return (int) erts_no_schedulers; + ERTS_THR_MEMORY_BARRIER; + return (int) erts_smp_atomic32_read(&schdlr_sspnd.active); +#else + return 1; +#endif +} + #ifdef ERTS_SMP static void @@ -2401,13 +2959,113 @@ susp_sched_resume_block(void *unused) } static void +scheduler_ix_resume_wake(Uint ix) +{ + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t oflgs; + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, 0, xflgs); + if (oflgs == xflgs) { + erts_sched_finish_poke(ssi, oflgs); + break; + } + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); +} + +static erts_aint32_t +sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) +{ + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t xflgs = xpct; + + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + + return oflgs; +} + +static erts_aint32_t +sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + erts_aint32_t flgs; + + do { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; +} + +static erts_aint32_t +sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) +{ + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + return oflgs; + } + xflgs = oflgs; + } +} + +static void suspend_scheduler(ErtsSchedulerData *esdp) { + erts_aint32_t flgs; + erts_aint32_t changing; long no = (long) esdp->no; - ErtsRunQueue *rq = esdp->run_queue; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; long active_schedulers; int curr_online = 1; int wake = 0; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + erts_aint32_t aux_work; +#endif /* * Schedulers may be suspended in two different ways: @@ -2424,126 +3082,145 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_runq_unlock(esdp->run_queue); - /* Unbind from cpu */ - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - if (scheduler2cpu_map[esdp->no].bound_id >= 0 - && erts_unbind_from_cpu(erts_cpuinfo) == 0) { - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; - } - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + erts_sched_check_cpu_bind_prep_suspend(esdp); if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) - schdlr_sspnd.changing = 0; - } - - while (1) { + flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; - erts_smp_runq_lock(esdp->run_queue); - check_children = esdp->check_children; - esdp->check_children = 0; - erts_smp_runq_unlock(esdp->run_queue); - if (check_children) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_check_children(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + active_schedulers = erts_smp_atomic32_dectest(&schdlr_sspnd.active); + ASSERT(active_schedulers >= 1); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { + if (active_schedulers == schdlr_sspnd.msb.wait_active) + wake = 1; + if (active_schedulers == 1) { + changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; + } } -#endif - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) { - int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; - curr_online = 0; - changed = 1; + while (1) { + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + int changed = 0; + if (no > schdlr_sspnd.online && curr_online) { + schdlr_sspnd.curr_online--; + curr_online = 0; + changed = 1; + } + else if (no <= schdlr_sspnd.online && !curr_online) { + schdlr_sspnd.curr_online++; + curr_online = 1; + changed = 1; + } + if (changed + && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) + wake = 1; + if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + } } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; - curr_online = 1; - changed = 1; + + if (wake) { + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + wake = 0; } - if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) - schdlr_sspnd.changing = 0; - } - if (wake) { - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - wake = 0; - } + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + blockable_aux_work: + blockable_aux_work(esdp, ssi, aux_work); +#endif + + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + while (1) { + erts_aint32_t flgs; +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (1) { + flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + break; -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->blocked_check_children) - break; + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto blockable_aux_work; + } #endif - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + } - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) - break; + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->blocked_check_children = 0; -#endif + active_schedulers = erts_smp_atomic32_inctest(&schdlr_sspnd.active); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && schdlr_sspnd.online == active_schedulers) { + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - } + ASSERT(no <= schdlr_sspnd.online); + ASSERT(!erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); - active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED - && schdlr_sspnd.online == active_schedulers) { - schdlr_sspnd.changing = 0; } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(curr_online); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - /* Make sure we check if we should bind to a cpu or not... */ - if (rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - erts_smp_atomic_set(&esdp->chk_cpu_bind, 1); - else - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_sched_check_cpu_bind_post_suspend(esdp); } #define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID) \ @@ -2558,7 +3235,7 @@ do { \ (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ - erts_smp_atomic_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED); \ + erts_smp_atomic32_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ (RQ)->procs.prio_info[pix__].max_len = 0; \ (RQ)->procs.prio_info[pix__].reds = 0; \ @@ -2600,8 +3277,10 @@ erts_schedulers_state(Uint *total, int yield_allowed) { int res; + erts_aint32_t changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (yield_allowed && schdlr_sspnd.changing) + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { *active = *online = schdlr_sspnd.online; @@ -2621,6 +3300,7 @@ erts_set_schedulers_online(Process *p, Sint *old_no) { int ix, res, no, have_unlocked_plocks; + erts_aint32_t changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; @@ -2630,7 +3310,8 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } else { @@ -2639,17 +3320,19 @@ erts_set_schedulers_online(Process *p, res = ERTS_SCHDLR_SSPND_DONE; } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); schdlr_sspnd.online = no; if (no > online) { int ix; schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) - /* No schedulers to resume */; + if (ongoing_multi_scheduling_block()) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } else if (erts_common_run_queue) { for (ix = online; ix < no; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 0); + scheduler_ix_resume_wake(ix); } else { if (plocks) { @@ -2663,6 +3346,7 @@ erts_set_schedulers_online(Process *p, erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* * Spread evacuation paths among all online @@ -2673,11 +3357,10 @@ erts_set_schedulers_online(Process *p, ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); evacuate_run_queue(from_rq, to_rq); } - erts_smp_atomic_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); res = ERTS_SCHDLR_SSPND_DONE; } else /* if (no < online) */ { @@ -2694,12 +3377,17 @@ erts_set_schedulers_online(Process *p, schdlr_sspnd.wait_curr_online = no+1; } - if (ongoing_multi_scheduling_block()) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - else if (erts_common_run_queue) { + if (ongoing_multi_scheduling_block()) { for (ix = no; ix < online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 1); + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } + else if (erts_common_run_queue) { + for (ix = no; ix < online; ix++) { + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_bor(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } wake_all_schedulers(); } else { @@ -2723,10 +3411,13 @@ erts_set_schedulers_online(Process *p, for (ix = erts_no_run_queues-1; ix >= no; ix--) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % no)); - erts_smp_atomic_set(&balance_info.used_runqs, no); + set_no_used_runqs(no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq, 0, 1); + } } } @@ -2740,6 +3431,13 @@ erts_set_schedulers_online(Process *p, susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic32_read(&schdlr_sspnd.changing))); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } } @@ -2754,37 +3452,41 @@ ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) { int ix, res, have_unlocked_plocks = 0; + erts_aint32_t changing; ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } else if (on) { /* ------ BLOCK ------ */ - if (erts_is_multi_scheduling_blocked()) { + if (schdlr_sspnd.msb.procs) { plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } else { + int online = schdlr_sspnd.online; p->flags |= F_HAVE_BLCKD_MSCHED; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1); - if (schdlr_sspnd.online == 1) { + ASSERT(0 == erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); + erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 1); + if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); if (p->scheduler_data->no == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; schdlr_sspnd.msb.wait_active = 1; @@ -2798,17 +3500,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } if (erts_common_run_queue) { - for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1); + for (ix = 1; ix < online; ix++) + erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); - erts_smp_atomic_set(&balance_info.used_runqs, 1); - for (ix = 0; ix < schdlr_sspnd.online; ix++) { + set_no_used_runqs(1); + for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); erts_smp_runq_unlock(rq); } @@ -2826,13 +3530,20 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) susp_sched_prep_block, susp_sched_resume_block, NULL); - while (erts_smp_atomic_read(&schdlr_sspnd.active) + while (erts_smp_atomic32_read(&schdlr_sspnd.active) != schdlr_sspnd.msb.wait_active) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); erts_smp_activity_end(ERTS_ACTIVITY_WAIT, susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic32_read(&schdlr_sspnd.changing))); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -2876,7 +3587,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.msb.procs) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); #ifdef DEBUG ERTS_FOREACH_RUNQ(rq, { @@ -2899,17 +3610,17 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) }); #endif p->flags &= ~F_HAVE_BLCKD_MSCHED; - erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 0); + erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 0); if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); - schdlr_sspnd.changing = 0; + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0); + erts_smp_atomic32_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } else { int online = schdlr_sspnd.online; @@ -2926,6 +3637,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* Spread evacuation paths among all online run queues */ @@ -2933,7 +3645,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % online)); - erts_smp_atomic_set(&balance_info.used_runqs, online); + set_no_used_runqs(online); /* Make sure that we balance soon... */ balance_info.forced_check_balance = 1; erts_smp_runq_lock(ERTS_RUNQ_IX(0)); @@ -2941,7 +3653,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -2958,7 +3669,7 @@ void erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) { if (return_value == am_blocked) { - long active = erts_smp_atomic_read(&schdlr_sspnd.active); + erts_aint32_t active = erts_smp_atomic32_read(&schdlr_sspnd.active); ASSERT(1 <= active && active <= 2); ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); } @@ -2968,8 +3679,11 @@ erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) int erts_is_multi_scheduling_blocked(void) { - return (erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) - && erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + int res; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + res = schdlr_sspnd.msb.procs != NULL; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + return res; } Eterm @@ -2978,7 +3692,7 @@ erts_multi_scheduling_blockers(Process *p) Eterm res = NIL; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (erts_is_multi_scheduling_blocked()) { + if (schdlr_sspnd.msb.procs) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; Uint max_size; @@ -3010,18 +3724,26 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { +#ifdef ERTS_SMP + Uint no = ((ErtsSchedulerData *) vesdp)->no; +#endif #ifdef ERTS_ENABLE_LOCK_CHECK { char buf[31]; - Uint no = ((ErtsSchedulerData *) vesdp)->no; - erts_snprintf(&buf[0], 31, "scheduler %bpu", no); + erts_snprintf(&buf[0], 31, "scheduler %beu", no); erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no); + erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP + + erts_sched_init_check_cpu_bind((ErtsSchedulerData *) vesdp); + erts_proc_lock_prepare_proc_lock_waiter(); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + + #endif erts_register_blockable_thread(); #ifdef HIPE @@ -3030,36 +3752,43 @@ sched_thread_func(void *vesdp) erts_thread_init_float(); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.changing) + & ERTS_SCHDLR_SSPND_CHNG_ONLN); - schdlr_sspnd.curr_online--; + if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (((ErtsSchedulerData *) vesdp)->no != 1) + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + } - if (((ErtsSchedulerData *) vesdp)->no != 1) { - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - schdlr_sspnd.changing = 0; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + if (((ErtsSchedulerData *) vesdp)->no == 1) { + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); } - } - else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - schdlr_sspnd.changing = 0; - else { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - ASSERT(!schdlr_sspnd.changing); + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC + ((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc + = erts_alloc_get_verify_unused_temp_alloc( + &((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc_data); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); +#endif + process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %bpu terminated\n", + erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", ((ErtsSchedulerData *) vesdp)->no); return NULL; } @@ -3089,11 +3818,7 @@ erts_start_schedulers(void) ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); actual++; ASSERT(actual == esdp->no); -#ifdef ERTS_ENABLE_LOCK_COUNT - res = erts_lcnt_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); -#else res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); -#endif if (res != 0) { actual--; break; @@ -3112,8 +3837,8 @@ erts_start_schedulers(void) erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); ASSERT(actual != wanted_no_schedulers); erts_dsprintf(dsbufp, - "Failed to create %bpu scheduler-threads (%s:%d); " - "only %bpu scheduler-thread%s created.\n", + "Failed to create %beu scheduler-threads (%s:%d); " + "only %beu scheduler-thread%s created.\n", wanted_no_schedulers, erl_errno_id(res), res, actual, actual == 1 ? " was" : "s were"); erts_send_error_to_logger_nogl(dsbufp); @@ -3122,1351 +3847,6 @@ erts_start_schedulers(void) #endif /* ERTS_SMP */ -static int -int_cmp(const void *vx, const void *vy) -{ - return *((int *) vx) - *((int *) vy); -} - -static int -cpu_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->core != y->core) - return x->core - y->core; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->node != y->node) - return x->node - y->node; - return 0; -} - -static int -cpu_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_thread_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - return 0; -} - -static int -cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->node != y->node) - return x->node - y->node; - if (x->core != y->core) - return x->core - y->core; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->core != y->core) - return x->core - y->core; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->core != y->core) - return x->core - y->core; - return 0; -} - -static int -cpu_no_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->thread != y->thread) - return x->thread - y->thread; - return 0; -} - -static ERTS_INLINE void -make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node) -{ - int ix; - int node = -1; - int processor = -1; - int processor_node = -1; - int processor_node_node = -1; - int core = -1; - int thread = -1; - int old_node = -1; - int old_processor = -1; - int old_processor_node = -1; - int old_core = -1; - int old_thread = -1; - - for (ix = 0; ix < size; ix++) { - if (!no_node || cpudata[ix].node >= 0) { - if (old_node == cpudata[ix].node) - cpudata[ix].node = node; - else { - old_node = cpudata[ix].node; - old_processor = processor = -1; - if (!no_node) - old_processor_node = processor_node = -1; - old_core = core = -1; - old_thread = thread = -1; - if (no_node || cpudata[ix].node >= 0) - cpudata[ix].node = ++node; - } - } - if (old_processor == cpudata[ix].processor) - cpudata[ix].processor = processor; - else { - old_processor = cpudata[ix].processor; - if (!no_node) - processor_node_node = old_processor_node = processor_node = -1; - old_core = core = -1; - old_thread = thread = -1; - cpudata[ix].processor = ++processor; - } - if (no_node && cpudata[ix].processor_node < 0) - old_processor_node = -1; - else { - if (old_processor_node == cpudata[ix].processor_node) { - if (no_node) - cpudata[ix].node = cpudata[ix].processor_node = node; - else { - if (processor_node_node >= 0) - cpudata[ix].node = processor_node_node; - cpudata[ix].processor_node = processor_node; - } - } - else { - old_processor_node = cpudata[ix].processor_node; - old_core = core = -1; - old_thread = thread = -1; - if (no_node) - cpudata[ix].node = cpudata[ix].processor_node = ++node; - else { - cpudata[ix].node = processor_node_node = ++node; - cpudata[ix].processor_node = ++processor_node; - } - } - } - if (!no_node && cpudata[ix].processor_node < 0) - cpudata[ix].processor_node = 0; - if (old_core == cpudata[ix].core) - cpudata[ix].core = core; - else { - old_core = cpudata[ix].core; - old_thread = thread = -1; - cpudata[ix].core = ++core; - } - if (old_thread == cpudata[ix].thread) - cpudata[ix].thread = thread; - else - old_thread = cpudata[ix].thread = ++thread; - } -} - -static void -cpu_bind_order_sort(erts_cpu_topology_t *cpudata, - int size, - ErtsCpuBindOrder bind_order, - int mk_seq) -{ - if (size > 1) { - int no_node = 0; - int (*cmp_func)(const void *, const void *); - switch (bind_order) { - case ERTS_CPU_BIND_SPREAD: - cmp_func = cpu_spread_order_cmp; - break; - case ERTS_CPU_BIND_PROCESSOR_SPREAD: - cmp_func = cpu_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_THREAD_SPREAD: - cmp_func = cpu_thread_spread_order_cmp; - break; - case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: - no_node = 1; - cmp_func = cpu_thread_no_node_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: - no_node = 1; - cmp_func = cpu_no_node_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: - no_node = 1; - cmp_func = cpu_no_node_thread_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_SPREAD: - cmp_func = cpu_no_spread_order_cmp; - break; - default: - cmp_func = NULL; - erl_exit(ERTS_ABORT_EXIT, - "Bad cpu bind type: %d\n", - (int) cpu_bind_order); - break; - } - - if (mk_seq) - make_cpudata_id_seq(cpudata, size, no_node); - - qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func); - } -} - -static int -processor_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->node != y->node) - return x->node - y->node; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->thread != y->thread) - return x->thread - y->thread; - return 0; -} - -static void -check_cpu_bind(ErtsSchedulerData *esdp) -{ - int res; - int cpu_id; - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - cpu_id = scheduler2cpu_map[esdp->no].bind_id; - if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { - res = erts_bind_to_cpu(erts_cpuinfo, cpu_id); - if (res == 0) - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id; - else { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n", - (int) esdp->no, cpu_id, erl_errno_id(-res)); - erts_send_error_to_logger_nogl(dsbufp); - if (scheduler2cpu_map[esdp->no].bound_id >= 0) - goto unbind; - } - } - else if (cpu_id < 0 && scheduler2cpu_map[esdp->no].bound_id >= 0) { - unbind: - /* Get rid of old binding */ - res = erts_unbind_from_cpu(erts_cpuinfo); - if (res == 0) - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; - else { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", - (int) esdp->no, cpu_id, erl_errno_id(-res)); - erts_send_error_to_logger_nogl(dsbufp); - } - } - erts_smp_runq_lock(esdp->run_queue); -#ifdef ERTS_SMP - if (erts_common_run_queue) - erts_smp_atomic_set(&esdp->chk_cpu_bind, 0); - else { - esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; - } -#endif - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - -} - -static void -signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) -{ - int s_ix = 1; - int cpu_ix; - - if (cpu_bind_order != ERTS_CPU_BIND_NONE) { - - cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1); - - for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++) - if (erts_is_cpu_available(erts_cpuinfo, cpudata[cpu_ix].logical)) - scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical; - } - - if (s_ix <= erts_no_schedulers) - for (; s_ix <= erts_no_schedulers; s_ix++) - scheduler2cpu_map[s_ix].bind_id = -1; - -#ifdef ERTS_SMP - if (erts_common_run_queue) { - for (s_ix = 0; s_ix < erts_no_schedulers; s_ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(s_ix)->chk_cpu_bind, 1); - wake_all_schedulers(); - } - else { - ERTS_FOREACH_RUNQ(rq, - { - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - wake_scheduler(rq, 0); - }); - } -#else - check_cpu_bind(erts_get_scheduler_data()); -#endif -} - -int -erts_init_scheduler_bind_type(char *how) -{ - if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) - return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; - - if (!system_cpudata && !user_cpudata) - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; - - if (sys_strcmp(how, "s") == 0) - cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (sys_strcmp(how, "ps") == 0) - cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "ts") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (sys_strcmp(how, "db") == 0 - || sys_strcmp(how, "tnnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "nnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "nnts") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (sys_strcmp(how, "ns") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (sys_strcmp(how, "u") == 0) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; - - return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; -} - -typedef struct { - int *id; - int used; - int size; -} ErtsCpuTopIdSeq; - -typedef struct { - ErtsCpuTopIdSeq logical; - ErtsCpuTopIdSeq thread; - ErtsCpuTopIdSeq core; - ErtsCpuTopIdSeq processor_node; - ErtsCpuTopIdSeq processor; - ErtsCpuTopIdSeq node; -} ErtsCpuTopEntry; - -static void -init_cpu_top_entry(ErtsCpuTopEntry *cte) -{ - int size = 10; - cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->logical.size = size; - cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->thread.size = size; - cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->core.size = size; - cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->processor_node.size = size; - cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->processor.size = size; - cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->node.size = size; -} - -static void -destroy_cpu_top_entry(ErtsCpuTopEntry *cte) -{ - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id); -} - -static int -get_cput_value_or_range(int *v, int *vr, char **str) -{ - long l; - char *c = *str; - errno = 0; - if (!isdigit((unsigned char)*c)) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; - l = strtol(c, &c, 10); - if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; - *v = (int) l; - if (*c == '-') { - c++; - if (!isdigit((unsigned char)*c)) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - l = strtol(c, &c, 10); - if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - *vr = (int) l; - } - *str = c; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str) -{ - int ix = 0; - int need_size = 0; - char *c = *str; - - while (1) { - int res; - int val; - int nids; - int val_range = -1; - res = get_cput_value_or_range(&val, &val_range, &c); - if (res != ERTS_INIT_CPU_TOPOLOGY_OK) - return res; - if (val_range < 0 || val_range == val) - nids = 1; - else { - if (val_range > val) - nids = val_range - val + 1; - else - nids = val - val_range + 1; - } - need_size += nids; - if (need_size > idseq->size) { - idseq->size = need_size + 10; - idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS, - idseq->id, - sizeof(int)*idseq->size); - } - if (nids == 1) - idseq->id[ix++] = val; - else if (val_range > val) { - for (; val <= val_range; val++) - idseq->id[ix++] = val; - } - else { - for (; val >= val_range; val--) - idseq->id[ix++] = val; - } - if (*c != ',') - break; - c++; - } - *str = c; - idseq->used = ix; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -get_cput_entry(ErtsCpuTopEntry *cput, char **str) -{ - int h; - char *c = *str; - - cput->logical.used = 0; - cput->thread.id[0] = 0; - cput->thread.used = 1; - cput->core.id[0] = 0; - cput->core.used = 1; - cput->processor_node.id[0] = -1; - cput->processor_node.used = 1; - cput->processor.id[0] = 0; - cput->processor.used = 1; - cput->node.id[0] = -1; - cput->node.used = 1; - - h = ERTS_TOPOLOGY_MAX_DEPTH; - while (*c != ':' && *c != '\0') { - int res; - ErtsCpuTopIdSeq *idseqp; - switch (*c++) { - case 'L': - if (h <= ERTS_TOPOLOGY_LOGICAL) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->logical; - h = ERTS_TOPOLOGY_LOGICAL; - break; - case 't': - case 'T': - if (h <= ERTS_TOPOLOGY_THREAD) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->thread; - h = ERTS_TOPOLOGY_THREAD; - break; - case 'c': - case 'C': - if (h <= ERTS_TOPOLOGY_CORE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->core; - h = ERTS_TOPOLOGY_CORE; - break; - case 'p': - case 'P': - if (h <= ERTS_TOPOLOGY_PROCESSOR) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->processor; - h = ERTS_TOPOLOGY_PROCESSOR; - break; - case 'n': - case 'N': - if (h <= ERTS_TOPOLOGY_PROCESSOR) { - do_node: - if (h <= ERTS_TOPOLOGY_NODE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->node; - h = ERTS_TOPOLOGY_NODE; - } - else { - int p_node = 0; - char *p_chk = c; - while (*p_chk != '\0' && *p_chk != ':') { - if (*p_chk == 'p' || *p_chk == 'P') { - p_node = 1; - break; - } - p_chk++; - } - if (!p_node) - goto do_node; - if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->processor_node; - h = ERTS_TOPOLOGY_PROCESSOR_NODE; - } - break; - default: - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE; - } - res = get_cput_id_seq(idseqp, &c); - if (res != ERTS_INIT_CPU_TOPOLOGY_OK) - return res; - } - - if (cput->logical.used < 1) - return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID; - - if (*c == ':') { - c++; - } - - if (cput->thread.used != 1 - && cput->thread.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->core.used != 1 - && cput->core.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->processor_node.used != 1 - && cput->processor_node.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->processor.used != 1 - && cput->processor.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->node.used != 1 - && cput->node.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - - *str = c; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -verify_topology(erts_cpu_topology_t *cpudata, int size) -{ - if (size > 0) { - int *logical; - int node, processor, no_nodes, i; - - /* Verify logical ids */ - logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); - - for (i = 0; i < user_cpudata_size; i++) - logical[i] = user_cpudata[i].logical; - - qsort(logical, user_cpudata_size, sizeof(int), int_cmp); - for (i = 0; i < user_cpudata_size-1; i++) { - if (logical[i] == logical[i+1]) { - erts_free(ERTS_ALC_T_TMP, logical); - return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; - } - } - - erts_free(ERTS_ALC_T_TMP, logical); - - qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp); - - /* Verify unique entities */ - - for (i = 1; i < user_cpudata_size; i++) { - if (user_cpudata[i-1].processor == user_cpudata[i].processor - && user_cpudata[i-1].node == user_cpudata[i].node - && (user_cpudata[i-1].processor_node - == user_cpudata[i].processor_node) - && user_cpudata[i-1].core == user_cpudata[i].core - && user_cpudata[i-1].thread == user_cpudata[i].thread) { - return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; - } - } - - /* Verify numa nodes */ - node = cpudata[0].node; - processor = cpudata[0].processor; - no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0; - for (i = 1; i < size; i++) { - if (no_nodes) { - if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - } - else { - if (cpudata[i].processor == processor && cpudata[i].node != node) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - node = cpudata[i].node; - processor = cpudata[i].processor; - if (node >= 0 && cpudata[i].processor_node >= 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - if (node < 0 && cpudata[i].processor_node < 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - } - } - } - - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -int -erts_init_cpu_topology(char *topology_str) -{ - ErtsCpuTopEntry cput; - int need_size; - char *c; - int ix; - int error = ERTS_INIT_CPU_TOPOLOGY_OK; - - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata_size = 10; - - user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - - init_cpu_top_entry(&cput); - - ix = 0; - need_size = 0; - - c = topology_str; - if (*c == '\0') { - error = ERTS_INIT_CPU_TOPOLOGY_MISSING; - goto fail; - } - do { - int r; - error = get_cput_entry(&cput, &c); - if (error != ERTS_INIT_CPU_TOPOLOGY_OK) - goto fail; - need_size += cput.logical.used; - if (user_cpudata_size < need_size) { - user_cpudata_size = need_size + 10; - user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, - user_cpudata, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - } - - ASSERT(cput.thread.used == 1 - || cput.thread.used == cput.logical.used); - ASSERT(cput.core.used == 1 - || cput.core.used == cput.logical.used); - ASSERT(cput.processor_node.used == 1 - || cput.processor_node.used == cput.logical.used); - ASSERT(cput.processor.used == 1 - || cput.processor.used == cput.logical.used); - ASSERT(cput.node.used == 1 - || cput.node.used == cput.logical.used); - - for (r = 0; r < cput.logical.used; r++) { - user_cpudata[ix].logical = cput.logical.id[r]; - user_cpudata[ix].thread = - cput.thread.id[cput.thread.used == 1 ? 0 : r]; - user_cpudata[ix].core = - cput.core.id[cput.core.used == 1 ? 0 : r]; - user_cpudata[ix].processor_node = - cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r]; - user_cpudata[ix].processor = - cput.processor.id[cput.processor.used == 1 ? 0 : r]; - user_cpudata[ix].node = - cput.node.id[cput.node.used == 1 ? 0 : r]; - ix++; - } - } while (*c != '\0'); - - if (user_cpudata_size != ix) { - user_cpudata_size = ix; - user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, - user_cpudata, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - } - - error = verify_topology(user_cpudata, user_cpudata_size); - if (error == ERTS_INIT_CPU_TOPOLOGY_OK) { - destroy_cpu_top_entry(&cput); - return ERTS_INIT_CPU_TOPOLOGY_OK; - } - - fail: - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata_size = 0; - destroy_cpu_top_entry(&cput); - return error; -} - -#define ERTS_GET_CPU_TOPOLOGY_ERROR -1 -#define ERTS_GET_USED_CPU_TOPOLOGY 0 -#define ERTS_GET_DETECTED_CPU_TOPOLOGY 1 -#define ERTS_GET_DEFINED_CPU_TOPOLOGY 2 - -static Eterm get_cpu_topology_term(Process *c_p, int type); - -Eterm -erts_set_cpu_topology(Process *c_p, Eterm term) -{ - erts_cpu_topology_t *cpudata = NULL; - int cpudata_size = 0; - Eterm res; - - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); - if (term == am_undefined) { - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata = NULL; - user_cpudata_size = 0; - - if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) { - cpudata_size = system_cpudata_size; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - - sys_memcpy((void *) cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*cpudata_size); - } - } - else if (is_not_list(term)) { - error: - res = THE_NON_VALUE; - goto done; - } - else { - Eterm list = term; - int ix = 0; - - cpudata_size = 100; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - - while (is_list(list)) { - Eterm *lp = list_val(list); - Eterm cpu = CAR(lp); - Eterm* tp; - Sint id; - - if (is_not_tuple(cpu)) - goto error; - - tp = tuple_val(cpu); - - if (arityval(tp[0]) != 7 || tp[1] != am_cpu) - goto error; - - if (ix >= cpudata_size) { - cpudata_size += 100; - cpudata = erts_realloc(ERTS_ALC_T_TMP, - cpudata, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - } - - id = signed_val(tp[2]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].node = (int) id; - - id = signed_val(tp[3]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].processor = (int) id; - - id = signed_val(tp[4]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].processor_node = (int) id; - - id = signed_val(tp[5]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].core = (int) id; - - id = signed_val(tp[6]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].thread = (int) id; - - id = signed_val(tp[7]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].logical = (int) id; - - list = CDR(lp); - ix++; - } - - if (is_not_nil(list)) - goto error; - - cpudata_size = ix; - - if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size)) - goto error; - - if (user_cpudata_size != cpudata_size) { - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - sizeof(erts_cpu_topology_t)*cpudata_size); - user_cpudata_size = cpudata_size; - } - - sys_memcpy((void *) user_cpudata, - (void *) cpudata, - sizeof(erts_cpu_topology_t)*cpudata_size); - } - - signal_schedulers_bind_change(cpudata, cpudata_size); - - done: - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - if (cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); - - return res; -} - -static Eterm -bound_schedulers_term(ErtsCpuBindOrder order) -{ - switch (order) { - case ERTS_CPU_BIND_SPREAD: { - ERTS_DECL_AM(spread); - return AM_spread; - } - case ERTS_CPU_BIND_PROCESSOR_SPREAD: { - ERTS_DECL_AM(processor_spread); - return AM_processor_spread; - } - case ERTS_CPU_BIND_THREAD_SPREAD: { - ERTS_DECL_AM(thread_spread); - return AM_thread_spread; - } - case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: { - ERTS_DECL_AM(thread_no_node_processor_spread); - return AM_thread_no_node_processor_spread; - } - case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: { - ERTS_DECL_AM(no_node_processor_spread); - return AM_no_node_processor_spread; - } - case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: { - ERTS_DECL_AM(no_node_thread_spread); - return AM_no_node_thread_spread; - } - case ERTS_CPU_BIND_NO_SPREAD: { - ERTS_DECL_AM(no_spread); - return AM_no_spread; - } - case ERTS_CPU_BIND_NONE: { - ERTS_DECL_AM(unbound); - return AM_unbound; - } - default: - ASSERT(0); - return THE_NON_VALUE; - } -} - -Eterm -erts_bound_schedulers_term(Process *c_p) -{ - ErtsCpuBindOrder order; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - order = cpu_bind_order; - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return bound_schedulers_term(order); -} - -static void -create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size) -{ - if (user_cpudata) { - *cpudata_size = user_cpudata_size; - *cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * (*cpudata_size))); - sys_memcpy((void *) *cpudata, - (void *) user_cpudata, - sizeof(erts_cpu_topology_t)*(*cpudata_size)); - } - else if (system_cpudata) { - *cpudata_size = system_cpudata_size; - *cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * (*cpudata_size))); - sys_memcpy((void *) *cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*(*cpudata_size)); - } - else { - *cpudata = NULL; - *cpudata_size = 0; - } -} - -static void -destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata) -{ - if (cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); -} - -Eterm -erts_bind_schedulers(Process *c_p, Eterm how) -{ - Eterm res; - erts_cpu_topology_t *cpudata; - int cpudata_size; - ErtsCpuBindOrder old_cpu_bind_order; - - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - - if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) { - ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP); - } - else { - - old_cpu_bind_order = cpu_bind_order; - - if (ERTS_IS_ATOM_STR("spread", how)) - cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (ERTS_IS_ATOM_STR("processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("default_bind", how) - || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("no_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (ERTS_IS_ATOM_STR("unbound", how)) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else { - cpu_bind_order = old_cpu_bind_order; - ERTS_BIF_PREP_ERROR(res, c_p, BADARG); - goto done; - } - - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - - if (!cpudata) { - cpu_bind_order = old_cpu_bind_order; - ERTS_BIF_PREP_ERROR(res, c_p, BADARG); - goto done; - } - - signal_schedulers_bind_change(cpudata, cpudata_size); - - destroy_tmp_cpu_topology_copy(cpudata); - - res = bound_schedulers_term(old_cpu_bind_order); - } - - done: - - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - return res; -} - -Eterm -erts_fake_scheduler_bindings(Process *p, Eterm how) -{ - ErtsCpuBindOrder fake_cpu_bind_order; - erts_cpu_topology_t *cpudata; - int cpudata_size; - Eterm res; - - if (ERTS_IS_ATOM_STR("spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (ERTS_IS_ATOM_STR("processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("default_bind", how) - || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("no_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (ERTS_IS_ATOM_STR("unbound", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NONE; - else { - ERTS_BIF_PREP_ERROR(res, p, BADARG); - return res; - } - - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - - if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) - ERTS_BIF_PREP_RET(res, am_false); - else { - int i; - Eterm *hp; - - cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1); - -#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA - - erts_fprintf(stderr, "node: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].node); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "processor: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].processor); - erts_fprintf(stderr, "\n"); - if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD - && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD - && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) { - erts_fprintf(stderr, "processor_node:"); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].processor_node); - erts_fprintf(stderr, "\n"); - } - erts_fprintf(stderr, "core: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].core); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "thread: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].thread); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "logical: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].logical); - erts_fprintf(stderr, "\n"); -#endif - - hp = HAlloc(p, cpudata_size+1); - ERTS_BIF_PREP_RET(res, make_tuple(hp)); - *hp++ = make_arityval((Uint) cpudata_size); - for (i = 0; i < cpudata_size; i++) - *hp++ = make_small((Uint) cpudata[i].logical); - } - - destroy_tmp_cpu_topology_copy(cpudata); - - return res; -} - -Eterm -erts_get_schedulers_binds(Process *c_p) -{ - int ix; - ERTS_DECL_AM(unbound); - Eterm *hp = HAlloc(c_p, erts_no_schedulers+1); - Eterm res = make_tuple(hp); - - *(hp++) = make_arityval(erts_no_schedulers); - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - for (ix = 1; ix <= erts_no_schedulers; ix++) - *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 - ? make_small(scheduler2cpu_map[ix].bound_id) - : AM_unbound); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return res; -} - -static Eterm -bld_topology_term(Eterm **hpp, - Uint *hszp, - erts_cpu_topology_t *cpudata, - int size) -{ - Eterm res = NIL; - int i; - - if (size == 0) - return am_undefined; - - for (i = size-1; i >= 0; i--) { - res = erts_bld_cons(hpp, - hszp, - erts_bld_tuple(hpp, - hszp, - 7, - am_cpu, - make_small(cpudata[i].node), - make_small(cpudata[i].processor), - make_small(cpudata[i].processor_node), - make_small(cpudata[i].core), - make_small(cpudata[i].thread), - make_small(cpudata[i].logical)), - res); - } - return res; -} - -static Eterm -get_cpu_topology_term(Process *c_p, int type) -{ -#ifdef DEBUG - Eterm *hp_end; -#endif - Eterm *hp; - Uint hsz; - Eterm res = THE_NON_VALUE; - erts_cpu_topology_t *cpudata = NULL; - int size = 0; - - switch (type) { - case ERTS_GET_USED_CPU_TOPOLOGY: - if (user_cpudata) - goto defined; - else - goto detected; - case ERTS_GET_DETECTED_CPU_TOPOLOGY: - detected: - if (!system_cpudata) - res = am_undefined; - else { - size = system_cpudata_size; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * size)); - sys_memcpy((void *) cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*size); - } - break; - case ERTS_GET_DEFINED_CPU_TOPOLOGY: - defined: - if (!user_cpudata) - res = am_undefined; - else { - size = user_cpudata_size; - cpudata = user_cpudata; - } - break; - default: - erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); - break; - } - - if (res == am_undefined) { - ASSERT(!cpudata); - return res; - } - - hsz = 0; - - bld_topology_term(NULL, &hsz, - cpudata, size); - - hp = HAlloc(c_p, hsz); - -#ifdef DEBUG - hp_end = hp + hsz; -#endif - - res = bld_topology_term(&hp, NULL, - cpudata, size); - - ASSERT(hp_end == hp); - - if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); - - return res; -} - -Eterm -erts_get_cpu_topology_term(Process *c_p, Eterm which) -{ - Eterm res; - int type; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - if (ERTS_IS_ATOM_STR("used", which)) - type = ERTS_GET_USED_CPU_TOPOLOGY; - else if (ERTS_IS_ATOM_STR("detected", which)) - type = ERTS_GET_DETECTED_CPU_TOPOLOGY; - else if (ERTS_IS_ATOM_STR("defined", which)) - type = ERTS_GET_DEFINED_CPU_TOPOLOGY; - else - type = ERTS_GET_CPU_TOPOLOGY_ERROR; - if (type == ERTS_GET_CPU_TOPOLOGY_ERROR) - res = THE_NON_VALUE; - else - res = get_cpu_topology_term(c_p, type); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return res; -} - -static void -early_cpu_bind_init(void) -{ - user_cpudata = NULL; - user_cpudata_size = 0; - - system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo); - system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(erts_cpu_topology_t) - * system_cpudata_size)); - - cpu_bind_order = ERTS_CPU_BIND_NONE; - - if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) - || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, - system_cpudata_size)) { - erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); - system_cpudata = NULL; - system_cpudata_size = 0; - } -} - -static void -late_cpu_bind_init(void) -{ - int ix; - - erts_smp_rwmtx_init(&erts_cpu_bind_rwmtx, "cpu_bind"); - - scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(ErtsCpuBindData) - * (erts_no_schedulers+1))); - for (ix = 1; ix <= erts_no_schedulers; ix++) { - scheduler2cpu_map[ix].bind_id = -1; - scheduler2cpu_map[ix].bound_id = -1; - } - - if (cpu_bind_order != ERTS_CPU_BIND_NONE) { - erts_cpu_topology_t *cpudata; - int cpudata_size; - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - ASSERT(cpudata); - signal_schedulers_bind_change(cpudata, cpudata_size); - destroy_tmp_cpu_topology_copy(cpudata); - } -} - #ifdef ERTS_SMP static void @@ -4481,7 +3861,7 @@ add_pend_suspend(Process *suspendee, sizeof(ErtsPendingSuspend)); psp->next = NULL; #ifdef DEBUG -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead; #else psp->end = (ErtsPendingSuspend *) 0xdeaddead; @@ -4572,21 +3952,9 @@ handle_pend_sync_suspend(Process *suspendee, } } -/* - * Like erts_pid2proc() but: - * - * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p. - * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid. - * * It also waits for proc to be in a state != running and garbing. - * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to - * yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been - * suspended. - */ - - -Process * -erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, - Eterm pid, ErtsProcLocks pid_locks) +static Process * +pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks, int suspend) { Process *rp; int unlock_c_p_status; @@ -4613,7 +3981,7 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, c_p->suspendee = NIL; ASSERT(c_p->flags & F_P2PNR_RESCHED); c_p->flags &= ~F_P2PNR_RESCHED; - if (rp) + if (!suspend && rp) resume_process(rp); } else { @@ -4677,6 +4045,8 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, } /* rp is not running and we got the locks we want... */ + if (suspend) + suspend_process(rp_rq, rp); } erts_smp_runqs_unlock(cp_rq, rp_rq); } @@ -4689,6 +4059,35 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, return rp; } + +/* + * Like erts_pid2proc() but: + * + * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p. + * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid. + * * It also waits for proc to be in a state != running and garbing. + * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to + * yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been + * suspended. + */ +Process * +erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 0); +} + +/* + * Like erts_pid2proc_not_running(), but hands over the process + * in a suspended state unless (c_p is looked up). + */ +Process * +erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 1); +} + /* * erts_pid2proc_nropt() is normally the same as * erts_pid2proc_not_running(). However it is only @@ -4802,6 +4201,21 @@ handle_pend_bif_async_suspend(Process *suspendee, } } +#else + +/* + * Non-smp version of erts_pid2proc_suspend(). + */ +Process * +erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks, + Eterm pid, ErtsProcLocks pid_locks) +{ + Process *rp = erts_pid2proc(c_p, c_p_locks, pid, pid_locks); + if (rp) + erts_suspend(rp, pid_locks, NULL); + return rp; +} + #endif /* ERTS_SMP */ /* @@ -5322,7 +4736,7 @@ dequeue_process(ErtsRunQueue *runq, Process *p) } /* schedule a process */ -static ERTS_INLINE void +static ERTS_INLINE ErtsRunQueue * internal_add_to_runq(ErtsRunQueue *runq, Process *p) { Uint32 prev_status = p->status; @@ -5333,12 +4747,12 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) - return; + return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - ASSERT(p->status != P_SUSPENDED); + ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - return; + return NULL; } ASSERT(!p->scheduler_data); #endif @@ -5346,9 +4760,8 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); #ifndef ERTS_SMP /* Never schedule a suspended process (ok in smp case) */ - ASSERT(p->status != P_SUSPENDED); + ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); add_runq = runq; - #else ASSERT(!p->bound_runq || p->bound_runq == p->run_queue); if (p->bound_runq) { @@ -5377,20 +4790,23 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) profile_runnable_proc(p, am_active); } - smp_notify_inc_runq(add_runq); - if (add_runq != runq) erts_smp_runq_unlock(add_runq); + + return add_runq; } void erts_add_to_runq(Process *p) { + ErtsRunQueue *notify_runq; ErtsRunQueue *runq = erts_get_runq_proc(p); erts_smp_runq_lock(runq); - internal_add_to_runq(runq, p); + notify_runq = internal_add_to_runq(runq, p); erts_smp_runq_unlock(runq); + smp_notify_inc_runq(notify_runq); + } /* Possibly remove a scheduled process we need to suspend */ @@ -5529,8 +4945,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, p->run_queue = to_rq; enqueue_process(to_rq, p); - smp_notify_inc_runq(to_rq); - return ERTS_MIGRATE_SUCCESS; } #endif /* ERTS_SMP */ @@ -5727,30 +5141,6 @@ erts_set_process_priority(Process *p, Eterm new_value) return old_value; } -#ifdef ERTS_SMP - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - return 0; -} - -#else - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - return !erts_port_task_have_outstanding_io_tasks(); -} - -#endif - /* note that P_RUNNING is only set so that we don't try to remove ** running processes from the schedule queue if they exit - a running ** process not being in the schedule queue!! @@ -5780,10 +5170,10 @@ Process *schedule(Process *p, int calls) { ErtsRunQueue *rq; ErtsRunPrioQueue *rpq; - long dt; + erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; - long fcalls; + int fcalls; int input_reductions; int actual_reds; int reds; @@ -5806,7 +5196,7 @@ Process *schedule(Process *p, int calls) esdp = erts_get_scheduler_data(); rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = erts_smp_atomic_read(&function_calls); + fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { @@ -5824,7 +5214,7 @@ Process *schedule(Process *p, int calls) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; - fcalls = erts_smp_atomic_addtest(&function_calls, reds); + fcalls = (int) erts_smp_atomic32_addtest(&function_calls, reds); ASSERT(esdp && esdp == erts_get_scheduler_data()); rq = erts_get_runq_current(esdp); @@ -5839,6 +5229,9 @@ Process *schedule(Process *p, int calls) } if (IS_TRACED(p)) { + if (IS_TRACED_FL(p, F_TRACE_CALLS) && p->status != P_FREE) { + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); + } switch (p->status) { case P_EXITING: if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) @@ -5868,7 +5261,7 @@ Process *schedule(Process *p, int calls) handle_pending_suspend(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) - || p->status != P_SUSPENDED); + || p->rcount == 0); } #endif erts_smp_runq_lock(rq); @@ -5882,8 +5275,11 @@ Process *schedule(Process *p, int calls) p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { + ErtsRunQueue *notify_runq; p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); + if (notify_runq != rq) + smp_notify_inc_runq(notify_runq); } #endif @@ -5919,10 +5315,10 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - dt = do_time_read_and_reset(); + dt = erts_do_time_read_and_reset(); if (dt) { erts_smp_runq_unlock(rq); - bump_timer(dt); + erts_bump_timer(dt); erts_smp_runq_lock(rq); } BM_STOP_TIMER(system); @@ -5951,21 +5347,33 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic_read(&esdp->chk_cpu_bind)) { - check_cpu_bind(esdp); + || erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) { + erts_sched_check_cpu_bind(esdp); } } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->check_children) { - esdp->check_children = 0; - erts_smp_runq_unlock(rq); - erts_check_children(); - erts_smp_runq_lock(rq); +#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) + { + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + erts_aint32_t aux_work = erts_smp_atomic32_read(&ssi->aux_work); + if (aux_work) { + erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + erts_smp_runq_lock(rq); + } } #endif @@ -5997,7 +5405,10 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; } @@ -6014,17 +5425,7 @@ Process *schedule(Process *p, int calls) } } - if (prepare_for_sys_schedule()) { - erts_smp_atomic_set(&function_calls, 0); - fcalls = 0; - sched_sys_wait(esdp->no, rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - else { - /* If all schedulers are waiting, one of them *should* - be waiting in erl_sys_schedule() */ - sched_cnd_wait(esdp->no, rq); - } + scheduler_wait(&fcalls, esdp, rq); non_empty_runq(rq); @@ -6048,19 +5449,21 @@ Process *schedule(Process *p, int calls) * Schedule system-level activities. */ - erts_smp_atomic_set(&function_calls, 0); + erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + #ifdef ERTS_SMP /* erts_sys_schedule_interrupt(0); */ #endif erts_smp_runq_unlock(rq); erl_sys_schedule(runnable); - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP erts_smp_runq_lock(rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); goto continue_check_activities_to_run; #else if (!runnable) @@ -6081,14 +5484,14 @@ Process *schedule(Process *p, int calls) if (rq->wakeup_other < 0) rq->wakeup_other = 0; } - else if (rq->wakeup_other < ERTS_WAKEUP_OTHER_LIMIT) + else if (rq->wakeup_other < wakeup_other_limit) rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC; else { if (erts_common_run_queue) { if (erts_common_run_queue->waiting) - wake_one_scheduler(); + wake_scheduler(erts_common_run_queue, 0, 1); } - else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) { + else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -6231,10 +5634,10 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); if (erts_sched_stat.enabled) { - Uint old = ERTS_PROC_SCHED_ID(p, + UWord old = ERTS_PROC_SCHED_ID(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_STATUS), - esdp->no); + (UWord) esdp->no); int migrated = old && old != esdp->no; erts_smp_spin_lock(&erts_sched_stat.lock); @@ -6275,7 +5678,11 @@ Process *schedule(Process *p, int calls) trace_virtual_sched(p, am_in); break; } + if (IS_TRACED_FL(p, F_TRACE_CALLS)) { + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN); + } } + if (p->status != P_EXITING) p->status = P_RUNNING; @@ -6378,6 +5785,15 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) ErtsRunQueue *rq = erts_get_runq_current(NULL); ErtsMiscOpList *molp = misc_op_list_alloc(); + if (!rq) { + /* + * This can only happen when the sys msg dispatcher + * thread schedules misc ops (this happens *very* + * seldom; only when trace drivers are unloaded). + */ + rq = ERTS_RUNQ_IX(0); + } + erts_smp_runq_lock(rq); while (rq->misc.evac_runq) { @@ -6397,8 +5813,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) else rq->misc.start = molp; rq->misc.end = molp; - smp_notify_inc_runq(rq); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(rq); } static void @@ -6532,7 +5948,7 @@ erts_test_next_pid(int set, Uint next) Uint erts_process_count(void) { - long res = erts_smp_atomic_read(&process_count); + erts_aint32_t res = erts_smp_atomic32_read(&process_count); ASSERT(res >= 0); return (Uint) res; } @@ -6581,7 +5997,7 @@ alloc_process(void) ASSERT(!process_tab[p_next]); process_tab[p_next] = p; - erts_smp_atomic_inc(&process_count); + erts_smp_atomic32_inc(&process_count); p->id = make_internal_pid(p_serial << p_serial_shift | p_next); if (p->id == ERTS_INVALID_PID) { /* Do not use the invalid pid; change serial */ @@ -6640,7 +6056,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { - ErtsRunQueue *rq; + ErtsRunQueue *rq, *notify_runq; Process *p; Sint arity; /* Number of arguments. */ #ifndef HYBRID @@ -6707,7 +6123,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; p->prio = PRIORITY_NORMAL; - p->max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + p->max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); } p->skipped = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); @@ -6719,11 +6135,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* * Must initialize binary lists here before copying binaries to process. */ - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; heap_need += @@ -6757,13 +6169,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; p->bin_old_vheap = 0; + p->bin_vheap_mature = 0; /* No need to initialize p->fcalls. */ p->current = p->initial+INITIAL_MOD; - p->i = (Eterm *) beam_apply; - p->cp = (Eterm *) beam_apply+1; + p->i = (BeamInstr *) beam_apply; + p->cp = (BeamInstr *) beam_apply+1; p->arg_reg = p->def_arg_reg; p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]); @@ -6813,7 +6226,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->group_leader = IS_CONST(parent->group_leader) ? parent->group_leader - : STORE_NC(&p->htop, &p->off_heap.externals, parent->group_leader); + : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); @@ -6957,10 +6370,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #endif p->status = P_WAITING; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(notify_runq); + res = p->id; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); @@ -7007,6 +6422,7 @@ void erts_init_empty_process(Process *p) p->bin_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; + p->bin_vheap_mature = 0; #ifdef ERTS_SMP p->u.ptimer = NULL; p->bound_runq = NULL; @@ -7014,11 +6430,7 @@ void erts_init_empty_process(Process *p) memset(&(p->u.tm), 0, sizeof(ErlTimer)); #endif p->next = NULL; - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; p->reg = NULL; p->heap_sz = 0; @@ -7165,11 +6577,7 @@ erts_debug_verify_clean_empty_process(Process* p) /* Thing that erts_cleanup_empty_process() cleans up */ - ASSERT(p->off_heap.mso == NULL); -#ifndef HYBRID /* FIND ME! */ - ASSERT(p->off_heap.funs == NULL); -#endif - ASSERT(p->off_heap.externals == NULL); + ASSERT(p->off_heap.first == NULL); ASSERT(p->off_heap.overhead == 0); ASSERT(p->mbuf == NULL); @@ -7180,25 +6588,16 @@ erts_debug_verify_clean_empty_process(Process* p) void erts_cleanup_empty_process(Process* p) { - ErlHeapFragment* mbufp; - /* We only check fields that are known to be used... */ erts_cleanup_offheap(&p->off_heap); - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; - mbufp = p->mbuf; - while (mbufp) { - ErlHeapFragment *next = mbufp->next; - free_message_buffer(mbufp); - mbufp = next; + if (p->mbuf != NULL) { + free_message_buffer(p->mbuf); + p->mbuf = NULL; } - p->mbuf = NULL; #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) erts_lcnt_proc_lock_destroy(p); #endif @@ -7214,7 +6613,6 @@ static void delete_process(Process* p) { ErlMessage* mp; - ErlHeapFragment* bp; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id)); @@ -7230,7 +6628,7 @@ delete_process(Process* p) * The mso list should not be used anymore, but if it is, make sure that * we'll notice. */ - p->off_heap.mso = (void *) 0x8DEFFACD; + p->off_heap.first = (void *) 0x8DEFFACD; if (p->arg_reg != p->def_arg_reg) { erts_free(ERTS_ALC_T_ARG_REG, p->arg_reg); @@ -7264,11 +6662,8 @@ delete_process(Process* p) /* * Free all pending message buffers. */ - bp = p->mbuf; - while (bp != NULL) { - ErlHeapFragment* next_bp = bp->next; - free_message_buffer(bp); - bp = next_bp; + if (p->mbuf != NULL) { + free_message_buffer(p->mbuf); } erts_erase_dicts(p); @@ -7348,7 +6743,7 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) p->freason = EXTAG_EXIT; KILL_CATCHES(p); cancel_timer(p); - p->i = (Eterm *) beam_exit; + p->i = (BeamInstr *) beam_exit; } @@ -7778,13 +7173,14 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_port_release(prt); } else if (is_internal_pid(mon->pid)) {/* local by name or pid */ Eterm watched; - Eterm lhp[3]; + DeclareTmpHeapNoproc(lhp,3); ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK | ERTS_PROC_LOCKS_MSG_SEND); rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); if (rp == NULL) { goto done; } + UseTmpHeapNoproc(3); rmon = erts_remove_monitor(&(rp->monitors),mon->ref); if (rmon) { erts_destroy_monitor(rmon); @@ -7795,6 +7191,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, watched, pcontext->reason); } + UnUseTmpHeapNoproc(3); /* else: demonitor while we exited, i.e. do nothing... */ erts_smp_proc_unlock(rp, rp_locks); } else { /* external by pid or name */ @@ -8025,8 +7422,13 @@ erts_do_exit_process(Process* p, Eterm reason) ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); #endif - if (IS_TRACED_FL(p,F_TRACE_PROCS)) - trace_proc(p, p, am_exit, reason); + if (IS_TRACED(p)) { + if (IS_TRACED_FL(p, F_TRACE_CALLS)) + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING); + + if (IS_TRACED_FL(p,F_TRACE_PROCS)) + trace_proc(p, p, am_exit, reason); + } erts_trace_check_exiting(p->id); @@ -8075,6 +7477,8 @@ continue_exit_process(Process *p Eterm reason = p->fvalue; DistEntry *dep; struct saved_calls *scb; + process_breakpoint_time_t *pbt; + #ifdef DEBUG int yield_allowed = 1; #endif @@ -8176,8 +7580,8 @@ continue_exit_process(Process *p p->status_flags = 0; #endif process_tab[pix] = NULL; /* Time of death! */ - ASSERT(erts_smp_atomic_read(&process_count) > 0); - erts_smp_atomic_dec(&process_count); + ASSERT(erts_smp_atomic32_read(&process_count) > 0); + erts_smp_atomic32_dec(&process_count); #ifdef ERTS_SMP erts_pix_unlock(pix_lock); @@ -8214,6 +7618,7 @@ continue_exit_process(Process *p ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) : NULL); scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); + pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); processes_busy--; @@ -8228,11 +7633,12 @@ continue_exit_process(Process *p * Pre-build the EXIT tuple if there are any links. */ if (lnk) { - Eterm tmp_heap[4]; + DeclareTmpHeap(tmp_heap,4,p); Eterm exit_tuple; Uint exit_tuple_sz; Eterm* hp; + UseTmpHeap(4,p); hp = &tmp_heap[0]; exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason); @@ -8243,16 +7649,21 @@ continue_exit_process(Process *p ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz}; erts_sweep_links(lnk, &doit_exit_link, &context); } + UnUseTmpHeap(4,p); } { ExitMonitorContext context = {reason, p}; - erts_sweep_monitors(mon,&doit_exit_monitor,&context); + erts_sweep_monitors(mon,&doit_exit_monitor,&context); /* Allocates TmpHeap, but we + have none here */ } if (scb) erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); + if (pbt) + erts_free(ERTS_ALC_T_BPD, (void *) pbt); + delete_process(p); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); @@ -8271,7 +7682,7 @@ continue_exit_process(Process *p ASSERT(p->status == P_EXITING); - p->i = (Eterm *) beam_continue_exit; + p->i = (BeamInstr *) beam_continue_exit; if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) { erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); @@ -8291,14 +7702,33 @@ continue_exit_process(Process *p static void timeout_proc(Process* p) { - p->i = (Eterm *) p->def_arg_reg[0]; + BeamInstr** pi = (BeamInstr **) p->def_arg_reg; + p->i = *pi; p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; - if (p->status == P_WAITING) - erts_add_to_runq(p); - if (p->status == P_SUSPENDED) + switch (p->status) { + case P_GARBING: + switch (p->gcstatus) { + case P_SUSPENDED: + goto suspended; + case P_WAITING: + goto waiting; + default: + break; + } + break; + case P_WAITING: + waiting: + erts_add_to_runq(p); + break; + case P_SUSPENDED: + suspended: p->rstatus = P_RUNABLE; /* MUST set resume status to runnable */ + break; + default: + break; + } } @@ -8310,7 +7740,7 @@ cancel_timer(Process* p) #ifdef ERTS_SMP erts_cancel_smp_ptimer(p->u.ptimer); #else - erl_cancel_timer(&p->u.tm); + erts_cancel_timer(&p->u.tm); #endif } @@ -8336,7 +7766,7 @@ set_timer(Process* p, Uint timeout) (ErlTimeoutProc) timeout_proc, timeout); #else - erl_set_timer(&p->u.tm, + erts_set_timer(&p->u.tm, (ErlTimeoutProc) timeout_proc, NULL, (void*) p, @@ -8384,15 +7814,15 @@ erts_program_counter_info(int to, void *to_arg, Process *p) * only cause problems. */ for (i = 0; i < p->arity; i++) - erts_print(to, to_arg, " %T\n", p->arg_reg[i]); + erts_print(to, to_arg, " %.*T\n", INT_MAX, p->arg_reg[i]); } } } static void -print_function_from_pc(int to, void *to_arg, Eterm* x) +print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - Eterm* addr = find_function_from_pc(x); + BeamInstr* addr = find_function_from_pc(x); if (addr == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); @@ -8426,7 +7856,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "Return addr %p (", (Eterm *) x); + erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x)); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; @@ -9255,8 +8685,8 @@ init_processes_bif(void) processes_trap_export.code[0] = am_erlang; processes_trap_export.code[1] = am_processes_trap; processes_trap_export.code[2] = 2; - processes_trap_export.code[3] = (Eterm) em_apply_bif; - processes_trap_export.code[4] = (Eterm) &processes_trap; + processes_trap_export.code[3] = (BeamInstr) em_apply_bif; + processes_trap_export.code[4] = (BeamInstr) &processes_trap; #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST erts_get_emu_time(&debug_tv_start); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index f58b6932b3..296acc7367 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,6 +28,12 @@ #define ERTS_INCLUDE_SCHEDULER_INTERNALS #endif +/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */ + +#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG) +# define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC +#endif + typedef struct process Process; #include "sys.h" @@ -174,12 +180,12 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \ ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO))) -#define ERTS_RUNQ_IFLG_SUSPENDED (((long) 1) << 0) -#define ERTS_RUNQ_IFLG_NONEMPTY (((long) 1) << 1) +#define ERTS_RUNQ_IFLG_SUSPENDED (((erts_aint32_t) 1) << 0) +#define ERTS_RUNQ_IFLG_NONEMPTY (((erts_aint32_t) 1) << 1) #ifdef DEBUG -# ifdef ARCH_64 +# if defined(ARCH_64) && !HALFWORD_HEAP # define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ (*((char **) &(RQP)) = (char *) (0xdeadbeefdead0003 | ((N) << 4))) # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ @@ -194,8 +200,8 @@ do { \ # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ do { \ ASSERT((RQP) != NULL); \ - ASSERT(((((Uint) (RQP)) & ((Uint) 1))) == ((Uint) 0)); \ - ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdead0000)); \ + ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ + ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ } while (0) # endif #else @@ -219,6 +225,49 @@ typedef enum { ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED } ErtsMigrateResult; +#define ERTS_SSI_FLG_SLEEPING (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_FLG_POLL_SLEEPING (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_FLG_TSE_SLEEPING (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4) + +#define ERTS_SSI_FLGS_SLEEP_TYPE \ + (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) + +#define ERTS_SSI_FLGS_SLEEP \ + (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE) + +#define ERTS_SSI_FLGS_ALL \ + (ERTS_SSI_FLGS_SLEEP \ + | ERTS_SSI_FLG_WAITING \ + | ERTS_SSI_FLG_SUSPENDED) + +#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 1) + +#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ + (ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ + | ERTS_SSI_AUX_WORK_MISC) +#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ + (0) + +typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; + +typedef struct { + erts_smp_spinlock_t lock; + ErtsSchedulerSleepInfo *list; +} ErtsSchedulerSleepList; + +struct ErtsSchedulerSleepInfo_ { + ErtsSchedulerSleepInfo *next; + ErtsSchedulerSleepInfo *prev; + erts_smp_atomic32_t flags; + erts_tse_t *event; + erts_smp_atomic32_t aux_work; +}; + /* times to reschedule low prio process before running */ #define RESCHEDULE_LOW 8 @@ -266,13 +315,14 @@ typedef struct { struct ErtsRunQueue_ { int ix; - erts_smp_atomic_t info_flags; + erts_smp_atomic32_t info_flags; erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - erts_smp_atomic_t spin_waiter; - erts_smp_atomic_t spin_wake; +#ifdef ERTS_SMP + ErtsSchedulerSleepList sleepers; +#endif ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ @@ -344,13 +394,25 @@ struct ErtsSchedulerData_ { * numbered registers as possible in the same cache * line). */ +#if !HALFWORD_HEAP Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers */ +#else + Eterm *save_reg; +#endif FloatDef freg[MAX_REG]; /* Floating point registers. */ ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ + ErtsSchedulerSleepInfo *ssi; Process *free_process; #endif +#if !HEAP_ON_C_STACK + Eterm tmp_heap[TMP_HEAP_SIZE]; + int num_tmp_heap_used; + Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE]; + Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE]; + Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; +#endif Process *current_process; Uint no; /* Scheduler number */ @@ -363,15 +425,22 @@ struct ErtsSchedulerData_ { #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; /* run queue mutex */ - int blocked_check_children; /* schdlr_sspnd mutex */ + erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */ #endif - erts_smp_atomic_t suspended; /* Only used when common run queue */ - erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */ + +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC + erts_alloc_verify_func_t verify_unused_temp_alloc; + Allctr_t *verify_unused_temp_alloc_data; #endif }; +typedef union { + ErtsSchedulerData esd; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; +} ErtsAlignedSchedulerData; + +extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; + #ifndef ERTS_SMP extern ErtsSchedulerData *erts_scheduler_data; #endif @@ -386,8 +455,9 @@ extern ErtsSchedulerData *erts_scheduler_data; #define ERTS_PSD_SAVED_CALLS_BUF 1 #define ERTS_PSD_SCHED_ID 2 #define ERTS_PSD_DIST_ENTRY 3 +#define ERTS_PSD_CALL_TIME_BP 4 -#define ERTS_PSD_SIZE 4 +#define ERTS_PSD_SIZE 5 typedef struct { void *data[ERTS_PSD_SIZE]; @@ -408,6 +478,9 @@ typedef struct { #define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN + typedef struct { ErtsProcLocks get_locks; ErtsProcLocks set_locks; @@ -479,6 +552,7 @@ struct ErtsPendingSuspend_ { # define MIN_VHEAP_SIZE(p) (p)->min_vheap_size # define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz +# define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap @@ -518,8 +592,8 @@ struct process { unsigned max_arg_reg; /* Maximum number of argument registers available. */ Eterm def_arg_reg[6]; /* Default array for argument registers. */ - Eterm* cp; /* Continuation pointer (for threaded code). */ - Eterm* i; /* Program counter for threaded code. */ + BeamInstr* cp; /* (untagged) Continuation pointer (for threaded code). */ + BeamInstr* i; /* Program counter for threaded code. */ Sint catches; /* Number of catches on stack */ Sint fcalls; /* * Number of reductions left to execute. @@ -566,11 +640,12 @@ struct process { Uint seq_trace_lastcnt; Eterm seq_trace_token; /* Sequential trace token (tuple size 5 see below) */ - Eterm initial[3]; /* Initial module(0), function(1), arity(2) */ - Eterm* current; /* Current Erlang function: + BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead + of pointer to funcinfo instruction, hence the BeamInstr datatype */ + BeamInstr* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; - * arity an untagged integer). + * arity an untagged integer). BeamInstr * because it references code */ /* @@ -595,9 +670,10 @@ struct process { Uint mbuf_sz; /* Size of all message buffers */ ErtsPSD *psd; /* Rarely used process specific data */ - Uint bin_vheap_sz; /* Virtual heap block size for binaries */ - Uint bin_old_vheap_sz; /* Virtual old heap block size for binaries */ - Uint bin_old_vheap; /* Virtual old heap size for binaries */ + Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ + Uint64 bin_vheap_mature; /* Virtual heap block size for binaries */ + Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ + Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ union { #ifdef ERTS_SMP @@ -753,13 +829,13 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) { ErlHeapFragment* hf = MBUF(p); - ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->size)); + ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size)); hf->used_size = hp - hf->mem; } #endif /* inline */ -Eterm* erts_heap_alloc(Process* p, Uint need); +Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); #ifdef CHECK_FOR_HOLES Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif @@ -812,13 +888,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_INSLPQUEUE (1 << 1) /* Set if in timer queue */ #define F_TIMO (1 << 2) /* Set if timeout */ #define F_HEAP_GROW (1 << 3) -#define F_NEED_FULLSWEEP (1 << 4) /* If process has old binaries & funs. */ +#define F_NEED_FULLSWEEP (1 << 4) #define F_USING_DB (1 << 5) /* If have created tables */ #define F_DISTRIBUTION (1 << 6) /* Process used in distribution */ #define F_USING_DDLL (1 << 7) /* Process has used the DDLL interface */ #define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */ #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ +#define F_HIBERNATE_SCHED (1 << 11) /* Schedule out after hibernate op */ /* process trace_flags */ #define F_SENSITIVE (1 << 0) @@ -946,26 +1023,12 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; (p)->flags &= ~F_TIMO; \ } while (0) - -#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS 0 -#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED 1 -#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY 2 -#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE 3 - -int erts_init_scheduler_bind_type(char *how); - -#define ERTS_INIT_CPU_TOPOLOGY_OK 0 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID 1 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE 2 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY 3 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE 4 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES 5 -#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID 6 -#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS 7 -#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES 8 -#define ERTS_INIT_CPU_TOPOLOGY_MISSING 9 - -int erts_init_cpu_topology(char *topology_str); +#define ERTS_RUNQ_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + &erts_aligned_run_queues[(IX)].runq) +#define ERTS_SCHEDULER_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &erts_aligned_scheduler_data[(IX)].esd) void erts_pre_init_process(void); void erts_late_init_process(void); @@ -976,9 +1039,12 @@ ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); int erts_proclist_same(ErtsProcList *, Process *); +int erts_sched_set_wakeup_limit(char *str); + #ifdef DEBUG void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); #endif +int erts_get_max_no_executing_schedulers(void); #ifdef ERTS_SMP ErtsSchedSuspendResult erts_schedulers_state(Uint *, Uint *, Uint *, int); @@ -993,9 +1059,15 @@ int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_smp_notify_check_children_needed(void); +void +erts_smp_schedule_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg); #endif +void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); -void erts_init_process(void); +void erts_init_process(int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); Uint erts_run_queues_len(Uint *); void erts_add_to_runq(Process *); @@ -1069,6 +1141,9 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks); void erts_deep_process_dump(int, void *); +Eterm erts_get_reader_groups_map(Process *c_p); +Eterm erts_debug_reader_groups_map(Process *c_p, int groups); + Sint erts_test_next_pid(int, Uint); Eterm erts_debug_processes(Process *c_p); Eterm erts_debug_processes_bif_info(Process *c_p); @@ -1082,6 +1157,20 @@ Uint erts_debug_nbalance(void); # define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data) #endif +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC +# define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P) \ +do { \ + ErtsSchedulerData *esdp__ = ((P) \ + ? ERTS_PROC_GET_SCHDATA((Process *) (P)) \ + : erts_get_scheduler_data()); \ + if (esdp__) \ + esdp__->verify_unused_temp_alloc( \ + esdp__->verify_unused_temp_alloc_data); \ +} while (0) +#else +# define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP) +#endif + #if defined(ERTS_SMP) || defined(USE_THREADS) ErtsSchedulerData *erts_get_scheduler_data(void); #else @@ -1184,7 +1273,7 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #endif #define ERTS_PROC_SCHED_ID(P, L, ID) \ - ((Uint) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) + ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) #define ERTS_PROC_GET_DIST_ENTRY(P) \ ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) @@ -1196,6 +1285,12 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB))) +#define ERTS_PROC_GET_CALL_TIME(P) \ + ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP)) +#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \ + ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) + + ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, @@ -1209,8 +1304,8 @@ erts_proc_get_error_handler(Process *p) if (!val) return am_error_handler; else { - ASSERT(is_atom(((Eterm) val))); - return (Eterm) val; + ASSERT(is_atom(((Eterm) (UWord) val))); + return (Eterm) (UWord) val; } } @@ -1220,13 +1315,13 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler) void *old_val; void *new_val; ASSERT(is_atom(handler)); - new_val = handler == am_error_handler ? NULL : (void *) handler; + new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler; old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val); if (!old_val) return am_error_handler; else { - ASSERT(is_atom(((Eterm) old_val))); - return (Eterm) old_val; + ASSERT(is_atom(((Eterm) (UWord) old_val))); + return (Eterm) (UWord) old_val; } } @@ -1440,6 +1535,10 @@ erts_get_atom_cache_map(Process *c_p) } #endif +Process *erts_pid2proc_suspend(Process *, + ErtsProcLocks, + Eterm, + ErtsProcLocks); #ifdef ERTS_SMP Process *erts_pid2proc_not_running(Process *, @@ -1487,29 +1586,32 @@ extern int erts_disable_proc_not_running_opt; #define ERTS_MIN_PROCESSES 16 #endif -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -ERTS_GLB_INLINE void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -void erts_smp_notify_inc_runq__(ErtsRunQueue *runq); -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ +void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF +#ifdef ERTS_SMP +void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); +ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS +#if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_smp_notify_inc_runq(ErtsRunQueue *runq) +erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - if (runq->waiting) - erts_smp_notify_inc_runq__(runq); -#endif + erts_aint32_t flags; + ERTS_THR_MEMORY_BARRIER; + flags = erts_smp_atomic32_read(&ssi->flags); + ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) + || (flags & ERTS_SSI_FLG_WAITING)); + if (flags & ERTS_SSI_FLG_SLEEPING) { + flags = erts_smp_atomic32_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + erts_sched_finish_poke(ssi, flags); + } } -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ - #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* #ifdef ERTS_SMP */ + #include "erl_process_lock.h" #undef ERTS_INCLUDE_SCHEDULER_INTERNALS diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 1666509c72..5410bcd495 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -45,16 +45,16 @@ static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep); static void dump_element_nl(int to, void *to_arg, Eterm x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg); -static void print_function_from_pc(int to, void *to_arg, Eterm* x); +static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static void heap_dump(int to, void *to_arg, Eterm x); static void dump_binaries(int to, void *to_arg, Binary* root); static void dump_externally(int to, void *to_arg, Eterm term); static Binary* all_binaries; -extern Eterm beam_apply[]; -extern Eterm beam_exit[]; -extern Eterm beam_continue_exit[]; +extern BeamInstr beam_apply[]; +extern BeamInstr beam_exit[]; +extern BeamInstr beam_continue_exit[]; void @@ -194,7 +194,7 @@ dump_element(int to, void *to_arg, Eterm x) } else if (is_pid(x)) { erts_print(to, to_arg, "P%T", x); } else if (is_port(x)) { - erts_print(to, to_arg, "p<%bpu.%bpu>", + erts_print(to, to_arg, "p<%beu.%beu>", port_channel_no(x), port_number(x)); } else if (is_nil(x)) { erts_putc(to, to_arg, 'N'); @@ -223,7 +223,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "SReturn addr 0x%X (", (Eterm *) x); + erts_print(to, to_arg, "SReturn addr 0x%X (", cp_val(x)); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; @@ -239,9 +239,9 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } static void -print_function_from_pc(int to, void *to_arg, Eterm* x) +print_function_from_pc(int to, void *to_arg, BeamInstr* x) { - Eterm* addr = find_function_from_pc(x); + BeamInstr* addr = find_function_from_pc(x); if (addr == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); @@ -261,139 +261,139 @@ print_function_from_pc(int to, void *to_arg, Eterm* x) static void heap_dump(int to, void *to_arg, Eterm x) { + DeclareTmpHeapNoproc(last,1); + Eterm* next = last; Eterm* ptr; - Eterm last = OUR_NIL; - Eterm* next = &last; if (is_immed(x) || is_CP(x)) { return; } - - again: - if (x == OUR_NIL) { /* We are done. */ - return; - } if (is_CP(x)) { - next = (Eterm *) x; - } else if (is_list(x)) { - ptr = list_val(x); - if (ptr[0] != OUR_NIL) { - erts_print(to, to_arg, ADDR_FMT ":l", ptr); - dump_element(to, to_arg, ptr[0]); - erts_putc(to, to_arg, '|'); - dump_element(to, to_arg, ptr[1]); - erts_putc(to, to_arg, '\n'); - if (is_immed(ptr[1])) { - ptr[1] = make_small(0); - } - x = ptr[0]; - ptr[0] = (Eterm) next; - next = ptr + 1; - goto again; - } - } else if (is_boxed(x)) { - Eterm hdr; - - ptr = boxed_val(x); - hdr = *ptr; - if (hdr != OUR_NIL) { /* If not visited */ - erts_print(to, to_arg, ADDR_FMT ":", ptr); - if (is_arity_value(hdr)) { - Uint i; - Uint arity = arityval(hdr); - - erts_print(to, to_arg, "t" WORD_FMT ":", arity); - for (i = 1; i <= arity; i++) { - dump_element(to, to_arg, ptr[i]); - if (is_immed(ptr[i])) { - ptr[i] = make_small(0); - } - if (i < arity) { - erts_putc(to, to_arg, ','); - } - } + UseTmpHeapNoproc(1); + *last = OUR_NIL; + + while (x != OUR_NIL) { + if (is_CP(x)) { + next = (Eterm *) EXPAND_POINTER(x); + } else if (is_list(x)) { + ptr = list_val(x); + if (ptr[0] != OUR_NIL) { + erts_print(to, to_arg, ADDR_FMT ":l", ptr); + dump_element(to, to_arg, ptr[0]); + erts_putc(to, to_arg, '|'); + dump_element(to, to_arg, ptr[1]); erts_putc(to, to_arg, '\n'); - if (arity == 0) { - ptr[0] = OUR_NIL; - } else { - x = ptr[arity]; - ptr[0] = (Eterm) next; - next = ptr + arity - 1; - goto again; + if (is_immed(ptr[1])) { + ptr[1] = make_small(0); } - } else if (hdr == HEADER_FLONUM) { - FloatDef f; - char sbuf[31]; - int i; - - GET_DOUBLE_DATA((ptr+1), f); - i = sys_double_to_chars(f.fd, (char*) sbuf); - sys_memset(sbuf+i, 0, 31-i); - erts_print(to, to_arg, "F%X:%s\n", i, sbuf); - *ptr = OUR_NIL; - } else if (_is_bignum_header(hdr)) { - erts_print(to, to_arg, "B%T\n", x); - *ptr = OUR_NIL; - } else if (is_binary_header(hdr)) { - Uint tag = thing_subtag(hdr); - Uint size = binary_size(x); - Uint i; - - if (tag == HEAP_BINARY_SUBTAG) { - byte* p; - - erts_print(to, to_arg, "Yh%X:", size); - p = binary_bytes(x); - for (i = 0; i < size; i++) { - erts_print(to, to_arg, "%02X", p[i]); + x = ptr[0]; + ptr[0] = (Eterm) COMPRESS_POINTER(next); + next = ptr + 1; + continue; + } + } else if (is_boxed(x)) { + Eterm hdr; + + ptr = boxed_val(x); + hdr = *ptr; + if (hdr != OUR_NIL) { /* If not visited */ + erts_print(to, to_arg, ADDR_FMT ":", ptr); + if (is_arity_value(hdr)) { + Uint i; + Uint arity = arityval(hdr); + + erts_print(to, to_arg, "t" WORD_FMT ":", arity); + for (i = 1; i <= arity; i++) { + dump_element(to, to_arg, ptr[i]); + if (is_immed(ptr[i])) { + ptr[i] = make_small(0); + } + if (i < arity) { + erts_putc(to, to_arg, ','); + } } - } else if (tag == REFC_BINARY_SUBTAG) { - ProcBin* pb = (ProcBin *) binary_val(x); - Binary* val = pb->val; - - if (erts_smp_atomic_xchg(&val->refc, 0) != 0) { - val->flags = (Uint) all_binaries; - all_binaries = val; + erts_putc(to, to_arg, '\n'); + if (arity == 0) { + ptr[0] = OUR_NIL; + } else { + x = ptr[arity]; + ptr[0] = (Eterm) COMPRESS_POINTER(next); + next = ptr + arity - 1; + continue; } - erts_print(to, to_arg, "Yc%X:%X:%X", val, - pb->bytes - (byte *)val->orig_bytes, - size); - } else if (tag == SUB_BINARY_SUBTAG) { - ErlSubBin* Sb = (ErlSubBin *) binary_val(x); - Eterm* real_bin = binary_val(Sb->orig); - void* val; - - if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) { - ProcBin* pb = (ProcBin *) real_bin; - val = pb->val; - } else { /* Heap binary */ - val = real_bin; + } else if (hdr == HEADER_FLONUM) { + FloatDef f; + char sbuf[31]; + int i; + + GET_DOUBLE_DATA((ptr+1), f); + i = sys_double_to_chars(f.fd, (char*) sbuf); + sys_memset(sbuf+i, 0, 31-i); + erts_print(to, to_arg, "F%X:%s\n", i, sbuf); + *ptr = OUR_NIL; + } else if (_is_bignum_header(hdr)) { + erts_print(to, to_arg, "B%T\n", x); + *ptr = OUR_NIL; + } else if (is_binary_header(hdr)) { + Uint tag = thing_subtag(hdr); + Uint size = binary_size(x); + Uint i; + + if (tag == HEAP_BINARY_SUBTAG) { + byte* p; + + erts_print(to, to_arg, "Yh%X:", size); + p = binary_bytes(x); + for (i = 0; i < size; i++) { + erts_print(to, to_arg, "%02X", p[i]); + } + } else if (tag == REFC_BINARY_SUBTAG) { + ProcBin* pb = (ProcBin *) binary_val(x); + Binary* val = pb->val; + + if (erts_smp_atomic_xchg(&val->refc, 0) != 0) { + val->flags = (UWord) all_binaries; + all_binaries = val; + } + erts_print(to, to_arg, "Yc%X:%X:%X", val, + pb->bytes - (byte *)val->orig_bytes, + size); + } else if (tag == SUB_BINARY_SUBTAG) { + ErlSubBin* Sb = (ErlSubBin *) binary_val(x); + Eterm* real_bin = binary_val(Sb->orig); + void* val; + + if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) { + ProcBin* pb = (ProcBin *) real_bin; + val = pb->val; + } else { /* Heap binary */ + val = real_bin; + } + erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size); } - erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size); + erts_putc(to, to_arg, '\n'); + *ptr = OUR_NIL; + } else if (is_external_pid_header(hdr)) { + erts_print(to, to_arg, "P%T\n", x); + *ptr = OUR_NIL; + } else if (is_external_port_header(hdr)) { + erts_print(to, to_arg, "p<%beu.%beu>\n", + port_channel_no(x), port_number(x)); + *ptr = OUR_NIL; + } else { + /* + * All other we dump in the external term format. + */ + dump_externally(to, to_arg, x); + erts_putc(to, to_arg, '\n'); + *ptr = OUR_NIL; } - erts_putc(to, to_arg, '\n'); - *ptr = OUR_NIL; - } else if (is_external_pid_header(hdr)) { - erts_print(to, to_arg, "P%T\n", x); - *ptr = OUR_NIL; - } else if (is_external_port_header(hdr)) { - erts_print(to, to_arg, "p<%bpu.%bpu>\n", - port_channel_no(x), port_number(x)); - *ptr = OUR_NIL; - } else { - /* - * All other we dump in the external term format. - */ - dump_externally(to, to_arg, x); - erts_putc(to, to_arg, '\n'); - *ptr = OUR_NIL; } } + x = *next; + *next = OUR_NIL; + next--; } - - x = *next; - *next = OUR_NIL; - next--; - goto again; + UnUseTmpHeapNoproc(1); } static void diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 52440fb635..72560aa124 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -71,9 +71,12 @@ const Process erts_proc_lock_busy; #ifdef ERTS_SMP -/*#define ERTS_PROC_LOCK_SPIN_ON_GATE*/ -#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 16000 +#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000 +#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000 +#define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50 + +#define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25 #ifdef ERTS_PROC_LOCK_DEBUG #define ERTS_PROC_LOCK_HARD_DEBUG @@ -83,32 +86,19 @@ const Process erts_proc_lock_busy; static void check_queue(erts_proc_lock_t *lck); #endif - -typedef struct erts_proc_lock_waiter_t_ erts_proc_lock_waiter_t; -struct erts_proc_lock_waiter_t_ { - erts_proc_lock_waiter_t *next; - erts_proc_lock_waiter_t *prev; - ErtsProcLocks wait_locks; - erts_smp_gate_t gate; - erts_proc_lock_queues_t *queues; -}; +#if SIZEOF_INT < 4 +#error "The size of the 'uflgs' field of the erts_tse_t type is too small" +#endif struct erts_proc_lock_queues_t_ { erts_proc_lock_queues_t *next; - erts_proc_lock_waiter_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; -}; - -struct erts_proc_lock_thr_spec_data_t_ { - erts_proc_lock_queues_t *qs; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; }; static erts_proc_lock_queues_t zeroqs = {0}; -static erts_smp_spinlock_t wtr_lock; -static erts_proc_lock_waiter_t *waiter_free_list; +static erts_smp_spinlock_t qs_lock; static erts_proc_lock_queues_t *queue_free_list; -static erts_tsd_key_t waiter_key; #ifdef ERTS_ENABLE_LOCK_CHECK static struct { @@ -122,122 +112,131 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; static int proc_lock_spin_count; -static int proc_lock_trans_spin_cost; +static int aux_thr_proc_lock_spin_count; -static void cleanup_waiter(void); +static void cleanup_tse(void); void -erts_init_proc_lock(void) +erts_init_proc_lock(int cpus) { int i; - int cpus; - erts_smp_spinlock_init(&wtr_lock, "proc_lck_wtr_alloc"); + erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { -#if ERTS_PROC_LOCK_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i)); -#else - erts_smp_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); -#endif -#else #ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, "pix_lock", make_small(i)); + erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, + "pix_lock", make_small(i)); #else erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock"); #endif -#endif } - waiter_free_list = NULL; queue_free_list = NULL; - erts_tsd_key_create(&waiter_key); - erts_thr_install_exit_handler(cleanup_waiter); + erts_thr_install_exit_handler(cleanup_tse); #ifdef ERTS_ENABLE_LOCK_CHECK lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif - cpus = erts_get_cpu_configured(erts_cpuinfo); - if (cpus > 1) - proc_lock_spin_count = (ERTS_PROC_LOCK_SPIN_COUNT_BASE - * ((int) erts_no_schedulers)); - else if (cpus == 1) - proc_lock_spin_count = 0; - else /* No of cpus unknown. Assume multi proc, but be conservative. */ + if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; - if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) - proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; - proc_lock_trans_spin_cost = proc_lock_spin_count/20; -} - -static ERTS_INLINE erts_proc_lock_waiter_t * -alloc_wtr(void) -{ - erts_proc_lock_waiter_t *wtr; - erts_smp_spin_lock(&wtr_lock); - wtr = waiter_free_list; - if (wtr) { - waiter_free_list = wtr->next; - ERTS_LC_ASSERT(queue_free_list); - wtr->queues = queue_free_list; - queue_free_list = wtr->queues->next; - erts_smp_spin_unlock(&wtr_lock); + proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC + * ((int) erts_no_schedulers)); + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT; } - else { - erts_smp_spin_unlock(&wtr_lock); - wtr = erts_alloc(ERTS_ALC_T_PROC_LCK_WTR, - sizeof(erts_proc_lock_waiter_t)); - erts_smp_gate_init(&wtr->gate); - wtr->wait_locks = (ErtsProcLocks) 0; - wtr->queues = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, - sizeof(erts_proc_lock_queues_t)); - sys_memcpy((void *) wtr->queues, - (void *) &zeroqs, - sizeof(erts_proc_lock_queues_t)); + else if (cpus == 1) { + proc_lock_spin_count = 0; + aux_thr_proc_lock_spin_count = 0; } - return wtr; + else { /* No of cpus unknown. Assume multi proc, but be conservative. */ + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2; + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2; + } + if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; } #ifdef ERTS_ENABLE_LOCK_CHECK static void -check_unused_waiter(erts_proc_lock_waiter_t *wtr) +check_unused_tse(erts_tse_t *wtr) { int i; - ERTS_LC_ASSERT(wtr->wait_locks == 0); + erts_proc_lock_queues_t *queues = wtr->udata; + ERTS_LC_ASSERT(wtr->uflgs == 0); for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - ERTS_LC_ASSERT(!wtr->queues->queue[i]); + ERTS_LC_ASSERT(!queues->queue[i]); } -#define CHECK_UNUSED_WAITER(W) check_unused_waiter((W)) +#define CHECK_UNUSED_TSE(W) check_unused_tse((W)) #else -#define CHECK_UNUSED_WAITER(W) +#define CHECK_UNUSED_TSE(W) #endif +static ERTS_INLINE erts_tse_t * +tse_fetch(erts_pix_lock_t *pix_lock) +{ + erts_tse_t *tse = erts_tse_fetch(); + if (!tse->udata) { + erts_proc_lock_queues_t *qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_unlock(pix_lock); +#endif + erts_smp_spin_lock(&qs_lock); + qs = queue_free_list; + if (qs) { + queue_free_list = queue_free_list->next; + erts_smp_spin_unlock(&qs_lock); + } + else { + erts_smp_spin_unlock(&qs_lock); + qs = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, + sizeof(erts_proc_lock_queues_t)); + sys_memcpy((void *) qs, + (void *) &zeroqs, + sizeof(erts_proc_lock_queues_t)); + } + tse->udata = qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_lock(pix_lock); +#endif + } + tse->uflgs = 0; + return tse; +} static ERTS_INLINE void -free_wtr(erts_proc_lock_waiter_t *wtr) +tse_return(erts_tse_t *tse, int force_free_q) { - CHECK_UNUSED_WAITER(wtr); - erts_smp_spin_lock(&wtr_lock); - wtr->next = waiter_free_list; - waiter_free_list = wtr; - wtr->queues->next = queue_free_list; - queue_free_list = wtr->queues; - erts_smp_spin_unlock(&wtr_lock); + CHECK_UNUSED_TSE(tse); + if (force_free_q || erts_tse_is_tmp(tse)) { + erts_proc_lock_queues_t *qs = tse->udata; + ASSERT(qs); + erts_smp_spin_lock(&qs_lock); + qs->next = queue_free_list; + queue_free_list = qs; + erts_smp_spin_unlock(&qs_lock); + tse->udata = NULL; + } + erts_tse_return(tse); } void erts_proc_lock_prepare_proc_lock_waiter(void) { - erts_tsd_set(waiter_key, (void *) alloc_wtr()); + tse_return(tse_fetch(NULL), 0); } static void -cleanup_waiter(void) +cleanup_tse(void) { - erts_proc_lock_waiter_t *wtr = erts_tsd_get(waiter_key); - if (wtr) - free_wtr(wtr); + erts_tse_t *tse = erts_tse_fetch(); + if (tse) { + if (tse->udata) + tse_return(tse, 1); + else + erts_tse_return(tse); + } } @@ -250,7 +249,7 @@ cleanup_waiter(void) static ERTS_INLINE void enqueue_waiter(erts_proc_lock_queues_t *qs, int ix, - erts_proc_lock_waiter_t *wtr) + erts_tse_t *wtr) { if (!qs->queue[ix]) { qs->queue[ix] = wtr; @@ -266,10 +265,10 @@ enqueue_waiter(erts_proc_lock_queues_t *qs, } } -static erts_proc_lock_waiter_t * +static erts_tse_t * dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) { - erts_proc_lock_waiter_t *wtr = qs->queue[ix]; + erts_tse_t *wtr = qs->queue[ix]; ERTS_LC_ASSERT(qs->queue[ix]); if (wtr->next == wtr) { ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr); @@ -295,10 +294,10 @@ dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) * lock. */ static ERTS_INLINE void -try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) +try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) { ErtsProcLocks got_locks = (ErtsProcLocks) 0; - ErtsProcLocks locks = wtr->wait_locks; + ErtsProcLocks locks = wtr->uflgs; int lock_no; ERTS_LC_ASSERT(lck->queues); @@ -334,7 +333,7 @@ try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) } } - wtr->wait_locks &= ~got_locks; + wtr->uflgs &= ~got_locks; } /* @@ -350,8 +349,8 @@ transfer_locks(Process *p, int unlock) { int transferred = 0; - erts_proc_lock_waiter_t *wake = NULL; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wake = NULL; + erts_tse_t *wtr; ErtsProcLocks unset_waiter = 0; ErtsProcLocks tlocks = trnsfr_lcks; int lock_no; @@ -377,11 +376,11 @@ transfer_locks(Process *p, ERTS_LC_ASSERT(wtr); if (!qs->queue[lock_no]) unset_waiter |= lock; - ERTS_LC_ASSERT(wtr->wait_locks & lock); - wtr->wait_locks &= ~lock; - if (wtr->wait_locks) + ERTS_LC_ASSERT(wtr->uflgs & lock); + wtr->uflgs &= ~lock; + if (wtr->uflgs) try_aquire(&p->lock, wtr); - if (!wtr->wait_locks) { + if (!wtr->uflgs) { /* * The other thread got all locks it needs; * need to wake it up. @@ -412,9 +411,10 @@ transfer_locks(Process *p, erts_pix_unlock(pix_lock); do { - erts_proc_lock_waiter_t *tmp = wake; + erts_tse_t *tmp = wake; wake = wake->next; - erts_smp_gate_let_through(&tmp->gate, 1); + erts_atomic32_set(&tmp->uaflgs, 0); + erts_tse_set(tmp); } while (wake); if (!unlock) @@ -462,26 +462,16 @@ wait_for_locks(Process *p, ErtsProcLocks olflgs) { erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); - int tsd; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; + erts_proc_lock_queues_t *qs; /* Acquire a waiter object on which this thread can wait. */ - wtr = erts_tsd_get(waiter_key); - if (wtr) - tsd = 1; - else { -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_unlock(pix_lock); -#endif - wtr = alloc_wtr(); - tsd = 0; -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_lock(pix_lock); -#endif - } + wtr = tse_fetch(pix_lock); /* Record which locks this waiter needs. */ - wtr->wait_locks = need_locks; + wtr->uflgs = need_locks; + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); @@ -489,14 +479,16 @@ wait_for_locks(Process *p, ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); + qs = wtr->udata; + ASSERT(qs); /* Provide the process with waiter queues, if it doesn't have one. */ if (!p->lock.queues) { - wtr->queues->next = NULL; - p->lock.queues = wtr->queues; + qs->next = NULL; + p->lock.queues = qs; } else { - wtr->queues->next = p->lock.queues->next; - p->lock.queues->next = wtr->queues; + qs->next = p->lock.queues->next; + p->lock.queues->next = qs; } #ifdef ERTS_PROC_LOCK_HARD_DEBUG @@ -506,46 +498,59 @@ wait_for_locks(Process *p, /* Try to aquire locks one at a time in lock order and set wait flag */ try_aquire(&p->lock, wtr); + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif - if (wtr->wait_locks) { /* We didn't get them all; need to wait... */ - /* Got to wait for locks... */ + if (wtr->uflgs) { + /* We didn't get them all; need to wait... */ + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + + erts_atomic32_set(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); - /* - * Wait for needed locks. When we return all needed locks have - * have been acquired by other threads and transfered to us. - */ -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - erts_smp_gate_swait(&wtr->gate, proc_lock_spin_count); -#else - erts_smp_gate_wait(&wtr->gate); -#endif + while (1) { + int res; + erts_tse_reset(wtr); + + if (erts_atomic32_read(&wtr->uaflgs) == 0) + break; + + /* + * Wait for needed locks. When we are woken all needed locks have + * have been acquired by other threads and transfered to us. + * However, we need to be prepared for spurious wakeups. + */ + do { + res = erts_tse_wait(wtr); /* might return EINTR */ + } while (res != 0); + } erts_pix_lock(pix_lock); + + ASSERT(wtr->uflgs == 0); } /* Recover some queues to store in the waiter. */ ERTS_LC_ASSERT(p->lock.queues); if (p->lock.queues->next) { - wtr->queues = p->lock.queues->next; - p->lock.queues->next = wtr->queues->next; + qs = p->lock.queues->next; + p->lock.queues->next = qs->next; } else { - wtr->queues = p->lock.queues; + qs = p->lock.queues; p->lock.queues = NULL; } + wtr->udata = qs; erts_pix_unlock(pix_lock); ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks)); - if (tsd) - CHECK_UNUSED_WAITER(wtr); - else - free_wtr(wtr); + tse_return(wtr, 0); } /* @@ -563,52 +568,57 @@ erts_proc_lock_failed(Process *p, ErtsProcLocks locks, ErtsProcLocks old_lflgs) { -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - int spin_count = 0; -#else - int spin_count = proc_lock_spin_count; -#endif - + int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + int thr_spin_count; + int spin_count; ErtsProcLocks need_locks = locks; ErtsProcLocks olflgs = old_lflgs; - while (need_locks != 0) - { - ErtsProcLocks can_grab = in_order_locks(olflgs, need_locks); + if (erts_thr_get_main_status()) + thr_spin_count = proc_lock_spin_count; + else + thr_spin_count = aux_thr_proc_lock_spin_count; + + spin_count = thr_spin_count; + + while (need_locks != 0) { + ErtsProcLocks can_grab; + + can_grab = in_order_locks(olflgs, need_locks); - if (can_grab == 0) - { + if (can_grab == 0) { /* Someone already has the lowest-numbered lock we want. */ - if (spin_count-- <= 0) - { + if (spin_count-- <= 0) { /* Too many retries, give up and sleep for the lock. */ wait_for_locks(p, pixlck, locks, need_locks, olflgs); return; } + ERTS_SPIN_BODY; + + if (--until_yield == 0) { + until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock); } - else - { + else { /* Try to grab all of the grabbable locks at once with cmpxchg. */ ErtsProcLocks grabbed = olflgs | can_grab; ErtsProcLocks nflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, grabbed, olflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs); - if (nflgs == olflgs) - { + if (nflgs == olflgs) { /* Success! We grabbed the 'can_grab' locks. */ olflgs = grabbed; need_locks &= ~can_grab; -#ifndef ERTS_PROC_LOCK_SPIN_ON_GATE /* Since we made progress, reset the spin count. */ - spin_count = proc_lock_spin_count; -#endif + spin_count = thr_spin_count; } - else - { + else { /* Compare-and-exchange failed, try again. */ olflgs = nflgs; } @@ -945,7 +955,7 @@ erts_proc_lock_init(Process *p) { /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic_init(&p->lock.flags, (long) ERTS_PROC_LOCKS_ALL); + erts_smp_atomic32_init(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL); #else p->lock.flags = ERTS_PROC_LOCKS_ALL; #endif @@ -964,7 +974,7 @@ erts_proc_lock_init(Process *p) { int i; for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - erts_smp_atomic_init(&p->lock.locked[i], (long) 1); + erts_smp_atomic32_init(&p->lock.locked[i], (erts_aint32_t) 1); } #endif } @@ -1407,7 +1417,7 @@ check_queue(erts_proc_lock_t *lck) wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT; if (lflgs & wtr) { int n; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]); wtr = lck->queues->queue[lock_no]; n = 0; diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index d71e5a0a6e..355179f084 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. + * Copyright Ericsson AB 2007-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,20 +54,20 @@ #define ERTS_PROC_LOCK_MAX_BIT 3 -typedef Uint32 ErtsProcLocks; +typedef erts_aint32_t ErtsProcLocks; typedef struct erts_proc_lock_queues_t_ erts_proc_lock_queues_t; typedef struct erts_proc_lock_t_ { #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic_t flags; + erts_smp_atomic32_t flags; #else ErtsProcLocks flags; #endif erts_proc_lock_queues_t *queues; - long refc; + Sint32 refc; #ifdef ERTS_PROC_LOCK_DEBUG - erts_smp_atomic_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; + erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_t lcnt_main; @@ -255,11 +255,7 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); typedef struct { union { -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_t mtx; -#else erts_smp_spinlock_t spnlck; -#endif char buf[64]; /* Try to get locks in different cache lines */ } u; } erts_pix_lock_t; @@ -274,14 +270,19 @@ typedef struct { #if ERTS_PROC_LOCK_ATOMIC_IMPL #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK))) + ((ErtsProcLocks) erts_smp_atomic32_band(&(L)->flags, (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK))) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic_cmpxchg(&(L)->flags, \ - (long) (NEW), (long) (EXPECTED))) + ((ErtsProcLocks) erts_smp_atomic32_bor(&(L)->flags, (erts_aint32_t) (MSK))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \ + (erts_aint32_t) (NEW), \ + (erts_aint32_t) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \ + (erts_aint32_t) (NEW), \ + (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ - ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags)) + ((ErtsProcLocks) erts_smp_atomic32_read(&(L)->flags)) #else /* no opt atomic ops */ @@ -289,6 +290,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *, ErtsProcLocks); ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *, ErtsProcLocks); +ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *, + ErtsProcLocks, + ErtsProcLocks); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -322,7 +326,9 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK)) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags) @@ -330,7 +336,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; -void erts_init_proc_lock(void); +void erts_init_proc_lock(int cpus); void erts_proc_lock_prepare_proc_lock_waiter(void); void erts_proc_lock_failed(Process *, erts_pix_lock_t *, @@ -348,9 +354,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *, - erts_pix_lock_t *, - ErtsProcLocks, - char *file, unsigned int line); + erts_pix_lock_t *, + ErtsProcLocks, + char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *, erts_pix_lock_t *, @@ -372,30 +378,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int); ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_lock(&pixlck->u.mtx); -#else erts_smp_spin_lock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_unlock(&pixlck->u.mtx); -#else erts_smp_spin_unlock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) { -#if ERTS_PROC_LOCK_MUTEX_IMPL - return erts_smp_lc_mtx_is_locked(&pixlck->u.mtx); -#else return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck); -#endif } /* @@ -417,9 +411,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) ErtsProcLocks expct_lflgs = 0; while (1) { - ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, - expct_lflgs | locks, - expct_lflgs); + ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, + expct_lflgs | locks, + expct_lflgs); if (ERTS_LIKELY(lflgs == expct_lflgs)) { /* We successfully grabbed all locks. */ return 0; @@ -535,7 +529,7 @@ erts_smp_proc_unlock__(Process *p, if (want_lflgs != old_lflgs) { ErtsProcLocks new_lflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, want_lflgs, old_lflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs); if (new_lflgs != old_lflgs) { /* cmpxchg failed, try again. */ @@ -627,13 +621,13 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) { ErtsProcLocks lock = ((ErtsProcLocks) 1) << i; if (locks & lock) { - long lock_count; + erts_aint32_t lock_count; if (locked) { - lock_count = erts_smp_atomic_inctest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_inctest(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 1); } else { - lock_count = erts_smp_atomic_dectest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_dectest(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 0); } } diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 03d2a586e3..287327bfe1 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -43,27 +43,48 @@ typedef erts_thr_init_data_t erts_smp_thr_init_data_t; typedef erts_tid_t erts_smp_tid_t; typedef erts_mtx_t erts_smp_mtx_t; typedef erts_cnd_t erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER +#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED +#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED +typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef erts_gate_t erts_smp_gate_t; -typedef ethr_atomic_t erts_smp_atomic_t; +typedef erts_atomic_t erts_smp_atomic_t; +typedef erts_atomic32_t erts_smp_atomic32_t; typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; -typedef erts_thr_timeval_t erts_smp_thr_timeval_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ #else /* #ifdef ERTS_SMP */ -#define ERTS_SMP_THR_OPTS_DEFAULT_INITER 0 +#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} typedef int erts_smp_thr_opts_t; typedef int erts_smp_thr_init_data_t; typedef int erts_smp_tid_t; typedef int erts_smp_mtx_t; typedef int erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_SMP_RWMTX_TYPE_NORMAL 0 +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_LONG_LIVED 0 +#define ERTS_SMP_RWMTX_SHORT_LIVED 0 +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef int erts_smp_gate_t; -typedef long erts_smp_atomic_t; +typedef SWord erts_smp_atomic_t; +typedef Uint32 erts_smp_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -72,11 +93,6 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t; typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t; #endif -typedef struct { - long tv_sec; - long tv_nsec; -} erts_smp_thr_timeval_t; - #endif /* #ifdef ERTS_SMP */ ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id); @@ -103,8 +119,6 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx); ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line); @@ -119,9 +133,17 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx); ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd); ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd); +ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx); @@ -138,23 +160,82 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, long i); -ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, long i); -ERTS_GLB_INLINE long erts_smp_atomic_read(erts_smp_atomic_t *var); -ERTS_GLB_INLINE long erts_smp_atomic_inctest(erts_smp_atomic_t *incp); -ERTS_GLB_INLINE long erts_smp_atomic_dectest(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp); ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp); ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE long erts_smp_atomic_addtest(erts_smp_atomic_t *addp, - long i); -ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, long i); -ERTS_GLB_INLINE long erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, - long new); -ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, - long new, - long expected); -ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); +ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -185,17 +266,10 @@ ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE void erts_smp_thr_time_now(erts_smp_thr_timeval_t *time); ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key); -ERTS_GLB_INLINE void erts_smp_gate_init(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_destroy(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_close(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_smp_gate_wait(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount); #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 @@ -331,22 +405,6 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_set_forksafe(mtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_unset_forksafe(mtx); -#endif -} - ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) { @@ -433,6 +491,25 @@ erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd) } ERTS_GLB_INLINE void +erts_smp_rwmtx_set_reader_group(int no) +{ +#ifdef ERTS_SMP + erts_rwmtx_set_reader_group(no); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt_x(rwmtx, opt, name, extra); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) { #ifdef ERTS_SMP @@ -441,6 +518,16 @@ erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt(rwmtx, opt, name); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name) { #ifdef ERTS_SMP @@ -568,7 +655,7 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_smp_atomic_init(erts_smp_atomic_t *var, long i) +erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_init(var, i); @@ -578,7 +665,7 @@ erts_smp_atomic_init(erts_smp_atomic_t *var, long i) } ERTS_GLB_INLINE void -erts_smp_atomic_set(erts_smp_atomic_t *var, long i) +erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_set(var, i); @@ -587,7 +674,7 @@ erts_smp_atomic_set(erts_smp_atomic_t *var, long i) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var) { #ifdef ERTS_SMP @@ -597,7 +684,7 @@ erts_smp_atomic_read(erts_smp_atomic_t *var) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp) { #ifdef ERTS_SMP @@ -607,7 +694,7 @@ erts_smp_atomic_inctest(erts_smp_atomic_t *incp) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp) { #ifdef ERTS_SMP @@ -637,8 +724,8 @@ erts_smp_atomic_dec(erts_smp_atomic_t *decp) #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i) { #ifdef ERTS_SMP return erts_atomic_addtest(addp, i); @@ -648,7 +735,7 @@ erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i) } ERTS_GLB_INLINE void -erts_smp_atomic_add(erts_smp_atomic_t *addp, long i) +erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_add(addp, i); @@ -657,58 +744,344 @@ erts_smp_atomic_add(erts_smp_atomic_t *addp, long i) #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, long new) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new) { #ifdef ERTS_SMP return erts_atomic_xchg(xchgp, new); #else - long old; + erts_aint_t old; old = *xchgp; *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long new, long expected) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected) { #ifdef ERTS_SMP return erts_atomic_cmpxchg(xchgp, new, expected); #else - long old = *xchgp; + erts_aint_t old = *xchgp; if (old == expected) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask) { #ifdef ERTS_SMP return erts_atomic_bor(var, mask); #else - long old; + erts_aint_t old; old = *var; *var |= mask; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_band(erts_smp_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask) { #ifdef ERTS_SMP return erts_atomic_band(var, mask); #else - long old; + erts_aint_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i) +{ +#ifdef ERTS_SMP + erts_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_init(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + return erts_atomic32_inctest(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + erts_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + return erts_atomic32_addtest(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef ERTS_SMP + return erts_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old; + old = *xchgp; + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg(xchgp, new, expected); +#else + erts_aint32_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_bor(var, mask); +#else + erts_aint32_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_band(var, mask); +#else + erts_aint32_t old; old = *var; *var &= mask; return old; #endif } +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { @@ -878,14 +1251,6 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_smp_thr_time_now(erts_smp_thr_timeval_t *time) -{ -#ifdef ERTS_SMP - erts_thr_time_now(time); -#endif -} - -ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp) { #ifdef ERTS_SMP @@ -919,54 +1284,6 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_smp_gate_init(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_init((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_destroy(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_destroy((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_close(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_close((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no) -{ -#ifdef ERTS_SMP - erts_gate_let_through((erts_gate_t *) gp, no); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_wait(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_wait((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount) -{ -#ifdef ERTS_SMP - erts_gate_swait((erts_gate_t *) gp, spincount); -#endif -} - #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 2924abbd51..f77e8b798f 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2000-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -58,9 +58,9 @@ do { \ #endif #if ET_DEBUG -unsigned tag_val_def_debug(Eterm x, const char *file, unsigned line) +unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line) #else -unsigned tag_val_def(Eterm x) +unsigned tag_val_def(Wterm x) #define file __FILE__ #define line __LINE__ #endif @@ -68,7 +68,9 @@ unsigned tag_val_def(Eterm x) static char msg[32]; switch (x & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: return LIST_DEF; + case TAG_PRIMARY_LIST: + ET_ASSERT(_list_precond(x),file,line); + return LIST_DEF; case TAG_PRIMARY_BOXED: { Eterm hdr = *boxed_val(x); ET_ASSERT(is_header(hdr),file,line); @@ -103,7 +105,7 @@ unsigned tag_val_def(Eterm x) break; } } - sprintf(msg, "tag_val_def: %#lx", x); + sprintf(msg, "tag_val_def: %#lx", (unsigned long) x); et_abort(msg, file, line); #undef file #undef line @@ -121,12 +123,12 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \ return _unchecked_##FUN(x); \ } -ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_aligned); +ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header); -ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,is_boxed); -ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_aligned); +ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond); +ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header); -ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,is_list); +ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond); ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small); ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom); @@ -134,37 +136,38 @@ ET_DEFINE_CHECKED(Uint,header_arity,Eterm,is_header); ET_DEFINE_CHECKED(Uint,arityval,Eterm,is_arity_value); ET_DEFINE_CHECKED(Uint,thing_arityval,Eterm,is_thing); ET_DEFINE_CHECKED(Uint,thing_subtag,Eterm,is_thing); -ET_DEFINE_CHECKED(Eterm*,binary_val,Eterm,is_binary); -ET_DEFINE_CHECKED(Eterm*,fun_val,Eterm,is_fun); +ET_DEFINE_CHECKED(Eterm*,binary_val,Wterm,is_binary); +ET_DEFINE_CHECKED(Eterm*,fun_val,Wterm,is_fun); ET_DEFINE_CHECKED(int,bignum_header_is_neg,Eterm,_is_bignum_header); ET_DEFINE_CHECKED(Eterm,bignum_header_neg,Eterm,_is_bignum_header); ET_DEFINE_CHECKED(Uint,bignum_header_arity,Eterm,_is_bignum_header); -ET_DEFINE_CHECKED(Eterm*,big_val,Eterm,is_big); -ET_DEFINE_CHECKED(Eterm*,float_val,Eterm,is_float); -ET_DEFINE_CHECKED(Eterm*,tuple_val,Eterm,is_tuple); +ET_DEFINE_CHECKED(Eterm*,big_val,Wterm,is_big); +ET_DEFINE_CHECKED(Eterm*,float_val,Wterm,is_float); +ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple); ET_DEFINE_CHECKED(Uint,internal_pid_data,Eterm,is_internal_pid); ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid); ET_DEFINE_CHECKED(Uint,internal_port_data,Eterm,is_internal_port); ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port); -ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Eterm,is_internal_ref); -ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Eterm,is_internal_ref); -ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Eterm,is_internal_ref); +ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref); +ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref); +ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Wterm,is_internal_ref); ET_DEFINE_CHECKED(struct erl_node_*,internal_ref_node,Eterm,is_internal_ref); -ET_DEFINE_CHECKED(Eterm*,external_val,Eterm,is_external); -ET_DEFINE_CHECKED(Uint,external_data_words,Eterm,is_external); -ET_DEFINE_CHECKED(Uint,external_pid_data_words,Eterm,is_external_pid); -ET_DEFINE_CHECKED(Uint,external_pid_data,Eterm,is_external_pid); -ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Eterm,is_external_pid); -ET_DEFINE_CHECKED(Uint,external_port_data_words,Eterm,is_external_port); -ET_DEFINE_CHECKED(Uint,external_port_data,Eterm,is_external_port); -ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Eterm,is_external_port); -ET_DEFINE_CHECKED(Uint,external_ref_data_words,Eterm,is_external_ref); -ET_DEFINE_CHECKED(Uint32*,external_ref_data,Eterm,is_external_ref); +ET_DEFINE_CHECKED(Eterm*,external_val,Wterm,is_external); +ET_DEFINE_CHECKED(Uint,external_data_words,Wterm,is_external); +ET_DEFINE_CHECKED(Uint,external_pid_data_words,Wterm,is_external_pid); +ET_DEFINE_CHECKED(Uint,external_pid_data,Wterm,is_external_pid); +ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Wterm,is_external_pid); +ET_DEFINE_CHECKED(Uint,external_port_data_words,Wterm,is_external_port); +ET_DEFINE_CHECKED(Uint,external_port_data,Wterm,is_external_port); +ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Wterm,is_external_port); +ET_DEFINE_CHECKED(Uint,external_ref_data_words,Wterm,is_external_ref); +ET_DEFINE_CHECKED(Uint32*,external_ref_data,Wterm,is_external_ref); ET_DEFINE_CHECKED(struct erl_node_*,external_ref_node,Eterm,is_external_ref); -ET_DEFINE_CHECKED(Eterm*,export_val,Eterm,is_export); +ET_DEFINE_CHECKED(Eterm*,export_val,Wterm,is_export); +ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr); -ET_DEFINE_CHECKED(Eterm,make_cp,Uint*,_is_aligned); -ET_DEFINE_CHECKED(Uint*,cp_val,Eterm,is_CP); +ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer); +ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP); ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch); ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg); ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index a6596558fa..1d75fa313c 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -20,6 +20,34 @@ #ifndef __ERL_TERM_H #define __ERL_TERM_H +#include "sys.h" /* defines HALFWORD_HEAP */ + +typedef UWord Wterm; /* Full word terms */ + +#if HALFWORD_HEAP +# define HEAP_ON_C_STACK 0 +# if HALFWORD_ASSERT +# ifdef ET_DEBUG +# undef ET_DEBUG +# endif +# define ET_DEBUG 1 +# endif +# if 1 +# define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL +# define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer)) +# define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm)) +# else +# define CHECK_POINTER_MASK 0x0UL +# define COMPRESS_POINTER(AnUint) (AnUint) +# define EXPAND_POINTER(APointer) (APointer) +# endif +#else +# define HEAP_ON_C_STACK 1 +# define CHECK_POINTER_MASK 0x0UL +# define COMPRESS_POINTER(AnUint) (AnUint) +# define EXPAND_POINTER(APointer) (APointer) +#endif + struct erl_node_; /* Declared in erl_node_tables.h */ /* @@ -158,8 +186,15 @@ struct erl_node_; /* Declared in erl_node_tables.h */ /* boxed object access methods */ +#if HALFWORD_HEAP +#define _is_taggable_pointer(x) (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0) +#define _boxed_precond(x) (is_boxed(x)) +#else +#define _is_taggable_pointer(x) (((Uint)(x) & 0x3) == 0) +#define _boxed_precond(x) (is_boxed(x)) +#endif #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) -#define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED) +#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) _ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 @@ -170,12 +205,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #else #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif -#define _unchecked_boxed_val(x) ((Eterm*)((x) - TAG_PRIMARY_BOXED)) -_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm) +#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED))) +_ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) #define boxed_val(x) _ET_APPLY(boxed_val,(x)) /* cons cell ("list") access methods */ -#define _unchecked_make_list(x) ((Uint)(x) + TAG_PRIMARY_LIST) +#define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) _ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 @@ -187,8 +222,13 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define is_list(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST) #define is_not_list(x) (!is_list((x))) #endif -#define _unchecked_list_val(x) ((Eterm*)((x) - TAG_PRIMARY_LIST)) -_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm) +#if HALFWORD_HEAP +#define _list_precond(x) (is_list(x)) +#else +#define _list_precond(x) (is_list(x)) +#endif +#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST)) +_ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define list_val(x) _ET_APPLY(list_val,(x)) #define CONS(hp, car, cdr) \ @@ -198,13 +238,15 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Eterm) #define CDR(x) ((x)[1]) /* generic tagged pointer (boxed or list) access methods */ -#define _unchecked_ptr_val(x) ((Eterm*)((x) & ~((Uint) 0x3))) +#define _unchecked_ptr_val(x) ((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3))) #define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/ #define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm))) #define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/ +#define _unchecked_byte_offset_ptr(x,byte_offs) ((x)+(offs)) +#define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs) /*XXX*/ /* fixnum ("small") access methods */ -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP #define SMALL_BITS (64-4) #define SMALL_DIGITS (17) #else @@ -267,6 +309,7 @@ _ET_DECLARE_CHECKED(Uint,arityval,Eterm) /* thing access methods */ #define is_thing(x) (is_header((x)) && header_is_thing((x))) +#define is_thing_ptr(t) (is_thing((t)->header)) #define _unchecked_thing_arityval(x) _unchecked_header_arity((x)) _ET_DECLARE_CHECKED(Uint,thing_arityval,Eterm) #define thing_arityval(x) _ET_APPLY(thing_arityval,(x)) @@ -301,7 +344,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) #define is_binary(x) (is_boxed((x)) && is_binary_header(*boxed_val((x)))) #define is_not_binary(x) (!is_binary((x))) #define _unchecked_binary_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,binary_val,Wterm) #define binary_val(x) _ET_APPLY(binary_val,(x)) /* process binaries stuff (special case of binaries) */ @@ -318,7 +361,7 @@ _ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm) #define is_fun(x) (is_boxed((x)) && is_fun_header(*boxed_val((x)))) #define is_not_fun(x) (!is_fun((x))) #define _unchecked_fun_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm) #define fun_val(x) _ET_APPLY(fun_val,(x)) /* export access methods */ @@ -326,10 +369,14 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm) #define is_export(x) (is_boxed((x)) && is_export_header(*boxed_val((x)))) #define is_not_export(x) (!is_export((x))) #define _unchecked_export_val(x) _unchecked_boxed_val(x) -_ET_DECLARE_CHECKED(Eterm*,export_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,export_val,Wterm) #define export_val(x) _ET_APPLY(export_val,(x)) #define is_export_header(x) ((x) == HEADER_EXPORT) +#if HALFWORD_HEAP +#define HEADER_EXPORT _make_header(2,_TAG_HEADER_EXPORT) +#else #define HEADER_EXPORT _make_header(1,_TAG_HEADER_EXPORT) +#endif /* bignum access methods */ #define make_pos_bignum_header(sz) _make_header((sz),_TAG_HEADER_POS_BIG) @@ -349,11 +396,11 @@ _ET_DECLARE_CHECKED(Uint,bignum_header_arity,Eterm) #define is_big(x) (is_boxed((x)) && _is_bignum_header(*boxed_val((x)))) #define is_not_big(x) (!is_big((x))) #define _unchecked_big_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,big_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,big_val,Wterm) #define big_val(x) _ET_APPLY(big_val,(x)) /* flonum ("float") access methods */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP #define HEADER_FLONUM _make_header(1,_TAG_HEADER_FLOAT) #else #define HEADER_FLONUM _make_header(2,_TAG_HEADER_FLOAT) @@ -362,7 +409,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Eterm) #define is_float(x) (is_boxed((x)) && *boxed_val((x)) == HEADER_FLONUM) #define is_not_float(x) (!is_float(x)) #define _unchecked_float_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,float_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,float_val,Wterm) #define float_val(x) _ET_APPLY(float_val,(x)) /* Float definition for byte and word access */ @@ -374,21 +421,22 @@ typedef union float_def byte fb[sizeof(ieee754_8)]; Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)]; Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)]; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP Uint fdw; #endif } FloatDef; -#ifdef ARCH_64 -#define GET_DOUBLE(x, f) (f).fdw = *(float_val(x)+1) +#if defined(ARCH_64) && !HALFWORD_HEAP + +#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1) #define PUT_DOUBLE(f, x) *(x) = HEADER_FLONUM, \ *((x)+1) = (f).fdw #define GET_DOUBLE_DATA(p, f) (f).fdw = *((Uint *) (p)) #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fdw #else -#define GET_DOUBLE(x, f) (f).fw[0] = *(float_val(x)+1), \ - (f).fw[1] = *(float_val(x)+2) +#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fw[0] = *((fval)+1), \ + (f).fw[1] = *((fval)+2) #define PUT_DOUBLE(f, x) *(x) = HEADER_FLONUM, \ *((x)+1) = (f).fw[0], \ @@ -398,6 +446,9 @@ typedef union float_def #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fw[0],\ *(((Uint *) (p))+1) = (f).fw[1] #endif + +#define GET_DOUBLE(x, f) FLOAT_VAL_GET_DOUBLE(float_val(x), f) + #define DOUBLE_DATA_WORDS (sizeof(ieee754_8)/sizeof(Eterm)) #define FLOAT_SIZE_OBJECT (DOUBLE_DATA_WORDS+1) @@ -409,7 +460,7 @@ typedef union float_def (is_boxed((x)) && *boxed_val((x)) == make_arityval((a))) #define is_not_tuple_arity(x, a) (!is_tuple_arity((x),(a))) #define _unchecked_tuple_val(x) _unchecked_boxed_val(x) -_ET_DECLARE_CHECKED(Eterm*,tuple_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) #define tuple_val(x) _ET_APPLY(tuple_val,(x)) #define TUPLE0(t) \ @@ -679,7 +730,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define ERTS_MAX_REF_NUMBERS 3 #define ERTS_REF_NUMBERS ERTS_MAX_REF_NUMBERS -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP # define ERTS_REF_WORDS (ERTS_REF_NUMBERS/2 + 1) # define ERTS_REF_32BIT_WORDS (ERTS_REF_NUMBERS+1) #else @@ -701,7 +752,7 @@ typedef struct { #define make_ref_thing_header(DW) \ _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF) -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP /* * Ref layout on a 64-bit little endian machine: @@ -748,21 +799,24 @@ do { \ ((RefThing*) internal_ref_val(x)) #define is_internal_ref(x) \ - (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x)))) + (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x)))) + #define is_not_internal_ref(x) \ (!is_internal_ref((x))) #define _unchecked_internal_ref_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm) #define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x)) +#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t))) #define _unchecked_internal_ref_data_words(x) \ (_unchecked_thing_arityval(*_unchecked_internal_ref_val(x))) -_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm) #define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x)) -#define _unchecked_internal_ref_data(x) (_unchecked_ref_thing_ptr(x)->data.ui32) -_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Eterm) +#define internal_thing_ref_data(thing) ((thing)->data.ui32) +#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x))) +_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm) #define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x)) #define _unchecked_internal_ref_node(x) erts_this_node @@ -779,10 +833,10 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |A A A A A A A A A A A A A A A A A A A A A A A A A A|t t t t|0 0| Thing * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - * |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next - * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E| ErlNode * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X| Data 0 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * . . . @@ -793,7 +847,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * t : External pid thing tag (1100) * t : External port thing tag (1101) * t : External ref thing tag (1110) - * N : Next (external thing) pointer + * N : Next (off_heap) pointer * E : ErlNode pointer * X : Type specific data * @@ -807,11 +861,19 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * */ +/* XXX:PaN - this structure is not perfect for halfword heap, it takes + a lot of memory due to padding, and the array will not begin at the end of the + structure, as otherwise expected. Be sure to access data.ui32 array and not try + to do pointer manipulation on an Eterm * to reach the actual data... + + XXX:Sverk - Problem made worse by "one off-heap list" when 'next' pointer + must align with 'next' in ProcBin, erl_fun_thing and erl_off_heap_header. +*/ typedef struct external_thing_ { /* ----+ */ Eterm header; /* | */ - struct external_thing_ *next; /* > External thing head */ - struct erl_node_ *node; /* | */ + struct erl_node_* node; /* > External thing head */ + struct erl_off_heap_header* next; /* | */ /* ----+ */ union { Uint32 ui32[1]; @@ -839,14 +901,14 @@ typedef struct external_thing_ { #define is_external_header(x) \ (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID) -#define is_external(x) \ - (is_boxed((x)) && is_external_header(*boxed_val((x)))) +#define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x)))) + #define is_external_pid(x) \ (is_boxed((x)) && is_external_pid_header(*boxed_val((x)))) #define is_external_port(x) \ - (is_boxed((x)) && is_external_port_header(*boxed_val((x)))) -#define is_external_ref(x) \ - (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x)))) + (is_boxed((x)) && is_external_port_header(*boxed_val((x)))) + +#define is_external_ref(x) (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x)))) #define _unchecked_is_external(x) \ (_unchecked_is_boxed((x)) && is_external_header(*_unchecked_boxed_val((x)))) @@ -864,17 +926,21 @@ typedef struct external_thing_ { #define make_external_ref make_external #define _unchecked_external_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,external_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,external_val,Wterm) #define external_val(x) _ET_APPLY(external_val,(x)) #define external_thing_ptr(x) ((ExternalThing *) external_val((x))) #define _unchecked_external_thing_ptr(x) \ ((ExternalThing *) _unchecked_external_val((x))) +#define _unchecked_external_thing_data_words(thing) \ + (_unchecked_thing_arityval((thing)->header) + (1 - EXTERNAL_THING_HEAD_SIZE)) +_ET_DECLARE_CHECKED(Uint,external_thing_data_words,ExternalThing*) +#define external_thing_data_words(thing) _ET_APPLY(external_thing_data_words,(thing)) + #define _unchecked_external_data_words(x) \ - (_unchecked_thing_arityval(_unchecked_external_thing_ptr((x))->header) \ - + (1 - EXTERNAL_THING_HEAD_SIZE)) -_ET_DECLARE_CHECKED(Uint,external_data_words,Eterm) + _unchecked_external_thing_data_words(_unchecked_external_thing_ptr((x))) +_ET_DECLARE_CHECKED(Uint,external_data_words,Wterm) #define external_data_words(x) _ET_APPLY(external_data_words,(x)) #define _unchecked_external_data(x) (_unchecked_external_thing_ptr((x))->data.ui) @@ -885,15 +951,15 @@ _ET_DECLARE_CHECKED(Uint,external_data_words,Eterm) #define _unchecked_external_pid_data_words(x) \ _unchecked_external_data_words((x)) -_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Wterm) #define external_pid_data_words(x) _ET_APPLY(external_pid_data_words,(x)) #define _unchecked_external_pid_data(x) _unchecked_external_data((x))[0] -_ET_DECLARE_CHECKED(Uint,external_pid_data,Eterm) +_ET_DECLARE_CHECKED(Uint,external_pid_data,Wterm) #define external_pid_data(x) _ET_APPLY(external_pid_data,(x)) #define _unchecked_external_pid_node(x) _unchecked_external_node((x)) -_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm) +_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Wterm) #define external_pid_node(x) _ET_APPLY(external_pid_node,(x)) #define external_pid_number(x) _GET_PID_NUM(external_pid_data((x))) @@ -901,27 +967,29 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm) #define _unchecked_external_port_data_words(x) \ _unchecked_external_data_words((x)) -_ET_DECLARE_CHECKED(Uint,external_port_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,external_port_data_words,Wterm) #define external_port_data_words(x) _ET_APPLY(external_port_data_words,(x)) #define _unchecked_external_port_data(x) _unchecked_external_data((x))[0] -_ET_DECLARE_CHECKED(Uint,external_port_data,Eterm) +_ET_DECLARE_CHECKED(Uint,external_port_data,Wterm) #define external_port_data(x) _ET_APPLY(external_port_data,(x)) #define _unchecked_external_port_node(x) _unchecked_external_node((x)) -_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Eterm) +_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Wterm) #define external_port_node(x) _ET_APPLY(external_port_node,(x)) #define external_port_number(x) _GET_PORT_NUM(external_port_data((x))) #define _unchecked_external_ref_data_words(x) \ _unchecked_external_data_words((x)) -_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Wterm) #define external_ref_data_words(x) _ET_APPLY(external_ref_data_words,(x)) +#define external_thing_ref_data_words(thing) external_thing_data_words(thing) #define _unchecked_external_ref_data(x) (_unchecked_external_thing_ptr((x))->data.ui32) -_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Eterm) +_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm) #define external_ref_data(x) _ET_APPLY(external_ref_data,(x)) +#define external_thing_ref_data(thing) ((thing)->data.ui32) #define _unchecked_external_ref_node(x) _unchecked_external_node((x)) _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) @@ -944,15 +1012,15 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #error "fix yer arch, like" #endif -#define _unchecked_make_cp(x) ((Eterm)(x)) -_ET_DECLARE_CHECKED(Eterm,make_cp,Uint*) +#define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x)) +_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*) #define make_cp(x) _ET_APPLY(make_cp,(x)) #define is_not_CP(x) ((x) & _CPMASK) #define is_CP(x) (!is_not_CP(x)) -#define _unchecked_cp_val(x) ((Uint*)(x)) -_ET_DECLARE_CHECKED(Uint*,cp_val,Eterm) +#define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x)) +_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm) #define cp_val(x) _ET_APPLY(cp_val,(x)) #define make_catch(x) (((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH) @@ -1033,10 +1101,10 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) #define SMALL_DEF 0xf #if ET_DEBUG -extern unsigned tag_val_def_debug(Eterm, const char*, unsigned); +extern unsigned tag_val_def_debug(Wterm, const char*, unsigned); #define tag_val_def(x) tag_val_def_debug((x),__FILE__,__LINE__) #else -extern unsigned tag_val_def(Eterm); +extern unsigned tag_val_def(Wterm); #endif #define not_eq_tags(X,Y) (tag_val_def((X)) ^ tag_val_def((Y))) @@ -1052,5 +1120,81 @@ extern unsigned tag_val_def(Eterm); #define FLOAT_BIG _NUMBER_CODE(FLOAT_DEF,BIG_DEF) #define FLOAT_FLOAT _NUMBER_CODE(FLOAT_DEF,FLOAT_DEF) +#if HALFWORD_HEAP +#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE))) +#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE)) + +#else /* HALFWORD_HEAP */ + +#define ptr2rel(PTR,BASE) (PTR) +#define rterm2wterm(REL,BASE) (REL) + +#endif /* !HALFWORD_HEAP */ + +#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE)) +#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE)) +#define make_fun_rel make_boxed_rel +#define make_binary_rel make_boxed_rel +#define make_tuple_rel make_boxed_rel +#define make_external_rel make_boxed_rel +#define make_internal_ref_rel make_boxed_rel + +#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE)) +#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE)) +#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE)) +#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE)) +#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE)) +#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE)) +#define big_val_rel(RTERM,BASE) big_val(rterm2wterm(RTERM,BASE)) +#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE)) +#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE)) + +#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE)) +#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE)) + +#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE)) +#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE)) + +#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE)) +#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE)) +#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE)) + +#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE)) +#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE)) +#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE)) +#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE)) +#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE)) +#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE)) + +#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f) + +#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE)) +#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE)) +#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE)) +#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE)) +#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE)) + +#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE)) + + +#if HALFWORD_HEAP +ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) +{ + /* If bases differ, assume a and b are on different "heaps", + ie can only be same if immed */ + ASSERT(a_base == b_base || is_immed(a) || is_immed(b) + || rterm2wterm(a,a_base) != rterm2wterm(b,b_base)); + + return a == b && (a_base == b_base || is_immed(a)); +} +#endif + +#else /* !HALFWORD_HEAP */ +#define is_same(A,A_BASE,B,B_BASE) ((A)==(B)) +#endif + #endif /* __ERL_TERM_H */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index d635916dd8..8c9cace0c5 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -25,6 +25,8 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +#define ERTS_SPIN_BODY ETHR_SPIN_BODY + #include "sys.h" #ifdef USE_THREADS @@ -34,6 +36,18 @@ #include "erl_lock_count.h" #include "erl_term.h" +#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4 +/* + * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have + * only seen this bug in glibc versions before 2.4. Note that condition + * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by + * this bug since these implementations may use mutexes internally. + */ +# define ERTS_THR_HAVE_BUSY_DESTROY_BUG +#endif + +#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER + #ifdef ERTS_ENABLE_LOCK_COUNT #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__) @@ -46,6 +60,7 @@ #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER typedef ethr_thr_opts erts_thr_opts_t; typedef ethr_init_data erts_thr_init_data_t; +typedef ethr_late_init_data erts_thr_late_init_data_t; typedef ethr_tid erts_tid_t; /* mutex */ @@ -71,9 +86,23 @@ typedef struct { erts_lcnt_lock_t lcnt; #endif } erts_rwmtx_t; + +#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER +#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL +#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED +#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED +#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED +typedef ethr_rwmutex_opt erts_rwmtx_opt_t; + typedef ethr_tsd_key erts_tsd_key_t; -typedef ethr_gate erts_gate_t; +typedef ethr_ts_event erts_tse_t; +typedef ethr_sint_t erts_aint_t; typedef ethr_atomic_t erts_atomic_t; +typedef ethr_sint32_t erts_aint32_t; +typedef ethr_atomic32_t erts_atomic32_t; /* spinlock */ typedef struct { @@ -97,41 +126,48 @@ typedef struct { #endif } erts_rwlock_t; -typedef ethr_timeval erts_thr_timeval_t; __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_REC_MTX_INITER \ - {ETHR_REC_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1,THE_NON_VALUE,ERTS_LC_FLG_LT_MUTEX)} -#define ERTS_MTX_INITER \ - {ETHR_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_MUTEX)} -#else -#define ERTS_REC_MTX_INITER {ETHR_REC_MUTEX_INITER} -#define ERTS_MTX_INITER {ETHR_MUTEX_INITER} -#endif -#define ERTS_CND_INITER ETHR_COND_INITER #define ERTS_THR_INIT_DATA_DEF_INITER ETHR_INIT_DATA_DEFAULT_INITER +#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \ + ETHR_LATE_INIT_DATA_DEFAULT_INITER #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT #endif - #else /* #ifdef USE_THREADS */ +#define ERTS_THR_MEMORY_BARRIER + #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; typedef int erts_thr_init_data_t; +typedef int erts_thr_late_init_data_t; typedef int erts_tid_t; typedef int erts_mtx_t; typedef int erts_cnd_t; +#define ERTS_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_RWMTX_TYPE_NORMAL 0 +#define ERTS_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_RWMTX_LONG_LIVED 0 +#define ERTS_RWMTX_SHORT_LIVED 0 +#define ERTS_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_rwmtx_opt_t; typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; -typedef int erts_gate_t; -typedef long erts_atomic_t; +typedef int erts_tse_t; +typedef SWord erts_aint_t; +typedef SWord erts_atomic_t; +typedef SWord erts_aint32_t; +typedef SWord erts_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -139,12 +175,7 @@ typedef struct { } erts_rwlock_t; typedef struct { int gcc_is_buggy; } erts_spinlock_t; typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif -typedef struct { - long tv_sec; - long tv_nsec; -} erts_thr_timeval_t; -#define ERTS_REC_MTX_INITER 0 #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -153,7 +184,13 @@ typedef struct { #endif /* #ifdef USE_THREADS */ +#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) +#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) +#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) +#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) + ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id); +ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id); ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts); ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res); @@ -162,9 +199,6 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void erts_rec_mtx_init(erts_mtx_t *mtx); -#endif ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, @@ -173,8 +207,6 @@ ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_set_forksafe(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_unset_forksafe(erts_mtx_t *mtx); ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line); @@ -188,9 +220,17 @@ ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd); +ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); @@ -207,23 +247,69 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i); -ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i); -ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var); -ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp); -ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp); +ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp); ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp); ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp); -ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp, - long i); -ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i); -ERTS_GLB_INLINE long erts_atomic_xchg(erts_atomic_t *xchgp, - long new); -ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, - long new, - long expected); -ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var); +ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp); +ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp); +ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp, + erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var); +ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, + char *name, + Eterm extra, + Uint16 opt); ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra); @@ -254,17 +340,20 @@ ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock); -ERTS_GLB_INLINE void erts_thr_time_now(erts_thr_timeval_t *time); ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); -ERTS_GLB_INLINE void erts_gate_init(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_destroy(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_close(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_let_through(erts_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_gate_wait(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_swait(erts_gate_t *gp, int spincount); +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); +ERTS_GLB_INLINE int erts_thr_get_main_status(void); +ERTS_GLB_INLINE void erts_thr_yield(void); #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 @@ -286,15 +375,21 @@ erts_thr_init(erts_thr_init_data_t *id) } ERTS_GLB_INLINE void +erts_thr_late_init(erts_thr_late_init_data_t *id) +{ +#ifdef USE_THREADS + int res = ethr_late_init(id); + if (res) + erts_thr_fatal_error(res, "complete initialization of thread library"); +#endif +} + +ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts) { #ifdef USE_THREADS -#ifdef ERTS_ENABLE_LOCK_COUNT - int res = erts_lcnt_thr_create(tid, func, arg, opts); -#else int res = ethr_thr_create(tid, func, arg, opts); -#endif if (res) erts_thr_fatal_error(res, "create thread"); #endif @@ -362,20 +457,6 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) #endif } - -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void -erts_rec_mtx_init(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_rec_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize recursive mutex"); -#endif -} -#endif - - ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { @@ -422,9 +503,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -463,9 +542,7 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -487,28 +564,16 @@ erts_mtx_destroy(erts_mtx_t *mtx) erts_lcnt_destroy_lock(&mtx->lcnt); #endif res = ethr_mutex_destroy(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "destroy mutex"); -#endif -} - -ERTS_GLB_INLINE void -erts_mtx_set_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_set_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "set mutex forksafe"); + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy mutex destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } #endif -} - -ERTS_GLB_INLINE void -erts_mtx_unset_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_unset_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "unset mutex forksafe"); + erts_thr_fatal_error(res, "destroy mutex"); + } #endif } @@ -531,11 +596,7 @@ erts_mtx_trylock(erts_mtx_t *mtx) #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, res); -#endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try lock mutex"); - +#endif return res; #else return 0; @@ -551,19 +612,16 @@ erts_mtx_lock(erts_mtx_t *mtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif - res = ethr_mutex_lock(&mtx->mtx); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&mtx->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "lock mutex"); #endif } @@ -571,16 +629,13 @@ ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&mtx->lcnt); #endif - res = ethr_mutex_unlock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "unlock mutex"); + ethr_mutex_unlock(&mtx->mtx); #endif } @@ -613,8 +668,16 @@ erts_cnd_destroy(erts_cnd_t *cnd) { #ifdef USE_THREADS int res = ethr_cond_destroy(cnd); - if (res) + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy cond destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif erts_thr_fatal_error(res, "destroy condition variable"); + } #endif } @@ -648,9 +711,7 @@ ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_signal(cnd); - if (res) - erts_thr_fatal_error(res, "signal on condition variable"); + ethr_cond_signal(cnd); #endif } @@ -659,19 +720,34 @@ ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_broadcast(cnd); - if (res) - erts_thr_fatal_error(res, "broadcast on condition variable"); + ethr_cond_broadcast(cnd); #endif } /* rwmutex */ ERTS_GLB_INLINE void -erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) +erts_rwmtx_set_reader_group(int no) +{ +#ifdef USE_THREADS + int res; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX); +#endif + res = ethr_rwmutex_set_reader_group(no); + if (res != 0) + erts_thr_fatal_error(res, "set reader group"); +#endif +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -684,10 +760,20 @@ erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void -erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, + char *name, + Eterm extra) +{ + erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra); +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -700,6 +786,12 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) } ERTS_GLB_INLINE void +erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +{ + erts_rwmtx_init_opt(rwmtx, NULL, name); +} + +ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS @@ -711,8 +803,16 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) erts_lcnt_destroy_lock(&rwmtx->lcnt); #endif res = ethr_rwmutex_destroy(&rwmtx->rwmtx); - if (res != 0) + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy rwmutex destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif erts_thr_fatal_error(res, "destroy rwmutex"); + } #endif } @@ -736,9 +836,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try read lock rwmutex"); return res; #else @@ -754,19 +851,16 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_rlock(&rwmtx->rwmtx); + ethr_rwmutex_rlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "read lock rwmutex"); #endif } @@ -774,16 +868,13 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_runlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "read unlock rwmutex"); + ethr_rwmutex_runlock(&rwmtx->rwmtx); #endif } @@ -808,9 +899,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try write lock rwmutex"); return res; #else @@ -826,19 +914,16 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwlock(&rwmtx->rwmtx); + ethr_rwmutex_rwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "write lock rwmutex"); #endif } @@ -846,16 +931,13 @@ ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwunlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "write unlock rwmutex"); + ethr_rwmutex_rwunlock(&rwmtx->rwmtx); #endif } @@ -914,66 +996,50 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_atomic_init(erts_atomic_t *var, long i) +erts_atomic_init(erts_atomic_t *var, erts_aint_t i) { #ifdef USE_THREADS - int res = ethr_atomic_init(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic init"); + ethr_atomic_init(var, i); #else *var = i; #endif } ERTS_GLB_INLINE void -erts_atomic_set(erts_atomic_t *var, long i) +erts_atomic_set(erts_atomic_t *var, erts_aint_t i) { #ifdef USE_THREADS - int res = ethr_atomic_set(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic set"); + ethr_atomic_set(var, i); #else *var = i; #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var) { #ifdef USE_THREADS - long i; - int res = ethr_atomic_read(var, &i); - if (res) - erts_thr_fatal_error(res, "perform atomic read"); - return i; + return ethr_atomic_read(var); #else return *var; #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_inctest(incp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic increment and test"); - return test; + return ethr_atomic_inc_read(incp); #else return ++(*incp); #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_dectest(decp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement and test"); - return test; + return ethr_atomic_dec_read(decp); #else return --(*decp); #endif @@ -983,9 +1049,7 @@ ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp) { #ifdef USE_THREADS - int res = ethr_atomic_inc(incp); - if (res) - erts_thr_fatal_error(res, "perform atomic increment"); + ethr_atomic_inc(incp); #else ++(*incp); #endif @@ -995,100 +1059,364 @@ ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp) { #ifdef USE_THREADS - int res = ethr_atomic_dec(decp); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement"); + ethr_atomic_dec(decp); #else --(*decp); #endif } -ERTS_GLB_INLINE long -erts_atomic_addtest(erts_atomic_t *addp, long i) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_addtest(erts_atomic_t *addp, erts_aint_t i) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_addtest(addp, i, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic addition and test"); - return test; + return ethr_atomic_add_read(addp, i); #else return *addp += i; #endif } ERTS_GLB_INLINE void -erts_atomic_add(erts_atomic_t *addp, long i) +erts_atomic_add(erts_atomic_t *addp, erts_aint_t i) { #ifdef USE_THREADS - int res = ethr_atomic_add(addp, i); - if (res) - erts_thr_fatal_error(res, "perform atomic addition"); + ethr_atomic_add(addp, i); #else *addp += i; #endif } -ERTS_GLB_INLINE long -erts_atomic_xchg(erts_atomic_t *xchgp, long new) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_xchg(xchgp, new, &old); - if (res) - erts_thr_fatal_error(res, "perform atomic exchange"); + return ethr_atomic_xchg(xchgp, new); #else - old = *xchgp; + erts_aint_t old = *xchgp; *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_cmpxchg(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg(xchgp, new, expected); +#else + erts_aint_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_bor(erts_atomic_t *var, erts_aint_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic_read_bor(var, mask); +#else + erts_aint_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_band(erts_atomic_t *var, erts_aint_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic_read_band(var, mask); +#else + erts_aint_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_acqb(erts_atomic_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i) +{ +#ifdef USE_THREADS + ethr_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_dec_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_dectest_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic_dec_read_relb(decp); +#else + return --(*decp); #endif +} + +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif } -ERTS_GLB_INLINE long -erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) { #ifdef USE_THREADS - long old; - int res = ethr_atomic_cmpxchg(xchgp, new, expected, &old); - if (ERTS_UNLIKELY(res != 0)) - erts_thr_fatal_error(res, "perform atomic exchange"); + return ethr_atomic_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif +} + +/* atomic32 */ + +ERTS_GLB_INLINE void +erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_init(var, i); #else - long old = *xchgp; + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_inctest(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + return ethr_atomic32_inc_read(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_inc(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + ethr_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef USE_THREADS + return ethr_atomic32_add_read(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef USE_THREADS + return ethr_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old = *xchgp; + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg(xchgp, new, expected); +#else + erts_aint32_t old = *xchgp; if (old == expected) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_atomic_bor(erts_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_or_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise or"); + return ethr_atomic32_read_bor(var, mask); #else + erts_aint32_t old; old = *var; *var |= mask; -#endif return old; +#endif } -ERTS_GLB_INLINE long -erts_atomic_band(erts_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_and_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise and"); + return ethr_atomic32_read_band(var, mask); #else + erts_aint32_t old; old = *var; *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_acqb(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; #endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif } /* spinlock */ @@ -1112,6 +1440,26 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, + Uint16 opt) +{ +#ifdef USE_THREADS + int res = ethr_spinlock_init(&lock->slck); + if (res) + erts_thr_fatal_error(res, "init spinlock"); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); +#endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra); +#endif +#else + (void)lock; +#endif +} + + +ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name) { #ifdef USE_THREADS @@ -1141,8 +1489,16 @@ erts_spinlock_destroy(erts_spinlock_t *lock) erts_lcnt_destroy_lock(&lock->lcnt); #endif res = ethr_spinlock_destroy(&lock->slck); - if (res) - erts_thr_fatal_error(res, "destroy spinlock"); + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy spinlock destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif + erts_thr_fatal_error(res, "destroy rwlock"); + } #else (void)lock; #endif @@ -1152,16 +1508,13 @@ ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&lock->lcnt); #endif - res = ethr_spin_unlock(&lock->slck); - if (res) - erts_thr_fatal_error(res, "release spin lock"); + ethr_spin_unlock(&lock->slck); #else (void)lock; #endif @@ -1175,19 +1528,16 @@ erts_spin_lock(erts_spinlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&lock->lcnt); #endif - res = ethr_spin_lock(&lock->slck); + ethr_spin_lock(&lock->slck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take spin lock"); #else (void)lock; #endif @@ -1257,8 +1607,16 @@ erts_rwlock_destroy(erts_rwlock_t *lock) erts_lcnt_destroy_lock(&lock->lcnt); #endif res = ethr_rwlock_destroy(&lock->rwlck); - if (res) + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy rwlock destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif erts_thr_fatal_error(res, "destroy rwlock"); + } #else (void)lock; #endif @@ -1268,16 +1626,13 @@ ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release read lock"); + ethr_read_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1291,19 +1646,16 @@ erts_read_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_lock(&lock->rwlck); + ethr_read_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take read lock"); #else (void)lock; #endif @@ -1313,16 +1665,13 @@ ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release write lock"); + ethr_write_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1336,19 +1685,16 @@ erts_write_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_lock(&lock->rwlck); + ethr_write_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take write lock"); #else (void)lock; #endif @@ -1383,16 +1729,6 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_thr_time_now(erts_thr_timeval_t *time) -{ -#ifdef USE_THREADS - int res = ethr_time_now(time); - if (res) - erts_thr_fatal_error(res, "get current time"); -#endif -} - -ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp) { #ifdef USE_THREADS @@ -1432,66 +1768,95 @@ erts_tsd_get(erts_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_gate_init(erts_gate_t *gp) +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void) { #ifdef USE_THREADS - int res = ethr_gate_init((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "initialize gate"); + return (erts_tse_t *) ethr_get_ts_event(); +#else + return (erts_tse_t *) NULL; #endif } -ERTS_GLB_INLINE void -erts_gate_destroy(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_destroy((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "destroy gate"); + ethr_leave_ts_event(ep); #endif } -ERTS_GLB_INLINE void -erts_gate_close(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_close((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "close gate"); + ethr_event_set(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_let_through(erts_gate_t *gp, unsigned no) +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_let_through((ethr_gate *) gp, no); - if (res != 0) - erts_thr_fatal_error(res, "let through gate"); + ethr_event_reset(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_wait(erts_gate_t *gp) +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) +{ +#ifdef USE_THREADS + return ethr_event_wait(&((ethr_ts_event *) ep)->event); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) +{ +#ifdef USE_THREADS + return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) +{ +#ifdef USE_THREADS + return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP; +#else + return 0; +#endif +} + +ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no) { #ifdef USE_THREADS - int res = ethr_gate_wait((ethr_gate *) gp); + int res = ethr_set_main_thr_status(on, no); if (res != 0) - erts_thr_fatal_error(res, "wait on gate"); + erts_thr_fatal_error(res, "set thread main status"); #endif } -ERTS_GLB_INLINE void -erts_gate_swait(erts_gate_t *gp, int spincount) +ERTS_GLB_INLINE int erts_thr_get_main_status(void) { #ifdef USE_THREADS - int res = ethr_gate_swait((ethr_gate *) gp, spincount); + int main_status; + int res = ethr_get_main_thr_status(&main_status); if (res != 0) - erts_thr_fatal_error(res, "swait on gate"); + erts_thr_fatal_error(res, "get thread main status"); + return main_status; +#else + return 1; #endif } +ERTS_GLB_INLINE void erts_thr_yield(void) +{ +#ifdef USE_THREADS + int res = ETHR_YIELD(); + if (res != 0) + erts_thr_fatal_error(res, "yield"); +#endif +} + + #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 6f6b971d34..d0ad73cd81 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -20,11 +20,15 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ +extern erts_smp_atomic_t do_time; /* set at clock interrupt */ +extern SysTimeval erts_first_emu_time; + /* ** Timer entry: */ typedef struct erl_timer { struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* prev; /* prev entry tiw slot or chain */ Uint slot; /* slot in timer wheel */ Uint count; /* number of loops remaining */ int active; /* 1=activated, 0=deactivated */ @@ -39,7 +43,6 @@ typedef void (*ErlTimeoutProc)(void*); typedef void (*ErlCancelProc)(void*); #ifdef ERTS_SMP - /* * Process and port timer */ @@ -61,7 +64,66 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, ErlTimeoutProc timeout_func, Uint timeout); void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer); +#endif + +/* timer-wheel api */ +void erts_init_time(void); +void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); +void erts_cancel_timer(ErlTimer*); +void erts_bump_timer(erts_aint_t); +Uint erts_timer_wheel_memory_size(void); +Uint erts_time_left(ErlTimer *); +erts_aint_t erts_next_time(void); + +#ifdef DEBUG +void erts_p_slpq(void); #endif +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void); +ERTS_GLB_INLINE void erts_do_time_add(long); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); } +ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); } + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +/* time_sup */ + +#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) +# ifndef HAVE_ERTS_NOW_CPU +# define HAVE_ERTS_NOW_CPU +# ifdef HAVE_GETHRVTIME +# define erts_start_now_cpu() sys_start_hrvtime() +# define erts_stop_now_cpu() sys_stop_hrvtime() +# endif +# endif +void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif + +void erts_get_timeval(SysTimeval *tv); +long erts_get_time(void); +void erts_get_emu_time(SysTimeval *); + +ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int +erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) +{ + if (t1p->tv_sec == t2p->tv_sec) { + if (t1p->tv_usec < t2p->tv_usec) + return -1; + else if (t1p->tv_usec > t2p->tv_usec) + return 1; + return 0; + } + return t1p->tv_sec < t2p->tv_sec ? -1 : 1; +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* ERL_TIME_H__ */ diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index c15f85f8f1..ca4b54188e 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -358,10 +358,6 @@ static int clock_resolution; ** instead of something like select. */ -#if defined(ERTS_TIMER_THREAD) -static ERTS_INLINE void init_erts_deliver_time(const SysTimeval *inittv) { } -static ERTS_INLINE void do_erts_deliver_time(const SysTimeval *current) { } -#else static SysTimeval last_delivered; static void init_erts_deliver_time(const SysTimeval *inittv) @@ -389,11 +385,10 @@ static void do_erts_deliver_time(const SysTimeval *current) this by simply pretend as if the time stood still. :) */ if (elapsed > 0) { - do_time_add(elapsed); + erts_do_time_add(elapsed); last_delivered = cur_time; } } -#endif int erts_init_time_sup(void) @@ -650,6 +645,22 @@ local_to_univ(Sint *year, Sint *month, Sint *day, t.tm_sec = *second; t.tm_isdst = isdst; the_clock = mktime(&t); + if (the_clock == -1) { + if (isdst) { + /* If this is a timezone without DST and the OS (correctly) + refuses to give us a DST time, we simulate the Linux/Solaris + behaviour of giving the same data as if is_dst was not set. */ + t.tm_isdst = 0; + the_clock = mktime(&t); + if (the_clock == -1) { + /* Failed anyway, something else is bad - will be a badarg */ + return 0; + } + } else { + /* Something else is the matter, badarg. */ + return 0; + } + } #ifdef HAVE_GMTIME_R gmtime_r(&the_clock, (tm = &tmbuf)); #else @@ -663,6 +674,10 @@ local_to_univ(Sint *year, Sint *month, Sint *day, *second = tm->tm_sec; return 1; } +#if defined(HAVE_POSIX2TIME) && defined(HAVE_DECL_POSIX2TIME) && \ + !HAVE_DECL_POSIX2TIME +extern time_t posix2time(time_t); +#endif int univ_to_local(Sint *year, Sint *month, Sint *day, @@ -766,7 +781,6 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) to a struct timeval representing current time (to save a gettimeofday() where possible) or NULL */ -#if !defined(ERTS_TIMER_THREAD) void erts_deliver_time(void) { SysTimeval now; @@ -777,7 +791,6 @@ void erts_deliver_time(void) { erts_smp_mtx_unlock(&erts_timeofday_mtx); } -#endif /* get *real* time (not ticks) remaining until next timeout - if there isn't one, give a "long" time, that is guaranteed @@ -786,14 +799,12 @@ void erts_deliver_time(void) { void erts_time_remaining(SysTimeval *rem_time) { int ticks; -#if !defined(ERTS_TIMER_THREAD) SysTimeval cur_time; -#endif long elapsed; - /* next_time() returns no of ticks to next timeout or -1 if none */ + /* erts_next_time() returns no of ticks to next timeout or -1 if none */ - if ((ticks = next_time()) == -1) { + if ((ticks = erts_next_time()) == -1) { /* timer queue empty */ /* this will cause at most 100000000 ticks */ rem_time->tv_sec = 100000; @@ -802,9 +813,6 @@ void erts_time_remaining(SysTimeval *rem_time) /* next timeout after ticks ticks */ ticks *= CLOCK_RESOLUTION; -#if defined(ERTS_TIMER_THREAD) - elapsed = 0; -#else erts_smp_mtx_lock(&erts_timeofday_mtx); get_tolerant_timeofday(&cur_time); @@ -819,7 +827,6 @@ void erts_time_remaining(SysTimeval *rem_time) rem_time->tv_sec = rem_time->tv_usec = 0; return; } -#endif rem_time->tv_sec = (ticks - elapsed) / 1000; rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000); } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 2842c2361a..8833137112 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -43,8 +43,9 @@ #undef DEBUG_PRINTOUTS #endif -extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* Pseudo export entries. Never filled in with data, only used to yield unique pointers of the correct type. */ @@ -397,11 +398,13 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, */ static void do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { - Eterm local_heap[4+5+5]; +#define LOCAL_HEAP_SIZE (4+5+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); Eterm message; Eterm *hp; Eterm mfarity; + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); ASSERT(is_pid(pid)); ASSERT(is_tuple(timestamp)); ASSERT(*tuple_val(timestamp) == make_arityval(3)); @@ -426,6 +429,8 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { pid, SYS_MSG_TYPE_UNDEFINED, message); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE } #endif @@ -442,7 +447,9 @@ send_to_port(Process *c_p, Eterm message, Eterm *tracer_pid, Uint *tracee_flags) { Port* trace_port; #ifndef ERTS_SMP - Eterm ts, local_heap[4], *hp; +#define LOCAL_HEAP_SIZE (4) + Eterm ts, *hp; + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); #endif ASSERT(is_internal_port(*tracer_pid)); @@ -486,6 +493,8 @@ send_to_port(Process *c_p, Eterm message, * (e.g. getting_linked) need not be the current process. That other * process might not have timestamps enabled. */ + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + if (*tracee_flags & F_TIMESTAMP) { ASSERT(is_tuple(message)); hp = tuple_val(message); @@ -522,6 +531,8 @@ send_to_port(Process *c_p, Eterm message, */ do_send_schedfix_to_port(trace_port, c_p->id, ts); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #endif } @@ -589,7 +600,10 @@ seq_trace_send_to_port(Process *c_p, { Port* trace_port; #ifndef ERTS_SMP - Eterm ts, local_heap[4], *hp; + Eterm ts, *hp; +#define LOCAL_HEAP_SIZE (4) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); #endif ASSERT(is_internal_port(seq_tracer)); @@ -607,6 +621,9 @@ seq_trace_send_to_port(Process *c_p, if (INVALID_TRACER_PORT(trace_port, seq_tracer)) { invalid_tracer_port: system_seq_tracer = am_false; +#ifndef ERTS_SMP + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#endif return; } @@ -620,6 +637,7 @@ seq_trace_send_to_port(Process *c_p, message); #ifndef ERTS_SMP + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); return; } /* Make a fake schedule only if the current process is traced @@ -660,6 +678,8 @@ seq_trace_send_to_port(Process *c_p, */ do_send_schedfix_to_port(trace_port, c_p->id, ts); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #endif } @@ -719,7 +739,8 @@ send_to_tracer(Process *tracee, static void trace_sched_aux(Process *p, Eterm what, int never_fake_sched) { - Eterm local_heap[5+4+1+TS_HEAP_WORDS]; +#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS) + DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); Eterm tmp, mess, *hp; ErlHeapFragment *bp = NULL; ErlOffHeap *off_heap; @@ -768,8 +789,10 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) curr_func = p->current != NULL; } + UseTmpHeap(LOCAL_HEAP_SIZE,p); + if (to_port) - hp = &local_heap[0]; + hp = local_heap; else { Uint size = 5; if (curr_func) @@ -802,6 +825,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) } send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched); + UnUseTmpHeap(LOCAL_HEAP_SIZE,p); +#undef LOCAL_HEAP_SIZE } /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp} @@ -848,7 +873,10 @@ trace_send(Process *p, Eterm to, Eterm msg) } if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[11]; +#define LOCAL_HEAP_SIZE (11) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE5(hp, am_trace, p->id, operation, msg, to); hp += 6; @@ -857,6 +885,8 @@ trace_send(Process *p, Eterm to, Eterm msg) hp = patch_ts(mess, hp); } send_to_port(p, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Uint need; @@ -908,7 +938,10 @@ trace_receive(Process *rp, Eterm msg) Eterm* hp; if (is_internal_port(rp->tracer_proc)) { - Eterm local_heap[10]; +#define LOCAL_HEAP_SIZE (10) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE4(hp, am_trace, rp->id, am_receive, msg); hp += 5; @@ -917,6 +950,8 @@ trace_receive(Process *rp, Eterm msg) hp = patch_ts(mess, hp); } send_to_port(rp, mess, &rp->tracer_proc, &rp->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Uint hsz; @@ -1018,7 +1053,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, } if (is_internal_port(seq_tracer)) { - Eterm local_heap[64]; +#define LOCAL_HEAP_SIZE (64) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; label = SEQ_TRACE_T_LABEL(token); lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token), @@ -1043,6 +1081,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, mess = TUPLE4(hp, am_seq_trace, label, mess, ts); seq_trace_send_to_port(process, seq_tracer, mess, ts); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { #ifndef ERTS_SMP @@ -1143,14 +1183,18 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, * or {trace, Pid, return_to, {Mod, Func, Arity}} */ void -erts_trace_return_to(Process *p, Uint *pc) +erts_trace_return_to(Process *p, BeamInstr *pc) { +#define LOCAL_HEAP_SIZE (4+5+5) Eterm* hp; Eterm mfa; Eterm mess; - Eterm local_heap[4+5+5]; + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + + BeamInstr *code_ptr = find_function_from_pc(pc); - Eterm *code_ptr = find_function_from_pc(pc); + + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; @@ -1196,6 +1240,8 @@ erts_trace_return_to(Process *p, Uint *pc) mess = copy_struct(mess, size, &hp, off_heap); ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp); } + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } @@ -1204,7 +1250,7 @@ erts_trace_return_to(Process *p, Uint *pc) * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval} */ void -erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) +erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) { Eterm* hp; Eterm mfa; @@ -1262,7 +1308,9 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) arity = fi[2]; if (is_internal_port(*tracer_pid)) { - Eterm local_heap[4+6+5]; +#define LOCAL_HEAP_SIZE (4+6+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; mfa = TUPLE3(hp, mod, name, make_small(arity)); hp += 4; @@ -1273,6 +1321,8 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) hp = patch_ts(mess, hp); } send_to_port(p, mess, tracer_pid, tracee_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -1331,7 +1381,7 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid) * Where Class is atomic but Value is any term. */ void -erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, +erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, Eterm *tracer_pid) { Eterm* hp; @@ -1385,21 +1435,26 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, } if (is_internal_port(*tracer_pid)) { - Eterm local_heap[4+3+6+5]; +#define LOCAL_HEAP_SIZE (4+3+6+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2])); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); hp += 4; cv = TUPLE2(hp, class, value); hp += 3; mess = TUPLE5(hp, am_trace, p->id, am_exception_from, mfa_tuple, cv); hp += 6; - ASSERT((hp - local_heap)*sizeof(*hp) <= sizeof(local_heap)); + ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE); erts_smp_mtx_lock(&smq_mtx); if (*tracee_flags & F_TIMESTAMP) { hp = patch_ts(mess, hp); /* hp += 5 */ - ASSERT((hp - local_heap)*sizeof(*hp) == sizeof(local_heap)); + ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE); } send_to_port(p, mess, tracer_pid, tracee_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -1431,7 +1486,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, * Build the trace tuple and put it into receive queue of the tracer process. */ - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2])); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm) mfa[2])); hp += 4; value = copy_struct(value, value_size, &hp, off_heap); cv = TUPLE2(hp, class, value); @@ -1468,7 +1523,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, * if it is a pid or port we do a meta trace. */ Uint32 -erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, +erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm* args, int local, Eterm *tracer_pid) { Eterm* hp; @@ -1483,7 +1538,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, Eterm tracee; #endif Eterm transformed_args[MAX_ARG]; - ErlSubBin sub_bin_heap; + DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p); ASSERT(tracer_pid); if (*tracer_pid == am_true) { @@ -1534,37 +1589,42 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, * such as size_object() and copy_struct(), we must make sure that we * temporarily convert any match contexts to sub binaries. */ - arity = mfa[2]; + arity = (Eterm) mfa[2]; + UseTmpHeap(ERL_SUB_BIN_SIZE,p); #ifdef DEBUG - sub_bin_heap.thing_word = 0; + sub_bin_heap->thing_word = 0; #endif for (i = 0; i < arity; i++) { Eterm arg = args[i]; if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg); ErlBinMatchBuffer* mb = &ms->mb; - ErlSubBin* sb = &sub_bin_heap; Uint bit_size; - ASSERT(sub_bin_heap.thing_word == 0); /* At most one of match context */ + ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */ bit_size = mb->size - mb->offset; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(bit_size); - sb->bitsize = BIT_OFFSET(bit_size); - sb->offs = BYTE_OFFSET(mb->offset); - sb->bitoffs = BIT_OFFSET(mb->offset); - sb->is_writable = 0; - sb->orig = mb->orig; - - arg = make_binary(sb); + sub_bin_heap->thing_word = HEADER_SUB_BIN; + sub_bin_heap->size = BYTE_OFFSET(bit_size); + sub_bin_heap->bitsize = BIT_OFFSET(bit_size); + sub_bin_heap->offs = BYTE_OFFSET(mb->offset); + sub_bin_heap->bitoffs = BIT_OFFSET(mb->offset); + sub_bin_heap->is_writable = 0; + sub_bin_heap->orig = mb->orig; + + arg = make_binary(sub_bin_heap); } transformed_args[i] = arg; } args = transformed_args; if (is_internal_port(*tracer_pid)) { +#if HEAP_ON_C_STACK Eterm local_heap[64+MAX_ARG]; +#else + Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*(64+MAX_ARG)); +#endif hp = local_heap; if (!erts_is_valid_tracer_port(*tracer_pid)) { @@ -1579,6 +1639,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, if (is_not_nil(tracee)) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1602,9 +1666,13 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, return_flags = 0; if (match_spec) { pam_result = erts_match_set_run(p, match_spec, args, arity, - &return_flags); + ERTS_PAM_TMP_RESULT, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } } @@ -1612,16 +1680,28 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, /* Meta trace */ if (pam_result == am_false) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } } else { /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } if (pam_result == am_false) { erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } if (local && (*tracee_flags & F_TRACE_RETURN_TO)) { @@ -1644,7 +1724,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); hp += 4; /* @@ -1664,6 +1744,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, send_to_port(p, mess, tracer_pid, tracee_flags); erts_smp_mtx_unlock(&smq_mtx); erts_match_set_release_result(p); +#if !HEAP_ON_C_STACK + erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); +#endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return *tracer_pid == NIL ? 0 : return_flags; } else { @@ -1706,6 +1790,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, if (is_not_nil(tracee)) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1728,9 +1813,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, return_flags = 0; if (match_spec) { pam_result = erts_match_set_run(p, match_spec, args, arity, - &return_flags); + ERTS_PAM_TMP_RESULT, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } } @@ -1738,16 +1824,19 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, /* Meta trace */ if (pam_result == am_false) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } } else { /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } if (pam_result == am_false) { erts_match_set_release_result(p); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } if (local && (*tracee_flags & F_TRACE_RETURN_TO)) { @@ -1798,7 +1887,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); hp += 4; /* @@ -1831,6 +1920,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, ASSERT(hp == limit); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); + UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } } @@ -1850,8 +1940,13 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) Eterm* hp; int need; + ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_is_system_blocked(0)); if (is_internal_port(t_p->tracer_proc)) { - Eterm local_heap[5+5]; +#define LOCAL_HEAP_SIZE (5+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + + hp = local_heap; mess = TUPLE4(hp, am_trace, t_p->id, what, data); hp += 5; @@ -1868,6 +1963,8 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) c_p, #endif mess, &t_p->tracer_proc, &t_p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Eterm tmp; @@ -1919,7 +2016,10 @@ trace_proc_spawn(Process *p, Eterm pid, Eterm* hp; if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[4+6+5]; +#define LOCAL_HEAP_SIZE (4+6+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mfa = TUPLE3(hp, mod, func, args); hp += 4; @@ -1930,6 +2030,8 @@ trace_proc_spawn(Process *p, Eterm pid, hp = patch_ts(mess, hp); } send_to_port(p, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { Eterm tmp; @@ -1991,7 +2093,7 @@ void save_calls(Process *p, Export *e) */ Eterm erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, Uint *I) + Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I) { Eterm result; int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META); @@ -2005,28 +2107,22 @@ erts_bif_trace(int bif_index, Process* p, * no tracing will occur. Doing the whole else branch will * also do nothing, only slower. */ - Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*) = bif_table[bif_index].f; + Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = bif_table[bif_index].f; result = func(p, arg1, arg2, arg3, I); } else { - Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*); + Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*); Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL); int local = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_LOCAL); + int time = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_CALL_TIME); Eterm meta_tracer_pid = NIL; int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif * is actually in the * export entry */ - Eterm *cp = p->cp; + BeamInstr *cp = p->cp; -#ifndef _OSE_ Eterm args[3] = {arg1, arg2, arg3}; -#else - Eterm args[3]; - args[0] = arg1; - args[1] = arg2; - args[2] = arg3; -#endif /* * Make continuation pointer OK, it is not during direct BIF calls, @@ -2043,6 +2139,17 @@ erts_bif_trace(int bif_index, Process* p, flags_meta = erts_bif_mtrace(p, ep->code+3, args, local, &meta_tracer_pid); } + if (time) { + BpDataTime *bdt = NULL; + BeamInstr *pc = (BeamInstr *)ep->code+3; + + bdt = (BpDataTime *) erts_get_time_break(p, pc); + ASSERT(bdt); + + if (!bdt->pause) { + erts_trace_time_break(p, pc, bdt, ERTS_BP_CALL_TIME_CALL); + } + } /* Restore original continuation pointer (if changed). */ p->cp = cp; @@ -2051,17 +2158,21 @@ erts_bif_trace(int bif_index, Process* p, result = func(p, arg1, arg2, arg3, I); if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { - Uint i_return_trace = beam_return_trace[0]; - Uint i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_time_trace = beam_return_time_trace[0]; Eterm *cpp; /* Maybe advance cp to skip trace stack frames */ for (cpp = p->stop; ; cp = cp_val(*cpp++)) { - ASSERT(is_CP((Eterm) cp)); - if (*cp_val((Eterm) cp) == i_return_trace) { + if (*cp == i_return_trace) { /* Skip stack frame variables */ while (is_not_CP(*cpp)) cpp++; cpp += 2; /* Skip return_trace parameters */ - } else if (*cp_val((Eterm) cp) == i_return_to_trace) { + } else if (*cp == i_return_time_trace) { + /* Skip stack frame variables */ + while (is_not_CP(*cpp)) cpp++; + cpp += 1; /* Skip return_time_trace parameters */ + } else if (*cp == i_return_to_trace) { /* A return_to trace message is going to be generated * by normal means, so we do not have to. */ @@ -2078,7 +2189,8 @@ erts_bif_trace(int bif_index, Process* p, if (reason != TRAP) { Eterm class; Eterm value = p->fvalue; - Eterm nocatch[3]; + DeclareTmpHeapNoproc(nocatch,3); + UseTmpHeapNoproc(3); /* Expand error value like in handle_error() */ if (reason & EXF_ARGLIST) { Eterm *tp; @@ -2126,6 +2238,7 @@ erts_bif_trace(int bif_index, Process* p, } } } + UnUseTmpHeapNoproc(3); if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); p->trace_flags |= F_EXCEPTION_TRACE; @@ -2213,15 +2326,19 @@ trace_gc(Process *p, Eterm what) BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p) }; - Eterm local_heap[(sizeof(values)/sizeof(Uint)) - *(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) - + 5/*4-tuple */ + TS_HEAP_WORDS]; +#define LOCAL_HEAP_SIZE \ + (sizeof(values)/sizeof(Eterm)) * \ + (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \ + 5/*4-tuple */ + TS_HEAP_WORDS + DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); #ifdef DEBUG Eterm* limit; #endif ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm)); + UseTmpHeap(LOCAL_HEAP_SIZE,p); + if (is_internal_port(p->tracer_proc)) { hp = local_heap; #ifdef DEBUG @@ -2252,7 +2369,7 @@ trace_gc(Process *p, Eterm what) #ifdef DEBUG limit = hp + size; - ASSERT(size <= sizeof(local_heap)/sizeof(Eterm)); + ASSERT(size <= LOCAL_HEAP_SIZE); #endif msg = erts_bld_atom_uint_2tup_list(&hp, @@ -2275,6 +2392,8 @@ trace_gc(Process *p, Eterm what) else ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, msg, bp); erts_smp_mtx_unlock(&smq_mtx); + UnUseTmpHeap(LOCAL_HEAP_SIZE,p); +#undef LOCAL_HEAP_SIZE } @@ -2465,7 +2584,9 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { Uint Ms, s, us; #ifndef ERTS_SMP - Eterm local_heap[4 + 7]; +#define LOCAL_HEAP_SIZE (4 + 7) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else ErlHeapFragment *bp; @@ -2498,6 +2619,8 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif @@ -2510,7 +2633,10 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M Eterm *hp, msg, timestamp; #ifndef ERTS_SMP - Eterm local_heap[4 + 7]; +#define LOCAL_HEAP_SIZE (4 + 7) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; #else ErlHeapFragment *bp; @@ -2528,6 +2654,8 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7; #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif @@ -2558,7 +2686,10 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { Eterm* hp; if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[5+6]; +#define LOCAL_HEAP_SIZE (5+6) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name); @@ -2569,6 +2700,8 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { } /* No fake schedule */ send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -2612,8 +2745,13 @@ trace_port(Port *t_p, Eterm what, Eterm data) { Eterm mess; Eterm* hp; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_is_system_blocked(0)); + if (is_internal_port(t_p->tracer_proc)) { - Eterm local_heap[5+5]; +#define LOCAL_HEAP_SIZE (5+5) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE4(hp, am_trace, t_p->id, what, data); hp += 5; @@ -2623,6 +2761,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) { } /* No fake schedule */ send_to_port(NULL, mess, &t_p->tracer_proc, &t_p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -2674,7 +2814,10 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { Eterm sched_id = am_undefined; if (is_internal_port(p->tracer_proc)) { - Eterm local_heap[5+6]; +#define LOCAL_HEAP_SIZE (5+6) + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) { @@ -2700,6 +2843,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { /* No fake scheduling */ send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); } else { ErlHeapFragment *bp; @@ -2750,7 +2895,11 @@ profile_runnable_port(Port *p, Eterm status) { Eterm count = make_small(0); #ifndef ERTS_SMP - Eterm local_heap[4 + 6]; +#define LOCAL_HEAP_SIZE (4 + 6) + + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; #else @@ -2771,6 +2920,8 @@ profile_runnable_port(Port *p, Eterm status) { #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif @@ -2785,7 +2936,11 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm where = am_undefined; #ifndef ERTS_SMP - Eterm local_heap[4 + 6 + 4]; +#define LOCAL_HEAP_SIZE (4 + 6 + 4) + + DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); + UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; #else ErlHeapFragment *bp; @@ -2818,6 +2973,8 @@ profile_runnable_proc(Process *p, Eterm status){ msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6; #ifndef ERTS_SMP profile_send(msg); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); +#undef LOCAL_HEAP_SIZE #else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); #endif diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index ab5811c70f..158eb361a4 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2008-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -30,6 +30,8 @@ #include "big.h" #include "erl_unicode.h" +#include "erl_unicode_normalize.h" + typedef struct _restart_context { byte *bytes; @@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p, Uint num_resulting_chars, int state, int left, Eterm tail); -static int analyze_utf8(byte *source, Uint size, - byte **err_pos, Uint *num_chars, int *left); -#define UTF8_OK 0 -#define UTF8_INCOMPLETE 1 -#define UTF8_ERROR 2 -#define UTF8_ANALYZE_MORE 3 - static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3); static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3); static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3); @@ -90,9 +85,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_utf8_trap",23); characters_to_utf8_trap_exp.code[2] = 3; characters_to_utf8_trap_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_utf8_trap_exp.code[4] = - (Eterm) &characters_to_utf8_trap; + (BeamInstr) &characters_to_utf8_trap; memset(&characters_to_list_trap_1_exp, 0, sizeof(Export)); characters_to_list_trap_1_exp.address = @@ -102,9 +97,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_1",25); characters_to_list_trap_1_exp.code[2] = 3; characters_to_list_trap_1_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_1_exp.code[4] = - (Eterm) &characters_to_list_trap_1; + (BeamInstr) &characters_to_list_trap_1; memset(&characters_to_list_trap_2_exp, 0, sizeof(Export)); characters_to_list_trap_2_exp.address = @@ -114,9 +109,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_2",25); characters_to_list_trap_2_exp.code[2] = 3; characters_to_list_trap_2_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_2_exp.code[4] = - (Eterm) &characters_to_list_trap_2; + (BeamInstr) &characters_to_list_trap_2; memset(&characters_to_list_trap_3_exp, 0, sizeof(Export)); @@ -127,9 +122,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_3",25); characters_to_list_trap_3_exp.code[2] = 3; characters_to_list_trap_3_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_3_exp.code[4] = - (Eterm) &characters_to_list_trap_3; + (BeamInstr) &characters_to_list_trap_3; memset(&characters_to_list_trap_4_exp, 0, sizeof(Export)); characters_to_list_trap_4_exp.address = @@ -139,9 +134,9 @@ void erts_init_unicode(void) am_atom_put("characters_to_list_trap_4",25); characters_to_list_trap_4_exp.code[2] = 1; characters_to_list_trap_4_exp.code[3] = - (Eterm) em_apply_bif; + (BeamInstr) em_apply_bif; characters_to_list_trap_4_exp.code[4] = - (Eterm) &characters_to_list_trap_4; + (BeamInstr) &characters_to_list_trap_4; c_to_b_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_binary_int,2); c_to_l_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_list_int,2); @@ -463,7 +458,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ } objp = list_val(ioterm); obj = CAR(objp); - if (!is_byte(obj)) + if (!is_small(obj)) break; } } else if (is_nil(obj)) { @@ -907,7 +902,6 @@ static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,int pos, static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3) { Eterm *real_bin; - Sint need; byte* bytes; Eterm rest_term; int left, sleft; @@ -923,7 +917,6 @@ static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3) ASSERT(is_binary(BIF_ARG_1)); real_bin = binary_val(BIF_ARG_1); ASSERT(*real_bin == HEADER_PROC_BIN); - need = ((ProcBin *) real_bin)->val->orig_size; pos = (int) binary_size(BIF_ARG_1); bytes = binary_bytes(BIF_ARG_1); sleft = left = allowed_iterations(BIF_P); @@ -970,11 +963,11 @@ static int is_valid_utf8(Eterm orig_bin) bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc); } size = binary_size(orig_bin); - ret = analyze_utf8(bytes, + ret = erts_analyze_utf8(bytes, size, &endpos,&numchar,NULL); erts_free_aligned_binary_bytes(temp_alloc); - return (ret == UTF8_OK); + return (ret == ERTS_UTF8_OK); } BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2) @@ -1084,14 +1077,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char hp += 2; rest_term = CONS(hp,leftover_bin,rest_term); } - BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL)); + BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL)); } else if (rest_term == NIL && num_leftovers != 0) { Eterm leftover_bin = new_binary(p, leftover, num_leftovers); if (check_leftovers(leftover,num_leftovers) != 0) { - BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR, + BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL)); } else { - BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE, + BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE, left, NIL)); } } else { /* All OK */ @@ -1107,11 +1100,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char rc.num_processed_bytes = 0; /* not used */ rc.num_bytes_to_process = pos; rc.num_resulting_chars = characters; - rc.state = UTF8_OK; /* not used */ + rc.state = ERTS_UTF8_OK; /* not used */ BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc), rest_term, latin1); } else { /* Success */ - BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL)); + BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL)); } } } @@ -1205,7 +1198,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2) * When input to characters_to_list is a plain binary and the format is 'unicode', we do * a faster analyze and size count with this function. */ -static int analyze_utf8(byte *source, Uint size, +int erts_analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left) { *err_pos = source; @@ -1216,60 +1209,60 @@ static int analyze_utf8(byte *source, Uint size, --size; } else if (((*source) & ((byte) 0xE0)) == 0xC0) { if (size < 2) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((*source) < 0xC2) /* overlong */) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 2; size -= 2; } else if (((*source) & ((byte) 0xF0)) == 0xE0) { if (size < 3) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if ((((*source) & ((byte) 0xF)) == 0xD) && ((source[1] & 0x20) != 0)) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if (((*source) == 0xEF) && (source[1] == 0xBF) && ((source[2] == 0xBE) || (source[2] == 0xBF))) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 3; size -= 3; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { if (size < 4) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || ((source[3] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if ((((*source) & ((byte)0x7)) > 0x4U) || ((((*source) & ((byte)0x7)) == 0x4U) && ((source[1] & ((byte)0x3F)) > 0xFU))) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 4; size -= 4; } else { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } ++(*num_chars); *err_pos = source; if (left && --(*left) <= 0) { - return UTF8_ANALYZE_MORE; + return ERTS_UTF8_ANALYZE_MORE; } } - return UTF8_OK; + return ERTS_UTF8_OK; } /* @@ -1304,7 +1297,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, } else if (((*source) & ((byte) 0xE0)) == 0xC0) { unipoint = (((Uint) ((*source) & ((byte) 0x1F))) << 6) | - ((Uint) (source[1] & ((byte) 0x3F))); + ((Uint) (source[1] & ((byte) 0x3F))); } else if (((*source) & ((byte) 0xF0)) == 0xE0) { unipoint = (((Uint) ((*source) & ((byte) 0xF))) << 12) | @@ -1330,6 +1323,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, return ret; } +static int is_candidate(Uint cp) +{ + int index,pos; + if (cp < 768) return 0; + if (cp > 4023) { + if (cp == 12441 || cp == 12442) return 1; + return 0; + } + index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET; + pos = cp % 32; + return !!(comp_candidate_map[index] & (1UL << pos)); +} + +static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c) +{ + int bucket = c % htab_size; + while (htab[bucket] != -1 && cv[htab[bucket]].c != c) + bucket = (bucket + 1) % htab_size; + return htab[bucket]; +} + +#define TRANSLATE_NO 0 +#define TRANSLATE_MAYBE -1 + +/* The s array is reversed */ +static int translate(Uint16 *s, int slen, Uint16 *res) +{ + /* Go backwards through buffer and match against tree */ + int pos = 0; + CompEntry *cv = compose_tab; + int *hc = hash_compose_tab; + int cvs = compose_tab_size; + int x; + while (pos < slen) { + x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]); + if (x < 0) { + return TRANSLATE_NO; + } + if (cv[x].res) { + *res = cv[x].res; + return pos; + } + cvs = cv[x].num_subs; + hc = cv[x].hash; + cv = cv[x].subs; + ++pos; + } + return TRANSLATE_MAYBE; +} + +static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint) +{ + /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/ + *numpointsp = 1; + savepoints[0] = (Uint16) unipoint; +} + +static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp) +{ + Eterm *hp = *hpp; + int res,i; + Uint16 newpoint; + Eterm ret = *retp; + + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } + } + *retp = ret; +} + +static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp) +{ + Eterm *hp = *hpp; + int numpoints = *numpointsp; + int res,i; + Uint16 newpoint; + Eterm ret = *retp; + + /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/ + if ((unipoint >> 16) == 0) { /* otherwise we're done here */ + savepoints[numpoints++] = (Uint16) unipoint; + res = translate(savepoints,numpoints,&newpoint); + if (res == TRANSLATE_NO) { + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else if (res > 0) { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } else { /* res < 0 */ + /* A "maybe", means we are not done yet */ + int j = 0; + while (i < numpoints) { + savepoints[j++] = savepoints[i++]; + } + numpoints = j; + goto breakaway; + } + } + numpoints = 0; + breakaway: + ; + } else if (res > 0) { + numpoints = 0; + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + } /* < 0 means go on */ + } else { + /* Unconditional rollup, this character is larger than 16 bit */ + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } + } + ret = CONS(hp,make_small(unipoint),ret); + hp += 2; + numpoints = 0; + } + *hpp = hp; + *numpointsp = numpoints; + *retp = ret; +} + +static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz) +{ + Eterm *hp,*hp_end; + Eterm ret; + byte *source; + Uint unipoint; + Uint16 savepoints[4]; + int numpoints = 0; + + ASSERT(num > 0); + + hp = HAlloc(p,num * 2); /* May be to much */ + hp_end = hp + num * 2; + ret = NIL; + source = bytes + sz; + while(--source >= bytes) { + if (((*source) & ((byte) 0x80)) == 0) { + unipoint = (Uint) *source; + } else if (((*source) & ((byte) 0xE0)) == 0xC0) { + unipoint = + (((Uint) ((*source) & ((byte) 0x1F))) << 6) | + ((Uint) (source[1] & ((byte) 0x3F))); + } else if (((*source) & ((byte) 0xF0)) == 0xE0) { + unipoint = + (((Uint) ((*source) & ((byte) 0xF))) << 12) | + (((Uint) (source[1] & ((byte) 0x3F))) << 6) | + ((Uint) (source[2] & ((byte) 0x3F))); + } else if (((*source) & ((byte) 0xF8)) == 0xF0) { + unipoint = + (((Uint) ((*source) & ((byte) 0x7))) << 18) | + (((Uint) (source[1] & ((byte) 0x3F))) << 12) | + (((Uint) (source[2] & ((byte) 0x3F))) << 6) | + ((Uint) (source[3] & ((byte) 0x3F))); + } else { + /* ignore 2#10XXXXXX */ + continue; + } + if (numpoints) { + handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret); + continue; + } + /* We are not building up any normalizations yet, look that we shouldn't start... */ + if (is_candidate(unipoint)) { + handle_first_norm(savepoints,&numpoints,unipoint); + continue; + } + ret = CONS(hp,make_small(unipoint),ret); + hp += 2; + } + /* so, we'we looped to the beginning, do we have anything saved? */ + if (numpoints) { + cleanup_norm(&hp,savepoints,numpoints,&ret); + } + if (hp_end != hp) { + HRelease(p,hp_end,hp); + } + return ret; +} + /* * The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way * as for characters_to_utf8). All sizes are known in advance and most data will be held in a @@ -1378,10 +1581,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p, */ free_restart(bytes); - if (state == UTF8_INCOMPLETE) { + if (state == ERTS_UTF8_INCOMPLETE) { hp = HAlloc(p,4); ret = TUPLE3(hp,am_incomplete,converted,rest); - } else if (state == UTF8_ERROR) { + } else if (state == ERTS_UTF8_ERROR) { hp = HAlloc(p,4); ret = TUPLE3(hp,am_error,converted,rest); } else { @@ -1408,7 +1611,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3) /* * Hooks into the process of decoding a binary depending on state. - * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process + * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process * and num_resulting_chars will grow * until we're done analyzing the binary. Then we'll eat * the bytes to process, lowering num_bytes_to_process and num_resulting_chars, @@ -1465,14 +1668,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, left = allowed_iterations(p); - if (state == UTF8_ANALYZE_MORE) { - state = analyze_utf8(bytes + num_bytes_to_process, + if (state == ERTS_UTF8_ANALYZE_MORE) { + state = erts_analyze_utf8(bytes + num_bytes_to_process, size - num_bytes_to_process, &endpos,&numchar,&left); cost_to_proc(p,numchar); num_resulting_chars += numchar; num_bytes_to_process = endpos - bytes; - if (state == UTF8_ANALYZE_MORE) { + if (state == ERTS_UTF8_ANALYZE_MORE) { Eterm epos = erts_make_integer(num_bytes_to_process,p); Eterm enumchar = erts_make_integer(num_resulting_chars,p); erts_free_aligned_binary_bytes(temp_alloc); @@ -1528,7 +1731,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, ErlSubBin *sb; Eterm orig; Uint offset; - ASSERT(state != UTF8_OK); + ASSERT(state != ERTS_UTF8_OK); hp = HAlloc(p, ERL_SUB_BIN_SIZE); sb = (ErlSubBin *) hp; ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize); @@ -1544,14 +1747,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, /* Done */ - if (state == UTF8_INCOMPLETE) { + if (state == ERTS_UTF8_INCOMPLETE) { if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes, b_sz) != 0) { goto error_return; } hp = HAlloc(p,4); ret = TUPLE3(hp,am_incomplete,converted,rest); - } else if (state == UTF8_ERROR) { + } else if (state == ERTS_UTF8_ERROR) { error_return: hp = HAlloc(p,4); ret = TUPLE3(hp,am_error,converted,rest); @@ -1589,7 +1792,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3) 0U, /* nothing processed yet */ num_bytes_to_process, num_resulting_chars, - UTF8_ANALYZE_MORE, /* always this state here */ + ERTS_UTF8_ANALYZE_MORE, /* always this state here */ NIL); /* Nothing built -> no tail yet */ } @@ -1642,7 +1845,7 @@ static BIF_RETTYPE utf8_to_list(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, - UTF8_ANALYZE_MORE,NIL); + ERTS_UTF8_ANALYZE_MORE,NIL); } @@ -1728,8 +1931,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist) Uint n; int reds_left = bin_size+1; /* Number of reductions left. */ - if (analyze_utf8(bytes, bin_size, &err_pos, - &n, &reds_left) == UTF8_OK) { + if (erts_analyze_utf8(bytes, bin_size, &err_pos, + &n, &reds_left) == ERTS_UTF8_OK) { /* * Correct UTF-8 encoding, but too many characters to * fit in an atom. @@ -1813,3 +2016,616 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2) { return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1); } + +/********************************************************** + * Simpler non-interruptable routines for UTF-8 and + * Windowish UTF-16 (restricted) + **********************************************************/ +/* + * This function is the heart of the Unicode support for + * open_port - spawn_executable. It converts both the name + * of the executable and the arguments according to the same rules + * as for filename conversion. That means as if your arguments are + * to be raw, you supply binaries, else unicode characters are allowed up to + * the encoding maximum (256 of the unicode max). + * Depending on the filename encoding standard, the vector is then + * converted to whatever is used, which might mean win_utf16 if on windows. + * Do not peek into the argument vector or filenam with ordinary + * string routines, that will certainly fail on some OS. + */ + +char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty) +{ + int encoding = erts_get_native_filename_encoding(); + char* name_buf = NULL; + + if (is_atom(name) || is_list(name) || (allow_empty && is_nil(name))) { + Sint need; + if ((need = erts_native_filename_need(name,encoding)) < 0) { + return NULL; + } + if (encoding == ERL_FILENAME_WIN_WCHAR) { + need += 2; + } else { + ++need; + } + name_buf = (char *) erts_alloc(alloc_type, need); + erts_native_filename_put(name,encoding,(byte *)name_buf); + name_buf[need-1] = 0; + if (encoding == ERL_FILENAME_WIN_WCHAR) { + name_buf[need-2] = 0; + } + } else if (is_binary(name)) { + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint size,num_chars; + + size = binary_size(name); + bytes = erts_get_aligned_binary_bytes(name, &temp_alloc); + if (encoding != ERL_FILENAME_WIN_WCHAR) { + /*Add 0 termination only*/ + name_buf = (char *) erts_alloc(alloc_type, size+1); + memcpy(name_buf,bytes,size); + name_buf[size]=0; + } else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || + erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + byte *p; + /* What to do now? Maybe latin1, so just take byte for byte instead */ + name_buf = (char *) erts_alloc(alloc_type, (size+1)*2); + p = (byte *) name_buf; + while (size--) { + *p++ = *bytes++; + *p++ = 0; + } + *p++ = 0; + *p++ = 0; + } else { /* WIN_WCHAR and valid UTF8 */ + name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2); + erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars); + name_buf[num_chars*2] = 0; + name_buf[num_chars*2+1] = 0; + } + erts_free_aligned_binary_bytes(temp_alloc); + } else { + return NULL; + } + return name_buf; +} + + +Sint erts_native_filename_need(Eterm ioterm, int encoding) +{ + Eterm *objp; + Eterm obj; + DECLARE_ESTACK(stack); + Sint need = 0; + + if (is_atom(ioterm)) { + Atom* ap; + int i; + ap = atom_tab(atom_val(ioterm)); + switch (encoding) { + case ERL_FILENAME_LATIN1: + need = ap->len; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + for (i = 0; i < ap->len; i++) { + need += (ap->name[i] >= 0x80) ? 2 : 1; + } + break; + case ERL_FILENAME_WIN_WCHAR: + need = 2*(ap->len); + break; + default: + need = -1; + } + DESTROY_ESTACK(stack); + return need; + } + + if (is_nil(ioterm)) { + DESTROY_ESTACK(stack); + return need; + } + if (!is_list(ioterm)) { + DESTROY_ESTACK(stack); + return (Sint) -1; + } + /* OK a list, needs to be processed in order, handling each flat list-level + as they occur, just like io_list_to_binary would */ + ESTACK_PUSH(stack,ioterm); + while (!ESTACK_ISEMPTY(stack)) { + ioterm = ESTACK_POP(stack); + if (is_nil(ioterm)) { + /* ignore empty lists */ + continue; + } + if(is_list(ioterm)) { +L_Again: /* Restart with sublist, old listend was pushed on stack */ + objp = list_val(ioterm); + obj = CAR(objp); + for(;;) { /* loop over one flat list of bytes and binaries + until sublist or list end is encountered */ + if (is_small(obj)) { /* Always small */ + for(;;) { + Uint x = unsigned_val(obj); + switch (encoding) { + case ERL_FILENAME_LATIN1: + if (x > 255) { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + need += 1; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + if (x < 0x80) { + need +=1; + } else if (x < 0x800) { + need += 2; + } else if (x < 0x10000) { + if ((x >= 0xD800 && x <= 0xDFFF) || + (x == 0xFFFE) || + (x == 0xFFFF)) { /* Invalid unicode range */ + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + need += 3; + } else if (x < 0x110000) { + need += 4; + } else { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + break; + case ERL_FILENAME_WIN_WCHAR: + if (x <= 0xffff) { + need += 2; + break; + } /* else fall throug to error */ + default: + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + + /* everything else will give badarg later + in the process, so we dont check */ + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + if (!is_small(obj)) + break; + } + } else if (is_nil(obj)) { + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + } else if (is_list(obj)) { + /* push rest of list for later processing, start + again with sublist */ + ESTACK_PUSH(stack,CDR(objp)); + ioterm = obj; + goto L_Again; + } else { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + if (is_nil(ioterm) || !is_list(ioterm)) { + break; + } + } /* for(;;) */ + } /* is_list(ioterm) */ + + if (!is_list(ioterm) && !is_nil(ioterm)) { + /* inproper list end */ + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + } /* while not estack empty */ + DESTROY_ESTACK(stack); + return need; +} + +void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) +{ + Eterm *objp; + Eterm obj; + DECLARE_ESTACK(stack); + + if (is_atom(ioterm)) { + Atom* ap; + int i; + ap = atom_tab(atom_val(ioterm)); + switch (encoding) { + case ERL_FILENAME_LATIN1: + for (i = 0; i < ap->len; i++) { + *p++ = ap->name[i]; + } + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + for (i = 0; i < ap->len; i++) { + if(ap->name[i] < 0x80) { + *p++ = ap->name[i]; + } else { + *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0)); + *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80)); + } + } + break; + case ERL_FILENAME_WIN_WCHAR: + for (i = 0; i < ap->len; i++) { + /* Little endian */ + *p++ = ap->name[i]; + *p++ = 0; + } + break; + default: + ASSERT(0); + } + DESTROY_ESTACK(stack); + return; + } + + if (is_nil(ioterm)) { + DESTROY_ESTACK(stack); + return; + } + ASSERT(is_list(ioterm)); + /* OK a list, needs to be processed in order, handling each flat list-level + as they occur, just like io_list_to_binary would */ + ESTACK_PUSH(stack,ioterm); + while (!ESTACK_ISEMPTY(stack)) { + ioterm = ESTACK_POP(stack); + if (is_nil(ioterm)) { + /* ignore empty lists */ + continue; + } + if(is_list(ioterm)) { +L_Again: /* Restart with sublist, old listend was pushed on stack */ + objp = list_val(ioterm); + obj = CAR(objp); + for(;;) { /* loop over one flat list of bytes and binaries + until sublist or list end is encountered */ + if (is_small(obj)) { /* Always small */ + for(;;) { + Uint x = unsigned_val(obj); + switch (encoding) { + case ERL_FILENAME_LATIN1: + ASSERT( x < 256); + *p++ = (byte) x; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + if (x < 0x80) { + *p++ = (byte) x; + } + else if (x < 0x800) { + *p++ = (((byte) (x >> 6)) | + ((byte) 0xC0)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } else if (x < 0x10000) { + ASSERT(!((x >= 0xD800 && x <= 0xDFFF) || + (x == 0xFFFE) || + (x == 0xFFFF))); + *p++ = (((byte) (x >> 12)) | + ((byte) 0xE0)); + *p++ = ((((byte) (x >> 6)) & 0x3F) | + ((byte) 0x80)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } else { + ASSERT(x < 0x110000); + *p++ = (((byte) (x >> 18)) | + ((byte) 0xF0)); + *p++ = ((((byte) (x >> 12)) & 0x3F) | + ((byte) 0x80)); + *p++ = ((((byte) (x >> 6)) & 0x3F) | + ((byte) 0x80)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } + break; + case ERL_FILENAME_WIN_WCHAR: + ASSERT(x <= 0xFFFF); + *p++ = (byte) (x & 0xFFU); + *p++ = (byte) ((x >> 8) & 0xFFU); + break; + default: + ASSERT(0); + } + + /* everything else will give badarg later + in the process, so we dont check */ + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + if (!is_small(obj)) + break; + } + } else if (is_nil(obj)) { + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + } else if (is_list(obj)) { + /* push rest of list for later processing, start + again with sublist */ + ESTACK_PUSH(stack,CDR(objp)); + ioterm = obj; + goto L_Again; + } else { + ASSERT(0); + } + if (is_nil(ioterm) || !is_list(ioterm)) { + break; + } + } /* for(;;) */ + } /* is_list(ioterm) */ + + ASSERT(is_list(ioterm) || is_nil(ioterm)); + } /* while not estack empty */ + DESTROY_ESTACK(stack); + return; +} +void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) +{ + Uint unipoint; + + while (num_chars--) { + if (((*bytes) & ((byte) 0x80)) == 0) { + unipoint = (Uint) *bytes; + ++bytes; + } else if (((*bytes) & ((byte) 0xE0)) == 0xC0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0x1F))) << 6) | + ((Uint) (bytes[1] & ((byte) 0x3F))); + bytes += 2; + } else if (((*bytes) & ((byte) 0xF0)) == 0xE0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0xF))) << 12) | + (((Uint) (bytes[1] & ((byte) 0x3F))) << 6) | + ((Uint) (bytes[2] & ((byte) 0x3F))); + bytes +=3; + } else if (((*bytes) & ((byte) 0xF8)) == 0xF0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0x7))) << 18) | + (((Uint) (bytes[1] & ((byte) 0x3F))) << 12) | + (((Uint) (bytes[2] & ((byte) 0x3F))) << 6) | + ((Uint) (bytes[3] & ((byte) 0x3F))); + bytes += 4; + } else { + erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1"); + } + *target++ = (byte) (unipoint & 0xFF); + *target++ = (byte) ((unipoint >> 8) & 0xFF); + } +} + +/* + * This internal bif converts a filename to whatever format is suitable for the file driver + * It also adds zero termination so that prim_file needn't bother with the character encoding + * of the file driver + */ +BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) +{ + int encoding = erts_get_native_filename_encoding(); + Sint need; + Eterm bin_term; + byte* bin_p; + /* Prim file explicitly does not allow atoms, although we could + very well cope with it. Instead of letting 'file' handle them, + it would probably be more efficient to handle them here. Subject to + change in R15. */ + if (is_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + if (is_binary(BIF_ARG_1)) { + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint size,num_chars; + /* Uninterpreted encoding except if windows widechar, in case we convert from + utf8 to win_wchar */ + size = binary_size(BIF_ARG_1); + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (encoding != ERL_FILENAME_WIN_WCHAR) { + /*Add 0 termination only*/ + bin_term = new_binary(BIF_P, NULL, size+1); + bin_p = binary_bytes(bin_term); + memcpy(bin_p,bytes,size); + bin_p[size]=0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } + /* In a wchar world, the emulator flags only affect how + binaries are interpreted when sent from the user. */ + /* Determine real length and create a new binary */ + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || + erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + /* What to do now? Maybe latin1, so just take byte for byte instead */ + bin_term = new_binary(BIF_P, 0, (size+1)*2); + bin_p = binary_bytes(bin_term); + while (size--) { + *bin_p++ = *bytes++; + *bin_p++ = 0; + } + *bin_p++ = 0; + *bin_p++ = 0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } + /* OK, UTF8 ok, number of characters is in num_chars */ + bin_term = new_binary(BIF_P, 0, (num_chars+1)*2); + bin_p = binary_bytes(bin_term); + erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars); + /* zero termination */ + bin_p[num_chars*2] = 0; + bin_p[num_chars*2+1] = 0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } /* binary */ + + + if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (encoding == ERL_FILENAME_WIN_WCHAR) { + need += 2; + } else { + ++need; + } + + bin_term = new_binary(BIF_P, 0, need); + bin_p = binary_bytes(bin_term); + erts_native_filename_put(BIF_ARG_1,encoding,bin_p); + bin_p[need-1] = 0; + if (encoding == ERL_FILENAME_WIN_WCHAR) { + bin_p[need-2] = 0; + } + BIF_RET(bin_term); +} + +BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1) +{ + Eterm real_bin; + Uint offset; + Uint size,num_chars; + Uint bitsize; + Uint bitoffs; + Eterm *hp; + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint num_built; /* characters */ + Uint num_eaten; /* bytes */ + Eterm ret; + int mac = 0; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + size = binary_size(BIF_ARG_1); + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (size == 0) { + BIF_RET(NIL); + } + switch (erts_get_native_filename_encoding()) { + case ERL_FILENAME_LATIN1: + hp = HAlloc(BIF_P, 2 * size); + bytes = binary_bytes(real_bin)+offset; + + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + case ERL_FILENAME_UTF8_MAC: + mac = 1; + case ERL_FILENAME_UTF8: + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { + erts_free_aligned_binary_bytes(temp_alloc); + goto noconvert; + } + num_built = 0; + num_eaten = 0; + if (mac) { + ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size); + } else { + ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL); + } + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); + case ERL_FILENAME_WIN_WCHAR: + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */ + size--; + hp = HAlloc(BIF_P, size+2); + ret = CONS(hp,make_small((Uint) bytes[size]),NIL); + hp += 2; + } else { + hp = HAlloc(BIF_P, size); + ret = NIL; + } + bytes += size-1; + while (size > 0) { + Uint x = ((Uint) *bytes--) << 8; + x |= ((Uint) *bytes--); + size -= 2; + ret = CONS(hp,make_small(x),ret); + hp += 2; + } + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); + default: + goto noconvert; + } + noconvert: + BIF_RET(BIF_ARG_1); +} + +BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1) +{ + Eterm real_bin; + Uint offset; + Uint size,num_chars; + Uint bitsize; + Uint bitoffs; + Eterm ret; + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + size = binary_size(BIF_ARG_1); + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (size == 0) { + BIF_RET(NIL); + } + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P,BADARG); + } + ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size); + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); +} + +BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0) +{ + switch (erts_get_native_filename_encoding()) { + case ERL_FILENAME_LATIN1: + BIF_RET(am_latin1); + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + BIF_RET(am_utf8); + case ERL_FILENAME_WIN_WCHAR: + if (erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + BIF_RET(am_latin1); + } else { + BIF_RET(am_utf8); + } + default: + BIF_RET(am_undefined); + } +} diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h new file mode 100644 index 0000000000..fb0a111ca2 --- /dev/null +++ b/erts/emulator/beam/erl_unicode_normalize.h @@ -0,0 +1,1687 @@ +/* +* %CopyrightBegin% +* +* Copyright Ericsson AB 1999-2010. All Rights Reserved. +* +* The contents of this file are subject to the Erlang Public License, +* Version 1.1, (the "License"); you may not use this file except in +* compliance with the License. You should have received a copy of the +* Erlang Public License along with this software. If not, it can be +* retrieved online at http://www.erlang.org/. +* +* Software distributed under the License is distributed on an "AS IS" +* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +* the License for the specific language governing rights and limitations +* under the License. +* +* %CopyrightEnd% +*/ +/* +* This file is automatically generated by dec.erl, do not edit manually +*/ +#define HASH_SIZE_FACTOR 2 +typedef struct _compose_entry { + Uint16 c; + Uint16 res; + Uint16 num_subs; + struct _compose_entry *subs; + int *hash; +} CompEntry; + +static int compose_tab_size = 61; +static int hash_compose_tab_0_15[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */ +static CompEntry compose_tab_0_15[] = { +{65, 7846, 0, NULL, NULL}, +{69, 7872, 0, NULL, NULL}, +{79, 7890, 0, NULL, NULL}, +{97, 7847, 0, NULL, NULL}, +{101, 7873, 0, NULL, NULL}, +{111, 7891, 0, NULL, NULL} +}; /* compose_tab_0_15 */ +static int hash_compose_tab_0_16[8] = +{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */ +static CompEntry compose_tab_0_16[] = { +{69, 7700, 0, NULL, NULL}, +{79, 7760, 0, NULL, NULL}, +{101, 7701, 0, NULL, NULL}, +{111, 7761, 0, NULL, NULL} +}; /* compose_tab_0_16 */ +static int hash_compose_tab_0_17[4] = +{-1,0,1,-1}; /* hash_compose_tab_0_17 */ +static CompEntry compose_tab_0_17[] = { +{65, 7856, 0, NULL, NULL}, +{97, 7857, 0, NULL, NULL} +}; /* compose_tab_0_17 */ +static int hash_compose_tab_0_18[8] = +{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */ +static CompEntry compose_tab_0_18[] = { +{85, 475, 0, NULL, NULL}, +{117, 476, 0, NULL, NULL}, +{953, 8146, 0, NULL, NULL}, +{965, 8162, 0, NULL, NULL} +}; /* compose_tab_0_18 */ +static int hash_compose_tab_0_19_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */ +static CompEntry compose_tab_0_19_0[] = { +{913, 8074, 0, NULL, NULL}, +{919, 8090, 0, NULL, NULL}, +{937, 8106, 0, NULL, NULL}, +{945, 8066, 0, NULL, NULL}, +{951, 8082, 0, NULL, NULL}, +{969, 8098, 0, NULL, NULL} +}; /* compose_tab_0_19_0 */ +static int hash_compose_tab_0_19[28] = +{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */ +static CompEntry compose_tab_0_19[] = { +{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0}, +{913, 7946, 0, NULL, NULL}, +{917, 7962, 0, NULL, NULL}, +{919, 7978, 0, NULL, NULL}, +{921, 7994, 0, NULL, NULL}, +{927, 8010, 0, NULL, NULL}, +{937, 8042, 0, NULL, NULL}, +{945, 7938, 0, NULL, NULL}, +{949, 7954, 0, NULL, NULL}, +{951, 7970, 0, NULL, NULL}, +{953, 7986, 0, NULL, NULL}, +{959, 8002, 0, NULL, NULL}, +{965, 8018, 0, NULL, NULL}, +{969, 8034, 0, NULL, NULL} +}; /* compose_tab_0_19 */ +static int hash_compose_tab_0_20_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */ +static CompEntry compose_tab_0_20_0[] = { +{913, 8075, 0, NULL, NULL}, +{919, 8091, 0, NULL, NULL}, +{937, 8107, 0, NULL, NULL}, +{945, 8067, 0, NULL, NULL}, +{951, 8083, 0, NULL, NULL}, +{969, 8099, 0, NULL, NULL} +}; /* compose_tab_0_20_0 */ +static int hash_compose_tab_0_20[30] = +{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5, + 12}; /* hash_compose_tab_0_20 */ +static CompEntry compose_tab_0_20[] = { +{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0}, +{913, 7947, 0, NULL, NULL}, +{917, 7963, 0, NULL, NULL}, +{919, 7979, 0, NULL, NULL}, +{921, 7995, 0, NULL, NULL}, +{927, 8011, 0, NULL, NULL}, +{933, 8027, 0, NULL, NULL}, +{937, 8043, 0, NULL, NULL}, +{945, 7939, 0, NULL, NULL}, +{949, 7955, 0, NULL, NULL}, +{951, 7971, 0, NULL, NULL}, +{953, 7987, 0, NULL, NULL}, +{959, 8003, 0, NULL, NULL}, +{965, 8019, 0, NULL, NULL}, +{969, 8035, 0, NULL, NULL} +}; /* compose_tab_0_20 */ +static int hash_compose_tab_0_21[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */ +static CompEntry compose_tab_0_21[] = { +{79, 7900, 0, NULL, NULL}, +{85, 7914, 0, NULL, NULL}, +{111, 7901, 0, NULL, NULL}, +{117, 7915, 0, NULL, NULL} +}; /* compose_tab_0_21 */ +static int hash_compose_tab_0_22[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */ +static CompEntry compose_tab_0_22[] = { +{945, 8114, 0, NULL, NULL}, +{951, 8130, 0, NULL, NULL}, +{969, 8178, 0, NULL, NULL} +}; /* compose_tab_0_22 */ +static int hash_compose_tab_0[78] = +{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9, + -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */ +static CompEntry compose_tab_0[] = { +{65, 192, 0, NULL, NULL}, +{69, 200, 0, NULL, NULL}, +{73, 204, 0, NULL, NULL}, +{79, 210, 0, NULL, NULL}, +{85, 217, 0, NULL, NULL}, +{87, 7808, 0, NULL, NULL}, +{89, 7922, 0, NULL, NULL}, +{97, 224, 0, NULL, NULL}, +{101, 232, 0, NULL, NULL}, +{105, 236, 0, NULL, NULL}, +{111, 242, 0, NULL, NULL}, +{117, 249, 0, NULL, NULL}, +{119, 7809, 0, NULL, NULL}, +{121, 7923, 0, NULL, NULL}, +{168, 8173, 0, NULL, NULL}, +{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15}, +{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16}, +{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17}, +{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18}, +{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19}, +{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20}, +{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21}, +{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22}, +{913, 8122, 0, NULL, NULL}, +{917, 8136, 0, NULL, NULL}, +{919, 8138, 0, NULL, NULL}, +{921, 8154, 0, NULL, NULL}, +{927, 8184, 0, NULL, NULL}, +{933, 8170, 0, NULL, NULL}, +{937, 8186, 0, NULL, NULL}, +{945, 8048, 0, NULL, NULL}, +{949, 8050, 0, NULL, NULL}, +{951, 8052, 0, NULL, NULL}, +{953, 8054, 0, NULL, NULL}, +{959, 8056, 0, NULL, NULL}, +{965, 8058, 0, NULL, NULL}, +{969, 8060, 0, NULL, NULL}, +{8127, 8141, 0, NULL, NULL}, +{8190, 8157, 0, NULL, NULL} +}; /* compose_tab_0 */ +static int hash_compose_tab_1_39[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */ +static CompEntry compose_tab_1_39[] = { +{65, 7844, 0, NULL, NULL}, +{69, 7870, 0, NULL, NULL}, +{79, 7888, 0, NULL, NULL}, +{97, 7845, 0, NULL, NULL}, +{101, 7871, 0, NULL, NULL}, +{111, 7889, 0, NULL, NULL} +}; /* compose_tab_1_39 */ +static int hash_compose_tab_1_40[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */ +static CompEntry compose_tab_1_40[] = { +{79, 7756, 0, NULL, NULL}, +{85, 7800, 0, NULL, NULL}, +{111, 7757, 0, NULL, NULL}, +{117, 7801, 0, NULL, NULL} +}; /* compose_tab_1_40 */ +static int hash_compose_tab_1_41[8] = +{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */ +static CompEntry compose_tab_1_41[] = { +{69, 7702, 0, NULL, NULL}, +{79, 7762, 0, NULL, NULL}, +{101, 7703, 0, NULL, NULL}, +{111, 7763, 0, NULL, NULL} +}; /* compose_tab_1_41 */ +static int hash_compose_tab_1_42[4] = +{-1,0,1,-1}; /* hash_compose_tab_1_42 */ +static CompEntry compose_tab_1_42[] = { +{65, 7854, 0, NULL, NULL}, +{97, 7855, 0, NULL, NULL} +}; /* compose_tab_1_42 */ +static int hash_compose_tab_1_43[12] = +{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */ +static CompEntry compose_tab_1_43[] = { +{73, 7726, 0, NULL, NULL}, +{85, 471, 0, NULL, NULL}, +{105, 7727, 0, NULL, NULL}, +{117, 472, 0, NULL, NULL}, +{953, 8147, 0, NULL, NULL}, +{965, 8163, 0, NULL, NULL} +}; /* compose_tab_1_43 */ +static int hash_compose_tab_1_44[4] = +{-1,0,1,-1}; /* hash_compose_tab_1_44 */ +static CompEntry compose_tab_1_44[] = { +{65, 506, 0, NULL, NULL}, +{97, 507, 0, NULL, NULL} +}; /* compose_tab_1_44 */ +static int hash_compose_tab_1_45_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */ +static CompEntry compose_tab_1_45_0[] = { +{913, 8076, 0, NULL, NULL}, +{919, 8092, 0, NULL, NULL}, +{937, 8108, 0, NULL, NULL}, +{945, 8068, 0, NULL, NULL}, +{951, 8084, 0, NULL, NULL}, +{969, 8100, 0, NULL, NULL} +}; /* compose_tab_1_45_0 */ +static int hash_compose_tab_1_45[28] = +{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */ +static CompEntry compose_tab_1_45[] = { +{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0}, +{913, 7948, 0, NULL, NULL}, +{917, 7964, 0, NULL, NULL}, +{919, 7980, 0, NULL, NULL}, +{921, 7996, 0, NULL, NULL}, +{927, 8012, 0, NULL, NULL}, +{937, 8044, 0, NULL, NULL}, +{945, 7940, 0, NULL, NULL}, +{949, 7956, 0, NULL, NULL}, +{951, 7972, 0, NULL, NULL}, +{953, 7988, 0, NULL, NULL}, +{959, 8004, 0, NULL, NULL}, +{965, 8020, 0, NULL, NULL}, +{969, 8036, 0, NULL, NULL} +}; /* compose_tab_1_45 */ +static int hash_compose_tab_1_46_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */ +static CompEntry compose_tab_1_46_0[] = { +{913, 8077, 0, NULL, NULL}, +{919, 8093, 0, NULL, NULL}, +{937, 8109, 0, NULL, NULL}, +{945, 8069, 0, NULL, NULL}, +{951, 8085, 0, NULL, NULL}, +{969, 8101, 0, NULL, NULL} +}; /* compose_tab_1_46_0 */ +static int hash_compose_tab_1_46[30] = +{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5, + 12}; /* hash_compose_tab_1_46 */ +static CompEntry compose_tab_1_46[] = { +{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0}, +{913, 7949, 0, NULL, NULL}, +{917, 7965, 0, NULL, NULL}, +{919, 7981, 0, NULL, NULL}, +{921, 7997, 0, NULL, NULL}, +{927, 8013, 0, NULL, NULL}, +{933, 8029, 0, NULL, NULL}, +{937, 8045, 0, NULL, NULL}, +{945, 7941, 0, NULL, NULL}, +{949, 7957, 0, NULL, NULL}, +{951, 7973, 0, NULL, NULL}, +{953, 7989, 0, NULL, NULL}, +{959, 8005, 0, NULL, NULL}, +{965, 8021, 0, NULL, NULL}, +{969, 8037, 0, NULL, NULL} +}; /* compose_tab_1_46 */ +static int hash_compose_tab_1_47[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */ +static CompEntry compose_tab_1_47[] = { +{79, 7898, 0, NULL, NULL}, +{85, 7912, 0, NULL, NULL}, +{111, 7899, 0, NULL, NULL}, +{117, 7913, 0, NULL, NULL} +}; /* compose_tab_1_47 */ +static int hash_compose_tab_1_48[4] = +{1,-1,-1,0}; /* hash_compose_tab_1_48 */ +static CompEntry compose_tab_1_48[] = { +{67, 7688, 0, NULL, NULL}, +{99, 7689, 0, NULL, NULL} +}; /* compose_tab_1_48 */ +static int hash_compose_tab_1_49[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */ +static CompEntry compose_tab_1_49[] = { +{945, 8116, 0, NULL, NULL}, +{951, 8132, 0, NULL, NULL}, +{959, 8180, 0, NULL, NULL} +}; /* compose_tab_1_49 */ +static int hash_compose_tab_1[140] = +{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7, + 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20, + 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66, + 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */ +static CompEntry compose_tab_1[] = { +{65, 193, 0, NULL, NULL}, +{67, 262, 0, NULL, NULL}, +{69, 201, 0, NULL, NULL}, +{71, 500, 0, NULL, NULL}, +{73, 205, 0, NULL, NULL}, +{75, 7728, 0, NULL, NULL}, +{76, 313, 0, NULL, NULL}, +{77, 7742, 0, NULL, NULL}, +{78, 323, 0, NULL, NULL}, +{79, 211, 0, NULL, NULL}, +{80, 7764, 0, NULL, NULL}, +{82, 340, 0, NULL, NULL}, +{83, 346, 0, NULL, NULL}, +{85, 218, 0, NULL, NULL}, +{87, 7810, 0, NULL, NULL}, +{89, 221, 0, NULL, NULL}, +{90, 377, 0, NULL, NULL}, +{97, 225, 0, NULL, NULL}, +{99, 263, 0, NULL, NULL}, +{101, 233, 0, NULL, NULL}, +{103, 501, 0, NULL, NULL}, +{105, 237, 0, NULL, NULL}, +{107, 7729, 0, NULL, NULL}, +{108, 314, 0, NULL, NULL}, +{109, 7743, 0, NULL, NULL}, +{110, 324, 0, NULL, NULL}, +{111, 243, 0, NULL, NULL}, +{112, 7765, 0, NULL, NULL}, +{114, 341, 0, NULL, NULL}, +{115, 347, 0, NULL, NULL}, +{117, 250, 0, NULL, NULL}, +{119, 7811, 0, NULL, NULL}, +{121, 253, 0, NULL, NULL}, +{122, 378, 0, NULL, NULL}, +{168, 8174, 0, NULL, NULL}, +{198, 508, 0, NULL, NULL}, +{216, 510, 0, NULL, NULL}, +{230, 509, 0, NULL, NULL}, +{248, 511, 0, NULL, NULL}, +{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39}, +{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40}, +{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41}, +{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42}, +{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43}, +{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44}, +{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45}, +{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46}, +{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47}, +{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48}, +{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49}, +{913, 8123, 0, NULL, NULL}, +{917, 8137, 0, NULL, NULL}, +{919, 8139, 0, NULL, NULL}, +{921, 8155, 0, NULL, NULL}, +{927, 8185, 0, NULL, NULL}, +{933, 8171, 0, NULL, NULL}, +{937, 8187, 0, NULL, NULL}, +{945, 8049, 0, NULL, NULL}, +{949, 8051, 0, NULL, NULL}, +{951, 8053, 0, NULL, NULL}, +{953, 8055, 0, NULL, NULL}, +{959, 8057, 0, NULL, NULL}, +{965, 8059, 0, NULL, NULL}, +{969, 8061, 0, NULL, NULL}, +{1043, 1027, 0, NULL, NULL}, +{1050, 1036, 0, NULL, NULL}, +{1075, 1107, 0, NULL, NULL}, +{1082, 1116, 0, NULL, NULL}, +{8127, 8142, 0, NULL, NULL}, +{8190, 8158, 0, NULL, NULL} +}; /* compose_tab_1 */ +static int hash_compose_tab_2_26[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */ +static CompEntry compose_tab_2_26[] = { +{65, 7852, 0, NULL, NULL}, +{69, 7878, 0, NULL, NULL}, +{79, 7896, 0, NULL, NULL}, +{97, 7853, 0, NULL, NULL}, +{101, 7879, 0, NULL, NULL}, +{111, 7897, 0, NULL, NULL} +}; /* compose_tab_2_26 */ +static int hash_compose_tab_2[54] = +{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1, + -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */ +static CompEntry compose_tab_2[] = { +{65, 194, 0, NULL, NULL}, +{67, 264, 0, NULL, NULL}, +{69, 202, 0, NULL, NULL}, +{71, 284, 0, NULL, NULL}, +{72, 292, 0, NULL, NULL}, +{73, 206, 0, NULL, NULL}, +{74, 308, 0, NULL, NULL}, +{79, 212, 0, NULL, NULL}, +{83, 348, 0, NULL, NULL}, +{85, 219, 0, NULL, NULL}, +{87, 372, 0, NULL, NULL}, +{89, 374, 0, NULL, NULL}, +{90, 7824, 0, NULL, NULL}, +{97, 226, 0, NULL, NULL}, +{99, 265, 0, NULL, NULL}, +{101, 234, 0, NULL, NULL}, +{103, 285, 0, NULL, NULL}, +{104, 293, 0, NULL, NULL}, +{105, 238, 0, NULL, NULL}, +{106, 309, 0, NULL, NULL}, +{111, 244, 0, NULL, NULL}, +{115, 349, 0, NULL, NULL}, +{117, 251, 0, NULL, NULL}, +{119, 373, 0, NULL, NULL}, +{121, 375, 0, NULL, NULL}, +{122, 7825, 0, NULL, NULL}, +{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26} +}; /* compose_tab_2 */ +static int hash_compose_tab_3_16[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */ +static CompEntry compose_tab_3_16[] = { +{65, 7850, 0, NULL, NULL}, +{69, 7876, 0, NULL, NULL}, +{79, 7894, 0, NULL, NULL}, +{97, 7851, 0, NULL, NULL}, +{101, 7877, 0, NULL, NULL}, +{111, 7895, 0, NULL, NULL} +}; /* compose_tab_3_16 */ +static int hash_compose_tab_3_17[4] = +{-1,0,1,-1}; /* hash_compose_tab_3_17 */ +static CompEntry compose_tab_3_17[] = { +{65, 7860, 0, NULL, NULL}, +{97, 7861, 0, NULL, NULL} +}; /* compose_tab_3_17 */ +static int hash_compose_tab_3_18[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */ +static CompEntry compose_tab_3_18[] = { +{79, 7904, 0, NULL, NULL}, +{85, 7918, 0, NULL, NULL}, +{111, 7905, 0, NULL, NULL}, +{117, 7919, 0, NULL, NULL} +}; /* compose_tab_3_18 */ +static int hash_compose_tab_3[38] = +{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0, + -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */ +static CompEntry compose_tab_3[] = { +{65, 195, 0, NULL, NULL}, +{69, 7868, 0, NULL, NULL}, +{73, 296, 0, NULL, NULL}, +{78, 209, 0, NULL, NULL}, +{79, 213, 0, NULL, NULL}, +{85, 360, 0, NULL, NULL}, +{86, 7804, 0, NULL, NULL}, +{89, 7928, 0, NULL, NULL}, +{97, 227, 0, NULL, NULL}, +{101, 7869, 0, NULL, NULL}, +{105, 297, 0, NULL, NULL}, +{110, 241, 0, NULL, NULL}, +{111, 245, 0, NULL, NULL}, +{117, 361, 0, NULL, NULL}, +{118, 7805, 0, NULL, NULL}, +{121, 7929, 0, NULL, NULL}, +{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16}, +{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17}, +{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18} +}; /* compose_tab_3 */ +static int hash_compose_tab_4_14[4] = +{-1,0,1,-1}; /* hash_compose_tab_4_14 */ +static CompEntry compose_tab_4_14[] = { +{65, 480, 0, NULL, NULL}, +{97, 481, 0, NULL, NULL} +}; /* compose_tab_4_14 */ +static int hash_compose_tab_4_15[8] = +{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */ +static CompEntry compose_tab_4_15[] = { +{65, 478, 0, NULL, NULL}, +{85, 469, 0, NULL, NULL}, +{97, 479, 0, NULL, NULL}, +{117, 470, 0, NULL, NULL} +}; /* compose_tab_4_15 */ +static int hash_compose_tab_4_16[8] = +{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */ +static CompEntry compose_tab_4_16[] = { +{76, 7736, 0, NULL, NULL}, +{82, 7772, 0, NULL, NULL}, +{108, 7737, 0, NULL, NULL}, +{114, 7773, 0, NULL, NULL} +}; /* compose_tab_4_16 */ +static int hash_compose_tab_4_17[4] = +{1,-1,-1,0}; /* hash_compose_tab_4_17 */ +static CompEntry compose_tab_4_17[] = { +{79, 492, 0, NULL, NULL}, +{111, 493, 0, NULL, NULL} +}; /* compose_tab_4_17 */ +static int hash_compose_tab_4[56] = +{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1, + 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1, + -1,10}; /* hash_compose_tab_4 */ +static CompEntry compose_tab_4[] = { +{65, 256, 0, NULL, NULL}, +{69, 274, 0, NULL, NULL}, +{71, 7712, 0, NULL, NULL}, +{73, 298, 0, NULL, NULL}, +{79, 332, 0, NULL, NULL}, +{85, 362, 0, NULL, NULL}, +{97, 257, 0, NULL, NULL}, +{101, 275, 0, NULL, NULL}, +{103, 7713, 0, NULL, NULL}, +{105, 299, 0, NULL, NULL}, +{111, 333, 0, NULL, NULL}, +{117, 363, 0, NULL, NULL}, +{198, 482, 0, NULL, NULL}, +{230, 483, 0, NULL, NULL}, +{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14}, +{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15}, +{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16}, +{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17}, +{913, 8121, 0, NULL, NULL}, +{921, 8153, 0, NULL, NULL}, +{933, 8169, 0, NULL, NULL}, +{945, 8113, 0, NULL, NULL}, +{953, 8145, 0, NULL, NULL}, +{965, 8161, 0, NULL, NULL}, +{1048, 1250, 0, NULL, NULL}, +{1059, 1262, 0, NULL, NULL}, +{1080, 1251, 0, NULL, NULL}, +{1091, 1263, 0, NULL, NULL} +}; /* compose_tab_4 */ +static int hash_compose_tab_5_12[4] = +{-1,0,1,-1}; /* hash_compose_tab_5_12 */ +static CompEntry compose_tab_5_12[] = { +{65, 7862, 0, NULL, NULL}, +{97, 7863, 0, NULL, NULL} +}; /* compose_tab_5_12 */ +static int hash_compose_tab_5_13[4] = +{-1,0,1,-1}; /* hash_compose_tab_5_13 */ +static CompEntry compose_tab_5_13[] = { +{69, 7708, 0, NULL, NULL}, +{101, 7709, 0, NULL, NULL} +}; /* compose_tab_5_13 */ +static int hash_compose_tab_5[60] = +{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21, + 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18, + -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */ +static CompEntry compose_tab_5[] = { +{65, 258, 0, NULL, NULL}, +{69, 276, 0, NULL, NULL}, +{71, 286, 0, NULL, NULL}, +{73, 300, 0, NULL, NULL}, +{79, 334, 0, NULL, NULL}, +{85, 364, 0, NULL, NULL}, +{97, 259, 0, NULL, NULL}, +{101, 277, 0, NULL, NULL}, +{103, 287, 0, NULL, NULL}, +{105, 301, 0, NULL, NULL}, +{111, 335, 0, NULL, NULL}, +{117, 365, 0, NULL, NULL}, +{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12}, +{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13}, +{913, 8120, 0, NULL, NULL}, +{921, 8152, 0, NULL, NULL}, +{933, 8168, 0, NULL, NULL}, +{945, 8112, 0, NULL, NULL}, +{953, 8144, 0, NULL, NULL}, +{965, 8160, 0, NULL, NULL}, +{1040, 1232, 0, NULL, NULL}, +{1045, 1238, 0, NULL, NULL}, +{1046, 1217, 0, NULL, NULL}, +{1048, 1049, 0, NULL, NULL}, +{1059, 1038, 0, NULL, NULL}, +{1072, 1233, 0, NULL, NULL}, +{1077, 1239, 0, NULL, NULL}, +{1078, 1218, 0, NULL, NULL}, +{1080, 1081, 0, NULL, NULL}, +{1091, 1118, 0, NULL, NULL} +}; /* compose_tab_5 */ +static int hash_compose_tab_6_36[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_36 */ +static CompEntry compose_tab_6_36[] = { +{83, 7780, 0, NULL, NULL}, +{115, 7781, 0, NULL, NULL} +}; /* compose_tab_6_36 */ +static int hash_compose_tab_6_38[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_38 */ +static CompEntry compose_tab_6_38[] = { +{83, 7782, 0, NULL, NULL}, +{115, 7783, 0, NULL, NULL} +}; /* compose_tab_6_38 */ +static int hash_compose_tab_6_39[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_39 */ +static CompEntry compose_tab_6_39[] = { +{83, 7784, 0, NULL, NULL}, +{115, 7785, 0, NULL, NULL} +}; /* compose_tab_6_39 */ +static int hash_compose_tab_6[80] = +{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24, + -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36, + -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9, + -1}; /* hash_compose_tab_6 */ +static CompEntry compose_tab_6[] = { +{66, 7682, 0, NULL, NULL}, +{67, 266, 0, NULL, NULL}, +{68, 7690, 0, NULL, NULL}, +{69, 278, 0, NULL, NULL}, +{70, 7710, 0, NULL, NULL}, +{71, 288, 0, NULL, NULL}, +{72, 7714, 0, NULL, NULL}, +{73, 304, 0, NULL, NULL}, +{77, 7744, 0, NULL, NULL}, +{78, 7748, 0, NULL, NULL}, +{80, 7766, 0, NULL, NULL}, +{82, 7768, 0, NULL, NULL}, +{83, 7776, 0, NULL, NULL}, +{84, 7786, 0, NULL, NULL}, +{87, 7814, 0, NULL, NULL}, +{88, 7818, 0, NULL, NULL}, +{89, 7822, 0, NULL, NULL}, +{90, 379, 0, NULL, NULL}, +{98, 7683, 0, NULL, NULL}, +{99, 267, 0, NULL, NULL}, +{100, 7691, 0, NULL, NULL}, +{101, 279, 0, NULL, NULL}, +{102, 7711, 0, NULL, NULL}, +{103, 289, 0, NULL, NULL}, +{104, 7715, 0, NULL, NULL}, +{109, 7745, 0, NULL, NULL}, +{110, 7749, 0, NULL, NULL}, +{112, 7767, 0, NULL, NULL}, +{114, 7769, 0, NULL, NULL}, +{115, 7777, 0, NULL, NULL}, +{116, 7787, 0, NULL, NULL}, +{119, 7815, 0, NULL, NULL}, +{120, 7819, 0, NULL, NULL}, +{121, 7823, 0, NULL, NULL}, +{122, 380, 0, NULL, NULL}, +{383, 7835, 0, NULL, NULL}, +{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36}, +{774, 784, 0, NULL, NULL}, +{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38}, +{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39} +}; /* compose_tab_6 */ +static int hash_compose_tab_7_23[4] = +{1,-1,-1,0}; /* hash_compose_tab_7_23 */ +static CompEntry compose_tab_7_23[] = { +{79, 7758, 0, NULL, NULL}, +{111, 7759, 0, NULL, NULL} +}; /* compose_tab_7_23 */ +static int hash_compose_tab_7_24[4] = +{-1,0,1,-1}; /* hash_compose_tab_7_24 */ +static CompEntry compose_tab_7_24[] = { +{85, 7802, 0, NULL, NULL}, +{117, 7803, 0, NULL, NULL} +}; /* compose_tab_7_24 */ +static int hash_compose_tab_7[100] = +{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1, + -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1, + -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1, + 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */ +static CompEntry compose_tab_7[] = { +{65, 196, 0, NULL, NULL}, +{69, 203, 0, NULL, NULL}, +{72, 7718, 0, NULL, NULL}, +{73, 207, 0, NULL, NULL}, +{79, 214, 0, NULL, NULL}, +{85, 220, 0, NULL, NULL}, +{87, 7812, 0, NULL, NULL}, +{88, 7820, 0, NULL, NULL}, +{89, 376, 0, NULL, NULL}, +{97, 228, 0, NULL, NULL}, +{101, 235, 0, NULL, NULL}, +{104, 7719, 0, NULL, NULL}, +{105, 239, 0, NULL, NULL}, +{111, 246, 0, NULL, NULL}, +{116, 7831, 0, NULL, NULL}, +{117, 252, 0, NULL, NULL}, +{119, 7813, 0, NULL, NULL}, +{120, 7821, 0, NULL, NULL}, +{121, 255, 0, NULL, NULL}, +{399, 1242, 0, NULL, NULL}, +{415, 1258, 0, NULL, NULL}, +{601, 1243, 0, NULL, NULL}, +{629, 1259, 0, NULL, NULL}, +{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23}, +{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24}, +{921, 938, 0, NULL, NULL}, +{933, 939, 0, NULL, NULL}, +{953, 970, 0, NULL, NULL}, +{965, 971, 0, NULL, NULL}, +{978, 980, 0, NULL, NULL}, +{1030, 1031, 0, NULL, NULL}, +{1040, 1234, 0, NULL, NULL}, +{1045, 1025, 0, NULL, NULL}, +{1046, 1244, 0, NULL, NULL}, +{1047, 1246, 0, NULL, NULL}, +{1048, 1252, 0, NULL, NULL}, +{1054, 1254, 0, NULL, NULL}, +{1059, 1264, 0, NULL, NULL}, +{1063, 1268, 0, NULL, NULL}, +{1067, 1272, 0, NULL, NULL}, +{1072, 1235, 0, NULL, NULL}, +{1077, 1105, 0, NULL, NULL}, +{1078, 1245, 0, NULL, NULL}, +{1079, 1247, 0, NULL, NULL}, +{1080, 1253, 0, NULL, NULL}, +{1086, 1255, 0, NULL, NULL}, +{1091, 1265, 0, NULL, NULL}, +{1095, 1269, 0, NULL, NULL}, +{1099, 1273, 0, NULL, NULL}, +{1110, 1111, 0, NULL, NULL} +}; /* compose_tab_7 */ +static int hash_compose_tab_8_12[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */ +static CompEntry compose_tab_8_12[] = { +{65, 7848, 0, NULL, NULL}, +{69, 7874, 0, NULL, NULL}, +{79, 7892, 0, NULL, NULL}, +{97, 7849, 0, NULL, NULL}, +{101, 7875, 0, NULL, NULL}, +{111, 7893, 0, NULL, NULL} +}; /* compose_tab_8_12 */ +static int hash_compose_tab_8_13[4] = +{-1,0,1,-1}; /* hash_compose_tab_8_13 */ +static CompEntry compose_tab_8_13[] = { +{65, 7858, 0, NULL, NULL}, +{97, 7859, 0, NULL, NULL} +}; /* compose_tab_8_13 */ +static int hash_compose_tab_8_14[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */ +static CompEntry compose_tab_8_14[] = { +{79, 7902, 0, NULL, NULL}, +{85, 7916, 0, NULL, NULL}, +{111, 7903, 0, NULL, NULL}, +{117, 7917, 0, NULL, NULL} +}; /* compose_tab_8_14 */ +static int hash_compose_tab_8[30] = +{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10, + -1,5}; /* hash_compose_tab_8 */ +static CompEntry compose_tab_8[] = { +{65, 7842, 0, NULL, NULL}, +{69, 7866, 0, NULL, NULL}, +{73, 7880, 0, NULL, NULL}, +{79, 7886, 0, NULL, NULL}, +{85, 7910, 0, NULL, NULL}, +{89, 7926, 0, NULL, NULL}, +{97, 7843, 0, NULL, NULL}, +{101, 7867, 0, NULL, NULL}, +{105, 7881, 0, NULL, NULL}, +{111, 7887, 0, NULL, NULL}, +{117, 7911, 0, NULL, NULL}, +{121, 7927, 0, NULL, NULL}, +{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12}, +{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13}, +{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14} +}; /* compose_tab_8 */ +static int hash_compose_tab_9[12] = +{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */ +static CompEntry compose_tab_9[] = { +{65, 197, 0, NULL, NULL}, +{85, 366, 0, NULL, NULL}, +{97, 229, 0, NULL, NULL}, +{117, 367, 0, NULL, NULL}, +{119, 7832, 0, NULL, NULL}, +{121, 7833, 0, NULL, NULL} +}; /* compose_tab_9 */ +static int hash_compose_tab_10[12] = +{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */ +static CompEntry compose_tab_10[] = { +{79, 336, 0, NULL, NULL}, +{85, 368, 0, NULL, NULL}, +{111, 337, 0, NULL, NULL}, +{117, 369, 0, NULL, NULL}, +{1059, 1266, 0, NULL, NULL}, +{1091, 1267, 0, NULL, NULL} +}; /* compose_tab_10 */ +static int hash_compose_tab_11_33[4] = +{-1,0,1,-1}; /* hash_compose_tab_11_33 */ +static CompEntry compose_tab_11_33[] = { +{85, 473, 0, NULL, NULL}, +{117, 474, 0, NULL, NULL} +}; /* compose_tab_11_33 */ +static int hash_compose_tab_11[68] = +{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1, + 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1, + -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */ +static CompEntry compose_tab_11[] = { +{65, 461, 0, NULL, NULL}, +{67, 268, 0, NULL, NULL}, +{68, 270, 0, NULL, NULL}, +{69, 282, 0, NULL, NULL}, +{71, 486, 0, NULL, NULL}, +{73, 463, 0, NULL, NULL}, +{75, 488, 0, NULL, NULL}, +{76, 317, 0, NULL, NULL}, +{78, 327, 0, NULL, NULL}, +{79, 465, 0, NULL, NULL}, +{82, 344, 0, NULL, NULL}, +{83, 352, 0, NULL, NULL}, +{84, 356, 0, NULL, NULL}, +{85, 467, 0, NULL, NULL}, +{90, 381, 0, NULL, NULL}, +{97, 462, 0, NULL, NULL}, +{99, 269, 0, NULL, NULL}, +{100, 271, 0, NULL, NULL}, +{101, 283, 0, NULL, NULL}, +{103, 487, 0, NULL, NULL}, +{105, 464, 0, NULL, NULL}, +{106, 496, 0, NULL, NULL}, +{107, 489, 0, NULL, NULL}, +{108, 318, 0, NULL, NULL}, +{110, 328, 0, NULL, NULL}, +{111, 466, 0, NULL, NULL}, +{114, 345, 0, NULL, NULL}, +{115, 353, 0, NULL, NULL}, +{116, 357, 0, NULL, NULL}, +{117, 468, 0, NULL, NULL}, +{122, 382, 0, NULL, NULL}, +{439, 494, 0, NULL, NULL}, +{658, 495, 0, NULL, NULL}, +{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33} +}; /* compose_tab_11 */ +static int hash_compose_tab_12_1[4] = +{-1,0,1,-1}; /* hash_compose_tab_12_1 */ +static CompEntry compose_tab_12_1[] = { +{953, 912, 0, NULL, NULL}, +{965, 944, 0, NULL, NULL} +}; /* compose_tab_12_1 */ +static int hash_compose_tab_12[34] = +{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16, + 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */ +static CompEntry compose_tab_12[] = { +{168, 901, 0, NULL, NULL}, +{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1}, +{913, 902, 0, NULL, NULL}, +{917, 904, 0, NULL, NULL}, +{919, 905, 0, NULL, NULL}, +{921, 906, 0, NULL, NULL}, +{927, 908, 0, NULL, NULL}, +{933, 910, 0, NULL, NULL}, +{937, 911, 0, NULL, NULL}, +{945, 940, 0, NULL, NULL}, +{949, 941, 0, NULL, NULL}, +{951, 942, 0, NULL, NULL}, +{953, 943, 0, NULL, NULL}, +{959, 972, 0, NULL, NULL}, +{965, 973, 0, NULL, NULL}, +{969, 974, 0, NULL, NULL}, +{978, 979, 0, NULL, NULL} +}; /* compose_tab_12 */ +static int hash_compose_tab_13[28] = +{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */ +static CompEntry compose_tab_13[] = { +{65, 512, 0, NULL, NULL}, +{69, 516, 0, NULL, NULL}, +{73, 520, 0, NULL, NULL}, +{79, 524, 0, NULL, NULL}, +{82, 528, 0, NULL, NULL}, +{85, 532, 0, NULL, NULL}, +{97, 513, 0, NULL, NULL}, +{101, 517, 0, NULL, NULL}, +{105, 521, 0, NULL, NULL}, +{111, 525, 0, NULL, NULL}, +{114, 529, 0, NULL, NULL}, +{117, 533, 0, NULL, NULL}, +{1140, 1142, 0, NULL, NULL}, +{1141, 1143, 0, NULL, NULL} +}; /* compose_tab_13 */ +static int hash_compose_tab_14[24] = +{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */ +static CompEntry compose_tab_14[] = { +{65, 514, 0, NULL, NULL}, +{69, 518, 0, NULL, NULL}, +{73, 522, 0, NULL, NULL}, +{79, 526, 0, NULL, NULL}, +{82, 530, 0, NULL, NULL}, +{85, 534, 0, NULL, NULL}, +{97, 515, 0, NULL, NULL}, +{101, 519, 0, NULL, NULL}, +{105, 523, 0, NULL, NULL}, +{111, 527, 0, NULL, NULL}, +{114, 531, 0, NULL, NULL}, +{117, 535, 0, NULL, NULL} +}; /* compose_tab_14 */ +static int hash_compose_tab_15_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */ +static CompEntry compose_tab_15_0[] = { +{913, 8072, 0, NULL, NULL}, +{919, 8088, 0, NULL, NULL}, +{937, 8104, 0, NULL, NULL}, +{945, 8064, 0, NULL, NULL}, +{951, 8080, 0, NULL, NULL}, +{969, 8096, 0, NULL, NULL} +}; /* compose_tab_15_0 */ +static int hash_compose_tab_15[30] = +{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5, + 11}; /* hash_compose_tab_15 */ +static CompEntry compose_tab_15[] = { +{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0}, +{913, 7944, 0, NULL, NULL}, +{917, 7960, 0, NULL, NULL}, +{919, 7976, 0, NULL, NULL}, +{921, 7992, 0, NULL, NULL}, +{927, 8008, 0, NULL, NULL}, +{937, 8040, 0, NULL, NULL}, +{945, 7936, 0, NULL, NULL}, +{949, 7952, 0, NULL, NULL}, +{951, 7968, 0, NULL, NULL}, +{953, 7984, 0, NULL, NULL}, +{959, 8000, 0, NULL, NULL}, +{961, 8164, 0, NULL, NULL}, +{965, 8016, 0, NULL, NULL}, +{969, 8032, 0, NULL, NULL} +}; /* compose_tab_15 */ +static int hash_compose_tab_16_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */ +static CompEntry compose_tab_16_0[] = { +{913, 8073, 0, NULL, NULL}, +{919, 8089, 0, NULL, NULL}, +{937, 8105, 0, NULL, NULL}, +{945, 8065, 0, NULL, NULL}, +{951, 8081, 0, NULL, NULL}, +{969, 8097, 0, NULL, NULL} +}; /* compose_tab_16_0 */ +static int hash_compose_tab_16[34] = +{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9, + -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */ +static CompEntry compose_tab_16[] = { +{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0}, +{913, 7945, 0, NULL, NULL}, +{917, 7961, 0, NULL, NULL}, +{919, 7977, 0, NULL, NULL}, +{921, 7993, 0, NULL, NULL}, +{927, 8009, 0, NULL, NULL}, +{929, 8172, 0, NULL, NULL}, +{933, 8025, 0, NULL, NULL}, +{937, 8041, 0, NULL, NULL}, +{945, 7937, 0, NULL, NULL}, +{949, 7953, 0, NULL, NULL}, +{951, 7969, 0, NULL, NULL}, +{953, 7985, 0, NULL, NULL}, +{959, 8001, 0, NULL, NULL}, +{961, 8165, 0, NULL, NULL}, +{965, 8017, 0, NULL, NULL}, +{969, 8033, 0, NULL, NULL} +}; /* compose_tab_16 */ +static int hash_compose_tab_17[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */ +static CompEntry compose_tab_17[] = { +{79, 416, 0, NULL, NULL}, +{85, 431, 0, NULL, NULL}, +{111, 417, 0, NULL, NULL}, +{117, 432, 0, NULL, NULL} +}; /* compose_tab_17 */ +static int hash_compose_tab_18_38[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */ +static CompEntry compose_tab_18_38[] = { +{79, 7906, 0, NULL, NULL}, +{85, 7920, 0, NULL, NULL}, +{111, 7907, 0, NULL, NULL}, +{117, 7921, 0, NULL, NULL} +}; /* compose_tab_18_38 */ +static int hash_compose_tab_18[78] = +{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1, + 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */ +static CompEntry compose_tab_18[] = { +{65, 7840, 0, NULL, NULL}, +{66, 7684, 0, NULL, NULL}, +{68, 7692, 0, NULL, NULL}, +{69, 7864, 0, NULL, NULL}, +{72, 7716, 0, NULL, NULL}, +{73, 7882, 0, NULL, NULL}, +{75, 7730, 0, NULL, NULL}, +{76, 7734, 0, NULL, NULL}, +{77, 7746, 0, NULL, NULL}, +{78, 7750, 0, NULL, NULL}, +{79, 7884, 0, NULL, NULL}, +{82, 7770, 0, NULL, NULL}, +{83, 7778, 0, NULL, NULL}, +{84, 7788, 0, NULL, NULL}, +{85, 7908, 0, NULL, NULL}, +{86, 7806, 0, NULL, NULL}, +{87, 7816, 0, NULL, NULL}, +{89, 7924, 0, NULL, NULL}, +{90, 7826, 0, NULL, NULL}, +{97, 7841, 0, NULL, NULL}, +{98, 7685, 0, NULL, NULL}, +{100, 7693, 0, NULL, NULL}, +{101, 7865, 0, NULL, NULL}, +{104, 7717, 0, NULL, NULL}, +{105, 7883, 0, NULL, NULL}, +{107, 7731, 0, NULL, NULL}, +{108, 7735, 0, NULL, NULL}, +{109, 7747, 0, NULL, NULL}, +{110, 7751, 0, NULL, NULL}, +{111, 7885, 0, NULL, NULL}, +{114, 7771, 0, NULL, NULL}, +{115, 7779, 0, NULL, NULL}, +{116, 7789, 0, NULL, NULL}, +{117, 7909, 0, NULL, NULL}, +{118, 7807, 0, NULL, NULL}, +{119, 7817, 0, NULL, NULL}, +{121, 7925, 0, NULL, NULL}, +{122, 7827, 0, NULL, NULL}, +{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38} +}; /* compose_tab_18 */ +static int hash_compose_tab_19[4] = +{-1,0,1,-1}; /* hash_compose_tab_19 */ +static CompEntry compose_tab_19[] = { +{85, 7794, 0, NULL, NULL}, +{117, 7795, 0, NULL, NULL} +}; /* compose_tab_19 */ +static int hash_compose_tab_20[4] = +{-1,0,1,-1}; /* hash_compose_tab_20 */ +static CompEntry compose_tab_20[] = { +{65, 7680, 0, NULL, NULL}, +{97, 7681, 0, NULL, NULL} +}; /* compose_tab_20 */ +static int hash_compose_tab_21[40] = +{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1, + -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */ +static CompEntry compose_tab_21[] = { +{67, 199, 0, NULL, NULL}, +{68, 7696, 0, NULL, NULL}, +{71, 290, 0, NULL, NULL}, +{72, 7720, 0, NULL, NULL}, +{75, 310, 0, NULL, NULL}, +{76, 315, 0, NULL, NULL}, +{78, 325, 0, NULL, NULL}, +{82, 342, 0, NULL, NULL}, +{83, 350, 0, NULL, NULL}, +{84, 354, 0, NULL, NULL}, +{99, 231, 0, NULL, NULL}, +{100, 7697, 0, NULL, NULL}, +{103, 291, 0, NULL, NULL}, +{104, 7721, 0, NULL, NULL}, +{107, 311, 0, NULL, NULL}, +{108, 316, 0, NULL, NULL}, +{110, 326, 0, NULL, NULL}, +{114, 343, 0, NULL, NULL}, +{115, 351, 0, NULL, NULL}, +{116, 355, 0, NULL, NULL} +}; /* compose_tab_21 */ +static int hash_compose_tab_22[20] = +{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */ +static CompEntry compose_tab_22[] = { +{65, 260, 0, NULL, NULL}, +{69, 280, 0, NULL, NULL}, +{73, 302, 0, NULL, NULL}, +{79, 490, 0, NULL, NULL}, +{85, 370, 0, NULL, NULL}, +{97, 261, 0, NULL, NULL}, +{101, 281, 0, NULL, NULL}, +{105, 303, 0, NULL, NULL}, +{111, 491, 0, NULL, NULL}, +{117, 371, 0, NULL, NULL} +}; /* compose_tab_22 */ +static int hash_compose_tab_23[24] = +{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */ +static CompEntry compose_tab_23[] = { +{68, 7698, 0, NULL, NULL}, +{69, 7704, 0, NULL, NULL}, +{76, 7740, 0, NULL, NULL}, +{78, 7754, 0, NULL, NULL}, +{84, 7792, 0, NULL, NULL}, +{85, 7798, 0, NULL, NULL}, +{100, 7699, 0, NULL, NULL}, +{101, 7705, 0, NULL, NULL}, +{108, 7741, 0, NULL, NULL}, +{110, 7755, 0, NULL, NULL}, +{116, 7793, 0, NULL, NULL}, +{117, 7799, 0, NULL, NULL} +}; /* compose_tab_23 */ +static int hash_compose_tab_24[4] = +{0,1,-1,-1}; /* hash_compose_tab_24 */ +static CompEntry compose_tab_24[] = { +{72, 7722, 0, NULL, NULL}, +{104, 7723, 0, NULL, NULL} +}; /* compose_tab_24 */ +static int hash_compose_tab_25[12] = +{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */ +static CompEntry compose_tab_25[] = { +{69, 7706, 0, NULL, NULL}, +{73, 7724, 0, NULL, NULL}, +{85, 7796, 0, NULL, NULL}, +{101, 7707, 0, NULL, NULL}, +{105, 7725, 0, NULL, NULL}, +{117, 7797, 0, NULL, NULL} +}; /* compose_tab_25 */ +static int hash_compose_tab_26[34] = +{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1, + -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */ +static CompEntry compose_tab_26[] = { +{66, 7686, 0, NULL, NULL}, +{68, 7694, 0, NULL, NULL}, +{75, 7732, 0, NULL, NULL}, +{76, 7738, 0, NULL, NULL}, +{78, 7752, 0, NULL, NULL}, +{82, 7774, 0, NULL, NULL}, +{84, 7790, 0, NULL, NULL}, +{90, 7828, 0, NULL, NULL}, +{98, 7687, 0, NULL, NULL}, +{100, 7695, 0, NULL, NULL}, +{104, 7830, 0, NULL, NULL}, +{107, 7733, 0, NULL, NULL}, +{108, 7739, 0, NULL, NULL}, +{110, 7753, 0, NULL, NULL}, +{114, 7775, 0, NULL, NULL}, +{116, 7791, 0, NULL, NULL}, +{122, 7829, 0, NULL, NULL} +}; /* compose_tab_26 */ +static int hash_compose_tab_27_1[4] = +{-1,0,1,-1}; /* hash_compose_tab_27_1 */ +static CompEntry compose_tab_27_1[] = { +{953, 8151, 0, NULL, NULL}, +{965, 8167, 0, NULL, NULL} +}; /* compose_tab_27_1 */ +static int hash_compose_tab_27_2_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */ +static CompEntry compose_tab_27_2_0[] = { +{913, 8078, 0, NULL, NULL}, +{919, 8094, 0, NULL, NULL}, +{937, 8110, 0, NULL, NULL}, +{945, 8070, 0, NULL, NULL}, +{951, 8086, 0, NULL, NULL}, +{969, 8102, 0, NULL, NULL} +}; /* compose_tab_27_2_0 */ +static int hash_compose_tab_27_2[20] = +{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */ +static CompEntry compose_tab_27_2[] = { +{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0}, +{913, 7950, 0, NULL, NULL}, +{919, 7982, 0, NULL, NULL}, +{921, 7998, 0, NULL, NULL}, +{937, 8046, 0, NULL, NULL}, +{945, 7942, 0, NULL, NULL}, +{951, 7974, 0, NULL, NULL}, +{953, 7990, 0, NULL, NULL}, +{965, 8022, 0, NULL, NULL}, +{969, 8038, 0, NULL, NULL} +}; /* compose_tab_27_2 */ +static int hash_compose_tab_27_3_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */ +static CompEntry compose_tab_27_3_0[] = { +{913, 8079, 0, NULL, NULL}, +{919, 8095, 0, NULL, NULL}, +{937, 8111, 0, NULL, NULL}, +{945, 8071, 0, NULL, NULL}, +{951, 8087, 0, NULL, NULL}, +{969, 8103, 0, NULL, NULL} +}; /* compose_tab_27_3_0 */ +static int hash_compose_tab_27_3[22] = +{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */ +static CompEntry compose_tab_27_3[] = { +{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0}, +{913, 7951, 0, NULL, NULL}, +{919, 7983, 0, NULL, NULL}, +{921, 7999, 0, NULL, NULL}, +{933, 8031, 0, NULL, NULL}, +{937, 8047, 0, NULL, NULL}, +{945, 7943, 0, NULL, NULL}, +{951, 7975, 0, NULL, NULL}, +{953, 7991, 0, NULL, NULL}, +{965, 8023, 0, NULL, NULL}, +{969, 8039, 0, NULL, NULL} +}; /* compose_tab_27_3 */ +static int hash_compose_tab_27_4[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */ +static CompEntry compose_tab_27_4[] = { +{945, 8119, 0, NULL, NULL}, +{951, 8135, 0, NULL, NULL}, +{969, 8183, 0, NULL, NULL} +}; /* compose_tab_27_4 */ +static int hash_compose_tab_27[24] = +{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */ +static CompEntry compose_tab_27[] = { +{168, 8129, 0, NULL, NULL}, +{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1}, +{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2}, +{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3}, +{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4}, +{945, 8118, 0, NULL, NULL}, +{951, 8134, 0, NULL, NULL}, +{953, 8150, 0, NULL, NULL}, +{965, 8166, 0, NULL, NULL}, +{969, 8182, 0, NULL, NULL}, +{8127, 8143, 0, NULL, NULL}, +{8190, 8159, 0, NULL, NULL} +}; /* compose_tab_27 */ +static int hash_compose_tab_28[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */ +static CompEntry compose_tab_28[] = { +{913, 8124, 0, NULL, NULL}, +{919, 8140, 0, NULL, NULL}, +{937, 8188, 0, NULL, NULL}, +{945, 8115, 0, NULL, NULL}, +{951, 8131, 0, NULL, NULL}, +{969, 8179, 0, NULL, NULL} +}; /* compose_tab_28 */ +static int hash_compose_tab_29[4] = +{0,-1,1,-1}; /* hash_compose_tab_29 */ +static CompEntry compose_tab_29[] = { +{1488, 64302, 0, NULL, NULL}, +{1522, 64287, 0, NULL, NULL} +}; /* compose_tab_29 */ +static int hash_compose_tab_30[2] = +{0,-1}; /* hash_compose_tab_30 */ +static CompEntry compose_tab_30[] = { +{1488, 64303, 0, NULL, NULL} +}; /* compose_tab_30 */ +static int hash_compose_tab_31[2] = +{-1,0}; /* hash_compose_tab_31 */ +static CompEntry compose_tab_31[] = { +{1493, 64331, 0, NULL, NULL} +}; /* compose_tab_31 */ +static int hash_compose_tab_32[44] = +{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */ +static CompEntry compose_tab_32[] = { +{1488, 64304, 0, NULL, NULL}, +{1489, 64305, 0, NULL, NULL}, +{1490, 64306, 0, NULL, NULL}, +{1491, 64307, 0, NULL, NULL}, +{1492, 64308, 0, NULL, NULL}, +{1493, 64309, 0, NULL, NULL}, +{1494, 64310, 0, NULL, NULL}, +{1496, 64312, 0, NULL, NULL}, +{1497, 64313, 0, NULL, NULL}, +{1498, 64314, 0, NULL, NULL}, +{1499, 64315, 0, NULL, NULL}, +{1500, 64316, 0, NULL, NULL}, +{1502, 64318, 0, NULL, NULL}, +{1504, 64320, 0, NULL, NULL}, +{1505, 64321, 0, NULL, NULL}, +{1507, 64323, 0, NULL, NULL}, +{1508, 64324, 0, NULL, NULL}, +{1510, 64326, 0, NULL, NULL}, +{1511, 64327, 0, NULL, NULL}, +{1512, 64328, 0, NULL, NULL}, +{1513, 64329, 0, NULL, NULL}, +{1514, 64330, 0, NULL, NULL} +}; /* compose_tab_32 */ +static int hash_compose_tab_33[6] = +{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */ +static CompEntry compose_tab_33[] = { +{1489, 64332, 0, NULL, NULL}, +{1499, 64333, 0, NULL, NULL}, +{1508, 64334, 0, NULL, NULL} +}; /* compose_tab_33 */ +static int hash_compose_tab_34_0[2] = +{-1,0}; /* hash_compose_tab_34_0 */ +static CompEntry compose_tab_34_0[] = { +{1513, 64300, 0, NULL, NULL} +}; /* compose_tab_34_0 */ +static int hash_compose_tab_34[4] = +{0,1,-1,-1}; /* hash_compose_tab_34 */ +static CompEntry compose_tab_34[] = { +{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0}, +{1513, 64298, 0, NULL, NULL} +}; /* compose_tab_34 */ +static int hash_compose_tab_35_0[2] = +{-1,0}; /* hash_compose_tab_35_0 */ +static CompEntry compose_tab_35_0[] = { +{1513, 64301, 0, NULL, NULL} +}; /* compose_tab_35_0 */ +static int hash_compose_tab_35[4] = +{0,1,-1,-1}; /* hash_compose_tab_35 */ +static CompEntry compose_tab_35[] = { +{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0}, +{1513, 64299, 0, NULL, NULL} +}; /* compose_tab_35 */ +static int hash_compose_tab_36[22] = +{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */ +static CompEntry compose_tab_36[] = { +{2325, 2392, 0, NULL, NULL}, +{2326, 2393, 0, NULL, NULL}, +{2327, 2394, 0, NULL, NULL}, +{2332, 2395, 0, NULL, NULL}, +{2337, 2396, 0, NULL, NULL}, +{2338, 2397, 0, NULL, NULL}, +{2344, 2345, 0, NULL, NULL}, +{2347, 2398, 0, NULL, NULL}, +{2351, 2399, 0, NULL, NULL}, +{2352, 2353, 0, NULL, NULL}, +{2355, 2356, 0, NULL, NULL} +}; /* compose_tab_36 */ +static int hash_compose_tab_37[8] = +{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */ +static CompEntry compose_tab_37[] = { +{2465, 2524, 0, NULL, NULL}, +{2466, 2525, 0, NULL, NULL}, +{2476, 2480, 0, NULL, NULL}, +{2479, 2527, 0, NULL, NULL} +}; /* compose_tab_37 */ +static int hash_compose_tab_38[2] = +{-1,0}; /* hash_compose_tab_38 */ +static CompEntry compose_tab_38[] = { +{2503, 2507, 0, NULL, NULL} +}; /* compose_tab_38 */ +static int hash_compose_tab_39[2] = +{-1,0}; /* hash_compose_tab_39 */ +static CompEntry compose_tab_39[] = { +{2503, 2508, 0, NULL, NULL} +}; /* compose_tab_39 */ +static int hash_compose_tab_40[10] = +{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */ +static CompEntry compose_tab_40[] = { +{2582, 2649, 0, NULL, NULL}, +{2583, 2650, 0, NULL, NULL}, +{2588, 2651, 0, NULL, NULL}, +{2593, 2652, 0, NULL, NULL}, +{2603, 2654, 0, NULL, NULL} +}; /* compose_tab_40 */ +static int hash_compose_tab_41[6] = +{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */ +static CompEntry compose_tab_41[] = { +{2849, 2908, 0, NULL, NULL}, +{2850, 2909, 0, NULL, NULL}, +{2863, 2911, 0, NULL, NULL} +}; /* compose_tab_41 */ +static int hash_compose_tab_42[2] = +{-1,0}; /* hash_compose_tab_42 */ +static CompEntry compose_tab_42[] = { +{2887, 2891, 0, NULL, NULL} +}; /* compose_tab_42 */ +static int hash_compose_tab_43[2] = +{-1,0}; /* hash_compose_tab_43 */ +static CompEntry compose_tab_43[] = { +{2887, 2888, 0, NULL, NULL} +}; /* compose_tab_43 */ +static int hash_compose_tab_44[2] = +{-1,0}; /* hash_compose_tab_44 */ +static CompEntry compose_tab_44[] = { +{2887, 2892, 0, NULL, NULL} +}; /* compose_tab_44 */ +static int hash_compose_tab_45[4] = +{-1,-1,0,1}; /* hash_compose_tab_45 */ +static CompEntry compose_tab_45[] = { +{3014, 3018, 0, NULL, NULL}, +{3015, 3019, 0, NULL, NULL} +}; /* compose_tab_45 */ +static int hash_compose_tab_46[4] = +{-1,-1,0,1}; /* hash_compose_tab_46 */ +static CompEntry compose_tab_46[] = { +{2962, 2964, 0, NULL, NULL}, +{3014, 3020, 0, NULL, NULL} +}; /* compose_tab_46 */ +static int hash_compose_tab_47[2] = +{0,-1}; /* hash_compose_tab_47 */ +static CompEntry compose_tab_47[] = { +{3142, 3144, 0, NULL, NULL} +}; /* compose_tab_47 */ +static int hash_compose_tab_48[2] = +{0,-1}; /* hash_compose_tab_48 */ +static CompEntry compose_tab_48[] = { +{3270, 3274, 0, NULL, NULL} +}; /* compose_tab_48 */ +static int hash_compose_tab_49_1[2] = +{0,-1}; /* hash_compose_tab_49_1 */ +static CompEntry compose_tab_49_1[] = { +{3270, 3275, 0, NULL, NULL} +}; /* compose_tab_49_1 */ +static int hash_compose_tab_49[6] = +{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */ +static CompEntry compose_tab_49[] = { +{3263, 3264, 0, NULL, NULL}, +{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1}, +{3270, 3271, 0, NULL, NULL} +}; /* compose_tab_49 */ +static int hash_compose_tab_50[2] = +{0,-1}; /* hash_compose_tab_50 */ +static CompEntry compose_tab_50[] = { +{3270, 3272, 0, NULL, NULL} +}; /* compose_tab_50 */ +static int hash_compose_tab_51[4] = +{-1,-1,0,1}; /* hash_compose_tab_51 */ +static CompEntry compose_tab_51[] = { +{3398, 3402, 0, NULL, NULL}, +{3399, 3403, 0, NULL, NULL} +}; /* compose_tab_51 */ +static int hash_compose_tab_52[2] = +{0,-1}; /* hash_compose_tab_52 */ +static CompEntry compose_tab_52[] = { +{3398, 3404, 0, NULL, NULL} +}; /* compose_tab_52 */ +static int hash_compose_tab_53[2] = +{-1,0}; /* hash_compose_tab_53 */ +static CompEntry compose_tab_53[] = { +{3661, 3635, 0, NULL, NULL} +}; /* compose_tab_53 */ +static int hash_compose_tab_54[2] = +{-1,0}; /* hash_compose_tab_54 */ +static CompEntry compose_tab_54[] = { +{3789, 3763, 0, NULL, NULL} +}; /* compose_tab_54 */ +static int hash_compose_tab_55_2[4] = +{-1,-1,0,1}; /* hash_compose_tab_55_2 */ +static CompEntry compose_tab_55_2[] = { +{4018, 3959, 0, NULL, NULL}, +{4019, 3961, 0, NULL, NULL} +}; /* compose_tab_55_2 */ +static int hash_compose_tab_55[6] = +{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */ +static CompEntry compose_tab_55[] = { +{3954, 3955, 0, NULL, NULL}, +{3956, 3957, 0, NULL, NULL}, +{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2} +}; /* compose_tab_55 */ +static int hash_compose_tab_56[4] = +{-1,-1,0,1}; /* hash_compose_tab_56 */ +static CompEntry compose_tab_56[] = { +{4018, 3958, 0, NULL, NULL}, +{4019, 3960, 0, NULL, NULL} +}; /* compose_tab_56 */ +static int hash_compose_tab_57[4] = +{0,1,-1,-1}; /* hash_compose_tab_57 */ +static CompEntry compose_tab_57[] = { +{3904, 3945, 0, NULL, NULL}, +{3984, 4025, 0, NULL, NULL} +}; /* compose_tab_57 */ +static int hash_compose_tab_58[20] = +{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */ +static CompEntry compose_tab_58[] = { +{3906, 3907, 0, NULL, NULL}, +{3916, 3917, 0, NULL, NULL}, +{3921, 3922, 0, NULL, NULL}, +{3926, 3927, 0, NULL, NULL}, +{3931, 3932, 0, NULL, NULL}, +{3986, 3987, 0, NULL, NULL}, +{3996, 3997, 0, NULL, NULL}, +{4001, 4002, 0, NULL, NULL}, +{4006, 4007, 0, NULL, NULL}, +{4011, 4012, 0, NULL, NULL} +}; /* compose_tab_58 */ +static int hash_compose_tab_59[96] = +{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19, + 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45, + 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1, + 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */ +static CompEntry compose_tab_59[] = { +{12358, 12436, 0, NULL, NULL}, +{12363, 12364, 0, NULL, NULL}, +{12365, 12366, 0, NULL, NULL}, +{12367, 12368, 0, NULL, NULL}, +{12369, 12370, 0, NULL, NULL}, +{12371, 12372, 0, NULL, NULL}, +{12373, 12374, 0, NULL, NULL}, +{12375, 12376, 0, NULL, NULL}, +{12377, 12378, 0, NULL, NULL}, +{12379, 12380, 0, NULL, NULL}, +{12381, 12382, 0, NULL, NULL}, +{12383, 12384, 0, NULL, NULL}, +{12385, 12386, 0, NULL, NULL}, +{12388, 12389, 0, NULL, NULL}, +{12390, 12391, 0, NULL, NULL}, +{12392, 12393, 0, NULL, NULL}, +{12399, 12400, 0, NULL, NULL}, +{12402, 12403, 0, NULL, NULL}, +{12405, 12406, 0, NULL, NULL}, +{12408, 12409, 0, NULL, NULL}, +{12411, 12412, 0, NULL, NULL}, +{12445, 12446, 0, NULL, NULL}, +{12454, 12532, 0, NULL, NULL}, +{12459, 12460, 0, NULL, NULL}, +{12461, 12462, 0, NULL, NULL}, +{12463, 12464, 0, NULL, NULL}, +{12465, 12466, 0, NULL, NULL}, +{12467, 12468, 0, NULL, NULL}, +{12469, 12470, 0, NULL, NULL}, +{12471, 12472, 0, NULL, NULL}, +{12473, 12474, 0, NULL, NULL}, +{12475, 12476, 0, NULL, NULL}, +{12477, 12478, 0, NULL, NULL}, +{12479, 12480, 0, NULL, NULL}, +{12481, 12482, 0, NULL, NULL}, +{12484, 12485, 0, NULL, NULL}, +{12486, 12487, 0, NULL, NULL}, +{12488, 12489, 0, NULL, NULL}, +{12495, 12496, 0, NULL, NULL}, +{12498, 12499, 0, NULL, NULL}, +{12501, 12502, 0, NULL, NULL}, +{12504, 12505, 0, NULL, NULL}, +{12507, 12508, 0, NULL, NULL}, +{12527, 12535, 0, NULL, NULL}, +{12528, 12536, 0, NULL, NULL}, +{12529, 12537, 0, NULL, NULL}, +{12530, 12538, 0, NULL, NULL}, +{12541, 12542, 0, NULL, NULL} +}; /* compose_tab_59 */ +static int hash_compose_tab_60[20] = +{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */ +static CompEntry compose_tab_60[] = { +{12399, 12401, 0, NULL, NULL}, +{12402, 12404, 0, NULL, NULL}, +{12405, 12407, 0, NULL, NULL}, +{12408, 12410, 0, NULL, NULL}, +{12411, 12413, 0, NULL, NULL}, +{12495, 12497, 0, NULL, NULL}, +{12498, 12500, 0, NULL, NULL}, +{12501, 12503, 0, NULL, NULL}, +{12504, 12506, 0, NULL, NULL}, +{12507, 12509, 0, NULL, NULL} +}; /* compose_tab_60 */ +static int hash_compose_tab[122] = +{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14, + 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45, + 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46, + 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */ +static CompEntry compose_tab[] = { +{768, 0, 39, compose_tab_0, hash_compose_tab_0}, +{769, 0, 70, compose_tab_1, hash_compose_tab_1}, +{770, 0, 27, compose_tab_2, hash_compose_tab_2}, +{771, 0, 19, compose_tab_3, hash_compose_tab_3}, +{772, 0, 28, compose_tab_4, hash_compose_tab_4}, +{774, 0, 30, compose_tab_5, hash_compose_tab_5}, +{775, 0, 40, compose_tab_6, hash_compose_tab_6}, +{776, 0, 50, compose_tab_7, hash_compose_tab_7}, +{777, 0, 15, compose_tab_8, hash_compose_tab_8}, +{778, 0, 6, compose_tab_9, hash_compose_tab_9}, +{779, 0, 6, compose_tab_10, hash_compose_tab_10}, +{780, 0, 34, compose_tab_11, hash_compose_tab_11}, +{781, 0, 17, compose_tab_12, hash_compose_tab_12}, +{783, 0, 14, compose_tab_13, hash_compose_tab_13}, +{785, 0, 12, compose_tab_14, hash_compose_tab_14}, +{787, 0, 15, compose_tab_15, hash_compose_tab_15}, +{788, 0, 17, compose_tab_16, hash_compose_tab_16}, +{795, 0, 4, compose_tab_17, hash_compose_tab_17}, +{803, 0, 39, compose_tab_18, hash_compose_tab_18}, +{804, 0, 2, compose_tab_19, hash_compose_tab_19}, +{805, 0, 2, compose_tab_20, hash_compose_tab_20}, +{807, 0, 20, compose_tab_21, hash_compose_tab_21}, +{808, 0, 10, compose_tab_22, hash_compose_tab_22}, +{813, 0, 12, compose_tab_23, hash_compose_tab_23}, +{814, 0, 2, compose_tab_24, hash_compose_tab_24}, +{816, 0, 6, compose_tab_25, hash_compose_tab_25}, +{817, 0, 17, compose_tab_26, hash_compose_tab_26}, +{834, 0, 12, compose_tab_27, hash_compose_tab_27}, +{837, 0, 6, compose_tab_28, hash_compose_tab_28}, +{1463, 0, 2, compose_tab_29, hash_compose_tab_29}, +{1464, 0, 1, compose_tab_30, hash_compose_tab_30}, +{1465, 0, 1, compose_tab_31, hash_compose_tab_31}, +{1468, 0, 22, compose_tab_32, hash_compose_tab_32}, +{1471, 0, 3, compose_tab_33, hash_compose_tab_33}, +{1473, 0, 2, compose_tab_34, hash_compose_tab_34}, +{1474, 0, 2, compose_tab_35, hash_compose_tab_35}, +{2364, 0, 11, compose_tab_36, hash_compose_tab_36}, +{2492, 0, 4, compose_tab_37, hash_compose_tab_37}, +{2494, 0, 1, compose_tab_38, hash_compose_tab_38}, +{2519, 0, 1, compose_tab_39, hash_compose_tab_39}, +{2620, 0, 5, compose_tab_40, hash_compose_tab_40}, +{2876, 0, 3, compose_tab_41, hash_compose_tab_41}, +{2878, 0, 1, compose_tab_42, hash_compose_tab_42}, +{2902, 0, 1, compose_tab_43, hash_compose_tab_43}, +{2903, 0, 1, compose_tab_44, hash_compose_tab_44}, +{3006, 0, 2, compose_tab_45, hash_compose_tab_45}, +{3031, 0, 2, compose_tab_46, hash_compose_tab_46}, +{3158, 0, 1, compose_tab_47, hash_compose_tab_47}, +{3266, 0, 1, compose_tab_48, hash_compose_tab_48}, +{3285, 0, 3, compose_tab_49, hash_compose_tab_49}, +{3286, 0, 1, compose_tab_50, hash_compose_tab_50}, +{3390, 0, 2, compose_tab_51, hash_compose_tab_51}, +{3415, 0, 1, compose_tab_52, hash_compose_tab_52}, +{3634, 0, 1, compose_tab_53, hash_compose_tab_53}, +{3762, 0, 1, compose_tab_54, hash_compose_tab_54}, +{3953, 0, 3, compose_tab_55, hash_compose_tab_55}, +{3968, 0, 2, compose_tab_56, hash_compose_tab_56}, +{4021, 0, 2, compose_tab_57, hash_compose_tab_57}, +{4023, 0, 10, compose_tab_58, hash_compose_tab_58}, +{12441, 0, 48, compose_tab_59, hash_compose_tab_59}, +{12442, 0, 10, compose_tab_60, hash_compose_tab_60} +}; /* compose_tab */ +#define COMP_CANDIDATE_MAP_OFFSET 24 +static Uint32 comp_candidate_map[] = { + 0x081ABFDFU, + 0x000361B8U, + 0x00000024U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x93800000U, + 0x00000006U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x10000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x50000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x10000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x50000000U, + 0x00C00000U, + 0x00000000U, + 0x00000000U, + 0x40000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00400000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00600004U, + 0x00000000U, + 0x00000000U, + 0x40000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00040000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00040000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00020000U, + 0x00000001U, + 0x00A00000U +}; diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 50b3e5b61c..e7fd144ec3 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -47,13 +47,23 @@ #define SEQ_TRACE 1 #define CONTEXT_REDS 2000 /* Swap process out after this number */ -#define MAX_ARG 256 /* Max number of arguments allowed */ +#define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ +/* Scheduler stores data for temporary heaps if + !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary + heap data on the C stack or if we use the buffers in the scheduler data. */ +#define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers + small heap for transient heap data */ +#define CMP_TMP_HEAP_SIZE 2 /* cmp wants its own tmp-heap... */ +#define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ +#define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ + /* * The new arithmetic operations need some extra X registers in the register array. + * so does the gc_bif's (i_gc_bif3 need 3 extra). */ -#define ERTS_X_REGS_ALLOCATED (MAX_REG+2) +#define ERTS_X_REGS_ALLOCATED (MAX_REG+3) #define INPUT_REDUCTIONS (2 * CONTEXT_REDS) @@ -74,6 +84,7 @@ #define ErtsHAllocLockCheck(P) \ ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks((P))) \ + || ((P)->id == ERTS_INVALID_PID) \ || ((P)->scheduler_data \ && (P) == (P)->scheduler_data->match_pseudo_process) \ || erts_is_system_blocked(0)) @@ -109,14 +120,15 @@ * Allocate heap memory, first on the ordinary heap; * failing that, in a heap fragment. */ -#define HAlloc(p, sz) \ +#define HAllocX(p, sz, xtra) \ (ASSERT_EXPR((sz) >= 0), \ ErtsHAllocLockCheck(p), \ (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ - ? erts_heap_alloc((p),(sz)) \ + ? erts_heap_alloc((p),(sz),(xtra)) \ : (INIT_HEAP_MEM(p,sz), \ HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) +#define HAlloc(P, SZ) HAllocX(P,SZ,0) #define HRelease(p, endp, ptr) \ if ((ptr) == (endp)) { \ @@ -130,8 +142,12 @@ #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) #if defined(DEBUG) || defined(CHECK_FOR_HOLES) +#if HALFWORD_HEAP +# define ERTS_HOLE_MARKER (0xaf5e78ccU) +#else # define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL) #endif +#endif /* * Allocate heap memory on the ordinary heap, NEVER in a heap @@ -184,6 +200,7 @@ extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ extern int erts_atom_table_size;/* Atom table size */ #define ORIG_CREATION 0 +#define INTERNAL_CREATION 255 /* macros for extracting bytes from uint16's */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 4930def4ed..ddc2c1396d 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -187,10 +187,10 @@ extern Eterm exception_tag[NUMBER_EXC_TAGS]; struct StackTrace { Eterm header; /* bignum header - must be first in struct */ Eterm freason; /* original exception reason is saved in the struct */ - Eterm* pc; - Eterm* current; + BeamInstr* pc; + BeamInstr* current; int depth; /* number of saved pointers in trace[] */ - Eterm *trace[1]; /* varying size - must be last in struct */ + BeamInstr *trace[1]; /* varying size - must be last in struct */ }; #endif /* __ERROR_H__ */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 271b40cf0f..5bc402fe22 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -43,11 +43,9 @@ static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. #define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock) #define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock) #define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock) -#define export_init_lock() erts_smp_rwmtx_init(&export_table_lock, \ - "export_tab") -extern Eterm* em_call_error_handler; -extern Uint* em_call_traced_function; +extern BeamInstr* em_call_error_handler; +extern BeamInstr* em_call_traced_function; void export_info(int to, void *to_arg) @@ -93,7 +91,7 @@ export_alloc(Export* tmpl) obj->code[2] = tmpl->code[2]; obj->slot.index = -1; obj->address = obj->code+3; - obj->code[3] = (Eterm) em_call_error_handler; + obj->code[3] = (BeamInstr) em_call_error_handler; obj->code[4] = 0; obj->match_prog_set = NULL; return obj; @@ -111,8 +109,12 @@ void init_export_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab"); - export_init_lock(); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; @@ -140,7 +142,7 @@ init_export_table(void) Export* erts_find_export_entry(Eterm m, Eterm f, unsigned int a) { - HashValue hval = EXPORT_HASH(m, f, a); + HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a); int ix; HashBucket* b; @@ -185,7 +187,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a) ep = hash_get(&export_table.htable, (void*) &e); if (ep != NULL && ep->address == ep->code+3 && - ep->code[3] != (Uint) em_call_traced_function) { + ep->code[3] != (BeamInstr) em_call_traced_function) { ep = NULL; } return ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index cd6af6dd85..c604fdf7c3 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -37,7 +37,7 @@ typedef struct export void* address; /* Pointer to code for function. */ struct binary* match_prog_set; /* Match program for tracing. */ - Eterm fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ + BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ /* * code[0]: Tagged atom for module. * code[1]: Tagged atom for function. @@ -52,7 +52,7 @@ typedef struct export * on_load function that has not been run yet. * Otherwise: 0. */ - Eterm code[5]; + BeamInstr code[5]; } Export; @@ -74,6 +74,6 @@ Export *export_get(Export*); #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ (((EntryPtr)->address == (EntryPtr)->code + 3) && \ - ((EntryPtr)->code[3] == (Uint) em_apply_bif)) + ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) #endif diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 099eddd195..1a102f7187 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -49,10 +49,8 @@ #define in_area(ptr,start,nbytes) ((Uint)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff -#define dec_set_creation(nodename,creat) \ - (((nodename) == erts_this_node->sysname && (creat) == ORIG_CREATION) \ - ? erts_this_node->creation \ - : (creat)) + +#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -65,11 +63,9 @@ # endif #endif -/* - * For backward compatibility reasons, only encode integers that - * fit in 28 bits (signed) using INTEGER_EXT. +/* Does Sint fit in Sint32? */ -#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) +#define IS_SSMALL32(x) (((Uint) (((x) >> (32-1)) + 1)) < 2) /* * Valid creations for nodes are 1, 2, or 3. 0 can also be sent @@ -85,14 +81,14 @@ * */ -static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32); +static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); -static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins); +static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins, int internal_tags); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); @@ -463,6 +459,12 @@ Uint erts_encode_ext_size(Eterm term) + 1 /* VERSION_MAGIC */; } +Uint erts_encode_ext_size_ets(Eterm term) +{ + return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS); +} + + void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp) { byte *ep = *ext; @@ -470,7 +472,7 @@ void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) #endif *ep++ = VERSION_MAGIC; - ep = enc_term(acmp, term, ep, flags); + ep = enc_term(acmp, term, ep, flags, NULL); if (!ep) erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_dist_ext(): Internal data structure error\n", @@ -482,7 +484,7 @@ void erts_encode_ext(Eterm term, byte **ext) { byte *ep = *ext; *ep++ = VERSION_MAGIC; - ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS); + ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", @@ -490,6 +492,12 @@ void erts_encode_ext(Eterm term, byte **ext) *ext = ep; } +byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off_heap) +{ + return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS, + off_heap); +} + ErtsDistExternal * erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) { @@ -504,7 +512,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) ASSERT(edep->ext_endp >= edep->extp); ext_sz = edep->ext_endp - edep->extp; - align_sz = ERTS_WORD_ALIGN_PAD_SZ(dist_ext_sz + ext_sz); + align_sz = ERTS_EXTRA_DATA_ALIGN_SZ(dist_ext_sz + ext_sz); new_edep = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dist_ext_sz + ext_sz + align_sz + xsize); @@ -815,7 +823,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins) goto fail; ep = edep->extp+1; } - res = decoded_size(ep, edep->ext_endp, no_refc_bins); + res = decoded_size(ep, edep->ext_endp, no_refc_bins, 0); if (res >= 0) return res; fail: @@ -827,9 +835,17 @@ Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins) { if (size == 0 || *ext != VERSION_MAGIC) return -1; - return decoded_size(ext+1, ext+size, no_refc_bins); + return decoded_size(ext+1, ext+size, no_refc_bins, 0); +} + +Sint erts_decode_ext_size_ets(byte *ext, Uint size) +{ + Sint sz = decoded_size(ext, ext+size, 0, 1); + ASSERT(sz >= 0); + return sz; } + /* ** hpp is set to either a &p->htop or ** a pointer to a memory pointer (form message buffers) @@ -889,7 +905,13 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) return obj; } - +Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +{ + Eterm obj; + ext = dec_term(NULL, hpp, ext, off_heap, &obj); + ASSERT(ext); + return obj; +} /**********************************************************************/ @@ -966,6 +988,7 @@ term_to_binary_1(Process* p, Eterm Term) return erts_term_to_binary(p, Term, 0, TERM_TO_BINARY_DFLAGS); } + Eterm term_to_binary_2(Process* p, Eterm Term, Eterm Flags) { @@ -1077,7 +1100,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) goto error; size = (Sint) dest_len; } - res = decoded_size(state->extp, state->extp + size, 0); + res = decoded_size(state->extp, state->extp + size, 0, 0); if (res < 0) goto error; return res; @@ -1185,7 +1208,8 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) opt = CAR(list_val(opts)); if (opt == am_safe) { fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE; - } else { + } + else { goto error; } opts = CDR(list_val(opts)); @@ -1240,7 +1264,7 @@ external_size_1(Process* p, Eterm Term) Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { - int size; + Uint size; Eterm bin; size_t real_size; byte* endp; @@ -1257,7 +1281,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) bytes = erts_alloc(ERTS_ALC_T_TMP, size); } - if ((endp = enc_term(NULL, Term, bytes, flags)) + if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { erl_exit(1, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -1302,7 +1326,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); bytes[0] = VERSION_MAGIC; - if ((endp = enc_term(NULL, Term, bytes+1, flags)) + if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { erl_exit(1, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -1332,6 +1356,21 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) ASSERT(is_atom(atom)); + if (dflags & DFLAGS_INTERNAL_TAGS) { + Uint aval = atom_val(atom); + ASSERT(aval < (1<<24)); + if (aval >= (1 << 16)) { + *ep++ = ATOM_INTERNAL_REF3; + put_int24(aval, ep); + ep += 3; + } + else { + *ep++ = ATOM_INTERNAL_REF2; + put_int16(aval, ep); + ep += 2; + } + return ep; + } /* * term_to_binary/1,2 and the initial distribution message * don't use the cache. @@ -1381,7 +1420,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - *ep++ = pid_creation(pid); + *ep++ = (is_internal_pid(pid) && (dflags & DFLAGS_INTERNAL_TAGS)) ? + INTERNAL_CREATION : pid_creation(pid); return ep; } @@ -1420,6 +1460,23 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) } ep += len; break; + case ATOM_INTERNAL_REF2: + n = get_int16(ep); + ep += 2; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + case ATOM_INTERNAL_REF3: + n = get_int24(ep); + ep += 3; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + default: error: *objp = NIL; /* Don't leave a hole in the heap */ @@ -1428,6 +1485,19 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) return ep; } +static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) +{ + switch (creation) { + case INTERNAL_CREATION: + return erts_this_node; + case ORIG_CREATION: + if (sysname == erts_this_node->sysname) { + creation = erts_this_node->creation; + } + } + return erts_find_or_insert_node(sysname,creation); +} + static byte* dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) { @@ -1451,18 +1521,20 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete ep += 4; if (ser > ERTS_MAX_PID_SERIAL) return NULL; - if ((cre = get_int8(ep)) >= MAX_CREATION) - return NULL; + cre = get_int8(ep); ep += 1; + if (!is_valid_creation(cre)) { + return NULL; + } + data = make_pid_data(ser, num); + /* * We are careful to create the node entry only after all * validity tests are done. */ - cre = dec_set_creation(sysname,cre); - node = erts_find_or_insert_node(sysname,cre); + node = dec_get_node(sysname, cre); - data = make_pid_data(ser, num); if(node == erts_this_node) { *objp = make_internal_pid(data); } else { @@ -1470,11 +1542,11 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete *hpp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -1487,22 +1559,31 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) static byte* -enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) +enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint n; Uint i; Uint j; Uint* ptr; Eterm val; FloatDef f; +#if HALFWORD_HEAP + UWord wobj; +#endif + goto L_jump_start; outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - switch (val = ESTACK_POP(s)) { + while (!WSTACK_ISEMPTY(s)) { +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); +#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; case ENC_ONE_CONS: @@ -1513,29 +1594,40 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) obj = CAR(cons); tl = CDR(cons); - ESTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - ESTACK_PUSH(s, tl); + WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); + WSTACK_PUSH(s, tl); } break; case ENC_PATCH_FUN_SIZE: { +#if HALFWORD_HEAP + byte* size_p = (byte *) wobj; +#else byte* size_p = (byte *) obj; - +#endif put_int32(ep - size_p, size_p); } goto outer_loop; case ENC_LAST_ARRAY_ELEMENT: { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; +#endif obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; +#endif obj = *ptr++; - ESTACK_PUSH(s, val-1); - ESTACK_PUSH(s, (Eterm) ptr); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, (UWord) ptr); } break; } @@ -1552,19 +1644,23 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) case SMALL_DEF: { + /* From R14B we no longer restrict INTEGER_EXT to 28 bits, + * as done earlier for backward compatibility reasons. */ Sint val = signed_val(obj); if ((Uint)val < 256) { *ep++ = SMALL_INTEGER_EXT; put_int8(val, ep); ep++; - } else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) { + } else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) { *ep++ = INTEGER_EXT; put_int32(val, ep); ep += 4; } else { - Eterm tmp_big[2]; - Eterm big = small_to_big(val, tmp_big); + DeclareTmpHeapNoproc(tmp_big,2); + Eterm big; + UseTmpHeapNoproc(2); + big = small_to_big(val, tmp_big); *ep++ = SMALL_BIG_EXT; n = big_bytes(big); ASSERT(n < 256); @@ -1572,23 +1668,38 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) ep += 1; *ep++ = big_sign(big); ep = big_to_bytes(big, ep); + UnUseTmpHeapNoproc(2); } } break; case BIG_DEF: - if ((n = big_bytes(obj)) < 256) { - *ep++ = SMALL_BIG_EXT; - put_int8(n, ep); - ep += 1; - } - else { - *ep++ = LARGE_BIG_EXT; - put_int32(n, ep); - ep += 4; + { + int sign = big_sign(obj); + n = big_bytes(obj); + if (sizeof(Sint)==4 && n<=4) { + Uint dig = big_digit(obj,0); + Sint val = sign ? -dig : dig; + if ((val<0) == sign) { + *ep++ = INTEGER_EXT; + put_int32(val, ep); + ep += 4; + break; + } + } + if (n < 256) { + *ep++ = SMALL_BIG_EXT; + put_int8(n, ep); + ep += 1; + } + else { + *ep++ = LARGE_BIG_EXT; + put_int32(n, ep); + ep += 4; + } + *ep++ = sign; + ep = big_to_bytes(obj, ep); } - *ep++ = big_sign(obj); - ep = big_to_bytes(obj, ep); break; case PID_DEF: @@ -1601,12 +1712,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) Uint32 *ref_num; ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); + *ep++ = NEW_REFERENCE_EXT; i = ref_no_of_numbers(obj); put_int16(i, ep); ep += 2; ep = enc_atom(acmp,ref_node_name(obj),ep,dflags); - *ep++ = ref_creation(obj); + *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_ref(obj)) ? + INTERNAL_CREATION : ref_creation(obj); ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -1622,7 +1735,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) j = port_number(obj); put_int32(j, ep); ep += 4; - *ep++ = port_creation(obj); + *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_port(obj)) ? + INTERNAL_CREATION : port_creation(obj); break; case LIST_DEF: @@ -1662,8 +1776,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) ep += 4; } if (i > 0) { - ESTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - ESTACK_PUSH(s, (Eterm) ptr); + WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); + WSTACK_PUSH(s, (UWord) ptr); } break; @@ -1702,6 +1816,41 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) byte* bytes; ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); + if (dflags & DFLAGS_INTERNAL_TAGS) { + ProcBin* pb = (ProcBin*) binary_val(obj); + Uint bytesize = pb->size; + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*)pb; + pb = (ProcBin*) binary_val(sub->orig); + ASSERT(bytesize == sub->size); + bytesize += (bitoffs + bitsize + 7) / 8; + } + if (pb->thing_word == HEADER_PROC_BIN + && heap_bin_size(bytesize) > PROC_BIN_SIZE) { + ProcBin tmp; + if (bitoffs || bitsize) { + *ep++ = BIT_BINARY_INTERNAL_REF; + *ep++ = bitoffs; + *ep++ = bitsize; + } + else { + *ep++ = BINARY_INTERNAL_REF; + } + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + erts_refc_inc(&pb->val->refc, 2); + + sys_memcpy(&tmp, pb, sizeof(ProcBin)); + tmp.next = *off_heap; + tmp.bytes = bytes; + tmp.size = bytesize; + sys_memcpy(ep, &tmp, sizeof(ProcBin)); + *off_heap = (struct erl_off_heap_header*) ep; + ep += sizeof(ProcBin); + break; + } + } if (bitsize == 0) { /* Plain old byte-sized binary. */ *ep++ = BINARY_EXT; @@ -1737,16 +1886,16 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) *ep++ = SMALL_INTEGER_EXT; *ep++ = bitsize; } - break; } + break; case EXPORT_DEF: { - Export* exp = (Export *) (export_val(obj))[1]; + Export* exp = *((Export **) (export_val(obj) + 1)); if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; ep = enc_atom(acmp, exp->code[0], ep, dflags); ep = enc_atom(acmp, exp->code[1], ep, dflags); - ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags); + ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap); } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -1770,8 +1919,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) int ei; *ep++ = NEW_FUN_EXT; - ESTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - ESTACK_PUSH(s, (Eterm) ep); /* Position for patching in size */ + WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); + WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ ep += 4; *ep = funp->arity; ep += 1; @@ -1782,14 +1931,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) put_int32(funp->num_free, ep); ep += 4; ep = enc_atom(acmp, funp->fe->module, ep, dflags); - ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags); - ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags); + ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); + ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); ep = enc_pid(acmp, funp->creator, ep, dflags); fun_env: for (ei = funp->num_free-1; ei > 0; ei--) { - ESTACK_PUSH(s, ENC_TERM); - ESTACK_PUSH(s, funp->env[ei]); + WSTACK_PUSH(s, ENC_TERM); + WSTACK_PUSH(s, (UWord) funp->env[ei]); } if (funp->num_free != 0) { obj = funp->env[0]; @@ -1832,11 +1981,12 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) break; } } - DESTROY_ESTACK(s); + DESTROY_WSTACK(s); return ep; } -static Uint +static +Uint is_external_string(Eterm list, int* p_is_string) { Uint len = 0; @@ -1874,69 +2024,30 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the lists. */ +/* Assumes that the ones to undo are preluding the list. */ static void undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) { const Uint area_sz = (end - start) * sizeof(Eterm); - struct proc_bin* mso; - struct proc_bin** mso_nextp = NULL; -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; - struct erl_fun_thing** funs_nextp = NULL; -#endif - struct external_thing_* ext; - struct external_thing_** ext_nextp = NULL; - - for (mso = off_heap->mso; ; mso=mso->next) { - if (!in_area(mso, start, area_sz)) { - if (mso_nextp != NULL) { - *mso_nextp = NULL; - erts_cleanup_mso(off_heap->mso); - off_heap->mso = mso; + struct erl_off_heap_header* hdr; + struct erl_off_heap_header** hdr_nextp = NULL; + + for (hdr = off_heap->first; ; hdr=hdr->next) { + if (!in_area(hdr, start, area_sz)) { + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(off_heap); + off_heap->first = hdr; } break; } - mso_nextp = &mso->next; + hdr_nextp = &hdr->next; } -#ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; ; funs=funs->next) { - if (!in_area(funs, start, area_sz)) { - if (funs_nextp != NULL) { - *funs_nextp = NULL; - erts_cleanup_funs(off_heap->funs); - off_heap->funs = funs; - } - break; - } - funs_nextp = &funs->next; - } -#endif - for (ext = off_heap->externals; ; ext=ext->next) { - if (!in_area(ext, start, area_sz)) { - if (ext_nextp != NULL) { - *ext_nextp = NULL; - erts_cleanup_externals(off_heap->externals); - off_heap->externals = ext; - } - break; - } - ext_nextp = &ext->next; - } - - /* Assert that the ones to undo were indeed preluding the lists. */ + /* Assert that the ones to undo were indeed preluding the list. */ #ifdef DEBUG - for (mso = off_heap->mso; mso != NULL; mso=mso->next) { - ASSERT(!in_area(mso, start, area_sz)); - } -# ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; funs != NULL; funs=funs->next) { - ASSERT(!in_area(funs, start, area_sz)); - } -# endif - for (ext = off_heap->externals; ext != NULL; ext=ext->next) { - ASSERT(!in_area(ext, start, area_sz)); + for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { + ASSERT(!in_area(hdr, start, area_sz)); } #endif /* DEBUG */ } @@ -1952,11 +2063,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et register Eterm* hp = *hpp; /* Please don't take the address of hp */ Eterm* next = objp; - *next = (Eterm) NULL; + *next = (Eterm) (UWord) NULL; while (next != NULL) { objp = next; - next = (Eterm *) (*objp); + next = (Eterm *) EXPAND_POINTER(*objp); switch (*ep++) { case INTEGER_EXT: @@ -1964,7 +2075,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et Sint sn = get_int32(ep); ep += 4; -#if defined(ARCH_64) +#if defined(ARCH_64) && !HALFWORD_HEAP *objp = make_small(sn); #else if (MY_IS_SSMALL(sn)) { @@ -2061,7 +2172,7 @@ dec_term_atom_common: hp += n; objp = hp - 1; while (n-- > 0) { - objp[0] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER(next); next = objp; objp--; } @@ -2079,12 +2190,12 @@ dec_term_atom_common: *objp = make_list(hp); hp += 2*n; objp = hp - 2; - objp[0] = (Eterm) (objp+1); - objp[1] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); + objp[1] = (Eterm) COMPRESS_POINTER(next); next = objp; objp -= 2; while (--n > 0) { - objp[0] = (Eterm) next; + objp[0] = (Eterm) COMPRESS_POINTER(next); objp[1] = make_list(objp + 2); next = objp; objp -= 2; @@ -2165,13 +2276,13 @@ dec_term_atom_common: goto error; } ep += 4; - if ((cre = get_int8(ep)) >= MAX_CREATION) { + cre = get_int8(ep); + ep++; + if (!is_valid_creation(cre)) { goto error; } - ep++; - cre = dec_set_creation(sysname,cre); - node = erts_find_or_insert_node(sysname, cre); + node = dec_get_node(sysname, cre); if(node == erts_this_node) { *objp = make_internal_port(num); } @@ -2180,11 +2291,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -2208,9 +2319,11 @@ dec_term_atom_common: goto error; ep += 4; - if ((cre = get_int8(ep)) >= MAX_CREATION) - goto error; + cre = get_int8(ep); ep += 1; + if (!is_valid_creation(cre)) { + goto error; + } goto ref_ext_common; case NEW_REFERENCE_EXT: @@ -2223,10 +2336,11 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &sysname)) == NULL) goto error; - if ((cre = get_int8(ep)) >= MAX_CREATION) - goto error; + cre = get_int8(ep); ep += 1; - + if (!is_valid_creation(cre)) { + goto error; + } r0 = get_int32(ep); ep += 4; if (r0 >= MAX_REFERENCE) @@ -2234,36 +2348,42 @@ dec_term_atom_common: ref_ext_common: - cre = dec_set_creation(sysname, cre); - node = erts_find_or_insert_node(sysname, cre); + node = dec_get_node(sysname, cre); if(node == erts_this_node) { RefThing *rtp = (RefThing *) hp; - hp += REF_THING_HEAD_SIZE; -#ifdef ARCH_64 + ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); + +#if defined(ARCH_64) && !HALFWORD_HEAP + hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; rtp->header = make_ref_thing_header(ref_words/2 + 1); #else + hp += REF_THING_HEAD_SIZE + ref_words; rtp->header = make_ref_thing_header(ref_words); #endif *objp = make_internal_ref(rtp); } else { ExternalThing *etp = (ExternalThing *) hp; - hp += EXTERNAL_THING_HEAD_SIZE; - -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP + hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; +#else + hp += EXTERNAL_THING_HEAD_SIZE + ref_words; +#endif + +#if defined(ARCH_64) && !HALFWORD_HEAP etp->header = make_external_ref_header(ref_words/2 + 1); #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); + ref_num = &(etp->data.ui32[0]); } - ref_num = (Uint32 *) hp; -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP *(ref_num++) = ref_words /* 32-bit arity */; #endif ref_num[0] = r0; @@ -2271,12 +2391,9 @@ dec_term_atom_common: ref_num[i] = get_int32(ep); ep += 4; } -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if ((1 + ref_words) % 2) ref_num[ref_words] = 0; - hp += ref_words/2 + 1; -#else - hp += ref_words; #endif break; } @@ -2304,8 +2421,8 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2341,8 +2458,8 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2398,7 +2515,12 @@ dec_term_atom_common: } *objp = make_export(hp); *hp++ = HEADER_EXPORT; +#if HALFWORD_HEAP + *((UWord *) (UWord) hp) = (UWord) erts_export_get_or_make_stub(mod, name, arity); + hp += 2; +#else *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); +#endif break; } break; @@ -2457,8 +2579,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, @@ -2474,11 +2596,11 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; + funp->env[i] = (Eterm) COMPRESS_POINTER(next); next = funp->env + i; } /* Creator */ - funp->creator = (Eterm) next; + funp->creator = (Eterm) COMPRESS_POINTER(next); next = &(funp->creator); break; } @@ -2535,8 +2657,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif old_uniq = unsigned_val(temp); @@ -2549,11 +2671,71 @@ dec_term_atom_common: /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; + funp->env[i] = (Eterm) COMPRESS_POINTER(next); next = funp->env + i; } break; } + case ATOM_INTERNAL_REF2: + n = get_int16(ep); + ep += 2; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + case ATOM_INTERNAL_REF3: + n = get_int24(ep); + ep += 3; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + + case BINARY_INTERNAL_REF: + { + ProcBin* pb = (ProcBin*) hp; + sys_memcpy(pb, ep, sizeof(ProcBin)); + ep += sizeof(ProcBin); + + erts_refc_inc(&pb->val->refc, 1); + hp += PROC_BIN_SIZE; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; + pb->flags = 0; + *objp = make_binary(pb); + break; + } + case BIT_BINARY_INTERNAL_REF: + { + Sint bitoffs = *ep++; + Sint bitsize = *ep++; + ProcBin* pb = (ProcBin*) hp; + ErlSubBin* sub; + sys_memcpy(pb, ep, sizeof(ProcBin)); + ep += sizeof(ProcBin); + + erts_refc_inc(&pb->val->refc, 1); + hp += PROC_BIN_SIZE; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; + pb->flags = 0; + + sub = (ErlSubBin*)hp; + sub->thing_word = HEADER_SUB_BIN; + sub->size = pb->size - (bitoffs + bitsize + 7)/8; + sub->offs = 0; + sub->bitoffs = bitoffs; + sub->bitsize = bitsize; + sub->is_writable = 0; + sub->orig = make_binary(pb); + + hp += ERL_SUB_BIN_SIZE; + *objp = make_binary(sub); + break; + } + default: error: /* UNDO: @@ -2580,26 +2762,35 @@ dec_term_atom_common: static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint m, i, arity; Uint result = 0; +#if HALFWORD_HEAP + UWord wobj = 0; +#endif goto L_jump_start; outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - + while (!WSTACK_ISEMPTY(s)) { +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); +#endif handle_popped_obj: - if (is_CP(obj)) { + if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */ +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else Eterm* ptr = (Eterm *) obj; - +#endif /* * Pointer into a tuple. */ obj = *ptr--; if (!is_header(obj)) { - ESTACK_PUSH(s, (Eterm)ptr); + WSTACK_PUSH(s, (UWord)ptr); } else { /* Reached tuple header */ ASSERT(header_is_arityval(obj)); @@ -2611,7 +2802,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) tl = CDR(cons); obj = CAR(cons); - ESTACK_PUSH(s, tl); + WSTACK_PUSH(s, tl); } else if (is_nil(obj)) { result++; goto outer_loop; @@ -2627,37 +2818,51 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case NIL_DEF: result++; break; - case ATOM_DEF: { - int alen = atom_tab(atom_val(obj))->len; - if ((MAX_ATOM_LENGTH <= 255 || alen <= 255) - && (dflags & DFLAG_SMALL_ATOM_TAGS)) { - /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */ - result += 1 + 1 + alen; + case ATOM_DEF: + if (dflags & DFLAGS_INTERNAL_TAGS) { + if (atom_val(obj) >= (1<<16)) { + result += 1 + 3; + } + else { + result += 1 + 2; + } } else { - /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */ - result += 1 + 2 + alen; + int alen = atom_tab(atom_val(obj))->len; + if ((MAX_ATOM_LENGTH <= 255 || alen <= 255) + && (dflags & DFLAG_SMALL_ATOM_TAGS)) { + /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */ + result += 1 + 1 + alen; + } + else { + /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */ + result += 1 + 2 + alen; + } + insert_acache_map(acmp, obj); } - insert_acache_map(acmp, obj); break; - } case SMALL_DEF: { Sint val = signed_val(obj); if ((Uint)val < 256) result += 1 + 1; /* SMALL_INTEGER_EXT */ - else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) + else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) result += 1 + 4; /* INTEGER_EXT */ else { - Eterm tmp_big[2]; + DeclareTmpHeapNoproc(tmp_big,2); + UseTmpHeapNoproc(2); i = big_bytes(small_to_big(val, tmp_big)); result += 1 + 1 + 1 + i; /* SMALL_BIG_EXT */ + UnUseTmpHeapNoproc(2); } } break; case BIG_DEF: - if ((i = big_bytes(obj)) < 256) + i = big_bytes(obj); + if (sizeof(Sint)==4 && i <= 4 && (big_digit(obj,0)-big_sign(obj)) < (1<<31)) + result += 1 + 4; /* INTEGER_EXT */ + else if (i < 256) result += 1 + 1 + 1 + i; /* tag,size,sign,digits */ else result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ @@ -2698,7 +2903,11 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) result += 1 + 4; } ptr += arity; +#if HALFWORD_HEAP + obj = (Eterm) (wobj = (UWord) ptr); +#else obj = (Eterm) ptr; +#endif goto handle_popped_obj; } break; @@ -2710,8 +2919,25 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } break; case BINARY_DEF: + if (dflags & DFLAGS_INTERNAL_TAGS) { + ProcBin* pb = (ProcBin*) binary_val(obj); + Uint sub_extra = 0; + Uint tot_bytes = pb->size; + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*) pb; + pb = (ProcBin*) binary_val(sub->orig); + sub_extra = 2; /* bitoffs and bitsize */ + tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8; + } + if (pb->thing_word == HEADER_PROC_BIN + && heap_bin_size(tot_bytes) > PROC_BIN_SIZE) { + + result += 1 + sub_extra + sizeof(ProcBin); + break; + } + } result += 1 + 4 + binary_size(obj) + - 5; /* For unaligned binary */ + 5; /* For unaligned binary */ break; case FUN_DEF: { @@ -2740,14 +2966,14 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) if (is_not_list(obj)) { /* Push any non-list terms on the stack */ - ESTACK_PUSH(s, obj); + WSTACK_PUSH(s, obj); } else { /* Lists must be handled specially. */ if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; } else { result += 5; - ESTACK_PUSH(s, obj); + WSTACK_PUSH(s, obj); } } } @@ -2760,8 +2986,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case EXPORT_DEF: { - Export* ep = (Export *) (export_val(obj))[1]; + Export* ep = *((Export **) (export_val(obj) + 1)); +#if HALFWORD_HEAP + result += 2; +#else result += 1; +#endif result += encode_size_struct2(acmp, ep->code[0], dflags); result += encode_size_struct2(acmp, ep->code[1], dflags); result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); @@ -2774,12 +3004,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } } - DESTROY_ESTACK(s); + DESTROY_WSTACK(s); return result; } static Sint -decoded_size(byte *ep, byte* endp, int no_refc_bins) +decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags) { int heap_size = 0; int terms; @@ -2886,7 +3116,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) ep += 2; atom_extra_skip = 1 + 4*id_words; /* In case it is an external ref */ -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; #else heap_size += EXTERNAL_THING_HEAD_SIZE + id_words; @@ -2961,7 +3191,11 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) break; case EXPORT_EXT: terms += 3; +#if HALFWORD_HEAP + heap_size += 3; +#else heap_size += 2; +#endif break; case NEW_FUN_EXT: { @@ -2985,6 +3219,29 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) heap_size += ERL_FUN_SIZE + num_free; break; } + case ATOM_INTERNAL_REF2: + SKIP(2+atom_extra_skip); + atom_extra_skip = 0; + break; + case ATOM_INTERNAL_REF3: + SKIP(3+atom_extra_skip); + atom_extra_skip = 0; + break; + + case BINARY_INTERNAL_REF: + if (!internal_tags) { + return -1; + } + SKIP(sizeof(ProcBin)); + heap_size += PROC_BIN_SIZE; + break; + case BIT_BINARY_INTERNAL_REF: + if (!internal_tags) { + return -1; + } + SKIP(2+sizeof(ProcBin)); + heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE; + break; default: return -1; } diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index eada6d4f27..d8287b96a4 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -54,6 +54,10 @@ #define DIST_HEADER 'D' #define ATOM_CACHE_REF 'R' +#define ATOM_INTERNAL_REF2 'I' +#define ATOM_INTERNAL_REF3 'K' +#define BINARY_INTERNAL_REF 'J' +#define BIT_BINARY_INTERNAL_REF 'L' #define COMPRESSED 'P' #if 0 @@ -156,7 +160,9 @@ Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *); void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *); Uint erts_encode_ext_size(Eterm); +Uint erts_encode_ext_size_ets(Eterm); void erts_encode_ext(Eterm, byte **); +byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap); #ifdef ERTS_WANT_EXTERNAL_TAGS ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint); @@ -172,7 +178,9 @@ Sint erts_decode_dist_ext_size(ErtsDistExternal *, int); Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint, int); +Sint erts_decode_ext_size_ets(byte*, Uint); Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); +Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); @@ -211,8 +219,8 @@ ERTS_GLB_INLINE void * erts_dist_ext_trailer(ErtsDistExternal *edep) { void *res = (void *) (edep->ext_endp - + ERTS_WORD_ALIGN_PAD_SZ(edep->ext_endp)); - ASSERT((((Uint) res) % sizeof(Uint)) == 0); + + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp)); + ASSERT((((UWord) res) % sizeof(Uint)) == 0); return res; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index cab249a53f..499bdd77ba 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -76,12 +76,6 @@ typedef struct line_buf { /* Buffer used in line oriented I/O */ The rest is the overflow buffer. */ } LineBuf; -/* Temporary object header, auto-deallocated when NIF returns. */ -struct enif_tmp_obj_t { - struct enif_tmp_obj_t* next; - void (*dtor)(struct enif_tmp_obj_t*); - /*char data[];*/ -}; struct enif_environment_t /* ErlNifEnv */ { struct erl_module_nif* mod_nif; @@ -189,7 +183,7 @@ struct port { process to get (line oriented I/O)*/ Uint32 status; /* Status and type flags */ int control_flags; /* Flags for port_control() */ - Uint32 snapshot; /* Next snapshot that port should be part of */ + erts_aint32_t snapshot; /* Next snapshot that port should be part of */ struct reg_proc *reg; ErlDrvPDL port_data_lock; @@ -403,7 +397,7 @@ extern Eterm erts_ddll_monitor_driver(Process *p, /* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */ #define ERTS_INTERNAL_BINARY_FIELDS \ - Uint flags; \ + UWord flags; \ erts_refc_t refc; \ ERTS_BINARY_STRUCT_ALIGNMENT @@ -470,7 +464,10 @@ typedef union { typedef struct proc_bin { Eterm thing_word; /* Subtag REFC_BINARY_SUBTAG. */ Uint size; /* Binary size in bytes. */ - struct proc_bin *next; /* Pointer to next ProcBin. */ +#if HALFWORD_HEAP + void* dummy_ptr_padding__; +#endif + struct erl_off_heap_header *next; Binary *val; /* Pointer to Binary structure. */ byte *bytes; /* Pointer to the actual data bytes. */ Uint flags; /* Flag word. */ @@ -500,8 +497,8 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*) pb; pb->val = mbp; pb->bytes = (byte *) mbp->orig_bytes; pb->flags = 0; @@ -518,13 +515,22 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \ && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC)) + +union erl_off_heap_ptr { + struct erl_off_heap_header* hdr; + ProcBin *pb; + struct erl_fun_thing* fun; + struct external_thing_* ext; + Eterm* ep; + void* voidp; +}; + /* arrays that get malloced at startup */ extern Port* erts_port; -extern erts_smp_atomic_t erts_ports_alive; extern Uint erts_max_ports; extern Uint erts_port_tab_index_mask; -extern erts_smp_atomic_t erts_ports_snapshot; +extern erts_smp_atomic32_t erts_ports_snapshot; extern erts_smp_atomic_t erts_dead_ports_ptr; ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); @@ -534,12 +540,12 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) { ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck)); - if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) { + if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, - -(long)sizeof(Eterm)); + -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1); + ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1); *tombstone = prt->id; } /*else no ongoing snapshot or port was already included or created after snapshot */ @@ -556,7 +562,7 @@ extern Uint display_items; /* no of items to display in traces etc */ extern Uint display_loads; /* print info about loaded modules */ extern int erts_backtrace_depth; -extern erts_smp_atomic_t erts_max_gen_gcs; +extern erts_smp_atomic32_t erts_max_gen_gcs; extern int erts_disable_tolerant_timeofday; @@ -723,6 +729,60 @@ do { \ #define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp))) +void erl_grow_wstack(UWord** start, UWord** sp, UWord** end); +#define WSTK_CONCAT(a,b) a##b +#define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i))) +#define DEF_WSTACK_SIZE (16) + +#define DECLARE_WSTACK(s) \ + UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE]; \ + UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack); \ + UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start); \ + UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE + +#define DESTROY_WSTACK(s) \ +do { \ + if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \ + erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start)); \ + } \ +} while(0) + +#define WSTACK_PUSH(s, x) \ +do { \ + if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) { \ + erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ + &WSTK_CONCAT(s,_end)); \ + } \ + *WSTK_CONCAT(s,_sp)++ = (x); \ +} while(0) + +#define WSTACK_PUSH2(s, x, y) \ +do { \ + if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) { \ + erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ + &WSTK_CONCAT(s,_end)); \ + } \ + *WSTK_CONCAT(s,_sp)++ = (x); \ + *WSTK_CONCAT(s,_sp)++ = (y); \ +} while(0) + +#define WSTACK_PUSH3(s, x, y, z) \ +do { \ + if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) { \ + erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ + &WSTK_CONCAT(s,_end)); \ + } \ + *WSTK_CONCAT(s,_sp)++ = (x); \ + *WSTK_CONCAT(s,_sp)++ = (y); \ + *WSTK_CONCAT(s,_sp)++ = (z); \ +} while(0) + +#define WSTACK_COUNT(s) (WSTK_CONCAT(s,_sp) - WSTK_CONCAT(s,_start)) + +#define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start)) +#define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp))) + + /* port status flags */ #define ERTS_PORT_SFLG_CONNECTED ((Uint32) (1 << 0)) @@ -773,9 +833,8 @@ do { \ void erts_emasculate_writable_binary(ProcBin* pb); Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, int); -Eterm new_binary(Process*, byte*, int); +Eterm new_binary(Process*, byte*, Uint); Eterm erts_realloc_binary(Eterm bin, size_t size); -void erts_cleanup_mso(ProcBin* pb); /* erl_bif_info.c */ @@ -802,7 +861,7 @@ void erts_system_profile_clear(Process *c_p); int erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, int size); void init_load(void); -Eterm* find_function_from_pc(Eterm* pc); +BeamInstr* find_function_from_pc(BeamInstr* pc); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); @@ -830,9 +889,33 @@ void erl_error(char*, va_list); /* copy.c */ void init_copy(void); Eterm copy_object(Eterm, Process*); + +#if HALFWORD_HEAP +Uint size_object_rel(Eterm, Eterm*); +# define size_object(A) size_object_rel(A,NULL) + +Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base); +# define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL) + +Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base); +# define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL) + +#else /* !HALFWORD_HEAP */ + Uint size_object(Eterm); +# define size_object_rel(A,B) size_object(A) + Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); +# define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH) + Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +# define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D) + +#endif + + +void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs); #ifdef HYBRID #define RRMA_DEFAULT_SIZE 256 @@ -965,7 +1048,7 @@ void print_pass_through(int, byte*, int); /* beam_emu.c */ int catchlevel(Process*); -void init_emulator(_VOID_); +void init_emulator(void); void process_main(void); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); @@ -1030,6 +1113,7 @@ Eterm erts_heap_sizes(Process* p); void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*); void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); +void erts_free_heap_frags(Process* p); #ifdef HYBRID int erts_global_garbage_collect(Process*, int, Eterm*, int); @@ -1143,13 +1227,12 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt) { #ifdef ERTS_SMP - long refc; + erts_aint_t refc; + erts_smp_mtx_unlock(prt->lock); refc = erts_smp_atomic_dectest(&prt->refc); ASSERT(refc >= 0); if (refc == 0) erts_port_cleanup(prt); - else - erts_smp_mtx_unlock(prt->lock); #endif } @@ -1171,7 +1254,7 @@ erts_smp_port_unlock(Port *prt) ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP) #define ERTS_PORT_SCHED_ID(P, ID) \ - ((Uint) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID))) + ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID))) #ifdef ERTS_SMP Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks); @@ -1363,84 +1446,6 @@ void erl_drv_thr_init(void); /* time.c */ -ERTS_GLB_INLINE long do_time_read_and_reset(void); -#ifdef ERTS_TIMER_THREAD -ERTS_GLB_INLINE int next_time(void); -ERTS_GLB_INLINE void bump_timer(long); -#else -int next_time(void); -void bump_timer(long); -extern erts_smp_atomic_t do_time; /* set at clock interrupt */ -ERTS_GLB_INLINE void do_time_add(long); -#endif - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -#ifdef ERTS_TIMER_THREAD -ERTS_GLB_INLINE long do_time_read_and_reset(void) { return 0; } -ERTS_GLB_INLINE int next_time(void) { return -1; } -ERTS_GLB_INLINE void bump_timer(long ignore) { } -#else -ERTS_GLB_INLINE long do_time_read_and_reset(void) -{ - return erts_smp_atomic_xchg(&do_time, 0L); -} -ERTS_GLB_INLINE void do_time_add(long elapsed) -{ - erts_smp_atomic_add(&do_time, elapsed); -} -#endif - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - -void init_time(void); -void erl_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); -void erl_cancel_timer(ErlTimer*); -Uint time_left(ErlTimer *); - -Uint erts_timer_wheel_memory_size(void); - -#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) -# ifndef HAVE_ERTS_NOW_CPU -# define HAVE_ERTS_NOW_CPU -# ifdef HAVE_GETHRVTIME -# define erts_start_now_cpu() sys_start_hrvtime() -# define erts_stop_now_cpu() sys_stop_hrvtime() -# endif -# endif -void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); -#endif - -void erts_get_timeval(SysTimeval *tv); -long erts_get_time(void); - -extern SysTimeval erts_first_emu_time; - -void erts_get_emu_time(SysTimeval *); - -ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE int -erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) -{ - if (t1p->tv_sec == t2p->tv_sec) { - if (t1p->tv_usec < t2p->tv_usec) - return -1; - else if (t1p->tv_usec > t2p->tv_usec) - return 1; - return 0; - } - return t1p->tv_sec < t2p->tv_sec ? -1 : 1; -} - -#endif - -#ifdef DEBUG -void p_slpq(_VOID_); -#endif - /* utils.c */ /* @@ -1449,7 +1454,6 @@ void p_slpq(_VOID_); void erts_silence_warn_unused_result(long unused); void erts_cleanup_offheap(ErlOffHeap *offheap); -void erts_cleanup_externals(ExternalThing *); Uint erts_fit_in_bits(Uint); int list_length(Eterm); @@ -1463,6 +1467,7 @@ Uint32 make_hash(Eterm); Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str); Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); +Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw); Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64); Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64); Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr); @@ -1481,7 +1486,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, Eterm atoms[], Uint uints1[], Uint uints2[]); Eterm store_external_or_ref_in_proc_(Process *, Eterm); -Eterm store_external_or_ref_(Uint **, ExternalThing **, Eterm); +Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); #define NC_HEAP_SIZE(NC) \ (ASSERT_EXPR(is_node_container((NC))), \ @@ -1499,23 +1504,39 @@ void erts_init_utils_mem(void); erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint); void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *); +#if HALFWORD_HEAP +int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); +# define eq(A,B) eq_rel(A,NULL,B,NULL) +#else int eq(Eterm, Eterm); +# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B) +#endif + #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) +#if HALFWORD_HEAP +Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*); +#define CMP(A,B) cmp_rel(A,NULL,B,NULL) +#else Sint cmp(Eterm, Eterm); -#define cmp_lt(a,b) (cmp((a),(b)) < 0) -#define cmp_le(a,b) (cmp((a),(b)) <= 0) -#define cmp_eq(a,b) (cmp((a),(b)) == 0) -#define cmp_ne(a,b) (cmp((a),(b)) != 0) -#define cmp_ge(a,b) (cmp((a),(b)) >= 0) -#define cmp_gt(a,b) (cmp((a),(b)) > 0) +#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B) +#define CMP(A,B) cmp(A,B) +#endif +#define cmp_lt(a,b) (CMP((a),(b)) < 0) +#define cmp_le(a,b) (CMP((a),(b)) <= 0) +#define cmp_eq(a,b) (CMP((a),(b)) == 0) +#define cmp_ne(a,b) (CMP((a),(b)) != 0) +#define cmp_ge(a,b) (CMP((a),(b)) >= 0) +#define cmp_gt(a,b) (CMP((a),(b)) > 0) #define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) #define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) #define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) #define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) +/* duplicates from big.h */ int term_to_Uint(Eterm term, Uint *up); +int term_to_UWord(Eterm, UWord*); #ifdef HAVE_ERTS_NOW_CPU extern int erts_cpu_timestamp; @@ -1525,9 +1546,26 @@ void erts_init_bif_chksum(void); /* erl_bif_re.c */ void erts_init_bif_re(void); Sint erts_re_set_loop_limit(Sint limit); +/* erl_bif_binary.c */ +void erts_init_bif_binary(void); +Sint erts_binary_set_loop_limit(Sint limit); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); + +void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ; +Sint erts_native_filename_need(Eterm ioterm, int encoding); +void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars); +int erts_analyze_utf8(byte *source, Uint size, + byte **err_pos, Uint *num_chars, int *left); +char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty); + +#define ERTS_UTF8_OK 0 +#define ERTS_UTF8_INCOMPLETE 1 +#define ERTS_UTF8_ERROR 2 +#define ERTS_UTF8_ANALYZE_MORE 3 + /* erl_trace.c */ void erts_init_trace(void); void erts_trace_check_exiting(Eterm exiting); @@ -1554,12 +1592,12 @@ void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *); void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); void trace_send(Process*, Eterm, Eterm); void trace_receive(Process*, Eterm); -Uint32 erts_call_trace(Process *p, Eterm mfa[], Binary *match_spec, Eterm* args, +Uint32 erts_call_trace(Process *p, BeamInstr mfa[], Binary *match_spec, Eterm* args, int local, Eterm *tracer_pid); -void erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid); -void erts_trace_exception(Process* p, Eterm mfa[], Eterm class, Eterm value, +void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid); +void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, Eterm *tracer); -void erts_trace_return_to(Process *p, Uint *pc); +void erts_trace_return_to(Process *p, BeamInstr *pc); void trace_sched(Process*, Eterm); void trace_proc(Process*, Process*, Eterm, Eterm); void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args); @@ -1589,7 +1627,7 @@ Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, Eterm *pTracer, int *pCpuTimestamp); Eterm erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, Uint *I); + Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I); #ifdef ERTS_SMP void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); @@ -1606,7 +1644,7 @@ void bin_write(int, void*, byte*, int); int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */ struct Sint_buf { -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP char s[22]; #else char s[12]; @@ -1614,12 +1652,16 @@ struct Sint_buf { }; char* Sint_to_buf(Sint, struct Sint_buf*); +#define ERTS_IOLIST_OK 0 +#define ERTS_IOLIST_OVERFLOW 1 +#define ERTS_IOLIST_TYPE 2 + Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */ int io_list_to_buf(Eterm, char*, int); int io_list_to_buf2(Eterm, char*, int); -int io_list_len(Eterm); +int erts_iolist_size(Eterm, Uint *); int is_string(Eterm); -void erl_at_exit(FUNCTION(void,(*),(void*)), void*); +void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); void dump_memory_to_fd(int); int dump_memory_data(const char *); @@ -1654,17 +1696,14 @@ Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live); +Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live); +Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live); Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(int to, void *arg, Process *c_p); -/* - * Interface to erl_init - */ -void erl_init(void); -void erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv); - +int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); #define seq_trace_output(token, msg, type, receiver, process) \ seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) #define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \ @@ -1683,8 +1722,10 @@ struct trace_pattern_flags { unsigned int local : 1; /* Local call trace breakpoint */ unsigned int meta : 1; /* Metadata trace breakpoint */ unsigned int call_count : 1; /* Fast call count breakpoint */ + unsigned int call_time : 1; /* Fast call time breakpoint */ }; extern const struct trace_pattern_flags erts_trace_pattern_flags_off; +extern int erts_call_time_breakpoint_tracing; int erts_set_trace_pattern(Eterm* mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags, @@ -1720,26 +1761,35 @@ do { \ extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr); Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); + +enum erts_pam_run_flags { + ERTS_PAM_TMP_RESULT=0, + ERTS_PAM_COPY_RESULT=1, + ERTS_PAM_CONTIGUOUS_TUPLE=2 +}; extern Eterm erts_match_set_run(Process *p, Binary *mpsp, Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, Uint32 *return_flags); extern Eterm erts_match_set_get_source(Binary *mpsp); extern void erts_match_prog_foreach_offheap(Binary *b, void (*)(ErlOffHeap *, void *), void *); -#define MATCH_SET_RETURN_TRACE 0x1 /* return trace requested */ -#define MATCH_SET_RETURN_TO_TRACE 0x2 /* Misleading name, it is not actually - set by the match program, but by the - breakpoint functions */ -#define MATCH_SET_EXCEPTION_TRACE 0x4 /* exception trace requested */ +#define MATCH_SET_RETURN_TRACE (0x1) /* return trace requested */ +#define MATCH_SET_RETURN_TO_TRACE (0x2) /* Misleading name, it is not actually + set by the match program, but by the + breakpoint functions */ +#define MATCH_SET_EXCEPTION_TRACE (0x4) /* exception trace requested */ #define MATCH_SET_RX_TRACE (MATCH_SET_RETURN_TRACE|MATCH_SET_EXCEPTION_TRACE) /* * Flag values when tracing bif + * Future note: flag field is 8 bits */ -#define BIF_TRACE_AS_LOCAL 0x1 -#define BIF_TRACE_AS_GLOBAL 0x2 -#define BIF_TRACE_AS_META 0x4 +#define BIF_TRACE_AS_LOCAL (0x1) +#define BIF_TRACE_AS_GLOBAL (0x2) +#define BIF_TRACE_AS_META (0x4) +#define BIF_TRACE_AS_CALL_TIME (0x8) extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; @@ -1781,7 +1831,7 @@ erts_alloc_message_heap(Uint size, #endif if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, "HUGE size (%bpu)\n", size); + erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size); if ( #if defined(ERTS_SMP) @@ -1839,4 +1889,67 @@ erts_alloc_message_heap(Uint size, #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif +#if !HEAP_ON_C_STACK +# if defined(DEBUG) +# define DeclareTmpHeap(VariableName,Size,Process) \ + Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process) +# define DeclareTmpHeapNoproc(VariableName,Size) \ + Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) +# define UseTmpHeap(Size,Proc) \ + do { \ + erts_debug_use_tmp_heap((Size),(Proc)); \ + } while (0) +# define UnUseTmpHeap(Size,Proc) \ + do { \ + erts_debug_unuse_tmp_heap((Size),(Proc)); \ + } while (0) +# define UseTmpHeapNoproc(Size) \ + do { \ + erts_debug_use_tmp_heap(Size,NULL); \ + } while (0) +# define UnUseTmpHeapNoproc(Size) \ + do { \ + erts_debug_unuse_tmp_heap(Size,NULL); \ + } while (0) +# else +# define DeclareTmpHeap(VariableName,Size,Process) \ + Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) +# define DeclareTmpHeapNoproc(VariableName,Size) \ + Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) +# define UseTmpHeap(Size,Proc) \ + do { \ + ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \ + } while (0) +# define UnUseTmpHeap(Size,Proc) \ + do { \ + ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \ + } while (0) +# define UseTmpHeapNoproc(Size) \ + do { \ + erts_get_scheduler_data()->num_tmp_heap_used += (Size); \ + } while (0) +# define UnUseTmpHeapNoproc(Size) \ + do { \ + erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \ + } while (0) + + +# endif + +#else +# define DeclareTmpHeap(VariableName,Size,Process) \ + Eterm VariableName[Size] +# define DeclareTypedTmpHeap(Type,VariableName,Process) \ + Type VariableName[1] +# define DeclareTmpHeapNoproc(VariableName,Size) \ + Eterm VariableName[Size] +# define UseTmpHeap(Size,Proc) /* Nothing */ +# define UnUseTmpHeap(Size,Proc) /* Nothing */ +# define UseTmpHeapNoproc(Size) /* Nothing */ +# define UnUseTmpHeapNoproc(Size) /* Nothing */ +#endif /* HEAP_ON_C_STACK */ +#endif /* !__GLOBAL_H__ */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 3309b77086..df5f8b22a3 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -56,7 +56,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o per thread basis (for BC interfaces) */ Port* erts_port; /* The port table */ -erts_smp_atomic_t erts_ports_alive; erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ @@ -72,6 +71,18 @@ erts_driver_t fd_driver; static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); static void pdl_init(void); +#ifdef ERTS_SMP +static void driver_monitor_lock_pdl(Port *p); +static void driver_monitor_unlock_pdl(Port *p); +#define DRV_MONITOR_LOCK_PDL(Port) driver_monitor_lock_pdl(Port) +#define DRV_MONITOR_UNLOCK_PDL(Port) driver_monitor_unlock_pdl(Port) +#else +#define DRV_MONITOR_LOCK_PDL(Port) /* nothing */ +#define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */ +#endif + +#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) +#define SMALL_WRITE_VEC 16 static ERTS_INLINE ErlIOQueue* drvport2ioq(ErlDrvPort drvport) @@ -181,7 +192,7 @@ typedef struct line_buf_context { static erts_smp_spinlock_t get_free_port_lck; static Uint last_port_num; static Uint port_num_mask; -erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ +erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ static ERTS_INLINE void @@ -271,10 +282,36 @@ erts_test_next_port(int set, Uint next) return res; } + +static void port_cleanup(Port *prt); + +#ifdef ERTS_SMP + +static void +sched_port_cleanup(void *vprt) +{ + Port *prt = (Port *) vprt; + erts_smp_mtx_lock(prt->lock); + port_cleanup(prt); +} + +#endif + void erts_port_cleanup(Port *prt) { #ifdef ERTS_SMP + if (erts_smp_mtx_trylock(prt->lock) == EBUSY) + erts_schedule_misc_op(sched_port_cleanup, (void *) prt); + else +#endif + port_cleanup(prt); +} + +void +port_cleanup(Port *prt) +{ +#ifdef ERTS_SMP Uint32 port_specific; erts_smp_mtx_t *mtx; #endif @@ -386,14 +423,13 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1); sys_strcpy(new_name, name); erts_smp_runq_lock(runq); - erts_smp_atomic_inc(&erts_ports_alive); erts_smp_port_state_lock(prt); prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; - prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot); + prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot); old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP - erts_smp_atomic_set(&prt->run_queue, (long) runq); + erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) runq); #endif ASSERT(!prt->drv_ptr); prt->drv_ptr = driver; @@ -635,7 +671,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ #ifdef ERTS_SMP erts_cancel_smp_ptimer(port->ptimer); #else - erl_cancel_timer(&(port->tm)); + erts_cancel_timer(&(port->tm)); #endif stopq(port); kill_port(port); @@ -919,13 +955,14 @@ do { \ int _bitoffs; \ int _bitsize; \ ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \ - ASSERT(_bitsize == 0); \ + if (_bitsize != 0) goto L_type_error; \ if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \ _bitoffs == 0) { \ b_size += _size; \ + if (b_size < _size) goto L_overflow_error; \ in_clist = 0; \ v_size++; \ - if (_size >= bin_limit) { \ + if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \ p_in_clist = 0; \ p_v_size++; \ } else { \ @@ -937,6 +974,7 @@ do { \ } \ } else { \ c_size += _size; \ + if (c_size < _size) goto L_overflow_error; \ if (!in_clist) { \ in_clist = 1; \ v_size++; \ @@ -951,29 +989,30 @@ do { \ /* -** Size of a io list in bytes -** return -1 if error -** returns: - Total size of io list -** vsize - SysIOVec size needed for a writev -** csize - Number of bytes not in binary (in the common binary) -** pvsize - SysIOVec size needed if packing small binaries -** pcsize - Number of bytes in the common binary if packing -*/ + * Returns 0 if successful and a non-zero value otherwise. + * + * Return values through pointers: + * *vsize - SysIOVec size needed for a writev + * *csize - Number of bytes not in binary (in the common binary) + * *pvsize - SysIOVec size needed if packing small binaries + * *pcsize - Number of bytes in the common binary if packing + * *total_size - Total size of iolist in bytes + */ static int -io_list_vec_len(Eterm obj, int* vsize, int* csize, - int bin_limit, /* small binaries limit */ - int * pvsize, int * pcsize) +io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, Uint* total_size) { DECLARE_ESTACK(s); Eterm* objp; - int v_size = 0; - int c_size = 0; - int b_size = 0; - int in_clist = 0; - int p_v_size = 0; - int p_c_size = 0; - int p_in_clist = 0; + Uint v_size = 0; + Uint c_size = 0; + Uint b_size = 0; + Uint in_clist = 0; + Uint p_v_size = 0; + Uint p_c_size = 0; + Uint p_in_clist = 0; + Uint total; goto L_jump_start; /* avoid a push */ @@ -987,6 +1026,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, if (is_byte(obj)) { c_size++; + if (c_size == 0) { + goto L_overflow_error; + } if (!in_clist) { in_clist = 1; v_size++; @@ -1026,32 +1068,31 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize, } } + total = c_size + b_size; + if (total < c_size) { + goto L_overflow_error; + } + *total_size = total; + DESTROY_ESTACK(s); - if (vsize != NULL) - *vsize = v_size; - if (csize != NULL) - *csize = c_size; - if (pvsize != NULL) - *pvsize = p_v_size; - if (pcsize != NULL) - *pcsize = p_c_size; - return c_size + b_size; + *vsize = v_size; + *csize = c_size; + *pvsize = p_v_size; + *pcsize = p_c_size; + return 0; L_type_error: + L_overflow_error: DESTROY_ESTACK(s); - return -1; + return 1; } -#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) -#define SMALL_WRITE_VEC 16 - - /* write data to a port */ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) { char *buf; erts_driver_t *drv = p->drv_ptr; - int size; + Uint size; int fpe_was_unmasked; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); @@ -1059,10 +1100,10 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) p->caller = caller_id; if (drv->outputv != NULL) { - int vsize; - int csize; - int pvsize; - int pcsize; + Uint vsize; + Uint csize; + Uint pvsize; + Uint pcsize; int blimit; SysIOVec iv[SMALL_WRITE_VEC]; ErlDrvBinary* bv[SMALL_WRITE_VEC]; @@ -1071,9 +1112,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) ErlDrvBinary* cbin; ErlIOVec ev; - if ((size = io_list_vec_len(list, &vsize, &csize, - ERL_SMALL_IO_BIN_LIMIT, - &pvsize, &pcsize)) < 0) { + if (io_list_vec_len(list, &vsize, &csize, + &pvsize, &pcsize, &size)) { goto bad_value; } /* To pack or not to pack (small binaries) ...? */ @@ -1148,7 +1188,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, buf); - if ((size = io_list_len(list)) < 0) { + if (erts_iolist_size(list, &size)) { goto bad_value; } @@ -1191,7 +1231,6 @@ void init_io(void) { int i; ErlDrvEntry** dp; - ErlDrvEntry* drv; char maxports[21]; /* enough for any 64-bit integer */ size_t maxportssize = sizeof(maxports); Uint ports_bits = ERTS_PORTS_BITS; @@ -1240,7 +1279,6 @@ void init_io(void) erts_smp_atomic_init(&erts_bytes_out, 0); erts_smp_atomic_init(&erts_bytes_in, 0); - erts_smp_atomic_init(&erts_ports_alive, 0); for (i = 0; i < erts_max_ports; i++) { erts_port_task_init_sched(&erts_port[i].sched); @@ -1262,7 +1300,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic_init(&erts_ports_snapshot, (long) 0); + erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -1274,10 +1312,8 @@ void init_io(void) init_driver(&fd_driver, &fd_driver_entry, NULL); init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); init_driver(&spawn_driver, &spawn_driver_entry, NULL); - for (dp = driver_tab; *dp != NULL; dp++) { - drv = *dp; + for (dp = driver_tab; *dp != NULL; dp++) erts_add_driver_entry(*dp, NULL, 1); - } erts_smp_tsd_set(driver_list_lock_status_key, NULL); erts_smp_mtx_unlock(&erts_driver_list_lock); @@ -1540,14 +1576,14 @@ static void deliver_read_message(Port* prt, Eterm to, pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; hp += PROC_BIN_SIZE; - ohp->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); listp = make_binary(pb); } @@ -1690,14 +1726,14 @@ deliver_vec_message(Port* prt, /* Port */ } pb->thing_word = HEADER_PROC_BIN; pb->size = iov->iov_len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = base; pb->flags = 0; hp += PROC_BIN_SIZE; - ohp->overhead += iov->iov_len / sizeof(Eterm); + OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm)); if (listp == NIL) { /* compatible with deliver_bin_message */ listp = make_binary(pb); @@ -1804,7 +1840,7 @@ terminate_port(Port *prt) #ifdef ERTS_SMP erts_cancel_smp_ptimer(prt->ptimer); #else - erl_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->tm); #endif drv = prt->drv_ptr; @@ -1998,12 +2034,13 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) p->nlinks = NULL; erts_sweep_links(lnk, &sweep_one_link, &sc); } + DRV_MONITOR_LOCK_PDL(p); { ErtsMonitor *moni = p->monitors; p->monitors = NULL; erts_sweep_monitors(moni, &sweep_one_monitor, NULL); } - + DRV_MONITOR_UNLOCK_PDL(p); if ((p->status & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) { erts_do_net_exits(p->dist_entry, rreason); @@ -2114,7 +2151,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) byte* to_port = NULL; /* Buffer to write to port. */ /* Initialization is for shutting up warning about use before set. */ - int to_len = 0; /* Length of buffer. */ + Uint to_len = 0; /* Length of buffer. */ int must_free = 0; /* True if the buffer should be freed. */ char port_result[ERL_ONHEAP_BIN_LIMIT]; /* Default buffer for result from port. */ char* port_resp; /* Pointer to result buffer. */ @@ -2159,7 +2196,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) } else { ASSERT(r == -1); /* Overflow */ erts_free(ERTS_ALC_T_TMP, (void *) to_port); - if ((to_len = io_list_len(iolist)) < 0) { /* Type error */ + if (erts_iolist_size(iolist, &to_len)) { /* Type error */ return THE_NON_VALUE; } must_free = 1; @@ -2223,12 +2260,12 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = dbin->orig_size; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(dbin); pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; - MSO(p).overhead += dbin->orig_size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm)); return make_binary(pb); } port_resp = dbin->orig_bytes; @@ -2384,7 +2421,7 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) if (len > (Uint) INT_MAX) erl_exit(ERTS_ABORT_EXIT, - "Absurdly large data buffer (%bpu bytes) passed to" + "Absurdly large data buffer (%beu bytes) passed to" "output callback of %s driver.\n", len, p->drv_ptr->name ? p->drv_ptr->name : "unknown"); @@ -2766,17 +2803,25 @@ driver_deliver_term(ErlDrvPort port, break; case ERL_DRV_INT: /* signed int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); +#if HALFWORD_HEAP + erts_bld_sint64(NULL, &need, (Sint64)ptr[0]); +#else /* check for bignum */ if (!IS_SSMALL((Sint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ +#endif ptr++; depth++; break; case ERL_DRV_UINT: /* unsigned int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); +#if HALFWORD_HEAP + erts_bld_uint64(NULL, &need, (Uint64)ptr[0]); +#else /* check for bignum */ if (!IS_USMALL(0, (Uint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ +#endif ptr++; depth++; break; @@ -2943,22 +2988,30 @@ driver_deliver_term(ErlDrvPort port, break; case ERL_DRV_INT: /* signed int argument */ +#if HALFWORD_HEAP + mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]); +#else if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); else { mess = small_to_big((Sint)ptr[0], hp); hp += BIG_UINT_HEAP_SIZE; } +#endif ptr++; break; case ERL_DRV_UINT: /* unsigned int argument */ +#if HALFWORD_HEAP + mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]); +#else if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); else { mess = uint_to_big((Uint)ptr[0], hp); hp += BIG_UINT_HEAP_SIZE; } +#endif ptr++; break; @@ -2997,14 +3050,14 @@ driver_deliver_term(ErlDrvPort port, driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; mess = make_binary(pb); hp += PROC_BIN_SIZE; - ohp->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); } ptr += 3; break; @@ -3036,12 +3089,12 @@ driver_deliver_term(ErlDrvPort port, hp += PROC_BIN_SIZE; pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->mso; - ohp->mso = pbp; + pbp->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; - ohp->overhead += (pbp->size / sizeof(Eterm)); + OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm)); mess = make_binary(pbp); } ptr += 2; @@ -3200,7 +3253,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -3235,7 +3288,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len) return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -3312,7 +3365,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip) /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + size)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + size)); deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size); return 0; } @@ -3356,25 +3409,25 @@ int len; * reference count on driver binaries... */ -long +ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_read(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_read(&bp->refc, 1); } -long +ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_inctest(&bp->refc, 2); + return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2); } -long +ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_dectest(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1); } @@ -3489,12 +3542,12 @@ pdl_init_refc(ErlDrvPDL pdl) erts_atomic_init(&pdl->refc, 1); } -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_read_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_read(&pdl->refc); + erts_aint_t refc = erts_atomic_read(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); - return refc; + return (ErlDrvSInt) refc; } static ERTS_INLINE void @@ -3504,12 +3557,12 @@ pdl_inc_refc(ErlDrvPDL pdl) ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1); } -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_inctest_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_inctest(&pdl->refc); + erts_aint_t refc = erts_atomic_inctest(&pdl->refc); ERTS_LC_ASSERT(refc > 1); - return refc; + return (ErlDrvSInt) refc; } #if 0 /* unused */ @@ -3521,12 +3574,12 @@ pdl_dec_refc(ErlDrvPDL pdl) } #endif -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_dectest_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_dectest(&pdl->refc); + erts_aint_t refc = erts_atomic_dectest(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); - return refc; + return (ErlDrvSInt) refc; } static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl) @@ -3536,6 +3589,32 @@ static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl) erts_free(ERTS_ALC_T_PORT_DATA_LOCK, pdl); } +#ifdef ERTS_SMP + +static void driver_monitor_lock_pdl(Port *p) { + if (p->port_data_lock) { + driver_pdl_lock(p->port_data_lock); + } + /* Now we either have the port lock or the port_data_lock */ + ERTS_LC_ASSERT(!p->port_data_lock + || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx))); + ERTS_SMP_LC_ASSERT(p->port_data_lock + || erts_lc_is_port_locked(p)); +} + +static void driver_monitor_unlock_pdl(Port *p) { + /* We should either have the port lock or the port_data_lock */ + ERTS_LC_ASSERT(!p->port_data_lock + || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx))); + ERTS_SMP_LC_ASSERT(p->port_data_lock + || erts_lc_is_port_locked(p)); + if (p->port_data_lock) { + driver_pdl_unlock(p->port_data_lock); + } +} + +#endif + /* * exported driver_pdl_* functions ... */ @@ -3571,7 +3650,7 @@ driver_pdl_lock(ErlDrvPDL pdl) void driver_pdl_unlock(ErlDrvPDL pdl) { - long refc; + ErlDrvSInt refc; #ifdef HARDDEBUG erts_fprintf(stderr, "driver_pdl_unlock(0x%08X)\r\n",(unsigned) pdl); #endif @@ -3581,28 +3660,30 @@ driver_pdl_unlock(ErlDrvPDL pdl) pdl_destroy(pdl); } -long +ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL pdl) { return pdl_read_refc(pdl); } -long +ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL pdl) { - long refc = pdl_inctest_refc(pdl); + ErlDrvSInt refc = pdl_inctest_refc(pdl); #ifdef HARDDEBUG - erts_fprintf(stderr, "driver_pdl_inc_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc); + erts_fprintf(stderr, "driver_pdl_inc_refc(%p) -> %bed\r\n", + pdl, refc); #endif return refc; } -long +ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL pdl) { - long refc = pdl_dectest_refc(pdl); + ErlDrvSInt refc = pdl_dectest_refc(pdl); #ifdef HARDDEBUG - erts_fprintf(stderr, "driver_pdl_dec_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc); + erts_fprintf(stderr, "driver_pdl_dec_refc(%p) -> %bpd\r\n", + pdl, refc); #endif if (!refc) pdl_destroy(pdl); @@ -3988,13 +4069,13 @@ drv_cancel_timer(Port *prt) #ifdef ERTS_SMP erts_cancel_smp_ptimer(prt->ptimer); #else - erl_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->tm); #endif if (erts_port_task_is_scheduled(&prt->timeout_task)) erts_port_task_abort(prt->id, &prt->timeout_task); } -int driver_set_timer(ErlDrvPort ix, Uint t) +int driver_set_timer(ErlDrvPort ix, UWord t) { Port* prt = erts_drvport2port(ix); @@ -4012,7 +4093,7 @@ int driver_set_timer(ErlDrvPort ix, Uint t) (ErlTimeoutProc) schedule_port_timeout, t); #else - erl_set_timer(&prt->tm, + erts_set_timer(&prt->tm, (ErlTimeoutProc) schedule_port_timeout, NULL, prt, @@ -4043,9 +4124,9 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SMP - *t = prt->ptimer ? time_left(&prt->ptimer->timer.tm) : 0; + *t = prt->ptimer ? erts_time_left(&prt->ptimer->timer.tm) : 0; #else - *t = time_left(&prt->tm); + *t = erts_time_left(&prt->tm); #endif return 0; } @@ -4053,12 +4134,16 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t) int driver_get_now(ErlDrvNowData *now_data) { + Uint mega,secs,micro; ERTS_SMP_CHK_NO_PROC_LOCKS; if (now_data == NULL) { return -1; } - get_now(&(now_data->megasecs),&(now_data->secs),&(now_data->microsecs)); + get_now(&mega,&secs,µ); + now_data->megasecs = (unsigned long) mega; + now_data->secs = (unsigned long) secs; + now_data->microsecs = (unsigned long) micro; return 0; } @@ -4072,14 +4157,15 @@ static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) memcpy(mon,refp,sizeof(RefThing)); } -int driver_monitor_process(ErlDrvPort port, - ErlDrvTermData process, - ErlDrvMonitor *monitor) + +static int do_driver_monitor_process(Port *prt, + Eterm *buf, + ErlDrvTermData process, + ErlDrvMonitor *monitor) { - Port *prt = erts_drvport2port(port); Process *rp; Eterm ref; - Eterm buf[REF_THING_SIZE]; + if (prt->drv_ptr->process_exit == NULL) { return -1; } @@ -4089,22 +4175,76 @@ int driver_monitor_process(ErlDrvPort port, if (!rp) { return 1; } + ref = erts_make_ref_in_buffer(buf); erts_add_monitor(&(prt->monitors), MON_ORIGIN, ref, rp->id, NIL); erts_add_monitor(&(rp->monitors), MON_TARGET, ref, prt->id, NIL); - + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); ref_to_driver_monitor(ref,monitor); return 0; } -int driver_demonitor_process(ErlDrvPort port, - const ErlDrvMonitor *monitor) +/* + * This can be called from a non scheduler thread iff a port_data_lock exists + */ +int driver_monitor_process(ErlDrvPort port, + ErlDrvTermData process, + ErlDrvMonitor *monitor) +{ + Port *prt; + int ret; + Uint32 status; + ErtsSchedulerData *sched = erts_get_scheduler_data(); + int ix = (int) port; + if (ix < 0 || erts_max_ports <= ix) { + return -1; + } + prt = &erts_port[ix]; + + DRV_MONITOR_LOCK_PDL(prt); + + if (sched) { + status = erts_port[ix].status; + } else { + erts_smp_port_state_lock(prt); + status = erts_port[ix].status; + erts_smp_port_state_unlock(prt); + } + + if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + DRV_MONITOR_UNLOCK_PDL(prt); + return -1; + } + + /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock + (if we're a driver thread) */ + ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); + +#if !HEAP_ON_C_STACK + if (!sched) { + /* Need a separate allocation for the ref :( */ + Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*REF_THING_SIZE); + ret = do_driver_monitor_process(prt,buf,process,monitor); + erts_free(ERTS_ALC_T_TEMP_TERM,buf); + } else +#endif + { + DeclareTmpHeapNoproc(buf,REF_THING_SIZE); + UseTmpHeapNoproc(REF_THING_SIZE); + ret = do_driver_monitor_process(prt,buf,process,monitor); + UnUseTmpHeapNoproc(REF_THING_SIZE); + } + DRV_MONITOR_UNLOCK_PDL(prt); + return ret; +} + +static int do_driver_demonitor_process(Port *prt, Eterm *buf, + const ErlDrvMonitor *monitor) { - Port *prt = erts_drvport2port(port); Process *rp; Eterm ref; - Eterm buf[REF_THING_SIZE]; ErtsMonitor *mon; Eterm to; @@ -4137,12 +4277,60 @@ int driver_demonitor_process(ErlDrvPort port, return 0; } -ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, +int driver_demonitor_process(ErlDrvPort port, + const ErlDrvMonitor *monitor) +{ + Port *prt; + int ret; + Uint32 status; + ErtsSchedulerData *sched = erts_get_scheduler_data(); + int ix = (int) port; + if (ix < 0 || erts_max_ports <= ix) { + return -1; + } + prt = &erts_port[ix]; + + DRV_MONITOR_LOCK_PDL(prt); + + if (sched) { + status = erts_port[ix].status; + } else { + erts_smp_port_state_lock(prt); + status = erts_port[ix].status; + erts_smp_port_state_unlock(prt); + } + + if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + DRV_MONITOR_UNLOCK_PDL(prt); + return -1; + } + + /* Now we should have either the port lock (if we have a scheduler) or the port data lock + (if we're a driver thread) */ + ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); +#if !HEAP_ON_C_STACK + if (!sched) { + /* Need a separate allocation for the ref :( */ + Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*REF_THING_SIZE); + ret = do_driver_demonitor_process(prt,buf,monitor); + erts_free(ERTS_ALC_T_TEMP_TERM,buf); + } else +#endif + { + DeclareTmpHeapNoproc(buf,REF_THING_SIZE); + UseTmpHeapNoproc(REF_THING_SIZE); + ret = do_driver_demonitor_process(prt,buf,monitor); + UnUseTmpHeapNoproc(REF_THING_SIZE); + } + DRV_MONITOR_UNLOCK_PDL(prt); + return ret; +} + +static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf, const ErlDrvMonitor *monitor) { - Port *prt = erts_drvport2port(port); Eterm ref; - Eterm buf[REF_THING_SIZE]; ErtsMonitor *mon; Eterm to; @@ -4158,6 +4346,59 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, return (ErlDrvTermData) to; } + +ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, + const ErlDrvMonitor *monitor) +{ + Port *prt; + ErlDrvTermData ret; + Uint32 status; + ErtsSchedulerData *sched = erts_get_scheduler_data(); + int ix = (int) port; + if (ix < 0 || erts_max_ports <= ix) { + return driver_term_nil; + } + prt = &erts_port[ix]; + + DRV_MONITOR_LOCK_PDL(prt); + + if (sched) { + status = erts_port[ix].status; + } else { + erts_smp_port_state_lock(prt); + status = erts_port[ix].status; + erts_smp_port_state_unlock(prt); + } + + if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + DRV_MONITOR_UNLOCK_PDL(prt); + return driver_term_nil; + } + + /* Now we should have either the port lock (if we have a scheduler) or the port data lock + (if we're a driver thread) */ + ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); + +#if !HEAP_ON_C_STACK + if (!sched) { + /* Need a separate allocation for the ref :( */ + Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, + sizeof(Eterm)*REF_THING_SIZE); + ret = do_driver_get_monitored_process(prt,buf,monitor); + erts_free(ERTS_ALC_T_TEMP_TERM,buf); + } else +#endif + { + DeclareTmpHeapNoproc(buf,REF_THING_SIZE); + UseTmpHeapNoproc(REF_THING_SIZE); + ret = do_driver_get_monitored_process(prt,buf,monitor); + UnUseTmpHeapNoproc(REF_THING_SIZE); + } + DRV_MONITOR_UNLOCK_PDL(prt); + return ret; +} + + int driver_compare_monitors(const ErlDrvMonitor *monitor1, const ErlDrvMonitor *monitor2) { @@ -4173,18 +4414,22 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(prt->drv_ptr != NULL); - + DRV_MONITOR_LOCK_PDL(prt); if (erts_lookup_monitor(prt->monitors,ref) == NULL) { + DRV_MONITOR_UNLOCK_PDL(prt); return; } callback = prt->drv_ptr->process_exit; ASSERT(callback != NULL); ref_to_driver_monitor(ref,&drv_monitor); + DRV_MONITOR_UNLOCK_PDL(prt); fpe_was_unmasked = erts_block_fpe(); (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor); erts_unblock_fpe(fpe_was_unmasked); + DRV_MONITOR_LOCK_PDL(prt); /* remove monitor *after* callback */ rmon = erts_remove_monitor(&(prt->monitors),ref); + DRV_MONITOR_UNLOCK_PDL(prt); if (rmon) { erts_destroy_monitor(rmon); } diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 87d13b3607..694e4ab72f 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -29,8 +29,8 @@ typedef struct erl_module { IndexSlot slot; /* Must be located at top of struct! */ int module; /* Atom index for module (not tagged). */ - Eterm* code; - Eterm* old_code; + BeamInstr* code; + BeamInstr* old_code; int code_length; /* Length of loaded code in bytes. */ int old_code_length; /* Length of old loaded code in bytes */ unsigned catches, old_catches; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index ce1df74f03..8a5763b4bb 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 1997-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 1997-2011. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -60,12 +60,17 @@ func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler # The undocumented and unsupported guard BIF is_constant/1 was removed -# in R13. The is_constant/2 operation is marked as obosolete in genop.tab, +# in R13. The is_constant/2 operation is marked as obsolete in genop.tab, # so the loader will automatically generate a too_old_compiler message # it is used, but we need to handle the is_constant/1 BIF specially here. bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler +# Since the constant pool was introduced in R12B, empty tuples ({}) +# are literals. Therefore we no longer need to allow put_tuple/2 +# with a tuple size of zero. + +put_tuple u==0 d => too_old_compiler # # All the other instructions. @@ -79,6 +84,8 @@ i_trace_breakpoint i_mtrace_breakpoint i_debug_breakpoint i_count_breakpoint +i_time_breakpoint +i_return_time_trace i_return_to_trace i_yield i_global_cons @@ -94,16 +101,16 @@ return %macro: test_heap TestHeap -pack allocate t t -allocate_heap I I I +allocate_heap t I t deallocate I init y allocate_zero t t -allocate_heap_zero I I I +allocate_heap_zero t I t trim N Remaining => i_trim N i_trim I -test_heap I I +test_heap I t allocate_heap S u==0 R => allocate S R allocate_heap_zero S u==0 R => allocate_zero S R @@ -115,15 +122,9 @@ init Y1 | init Y2 => init2 Y1 Y2 %macro: init2 Init2 -pack %macro: init3 Init3 -pack -# -# Warning: The put_string instruction is specially treated in the loader. -# Don't change the instruction format unless you change the loader too. -# -put_string I I d - # Selecting values -select_val S=q Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest) +select_val S=aiq Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest) select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ gen_jump_tab(S, Fail, Size, Rest) @@ -131,34 +132,59 @@ select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ gen_jump_tab(S, Fail, Size, Rest) +is_integer TypeFail=f S | select_val S=s Fail=f Size=u Rest=* | \ + mixed_types(Size, Rest) => \ + gen_split_values(S, TypeFail, Fail, Size, Rest) + select_val S=s Fail=f Size=u Rest=* | mixed_types(Size, Rest) => \ - gen_split_values(S, Fail, Size, Rest) + gen_split_values(S, Fail, Fail, Size, Rest) -is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | \ +is_integer Fail=f S | select_val S=d Fail=f Size=u Rest=* | \ fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest) -is_atom Fail=f S | select_val S=s Fail=f Size=u Rest=* | \ +is_atom Fail=f S | select_val S=d Fail=f Size=u Rest=* | \ fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest) -select_val S=s Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \ - gen_select_val(S, Fail, Size, Rest) +select_val S=s Fail=f Size=u Rest=* | floats_or_bignums(Size, Rest) => \ + gen_select_literals(S, Fail, Size, Rest) -select_val S=s Fail=f Size=u Rest=* | all_values_are_big(Size, Rest) => \ - gen_select_big(S, Fail, Size, Rest) +select_val S=d Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \ + gen_select_val(S, Fail, Size, Rest) -is_tuple Fail=f S | select_tuple_arity S=s Fail=f Size=u Rest=* => \ +is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -select_tuple_arity S=s Fail=f Size=u Rest=* => \ +select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val s f I -i_select_tuple_arity s f I -i_select_big s f -i_select_float s f I +i_select_val r f I +i_select_val x f I +i_select_val y f I + +i_select_val2 r f c f c f +i_select_val2 x f c f c f +i_select_val2 y f c f c f + +i_select_tuple_arity2 r f A f A f +i_select_tuple_arity2 x f A f A f +i_select_tuple_arity2 y f A f A f + +i_select_tuple_arity r f I +i_select_tuple_arity x f I +i_select_tuple_arity y f I -i_jump_on_val_zero s f I -i_jump_on_val s f I I +i_jump_on_val_zero r f I +i_jump_on_val_zero x f I +i_jump_on_val_zero y f I + +i_jump_on_val r f I I +i_jump_on_val x f I I +i_jump_on_val y f I I + +jump Target | label Lbl | same_label(Target, Lbl) => label Lbl + +is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \ + is_eq_exact Fail S1 S2 | label L2 %macro: get_list GetList -pack get_list x x x @@ -233,11 +259,17 @@ is_number Fail Literal=q => move Literal x | is_number Fail x jump f -case_end Literal=q => move Literal x | case_end x -badmatch Literal=q => move Literal x | badmatch x +case_end Literal=cq => move Literal x | case_end x +badmatch Literal=cq => move Literal x | badmatch x + +case_end r +case_end x +case_end y + +badmatch r +badmatch x +badmatch y -case_end s -badmatch s if_end raise s s @@ -247,12 +279,33 @@ system_limit j move R R => +move C=cxy r | jump Lbl => move_jump Lbl C + +%macro: move_jump MoveJump -nonext +move_jump f n +move_jump f c +move_jump f x +move_jump f y + move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 +move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 + +move C=aiq X=x==1 => move_x1 C +move C=aiq X=x==2 => move_x2 C + +move_x1 c +move_x2 c %macro: move2 Move2 -pack move2 x y x y move2 y x y x +move2 x x x x + +# The compiler almost never generates a "move Literal y(Y)" instruction, +# so let's cheat if we encounter one. +move S=n D=y => init D +move S=c D=y => move S x | move x D %macro:move Move -pack -gen_dest move x x @@ -264,15 +317,10 @@ move r x move r y move c r move c x -move c y move n x move n r move y y -%cold -move s d -%hot - # Receive operations. loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src @@ -305,58 +353,78 @@ i_wait_error_locked send # -# Comparisions. +# Optimized comparisons with one immediate/literal operand. +# + +is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C + +is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C +is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C + +%macro: i_is_eq_exact_immed EqualImmed -fail_action +i_is_eq_exact_immed f r c +i_is_eq_exact_immed f x c +i_is_eq_exact_immed f y c + +i_is_eq_exact_literal r f c +i_is_eq_exact_literal x f c +i_is_eq_exact_literal y f c + +%macro: i_is_ne_exact_immed NotEqualImmed -fail_action +i_is_ne_exact_immed f r c +i_is_ne_exact_immed f x c +i_is_ne_exact_immed f y c + +i_is_ne_exact_literal r f c +i_is_ne_exact_literal x f c +i_is_ne_exact_literal y f c + +# +# All other comparisons. # -is_eq_exact Lbl=f R=rxy C=ian => i_is_eq_immed Lbl R C -is_eq Lbl=f R=rxy C=an => i_is_eq_immed Lbl R C +is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl +is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl -is_eq_exact Lbl=f S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl -is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl - +i_is_eq_exact f +i_is_ne_exact f i_is_lt f i_is_ge f i_is_eq f i_is_ne f -i_is_eq_exact f -i_is_ne_exact f - -%macro: i_is_eq_immed EqualImmed -fail_action -i_is_eq_immed f r c -i_is_eq_immed f x c -i_is_eq_immed f y c # # Putting things. # -put_tuple u==0 Dst => i_put_tuple_only u Dst -put_tuple Arity Dst | put V => i_put_tuple Arity V Dst +put_tuple Arity Dst => i_put_tuple Dst u -i_put_tuple_only A d +i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \ + put S3 | put S4 | put S5 => \ + tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5) -%macro: i_put_tuple PutTuple -pack -i_put_tuple A x x -i_put_tuple A y x -i_put_tuple A r x -i_put_tuple A n x -i_put_tuple A c x -i_put_tuple A x y -i_put_tuple A x r -i_put_tuple A y r -i_put_tuple A n r -i_put_tuple A c r +i_put_tuple Dst Arity Puts=* | put S => \ + tuple_append_put(Arity, Dst, Puts, S) -%cold -i_put_tuple A r y -i_put_tuple A y y -i_put_tuple A c y -%hot +i_put_tuple/2 + +%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple +i_put_tuple r I +i_put_tuple x I +i_put_tuple y I + +# +# The instruction "put_list Const [] Dst" will not be generated by +# the current BEAM compiler. But until R15A, play it safe by handling +# that instruction with the following transformation. +# +put_list Const=c n Dst => move Const x | put_list x n Dst %macro:put_list PutList -pack -gen_dest @@ -364,10 +432,8 @@ put_list x n x put_list y n x put_list x x x put_list y x x -put_list c n x put_list x x r put_list y r r -put_list c n r put_list y y x put_list x y x @@ -378,6 +444,13 @@ put_list y y r put_list y r x put_list r n x +put_list x r x +put_list x y r +put_list y x r +put_list y x x + +put_list x r r + # put_list SrcReg Constant Dst put_list r c r put_list r c x @@ -405,17 +478,9 @@ put_list c y x put_list c y y %cold -put_list x r r put_list s s d %hot -%macro: put Put -put x -put r -put y -put c -put n - %macro: i_fetch FetchArgs -pack i_fetch c c i_fetch c r @@ -466,19 +531,20 @@ move_return n r move S r | deallocate D | return => move_deallocate_return S r D -%macro: move_deallocate_return MoveDeallocateReturn -nonext -move_deallocate_return x r P -move_deallocate_return y r P -move_deallocate_return c r P -move_deallocate_return n r P +%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext +move_deallocate_return x r Q +move_deallocate_return y r Q +move_deallocate_return c r Q +move_deallocate_return n r Q deallocate D | return => deallocate_return D %macro: deallocate_return DeallocateReturn -nonext -deallocate_return P +deallocate_return Q test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y +%macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y # Test tuple & arity (head) @@ -578,14 +644,14 @@ is_list f y is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs -%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -is_nonempty_list_allocate f x I I -is_nonempty_list_allocate f r I I +%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack +is_nonempty_list_allocate f x I t +is_nonempty_list_allocate f r I t is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2 -%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -is_non_empty_list_test_heap f r I I +%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack +is_non_empty_list_test_heap f r I t %macro: is_nonempty_list IsNonemptyList -fail_action is_nonempty_list f x @@ -914,8 +980,13 @@ node x node y %hot -i_fast_element j I s d -i_element j s s d +i_fast_element r j I d +i_fast_element x j I d +i_fast_element y j I d + +i_element r j s d +i_element x j s d +i_element y j s d bif1 f b s d bif1_body b s d @@ -942,11 +1013,11 @@ move S r | call_last Ar P=f D => move_call_last S r P D i_move_call_last f P c r -%macro:move_call_last MoveCallLast -arg_f -nonext +%macro:move_call_last MoveCallLast -arg_f -nonext -pack move_call_last/4 -move_call_last x r f P -move_call_last y r f P +move_call_last x r f Q +move_call_last y r f Q move S=c r | call_only Ar P=f => i_move_call_only P S r move S=x r | call_only Ar P=f => move_call_only S r P @@ -995,7 +1066,7 @@ is_function f y is_function f r is_function Fail=f c => jump Fail -func_info M=a F=a A=u | label L => gen_func_info(M, F, A, L) +func_info M F A => i_func_info u M F A # ================================================================ # New bit syntax matching (R11B). @@ -1184,12 +1255,6 @@ i_bs_init_bits_fail_heap I j I d i_bs_init_bits I I d i_bs_init_bits_heap I I I d -bs_bits_to_bytes Fail Src Dst => i_bs_bits_to_bytes Src Fail Dst - -i_bs_bits_to_bytes r j d -i_bs_bits_to_bytes x j d -i_bs_bits_to_bytes y j d - bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D @@ -1311,10 +1376,12 @@ fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3 fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3 fnegate p FR1 FR2 => i_fnegate FR1 FR2 -fconv Int=iq Dst=l => move Int x | fconv x Dst +fconv Arg=iqan Dst=l => move Arg x | fconv x Dst fmove q l fmove d l +fmove l d + fconv d l i_fadd l l l @@ -1330,12 +1397,6 @@ fcheckerror p => i_fcheckerror i_fcheckerror fclearerror -fmove FR=l Dst=d | new_float_allocation() => fmove_new FR Dst - -# The new instruction for moving a float out of a floating point register. -# (No allocation.) -fmove_new l d - # # New apply instructions in R10B. # @@ -1344,7 +1405,21 @@ apply I apply_last I P # -# New GCing arithmetic instructions. +# Optimize addition and subtraction of small literals using +# the i_increment/4 instruction (in bodies, not in guards). +# + +gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \ + gen_increment(Reg, Int, Live, Dst) +gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \ + gen_increment(Reg, Int, Live, Dst) + +gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ + negation_is_small(Int) => \ + gen_increment_from_minus(Reg, Int, Live, Dst) + +# +# GCing arithmetic instructions. # gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst @@ -1367,6 +1442,10 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst +i_increment r I I d +i_increment x I I d +i_increment y I I d + i_plus j I d i_minus j I d i_times j I d @@ -1397,34 +1476,60 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler # Guard BIFs. # gc_bif1 Fail I Bif=u$bif:erlang:length/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:bit_size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:byte_size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:abs/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:float/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:round/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:trunc/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) + +gc_bif2 Fail I Bif=u$bif:erlang:binary_part/2 S1 S2 Dst=d => \ + gen_guard_bif2(Fail, I, Bif, S1, S2, Dst) + +gc_bif3 Fail I Bif=u$bif:erlang:binary_part/3 S1 S2 S3 Dst=d => \ + gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D i_gc_bif1 j I s I d +ii_gc_bif2/6 + +ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D + +i_gc_bif2 j I I d + +ii_gc_bif3/7 + +ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D + +i_gc_bif3 j I s I d # # R13B03 # on_load + +# +# R14A. +# +recv_mark f + +recv_set Fail | label Lbl | loop_rec Lf Reg => \ + i_recv_set | label Lbl | loop_rec Lf Reg +i_recv_set diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index 8c8029d450..a66d60aa22 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -47,11 +47,6 @@ (((unsigned char*) (s))[1] << 8) | \ (((unsigned char*) (s))[0])) -#define put_int24(s, x) ((((unsigned char*)(s))[0] = ((x) >> 16) & 0xff), \ - (((unsigned char*)(s))[1] = ((x) >> 8) & 0xff), \ - (((unsigned char*)(s))[2] = (x) & 0xff)) - - #if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP) #define STRNCASECMP my_strncasecmp @@ -679,7 +674,7 @@ int packet_parse_http(const char* buf, int len, int* statep, while (n && SP(ptr)) { ptr++; n--; } - if (ptr==p0) return -1; + if (ptr==p0 && n>0) return -1; /* NOTE: the syntax allows empty reason phrases */ (*statep) = !0; @@ -833,7 +828,7 @@ int packet_parse_ssl(const char* buf, int len, char prefix[4]; /* <<1:8,Length:24,Data/binary>> */ prefix[0] = 1; - put_int24(&prefix[1],len-3); + put_int24(len-3,&prefix[1]); return pcb->ssl_tls(arg, 22, major, minor, buf+3, len-3, prefix, sizeof(prefix)); } else { diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 964c10a380..26d64887d0 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -39,8 +39,6 @@ static Hash process_reg; static erts_smp_rwmtx_t regtab_rwmtx; -#define reg_lock_init() erts_smp_rwmtx_init(®tab_rwmtx, \ - "reg_tab") #define reg_try_read_lock() erts_smp_rwmtx_tryrlock(®tab_rwmtx) #define reg_try_write_lock() erts_smp_rwmtx_tryrwlock(®tab_rwmtx) #define reg_read_lock() erts_smp_rwmtx_rlock(®tab_rwmtx) @@ -147,8 +145,11 @@ static void reg_free(RegProc *obj) void init_register_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - reg_lock_init(); + erts_smp_rwmtx_init_opt(®tab_rwmtx, &rwmtx_opt, "reg_tab"); f.hash = (H_FUN) reg_hash; f.cmp = (HCMP_FUN) reg_cmp; @@ -476,8 +477,9 @@ int erts_unregister_name(Process *c_p, * on c_prt. */ - if (!c_p) + if (!c_p) { c_p_locks = 0; + } current_c_p_locks = c_p_locks; restart: @@ -489,9 +491,15 @@ int erts_unregister_name(Process *c_p, if (is_non_value(name)) { /* Unregister current process name */ ASSERT(c_p); - if (c_p->reg) +#ifdef ERTS_SMP + if (current_c_p_locks != c_p_locks) { + erts_smp_proc_lock(c_p, c_p_locks); + current_c_p_locks = c_p_locks; + } +#endif + if (c_p->reg) { r.name = c_p->reg->name; - else { + } else { /* Name got unregistered while main lock was released */ res = 0; goto done; @@ -533,24 +541,25 @@ int erts_unregister_name(Process *c_p, } } else if (rp->p) { - Process* p = rp->p; + #ifdef ERTS_SMP erts_proc_safelock(c_p, current_c_p_locks, c_p_locks, rp->p, - 0, + (c_p == rp->p) ? current_c_p_locks : 0, ERTS_PROC_LOCK_MAIN); current_c_p_locks = c_p_locks; #endif - p->reg = NULL; + rp->p->reg = NULL; + if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) { + trace_proc(c_p, rp->p, am_unregister, r.name); + } #ifdef ERTS_SMP - if (rp->p != c_p) + if (rp->p != c_p) { erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN); -#endif - if (IS_TRACED_FL(p, F_TRACE_PROCS)) { - trace_proc(c_p, p, am_unregister, r.name); } +#endif } hash_erase(&process_reg, (void*) &r); res = 1; @@ -560,14 +569,17 @@ int erts_unregister_name(Process *c_p, reg_write_unlock(); if (c_prt != port) { - if (port) + if (port) { erts_smp_port_unlock(port); - if (c_prt) + } + if (c_prt) { erts_smp_port_lock(c_prt); + } } #ifdef ERTS_SMP - if (c_p && !current_c_p_locks) + if (c_p && !current_c_p_locks) { erts_smp_proc_lock(c_p, c_p_locks); + } #endif return res; } diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 21d6ce9304..4c54e19cdb 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit) erts_free(h->type, (void *) old_tab); } /*else already done */ - erts_smp_atomic_set(&h->is_rehashing, 0); + erts_smp_atomic_set_relb(&h->is_rehashing, 0); } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 4b949523fa..e64c43de6e 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,37 +1,30 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifndef __SYS_H__ #define __SYS_H__ + #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) # define NO_FPE_SIGNALS #endif -/* Never use elib-malloc when purify-memory-tracing */ -#if defined(PURIFY) -#undef ENABLE_ELIB_MALLOC -#undef ELIB_HEAP_SBRK -#undef ELIB_ALLOC_IS_CLIB -#endif - - /* xxxP __VXWORKS__ */ #ifdef VXWORKS #include <vxWorks.h> @@ -46,19 +39,10 @@ #define ENABLE_CHILD_WAITER_THREAD 1 #endif -/* The ERTS_TIMER_TREAD #define must be visible to the - erl_${OS}_sys.h #include files: it controls whether - certain optional facilities should be defined or not. */ -#if defined(ERTS_SMP) && 0 -#define ERTS_TIMER_THREAD -#endif - #if defined (__WIN32__) # include "erl_win_sys.h" #elif defined (VXWORKS) # include "erl_vxworks_sys.h" -#elif defined (_OSE_) -# include "erl_ose_sys.h" #else # include "erl_unix_sys.h" #ifndef UNIX @@ -172,23 +156,6 @@ void erl_assert_error(char* expr, char* file, int line); #include <stdarg.h> -#if defined(__STDC__) || defined(_MSC_VER) -# define EXTERN_FUNCTION(t, f, x) extern t f x -# define FUNCTION(t, f, x) t f x -# define _DOTS_ ... -# define _VOID_ void -#elif defined(__cplusplus) -# define EXTERN_FUNCTION(f, x) extern "C" { f x } -# define FUNCTION(t, f, x) t f x -# define _DOTS_ ... -# define _VOID_ void -#else -# define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/) -# define FUNCTION(t, f, x) t f (/*x*/) -# define _DOTS_ -# define _VOID_ -#endif - /* This isn't sys-dependent, but putting it here benefits sys.c and drivers - allow use of 'const' regardless of compiler */ @@ -198,7 +165,7 @@ void erl_assert_error(char* expr, char* file, int line); #ifdef VXWORKS /* Replace VxWorks' printf with a real one that does fprintf(stdout, ...) */ -EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); +int real_printf(const char *fmt, ...); # define printf real_printf #endif @@ -230,9 +197,14 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); ** Data types: ** ** Eterm: A tagged erlang term (possibly 64 bits) +** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller. ** UInt: An unsigned integer exactly as large as an Eterm. ** SInt: A signed integer exactly as large as an eterm and therefor large ** enough to hold the return value of the signed_val() macro. +** UWord: An unsigned integer at least as large as a void * and also as large +** or larger than an Eterm +** SWord: A signed integer at least as large as a void * and also as large +** or larger than an Eterm ** Uint32: An unsigned integer of 32 bits exactly ** Sint32: A signed integer of 32 bits exactly ** Uint16: An unsigned integer of 16 bits exactly @@ -253,11 +225,43 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); #else #error Neither 32 nor 64 bit architecture #endif +#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR) +# define HALFWORD_HEAP 1 +# define HALFWORD_ASSERT 0 +# define ASSERT_HALFWORD(COND) ASSERT(COND) +#else +# define HALFWORD_HEAP 0 +# define HALFWORD_ASSERT 0 +# define ASSERT_HALFWORD(COND) +#endif #if SIZEOF_VOID_P != SIZEOF_SIZE_T #error sizeof(void*) != sizeof(size_t) #endif +#if HALFWORD_HEAP + +#if SIZEOF_INT == 4 +typedef unsigned int Eterm; +typedef unsigned int Uint; +typedef int Sint; +#define ERTS_SIZEOF_ETERM SIZEOF_INT +#else +#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' +#endif + +#if SIZEOF_VOID_P == SIZEOF_LONG +typedef unsigned long UWord; +typedef long SWord; +#elif SIZEOF_VOID_P == SIZEOF_INT +typedef unsigned int UWord; +typedef int SWord; +#else +#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' +#endif + +#else /* !HALFWORD_HEAP */ + #if SIZEOF_VOID_P == SIZEOF_LONG typedef unsigned long Eterm; typedef unsigned long Uint; @@ -272,6 +276,13 @@ typedef int Sint; #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' #endif +typedef Uint UWord; +typedef Sint SWord; + +#endif /* HALFWORD_HEAP */ + +typedef UWord BeamInstr; + #ifndef HAVE_INT64 #if SIZEOF_LONG == 8 #define HAVE_INT64 1 @@ -316,23 +327,20 @@ typedef unsigned char byte; #error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found #endif -#if defined(ARCH_64) -# define ERTS_WORD_ALIGN_PAD_SZ(X) \ +# define ERTS_EXTRA_DATA_ALIGN_SZ(X) \ (((size_t) 8) - (((size_t) (X)) & ((size_t) 7))) -#elif defined(ARCH_32) -# define ERTS_WORD_ALIGN_PAD_SZ(X) \ - (((size_t) 4) - (((size_t) (X)) & ((size_t) 3))) -#else -#error "Not supported..." -#endif #include "erl_lock_check.h" + +/* needed by erl_smp.h */ +int erts_send_warning_to_logger_str_nogl(char *); + #include "erl_smp.h" #ifdef ERTS_WANT_BREAK_HANDLING # ifdef ERTS_SMP -extern erts_smp_atomic_t erts_break_requested; -# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic_read(&erts_break_requested)) +extern erts_smp_atomic32_t erts_break_requested; +# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic32_read(&erts_break_requested)) # else extern volatile int erts_break_requested; # define ERTS_BREAK_REQUESTED erts_break_requested @@ -345,8 +353,8 @@ void erts_do_break_handling(void); # define ERTS_GOT_SIGUSR1 0 # else # ifdef ERTS_SMP -extern erts_smp_atomic_t erts_got_sigusr1; -# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic_read(&erts_got_sigusr1)) +extern erts_smp_atomic32_t erts_got_sigusr1; +# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read(&erts_got_sigusr1)) # else extern volatile int erts_got_sigusr1; # define ERTS_GOT_SIGUSR1 erts_got_sigusr1 @@ -413,13 +421,6 @@ extern volatile int erts_writing_erl_crash_dump; in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */ -# ifdef _OSE_ -static const int zero_value = 0, one_value = 1; -# define SET_BLOCKING(fd) ioctl((fd), FIONBIO, (char*)&zero_value) -# define SET_NONBLOCKING(fd) ioctl((fd), FIONBIO, (char*)&one_value) -# define ERRNO_BLOCK EWOULDBLOCK -# else - # ifdef __WIN32__ static unsigned long zero_value = 0, one_value = 1; @@ -460,11 +461,8 @@ static const int zero_value = 0, one_value = 1; # endif /* !NB_FIONBIO */ # endif /* _WXWORKS_ */ # endif /* !__WIN32__ */ -# endif /* _OSE_ */ #endif /* WANT_NONBLOCKING */ -extern erts_cpu_info_t *erts_cpuinfo; /* erl_init.c */ - __decl_noreturn void __noreturn erl_exit(int n, char*, ...); /* Some special erl_exit() codes: */ @@ -523,7 +521,8 @@ int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_info_to_logger_str_nogl(char *); -int erts_send_warning_to_logger_str_nogl(char *); +/* needed by erl_smp.h (declared above) + int erts_send_warning_to_logger_str_nogl(char *); */ int erts_send_error_to_logger_str_nogl(char *); typedef struct preload { @@ -538,11 +537,9 @@ typedef struct preload { * None of the drivers use all of the fields. */ -/* OSE: Want process_type and priority in here as well! Needs updates in erl_bif_ports.c! */ - typedef struct _SysDriverOpts { - int ifd; /* Input file descriptor (fd driver). */ - int ofd; /* Outputfile descriptor (fd driver). */ + Uint ifd; /* Input file descriptor (fd driver). */ + Uint ofd; /* Outputfile descriptor (fd driver). */ int packet_bytes; /* Number of bytes in packet header. */ int read_write; /* Read and write bits. */ int use_stdio; /* Use standard I/O: TRUE or FALSE. */ @@ -556,12 +553,6 @@ typedef struct _SysDriverOpts { char *wd; /* Working directory. */ unsigned spawn_type; /* Bitfield of ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXTERNAL | both*/ - -#ifdef _OSE_ - enum PROCESS_TYPE process_type; - OSPRIORITY priority; -#endif /* _OSE_ */ - } SysDriverOpts; extern char *erts_default_arg0; @@ -569,11 +560,7 @@ extern char *erts_default_arg0; extern char os_type[]; extern int sys_init_time(void); -#if defined(ERTS_TIMER_THREAD) -#define erts_deliver_time() -#else extern void erts_deliver_time(void); -#endif extern void erts_time_remaining(SysTimeval *); extern int erts_init_time_sup(void); extern void erts_sys_init_float(void); @@ -621,7 +608,7 @@ extern char *erts_sys_ddll_error(int code); /* - * System interfaces for startup/sae code (functions found in respective sys.c) + * System interfaces for startup. */ @@ -638,18 +625,14 @@ extern void erts_sys_pre_init(void); extern void erl_sys_init(void); extern void erl_sys_args(int *argc, char **argv); extern void erl_sys_schedule(int); -#ifdef _OSE_ -extern void erl_sys_init_final(void); -#else -void sys_tty_reset(void); -#endif +void sys_tty_reset(int); -EXTERN_FUNCTION(int, sys_max_files, (_VOID_)); +int sys_max_files(void); void sys_init_io(void); Preload* sys_preloaded(void); -EXTERN_FUNCTION(unsigned char*, sys_preload_begin, (Preload*)); -EXTERN_FUNCTION(void, sys_preload_end, (Preload*)); -EXTERN_FUNCTION(int, sys_get_key, (int)); +unsigned char* sys_preload_begin(Preload*); +void sys_preload_end(Preload*); +int sys_get_key(int); void elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, unsigned long *ms_user_diff, unsigned long *ms_sys_diff); void wall_clock_elapsed_time_both(unsigned long *ms_total, @@ -666,7 +649,7 @@ int local_to_univ(Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second, int isdst); void get_now(Uint*, Uint*, Uint*); void get_sys_now(Uint*, Uint*, Uint*); -EXTERN_FUNCTION(void, set_break_quit, (void (*)(void), void (*)(void))); +void set_break_quit(void (*)(void), void (*)(void)); void os_flavor(char*, unsigned); void os_version(int*, int*, int*); @@ -706,7 +689,7 @@ int erts_write_env(char *key, char *value); #define ERTS_DEFAULT_MMAP_THRESHOLD (128 * 1024) #define ERTS_DEFAULT_MMAP_MAX 64 -EXTERN_FUNCTION(int, sys_alloc_opt, (int, int)); +int sys_alloc_opt(int, int); typedef struct { Sint trim_threshold; @@ -715,7 +698,7 @@ typedef struct { Sint mmap_max; } SysAllocStat; -EXTERN_FUNCTION(void, sys_alloc_stat, (SysAllocStat *)); +void sys_alloc_stat(SysAllocStat *); /* Block the whole system... */ @@ -739,11 +722,11 @@ typedef enum { } erts_activity_error_t; typedef struct { - erts_smp_atomic_t do_block; + erts_smp_atomic32_t do_block; struct { - erts_smp_atomic_t wait; - erts_smp_atomic_t gc; - erts_smp_atomic_t io; + erts_smp_atomic32_t wait; + erts_smp_atomic32_t gc; + erts_smp_atomic32_t io; } in_activity; } erts_system_block_state_t; @@ -894,7 +877,7 @@ ERTS_GLB_INLINE int erts_smp_pending_system_block(void) { #ifdef ERTS_SMP - return erts_smp_atomic_read(&erts_system_block_state.do_block); + return (int) erts_smp_atomic32_read(&erts_system_block_state.do_block); #else return 0; #endif @@ -930,7 +913,7 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.wait); if (locked) { /* You are not allowed to leave activity waiting * without supplying the possibility to block @@ -941,10 +924,10 @@ erts_smp_set_activity(erts_activity_t old_activity, } break; case ERTS_ACTIVITY_GC: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, @@ -960,13 +943,13 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.wait); break; case ERTS_ACTIVITY_GC: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, @@ -1001,27 +984,31 @@ erts_smp_set_activity(erts_activity_t old_activity, typedef erts_smp_atomic_t erts_refc_t; -ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, long val); -ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE long erts_refc_inctest(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE long erts_refc_dectest(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, long diff, long min_val); -ERTS_GLB_INLINE long erts_refc_read(erts_refc_t *refcp, long min_val); +ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val); +ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, + erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, + erts_aint_t min_val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_refc_init(erts_refc_t *refcp, long val) +erts_refc_init(erts_refc_t *refcp, erts_aint_t val) { erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val); } ERTS_GLB_INLINE void -erts_refc_inc(erts_refc_t *refcp, long min_val) +erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", @@ -1031,10 +1018,10 @@ erts_refc_inc(erts_refc_t *refcp, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_inctest(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1045,10 +1032,10 @@ erts_refc_inctest(erts_refc_t *refcp, long min_val) } ERTS_GLB_INLINE void -erts_refc_dec(erts_refc_t *refcp, long min_val) +erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", @@ -1058,10 +1045,10 @@ erts_refc_dec(erts_refc_t *refcp, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_dectest(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1072,10 +1059,10 @@ erts_refc_dectest(erts_refc_t *refcp, long min_val) } ERTS_GLB_INLINE void -erts_refc_add(erts_refc_t *refcp, long diff, long min_val) +erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); + erts_aint_t val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", @@ -1085,10 +1072,10 @@ erts_refc_add(erts_refc_t *refcp, long diff, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_read(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1104,13 +1091,10 @@ erts_refc_read(erts_refc_t *refcp, long min_val) extern int erts_use_kernel_poll; #endif -void elib_ensure_initialized(void); - - -#if (defined(VXWORKS) || defined(_OSE_)) +#if defined(VXWORKS) /* NOTE! sys_calloc2 does not exist on other platforms than VxWorks and OSE */ -EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); +void* sys_calloc2(Uint, Uint); #endif /* VXWORKS || OSE */ @@ -1150,14 +1134,14 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); /* Standard set of integer macros .. */ -#define get_int64(s) ((((unsigned char*) (s))[0] << 56) | \ - (((unsigned char*) (s))[1] << 48) | \ - (((unsigned char*) (s))[2] << 40) | \ - (((unsigned char*) (s))[3] << 32) | \ - (((unsigned char*) (s))[4] << 24) | \ - (((unsigned char*) (s))[5] << 16) | \ - (((unsigned char*) (s))[6] << 8) | \ - (((unsigned char*) (s))[7])) +#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \ + (((Uint64)((unsigned char*) (s))[1]) << 48) | \ + (((Uint64)((unsigned char*) (s))[2]) << 40) | \ + (((Uint64)((unsigned char*) (s))[3]) << 32) | \ + (((Uint64)((unsigned char*) (s))[4]) << 24) | \ + (((Uint64)((unsigned char*) (s))[5]) << 16) | \ + (((Uint64)((unsigned char*) (s))[6]) << 8) | \ + (((Uint64)((unsigned char*) (s))[7]))) #define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\ ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\ @@ -1180,6 +1164,15 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); ((char*)(s))[3] = (char)(i) & 0xff;} \ while (0) +#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \ + (((unsigned char*) (s))[1] << 8) | \ + (((unsigned char*) (s))[2])) + +#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff; \ + ((char*)(s))[1] = (char)((i) >> 8) & 0xff; \ + ((char*)(s))[2] = (char)(i) & 0xff;} \ + while (0) + #define get_int16(s) ((((unsigned char*) (s))[0] << 8) | \ (((unsigned char*) (s))[1])) @@ -1193,6 +1186,7 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0) + /* * Use DEBUGF as you would use printf, but use double parentheses: * @@ -1202,8 +1196,8 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); */ #ifdef DEBUG -EXTERN_FUNCTION(void, erl_debug, (char* format, ...)); -EXTERN_FUNCTION(void, erl_bin_write, (unsigned char *, int, int)); +void erl_debug(char* format, ...); +void erl_bin_write(unsigned char *, int, int); # define DEBUGF(x) erl_debug x #else @@ -1257,6 +1251,22 @@ char* win32_errorstr(int); #endif +/************************************************************************ + * Find out the native filename encoding of the process (look at locale of + * Unix processes and just do UTF16 on windows + ************************************************************************/ +#define ERL_FILENAME_UNKNOWN 0 +#define ERL_FILENAME_LATIN1 1 +#define ERL_FILENAME_UTF8 2 +#define ERL_FILENAME_UTF8_MAC 3 +#define ERL_FILENAME_WIN_WCHAR 4 + +int erts_get_native_filename_encoding(void); +/* The set function is only to be used by erl_init! */ +void erts_set_user_requested_filename_encoding(int encoding); +int erts_get_user_requested_filename_encoding(void); + +void erts_init_sys_common_misc(void); #endif diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index a07d6a5327..a00faff912 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -83,24 +83,8 @@ #define ASSERT_NO_LOCKED_LOCKS #endif +static erts_smp_mtx_t tiw_lock; -#if defined(ERTS_TIMER_THREAD) || 1 -/* I don't yet know why, but using a mutex instead of a spinlock - or spin-based rwlock avoids excessive delays at startup. */ -static erts_smp_rwmtx_t tiw_lock; -#define tiw_read_lock() erts_smp_rwmtx_rlock(&tiw_lock) -#define tiw_read_unlock() erts_smp_rwmtx_runlock(&tiw_lock) -#define tiw_write_lock() erts_smp_rwmtx_rwlock(&tiw_lock) -#define tiw_write_unlock() erts_smp_rwmtx_rwunlock(&tiw_lock) -#define tiw_init_lock() erts_smp_rwmtx_init(&tiw_lock, "timer_wheel") -#else -static erts_smp_rwlock_t tiw_lock; -#define tiw_read_lock() erts_smp_read_lock(&tiw_lock) -#define tiw_read_unlock() erts_smp_read_unlock(&tiw_lock) -#define tiw_write_lock() erts_smp_write_lock(&tiw_lock) -#define tiw_write_unlock() erts_smp_write_unlock(&tiw_lock) -#define tiw_init_lock() erts_smp_rwlock_init(&tiw_lock, "timer_wheel") -#endif /* BEGIN tiw_lock protected variables ** @@ -115,80 +99,37 @@ static erts_smp_rwlock_t tiw_lock; static ErlTimer** tiw; /* the timing wheel, allocated in init_time() */ static Uint tiw_pos; /* current position in wheel */ static Uint tiw_nto; /* number of timeouts in wheel */ +static Uint tiw_min; +static ErlTimer *tiw_min_ptr; /* END tiw_lock protected variables */ /* Actual interval time chosen by sys_init_time() */ static int itime; /* Constant after init */ -#if defined(ERTS_TIMER_THREAD) -static SysTimeval time_start; /* start of current time interval */ -static long ticks_end; /* time_start+ticks_end == time_wakeup */ -static long ticks_latest; /* delta from time_start at latest time update*/ - -static ERTS_INLINE long time_gettimeofday(SysTimeval *now) -{ - long elapsed; - - erts_get_timeval(now); - now->tv_usec = 1000 * (now->tv_usec / 1000); /* ms resolution */ - elapsed = (1000 * (now->tv_sec - time_start.tv_sec) + - (now->tv_usec - time_start.tv_usec) / 1000); - // elapsed /= CLOCK_RESOLUTION; - return elapsed; -} - -static long do_time_update(void) -{ - SysTimeval now; - long elapsed; - - elapsed = time_gettimeofday(&now); - ticks_latest = elapsed; - return elapsed; -} - -static ERTS_INLINE long do_time_read(void) -{ - return ticks_latest; -} - -static long do_time_reset(void) -{ - SysTimeval now; - long elapsed; - - elapsed = time_gettimeofday(&now); - time_start = now; - ticks_end = LONG_MAX; - ticks_latest = 0; - return elapsed; -} - -static ERTS_INLINE void do_time_init(void) -{ - (void)do_time_reset(); -} - -#else erts_smp_atomic_t do_time; /* set at clock interrupt */ -static ERTS_INLINE long do_time_read(void) { return erts_smp_atomic_read(&do_time); } -static ERTS_INLINE long do_time_update(void) { return do_time_read(); } +static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); } +static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); } static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); } -#endif /* get the time (in units of itime) to the next timeout, or -1 if there are no timeouts */ -static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ +static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ { int i, tm, nto; unsigned int min; ErlTimer* p; - long dt; + erts_aint_t dt; if (tiw_nto == 0) return -1; /* no timeouts in wheel */ + + if (tiw_min_ptr) { + min = tiw_min; + dt = do_time_read(); + return ((min >= dt) ? (min - dt) : 0); + } /* start going through wheel to find next timeout */ tm = nto = 0; @@ -201,11 +142,17 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ if (p->count == 0) { /* found next timeout */ dt = do_time_read(); + /* p->count is zero */ + tiw_min_ptr = p; + tiw_min = tm; return ((tm >= dt) ? (tm - dt) : 0); } else { /* keep shortest time in 'min' */ - if (tm + p->count*TIW_SIZE < min) + if (tm + p->count*TIW_SIZE < min) { min = tm + p->count*TIW_SIZE; + tiw_min_ptr = p; + tiw_min = min; + } } p = p->next; } @@ -218,30 +165,53 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ return ((min >= dt) ? (min - dt) : 0); } -#if !defined(ERTS_TIMER_THREAD) +static void remove_timer(ErlTimer *p) { + /* first */ + if (!p->prev) { + tiw[p->slot] = p->next; + if(p->next) + p->next->prev = NULL; + } else { + p->prev->next = p->next; + } + + /* last */ + if (!p->next) { + if (p->prev) + p->prev->next = NULL; + } else { + p->next->prev = p->prev; + } + + p->next = NULL; + p->prev = NULL; + /* Make sure cancel callback isn't called */ + p->active = 0; + tiw_nto--; +} + /* Private export to erl_time_sup.c */ -int next_time(void) +erts_aint_t erts_next_time(void) { - int ret; + erts_aint_t ret; - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); (void)do_time_update(); ret = next_time_internal(); - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return ret; } -#endif -static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-locked */ +static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is write-locked */ { Uint keep_pos; Uint count; ErlTimer *p, **prev, *timeout_head, **timeout_tail; - Uint dtime = (unsigned long)dt; + Uint dtime = (Uint) dt; /* no need to bump the position if there aren't any timeouts */ if (tiw_nto == 0) { - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return; } @@ -258,12 +228,16 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l if (tiw_pos == keep_pos) count--; prev = &tiw[tiw_pos]; while ((p = *prev) != NULL) { + ASSERT( p != p->next); if (p->count < count) { /* we have a timeout */ - *prev = p->next; /* Remove from list */ - tiw_nto--; - p->next = NULL; - p->active = 0; /* Make sure cancel callback - isn't called */ + /* remove min time */ + if (tiw_min_ptr == p) { + tiw_min_ptr = NULL; + tiw_min = 0; + } + + /* Remove from list */ + remove_timer(p); *timeout_tail = p; /* Insert in timeout queue */ timeout_tail = &p->next; } @@ -277,8 +251,10 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l dtime--; } tiw_pos = keep_pos; + if (tiw_min_ptr) + tiw_min -= dt; - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); /* Call timedout timers callbacks */ while (timeout_head) { @@ -291,24 +267,17 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l * callback is called. */ p->next = NULL; + p->prev = NULL; p->slot = 0; (*p->timeout)(p->arg); } } -#if defined(ERTS_TIMER_THREAD) -static void timer_thread_bump_timer(void) +void erts_bump_timer(erts_aint_t dt) /* dt is value from do_time */ { - tiw_write_lock(); - bump_timer_internal(do_time_reset()); -} -#else -void bump_timer(long dt) /* dt is value from do_time */ -{ - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); bump_timer_internal(dt); } -#endif Uint erts_timer_wheel_memory_size(void) @@ -316,82 +285,10 @@ erts_timer_wheel_memory_size(void) return (Uint) TIW_SIZE * sizeof(ErlTimer*); } -#if defined(ERTS_TIMER_THREAD) -static struct erts_iwait *timer_thread_iwait; - -static int timer_thread_setup_delay(SysTimeval *rem_time) -{ - long elapsed; - int ticks; - - tiw_write_lock(); - elapsed = do_time_update(); - ticks = next_time_internal(); - if (ticks == -1) /* timer queue empty */ - ticks = 100*1000*1000; - if (elapsed > ticks) - elapsed = ticks; - ticks -= elapsed; - //ticks *= CLOCK_RESOLUTION; - rem_time->tv_sec = ticks / 1000; - rem_time->tv_usec = 1000 * (ticks % 1000); - ticks_end = ticks; - tiw_write_unlock(); - return ticks; -} - -static void *timer_thread_start(void *ignore) -{ - SysTimeval delay; - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_set_thread_name("timer"); -#endif - erts_register_blockable_thread(); - - for(;;) { - if (timer_thread_setup_delay(&delay)) { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - ASSERT_NO_LOCKED_LOCKS; - erts_iwait_wait(timer_thread_iwait, &delay); - ASSERT_NO_LOCKED_LOCKS; - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - } - else - erts_smp_chk_system_block(NULL, NULL, NULL); - timer_thread_bump_timer(); - ASSERT_NO_LOCKED_LOCKS; - } - /*NOTREACHED*/ - return NULL; -} - -static ERTS_INLINE void timer_thread_post_insert(Uint ticks) -{ - if ((Sint)ticks < ticks_end) - erts_iwait_interrupt(timer_thread_iwait); -} - -static void timer_thread_init(void) -{ - erts_thr_opts_t opts = ERTS_THR_OPTS_DEFAULT_INITER; - erts_tid_t tid; - - opts->detached = 1; - - timer_thread_iwait = erts_iwait_init(); - erts_thr_create(&tid, timer_thread_start, NULL, &opts); -} - -#else -static ERTS_INLINE void timer_thread_post_insert(Uint ticks) { } -static ERTS_INLINE void timer_thread_init(void) { } -#endif - /* this routine links the time cells into a free list at the start and sets the time queue as empty */ void -init_time(void) +erts_init_time(void) { int i; @@ -399,7 +296,7 @@ init_time(void) if timer thread is enabled */ itime = erts_init_time_sup(); - tiw_init_lock(); + erts_smp_mtx_init(&tiw_lock, "timer_wheel"); tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL, TIW_SIZE * sizeof(ErlTimer*)); @@ -407,10 +304,13 @@ init_time(void) tiw[i] = NULL; do_time_init(); tiw_pos = tiw_nto = 0; - - timer_thread_init(); + tiw_min_ptr = NULL; + tiw_min = 0; } + + + /* ** Insert a process into the time queue, with a timeout 't' */ @@ -440,20 +340,35 @@ insert_timer(ErlTimer* p, Uint t) /* insert at head of list at slot */ p->next = tiw[tm]; + p->prev = NULL; + if (p->next != NULL) + p->next->prev = p; tiw[tm] = p; - tiw_nto++; - timer_thread_post_insert(ticks); + + /* insert min time */ + if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) { + tiw_min = ticks; + tiw_min_ptr = p; + } + if ((tiw_min_ptr == p) && (ticks > tiw_min)) { + /* some other timer might be 'min' now */ + tiw_min = 0; + tiw_min_ptr = NULL; + } + + tiw_nto++; } void -erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, +erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, void* arg, Uint t) { + erts_deliver_time(); - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); if (p->active) { /* XXX assert ? */ - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return; } p->timeout = timeout; @@ -461,45 +376,37 @@ erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, p->arg = arg; p->active = 1; insert_timer(p, t); - tiw_write_unlock(); -#if defined(ERTS_SMP) && !defined(ERTS_TIMER_THREAD) + erts_smp_mtx_unlock(&tiw_lock); +#if defined(ERTS_SMP) if (t <= (Uint) LONG_MAX) erts_sys_schedule_interrupt_timed(1, (long) t); #endif } void -erl_cancel_timer(ErlTimer* p) +erts_cancel_timer(ErlTimer* p) { - ErlTimer *tp; - ErlTimer **prev; - - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); if (!p->active) { /* allow repeated cancel (drivers) */ - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return; } - /* find p in linked list at slot p->slot and remove it */ - prev = &tiw[p->slot]; - while ((tp = *prev) != NULL) { - if (tp == p) { - *prev = p->next; /* Remove from list */ - tiw_nto--; - p->next = NULL; - p->slot = p->count = 0; - p->active = 0; - if (p->cancel != NULL) { - tiw_write_unlock(); - (*p->cancel)(p->arg); - } else { - tiw_write_unlock(); - } - return; - } else { - prev = &tp->next; - } + + /* is it the 'min' timer, remove min */ + if (p == tiw_min_ptr) { + tiw_min_ptr = NULL; + tiw_min = 0; } - tiw_write_unlock(); + + remove_timer(p); + p->slot = p->count = 0; + + if (p->cancel != NULL) { + erts_smp_mtx_unlock(&tiw_lock); + (*p->cancel)(p->arg); + return; + } + erts_smp_mtx_unlock(&tiw_lock); } /* @@ -509,15 +416,15 @@ erl_cancel_timer(ErlTimer* p) immediately if it hadn't been cancelled). */ Uint -time_left(ErlTimer *p) +erts_time_left(ErlTimer *p) { Uint left; - long dt; + erts_aint_t dt; - tiw_read_lock(); + erts_smp_mtx_lock(&tiw_lock); if (!p->active) { - tiw_read_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return 0; } @@ -531,19 +438,18 @@ time_left(ErlTimer *p) else left -= dt; - tiw_read_unlock(); + erts_smp_mtx_unlock(&tiw_lock); - return left * itime; + return (Uint) left * itime; } #ifdef DEBUG - -void p_slpq() +void erts_p_slpq() { int i; ErlTimer* p; - tiw_read_lock(); + erts_smp_mtx_lock(&tiw_lock); /* print the whole wheel, starting at the current position */ erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto); @@ -565,7 +471,6 @@ void p_slpq() } } - tiw_read_unlock(); + erts_smp_mtx_unlock(&tiw_lock); } - #endif /* DEBUG */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 31efddc0f2..a17de717bc 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -32,6 +32,7 @@ #include "erl_binary.h" #include "erl_bits.h" #include "packet_parser.h" +#include "erl_gc.h" #define ERTS_WANT_DB_INTERNAL__ #include "erl_db.h" #include "erl_threads.h" @@ -47,24 +48,17 @@ #undef M_MMAP_THRESHOLD #undef M_MMAP_MAX -#if !defined(ELIB_ALLOC_IS_CLIB) && defined(__GLIBC__) && defined(HAVE_MALLOC_H) +#if defined(__GLIBC__) && defined(HAVE_MALLOC_H) #include <malloc.h> #endif -#if defined(ELIB_ALLOC_IS_CLIB) || !defined(HAVE_MALLOPT) +#if !defined(HAVE_MALLOPT) #undef HAVE_MALLOPT #define HAVE_MALLOPT 0 #endif /* profile_scheduler mini message queue */ -#ifdef ERTS_TIMER_THREAD -/* A timer thread is not welcomed with this lock violation work around. - * - Bj�rn-Egil - */ -#error Timer thread may not be enabled due to lock violation. -#endif - typedef struct { Uint scheduler_id; Uint no_schedulers; @@ -97,7 +91,7 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq) Eterm* -erts_heap_alloc(Process* p, Uint need) +erts_heap_alloc(Process* p, Uint need, Uint xtra) { ErlHeapFragment* bp; Eterm* htop; @@ -123,9 +117,9 @@ erts_heap_alloc(Process* p, Uint need) p->space_verified_from = NULL; #endif /* FORCE_HEAP_FRAGS */ - n = need; + n = need + xtra; bp = MBUF(p); - if (bp != NULL && need <= (bp->size - bp->used_size)) { + if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) { Eterm* ret = bp->mem + bp->used_size; bp->used_size += need; return ret; @@ -158,16 +152,11 @@ erts_heap_alloc(Process* p, Uint need) bp->next = MBUF(p); MBUF(p) = bp; - bp->size = n; - bp->used_size = n; + bp->alloc_size = n; + bp->used_size = need; MBUF_SIZE(p) += n; - bp->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; bp->off_heap.overhead = 0; - return bp->mem; } @@ -204,6 +193,25 @@ erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end) *end = *start + new_size; *sp = *start + sp_offs; } +/* + * Helper function for the ESTACK macros defined in global.h. + */ +void +erl_grow_wstack(UWord** start, UWord** sp, UWord** end) +{ + Uint old_size = (*end - *start); + Uint new_size = old_size * 2; + Uint sp_offs = *sp - *start; + if (new_size > 2 * DEF_ESTACK_SIZE) { + *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(UWord)); + } else { + UWord* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(UWord)); + sys_memcpy(new_ptr, *start, old_size*sizeof(UWord)); + *start = new_ptr; + } + *end = *start + new_size; + *sp = *start + sp_offs; +} /* CTYPE macros */ @@ -354,6 +362,31 @@ erts_bld_uint(Uint **hpp, Uint *szp, Uint ui) return res; } +/* + * Erts_bld_uword is more or less similar to erts_bld_uint, but a pointer + * can safely be passed. + */ + +Eterm +erts_bld_uword(Uint **hpp, Uint *szp, UWord uw) +{ + Eterm res = THE_NON_VALUE; + if (IS_USMALL(0, uw)) { + if (hpp) + res = make_small((Uint) uw); + } + else { + if (szp) + *szp += BIG_UWORD_HEAP_SIZE(uw); + if (hpp) { + res = uword_to_big(uw, *hpp); + *hpp += BIG_UWORD_HEAP_SIZE(uw); + } + } + return res; +} + + Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64) { @@ -364,7 +397,7 @@ erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64) } else { if (szp) - *szp = ERTS_UINT64_HEAP_SIZE(ui64); + *szp += ERTS_UINT64_HEAP_SIZE(ui64); if (hpp) res = erts_uint64_to_big(ui64, hpp); } @@ -381,7 +414,7 @@ erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64) } else { if (szp) - *szp = ERTS_SINT64_HEAP_SIZE(si64); + *szp += ERTS_SINT64_HEAP_SIZE(si64); if (hpp) res = erts_sint64_to_big(si64, hpp); } @@ -465,7 +498,7 @@ erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len) if (hpp) { res = NIL; while (--i >= 0) { - res = CONS(*hpp, make_small(str[i]), res); + res = CONS(*hpp, make_small((byte) str[i]), res); *hpp += 2; } } @@ -711,7 +744,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) Uint32 make_hash(Eterm term_arg) { - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); Eterm term = term_arg; Eterm hash = 0; unsigned op; @@ -770,7 +803,7 @@ tail_recur: Uint y2 = y1 < 0 ? -(Uint)y1 : y1; UINT32_HASH_STEP(y2, FUNNY_NUMBER2); -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (y2 >> 32) UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2); #endif @@ -787,7 +820,7 @@ tail_recur: } case EXPORT_DEF: { - Export* ep = (Export *) (export_val(term))[1]; + Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; hash = hash*FUNNY_NUMBER1 + @@ -809,7 +842,7 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { - ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); + WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); } term = funp->env[0]; goto tail_recur; @@ -837,9 +870,9 @@ tail_recur: } case MAKE_HASH_CDR_PRE_OP: - term = ESTACK_POP(stack); + term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { - ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); + WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP); goto tail_recur; } /* fall through */ @@ -854,13 +887,13 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + unsigned_val(*list); if (is_not_list(CDR(list))) { - ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); + WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); term = CDR(list); goto tail_recur; } list = list_val(CDR(list)); } - ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); + WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); term = CAR(list); goto tail_recur; } @@ -888,7 +921,7 @@ tail_recur: } d = BIG_DIGIT(ptr, k); k = sizeof(ErtsDigit); -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP if (!(d >> 32)) k /= 2; #endif @@ -904,21 +937,21 @@ tail_recur: Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); - ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity); + WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity); op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_FUN_OP: { - Uint i = ESTACK_POP(stack); - Eterm* ptr = (Eterm*) ESTACK_POP(stack); + Uint i = (Uint) WSTACK_POP(stack); + Eterm* ptr = (Eterm*) WSTACK_POP(stack); if (i != 0) { term = *ptr; - ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op); + WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op); goto tail_recur; } if (op == MAKE_HASH_TUPLE_OP) { - Uint32 arity = ESTACK_POP(stack); + Uint32 arity = (Uint32) WSTACK_POP(stack); hash = hash*FUNNY_NUMBER9 + arity; } break; @@ -928,10 +961,10 @@ tail_recur: erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; } - if (ESTACK_ISEMPTY(stack)) break; - op = ESTACK_POP(stack); + if (WSTACK_ISEMPTY(stack)) break; + op = WSTACK_POP(stack); } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return hash; #undef UINT32_HASH_STEP @@ -1002,7 +1035,7 @@ Uint32 make_hash2(Eterm term) { Uint32 hash; - Eterm tmp_big[2]; + DeclareTmpHeapNoproc(tmp_big,2); /* (HCONST * {2, ..., 14}) mod 2^32 */ #define HCONST_2 0x3c6ef372UL @@ -1041,7 +1074,6 @@ make_hash2(Eterm term) } while(0) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) - /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { switch (term & _TAG_IMMED1_MASK) { @@ -1070,6 +1102,7 @@ make_hash2(Eterm term) Eterm tmp; DECLARE_ESTACK(s); + UseTmpHeapNoproc(2); hash = 0; for (;;) { switch (primary_tag(term)) { @@ -1123,7 +1156,7 @@ make_hash2(Eterm term) break; case EXPORT_SUBTAG: { - Export* ep = (Export *) (export_val(term))[1]; + Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 (ep->code[2], @@ -1314,6 +1347,7 @@ make_hash2(Eterm term) hash2_common: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + UnUseTmpHeapNoproc(2); return hash; } term = ESTACK_POP(s); @@ -1332,7 +1366,7 @@ make_hash2(Eterm term) Uint32 make_broken_hash(Eterm term) { Uint32 hash = 0; - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); unsigned op; tail_recur: op = tag_val_def(term); @@ -1346,7 +1380,7 @@ tail_recur: (atom_tab(atom_val(term))->slot.bucket.hvalue); break; case SMALL_DEF: -#ifdef ARCH_64 +#if defined(ARCH_64) && !HALFWORD_HEAP { Sint y1 = signed_val(term); Uint y2 = y1 < 0 ? -(Uint)y1 : y1; @@ -1399,7 +1433,7 @@ tail_recur: case EXPORT_DEF: { - Export* ep = (Export *) (export_val(term))[1]; + Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; hash = hash*FUNNY_NUMBER1 + @@ -1421,7 +1455,7 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { - ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); + WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); } term = funp->env[0]; goto tail_recur; @@ -1456,16 +1490,17 @@ tail_recur: break; case MAKE_HASH_CDR_PRE_OP: - term = ESTACK_POP(stack); + term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { - ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); + WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP); goto tail_recur; } /*fall through*/ case LIST_DEF: { Eterm* list = list_val(term); - ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); + WSTACK_PUSH2(stack, (UWord) CDR(list), + (UWord) MAKE_HASH_CDR_PRE_OP); term = CAR(list); goto tail_recur; } @@ -1538,21 +1573,21 @@ tail_recur: Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); - ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity); + WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity); op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_FUN_OP: { - Uint i = ESTACK_POP(stack); - Eterm* ptr = (Eterm*) ESTACK_POP(stack); + Uint i = (Uint) WSTACK_POP(stack); + Eterm* ptr = (Eterm*) WSTACK_POP(stack); if (i != 0) { term = *ptr; - ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op); + WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op); goto tail_recur; } if (op == MAKE_HASH_TUPLE_OP) { - Uint32 arity = ESTACK_POP(stack); + Uint32 arity = (UWord) WSTACK_POP(stack); hash = hash*FUNNY_NUMBER9 + arity; } break; @@ -1562,11 +1597,11 @@ tail_recur: erl_exit(1, "Invalid tag in make_broken_hash\n"); return 0; } - if (ESTACK_ISEMPTY(stack)) break; - op = ESTACK_POP(stack); + if (WSTACK_ISEMPTY(stack)) break; + op = (Uint) WSTACK_POP(stack); } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return hash; #undef MAKE_HASH_TUPLE_OP @@ -1859,42 +1894,44 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp) erts_free(ERTS_ALC_T_TMP_DSBUF, (void *) dsbufp); } - /* eq and cmp are written as separate functions a eq is a little faster */ /* * Test for equality of two terms. * Returns 0 if not equal, or a non-zero value otherwise. */ - +#if HALFWORD_HEAP +int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) +#else int eq(Eterm a, Eterm b) +#endif { - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); Sint sz; Eterm* aa; - Eterm* bb; + Eterm* bb; tailrecur: - if (a == b) goto pop_next; + if (is_same(a, a_base, b, b_base)) goto pop_next; tailrecur_ne: switch (primary_tag(a)) { case TAG_PRIMARY_LIST: if (is_list(b)) { - Eterm* aval = list_val(a); - Eterm* bval = list_val(b); + Eterm* aval = list_val_rel(a, a_base); + Eterm* bval = list_val_rel(b, b_base); while (1) { Eterm atmp = CAR(aval); Eterm btmp = CAR(bval); - if (atmp != btmp) { - ESTACK_PUSH2(stack,CDR(bval),CDR(aval)); + if (!is_same(atmp,a_base,btmp,b_base)) { + WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval)); a = atmp; b = btmp; goto tailrecur_ne; } atmp = CDR(aval); btmp = CDR(bval); - if (atmp == btmp) { + if (is_same(atmp,a_base,btmp,b_base)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -1902,22 +1939,22 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aval = list_val(atmp); - bval = list_val(btmp); + aval = list_val_rel(atmp, a_base); + bval = list_val_rel(btmp, b_base); } } break; /* not equal */ case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(a); + Eterm hdr = *boxed_val_rel(a,a_base); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { - aa = tuple_val(a); - if (!is_boxed(b) || *boxed_val(b) != *aa) + aa = tuple_val_rel(a, a_base); + if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) goto not_equal; - bb = tuple_val(b); + bb = tuple_val_rel(b,b_base); if ((sz = arityval(*aa)) == 0) goto pop_next; ++aa; ++bb; @@ -1936,16 +1973,16 @@ tailrecur_ne: Uint a_bitoffs; Uint b_bitoffs; - if (is_not_binary(b)) { + if (!is_binary_rel(b,b_base)) { goto not_equal; } - a_size = binary_size(a); - b_size = binary_size(b); + a_size = binary_size_rel(a,a_base); + b_size = binary_size_rel(b,b_base); if (a_size != b_size) { goto not_equal; } - ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); - ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); + ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); + ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next; } else if (a_bitsize == b_bitsize) { @@ -1956,9 +1993,9 @@ tailrecur_ne: } case EXPORT_SUBTAG: { - if (is_export(b)) { - Export* a_exp = (Export *) (export_val(a))[1]; - Export* b_exp = (Export *) (export_val(b))[1]; + if (is_export_rel(b,b_base)) { + Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); + Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); if (a_exp == b_exp) goto pop_next; } break; /* not equal */ @@ -1968,10 +2005,10 @@ tailrecur_ne: ErlFunThing* f1; ErlFunThing* f2; - if (is_not_fun(b)) + if (!is_fun_rel(b,b_base)) goto not_equal; - f1 = (ErlFunThing *) fun_val(a); - f2 = (ErlFunThing *) fun_val(b); + f1 = (ErlFunThing *) fun_val_rel(a,a_base); + f2 = (ErlFunThing *) fun_val_rel(b,b_base); if (f1->fe->module != f2->fe->module || f1->fe->old_index != f2->fe->old_index || f1->fe->old_uniq != f2->fe->old_uniq || @@ -1989,15 +2026,15 @@ tailrecur_ne: ExternalThing *ap; ExternalThing *bp; - if(is_not_external(b)) + if(!is_external_rel(b,b_base)) goto not_equal; - ap = external_thing_ptr(a); - bp = external_thing_ptr(b); + ap = external_thing_ptr_rel(a,a_base); + bp = external_thing_ptr_rel(b,b_base); if(ap->header == bp->header && ap->node == bp->node) { - ASSERT(1 == external_data_words(a)); - ASSERT(1 == external_data_words(b)); + ASSERT(1 == external_data_words_rel(a,a_base)); + ASSERT(1 == external_data_words_rel(b,b_base)); if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next; } @@ -2015,27 +2052,36 @@ tailrecur_ne: Uint alen; Uint blen; Uint i; + ExternalThing* athing; + ExternalThing* bthing; - if(is_not_external_ref(b)) + if(!is_external_ref_rel(b,b_base)) goto not_equal; - if(external_node(a) != external_node(b)) + athing = external_thing_ptr_rel(a,a_base); + bthing = external_thing_ptr_rel(b,b_base); + + if(athing->node != bthing->node) goto not_equal; - anum = external_ref_numbers(a); - bnum = external_ref_numbers(b); - alen = external_ref_no_of_numbers(a); - blen = external_ref_no_of_numbers(b); + anum = external_thing_ref_numbers(athing); + bnum = external_thing_ref_numbers(bthing); + alen = external_thing_ref_no_of_numbers(athing); + blen = external_thing_ref_no_of_numbers(bthing); goto ref_common; case REF_SUBTAG: - - if (is_not_internal_ref(b)) + if (!is_internal_ref_rel(b,b_base)) goto not_equal; - alen = internal_ref_no_of_numbers(a); - blen = internal_ref_no_of_numbers(b); - anum = internal_ref_numbers(a); - bnum = internal_ref_numbers(b); + + { + RefThing* athing = ref_thing_ptr_rel(a,a_base); + RefThing* bthing = ref_thing_ptr_rel(b,b_base); + alen = internal_thing_ref_no_of_numbers(athing); + blen = internal_thing_ref_no_of_numbers(bthing); + anum = internal_thing_ref_numbers(athing); + bnum = internal_thing_ref_numbers(bthing); + } ref_common: ASSERT(alen > 0 && blen > 0); @@ -2080,10 +2126,10 @@ tailrecur_ne: { int i; - if (is_not_big(b)) + if (!is_big_rel(b,b_base)) goto not_equal; - aa = big_val(a); /* get pointer to thing */ - bb = big_val(b); + aa = big_val_rel(a,a_base); + bb = big_val_rel(b,b_base); if (*aa != *bb) goto not_equal; i = BIG_ARITY(aa); @@ -2098,9 +2144,9 @@ tailrecur_ne: FloatDef af; FloatDef bf; - if (is_float(b)) { - GET_DOUBLE(a, af); - GET_DOUBLE(b, bf); + if (is_float_rel(b,b_base)) { + GET_DOUBLE_REL(a, af, a_base); + GET_DOUBLE_REL(b, bf, b_base); if (af.fd == bf.fd) goto pop_next; } break; /* not equal */ @@ -2119,7 +2165,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ Eterm* bp = bb; Sint i = sz; for (;;) { - if (*ap != *bp) break; + if (!is_same(*ap,a_base,*bp,b_base)) break; if (--i == 0) goto pop_next; ++ap; ++bp; @@ -2130,32 +2176,32 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ goto not_equal; } if (i > 1) { /* push the rest */ - ESTACK_PUSH3(stack, i-1, (Eterm)(bp+1), - ((Eterm)(ap+1)) | TAG_PRIMARY_HEADER); + WSTACK_PUSH3(stack, i-1, (UWord)(bp+1), + ((UWord)(ap+1)) | TAG_PRIMARY_HEADER); /* We (ab)use TAG_PRIMARY_HEADER to recognize a term_array */ } goto tailrecur_ne; } pop_next: - if (!ESTACK_ISEMPTY(stack)) { - Eterm something = ESTACK_POP(stack); - if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */ + if (!WSTACK_ISEMPTY(stack)) { + UWord something = WSTACK_POP(stack); + if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */ aa = (Eterm*) something; - bb = (Eterm*) ESTACK_POP(stack); - sz = ESTACK_POP(stack); + bb = (Eterm*) WSTACK_POP(stack); + sz = WSTACK_POP(stack); goto term_array; } a = something; - b = ESTACK_POP(stack); + b = WSTACK_POP(stack); goto tailrecur; } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return 1; not_equal: - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return 0; } @@ -2208,9 +2254,13 @@ static int cmp_atoms(Eterm a, Eterm b) bb->name+3, bb->len-3); } +#if HALFWORD_HEAP +Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) +#else Sint cmp(Eterm a, Eterm b) +#endif { - DECLARE_ESTACK(stack); + DECLARE_WSTACK(stack); Eterm* aa; Eterm* bb; int i; @@ -2242,7 +2292,7 @@ Sint cmp(Eterm a, Eterm b) tailrecur: - if (a == b) { /* Equal values or pointers. */ + if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */ goto pop_next; } tailrecur_ne: @@ -2268,9 +2318,9 @@ tailrecur_ne: if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port(b)) { - bnode = external_port_node(b); - bdata = external_port_data(b); + } else if (is_external_port_rel(b,b_base)) { + bnode = external_port_node_rel(b,b_base); + bdata = external_port_data_rel(b,b_base); } else { a_tag = PORT_DEF; goto mixed_types; @@ -2286,9 +2336,9 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid(b)) { - bnode = external_pid_node(b); - bdata = external_pid_data(b); + } else if (is_external_pid_rel(b,b_base)) { + bnode = external_pid_node_rel(b,b_base); + bdata = external_pid_data_rel(b,b_base); } else { a_tag = PID_DEF; goto mixed_types; @@ -2321,20 +2371,20 @@ tailrecur_ne: a_tag = LIST_DEF; goto mixed_types; } - aa = list_val(a); - bb = list_val(b); + aa = list_val_rel(a,a_base); + bb = list_val_rel(b,b_base); while (1) { Eterm atmp = CAR(aa); Eterm btmp = CAR(bb); - if (atmp != btmp) { - ESTACK_PUSH2(stack,CDR(bb),CDR(aa)); + if (!is_same(atmp,a_base,btmp,b_base)) { + WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa)); a = atmp; b = btmp; goto tailrecur_ne; } atmp = CDR(aa); btmp = CDR(bb); - if (atmp == btmp) { + if (is_same(atmp,a_base,btmp,b_base)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -2342,20 +2392,20 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aa = list_val(atmp); - bb = list_val(btmp); + aa = list_val_rel(atmp,a_base); + bb = list_val_rel(btmp,b_base); } case TAG_PRIMARY_BOXED: { - Eterm ahdr = *boxed_val(a); + Eterm ahdr = *boxed_val_rel(a,a_base); switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): - if (is_not_tuple(b)) { + if (!is_tuple_rel(b,b_base)) { a_tag = TUPLE_DEF; goto mixed_types; } - aa = tuple_val(a); - bb = tuple_val(b); + aa = tuple_val_rel(a,a_base); + bb = tuple_val_rel(b,b_base); /* compare the arities */ i = arityval(ahdr); /* get the arity*/ if (i != arityval(*bb)) { @@ -2369,31 +2419,31 @@ tailrecur_ne: goto term_array; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): - if (is_not_float(b)) { + if (!is_float_rel(b,b_base)) { a_tag = FLOAT_DEF; goto mixed_types; } else { FloatDef af; FloatDef bf; - GET_DOUBLE(a, af); - GET_DOUBLE(b, bf); + GET_DOUBLE_REL(a, af, a_base); + GET_DOUBLE_REL(b, bf, b_base); ON_CMP_GOTO(float_comp(af.fd, bf.fd)); } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - if (is_not_big(b)) { + if (!is_big_rel(b,b_base)) { a_tag = BIG_DEF; goto mixed_types; } - ON_CMP_GOTO(big_comp(a, b)); + ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base))); case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): - if (is_not_export(b)) { + if (!is_export_rel(b,b_base)) { a_tag = EXPORT_DEF; goto mixed_types; } else { - Export* a_exp = (Export *) (export_val(a))[1]; - Export* b_exp = (Export *) (export_val(b))[1]; + Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); + Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { RETURN_NEQ(j); @@ -2405,12 +2455,12 @@ tailrecur_ne: } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): - if (is_not_fun(b)) { + if (!is_fun_rel(b,b_base)) { a_tag = FUN_DEF; goto mixed_types; } else { - ErlFunThing* f1 = (ErlFunThing *) fun_val(a); - ErlFunThing* f2 = (ErlFunThing *) fun_val(b); + ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base); + ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base); Sint diff; diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name, @@ -2442,51 +2492,57 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid(b)) { - bnode = external_pid_node(b); - bdata = external_pid_data(b); + } else if (is_external_pid_rel(b,b_base)) { + bnode = external_pid_node_rel(b,b_base); + bdata = external_pid_data_rel(b,b_base); } else { a_tag = EXTERNAL_PID_DEF; goto mixed_types; } - anode = external_pid_node(a); - adata = external_pid_data(a); + anode = external_pid_node_rel(a,a_base); + adata = external_pid_data_rel(a,a_base); goto pid_common; case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port(b)) { - bnode = external_port_node(b); - bdata = external_port_data(b); + } else if (is_external_port_rel(b,b_base)) { + bnode = external_port_node_rel(b,b_base); + bdata = external_port_data_rel(b,b_base); } else { a_tag = EXTERNAL_PORT_DEF; goto mixed_types; } - anode = external_port_node(a); - adata = external_port_data(a); + anode = external_port_node_rel(a,a_base); + adata = external_port_data_rel(a,a_base); goto port_common; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): /* * Note! When comparing refs we need to compare ref numbers * (32-bit words), *not* ref data words. */ + - if (is_internal_ref(b)) { + if (is_internal_ref_rel(b,b_base)) { + RefThing* bthing = ref_thing_ptr_rel(b,b_base); bnode = erts_this_node; - bnum = internal_ref_numbers(b); - blen = internal_ref_no_of_numbers(b); - } else if(is_external_ref(b)) { - bnode = external_ref_node(b); - bnum = external_ref_numbers(b); - blen = external_ref_no_of_numbers(b); + bnum = internal_thing_ref_numbers(bthing); + blen = internal_thing_ref_no_of_numbers(bthing); + } else if(is_external_ref_rel(b,b_base)) { + ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + bnode = bthing->node; + bnum = external_thing_ref_numbers(bthing); + blen = external_thing_ref_no_of_numbers(bthing); } else { a_tag = REF_DEF; goto mixed_types; } - anode = erts_this_node; - anum = internal_ref_numbers(a); - alen = internal_ref_no_of_numbers(a); + { + RefThing* athing = ref_thing_ptr_rel(a,a_base); + anode = erts_this_node; + anum = internal_thing_ref_numbers(athing); + alen = internal_thing_ref_no_of_numbers(athing); + } ref_common: CMP_NODES(anode, bnode); @@ -2515,31 +2571,36 @@ tailrecur_ne: RETURN_NEQ((Sint32) (anum[i] - bnum[i])); goto pop_next; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): - if (is_internal_ref(b)) { + if (is_internal_ref_rel(b,b_base)) { + RefThing* bthing = ref_thing_ptr_rel(b,b_base); bnode = erts_this_node; - bnum = internal_ref_numbers(b); - blen = internal_ref_no_of_numbers(b); - } else if (is_external_ref(b)) { - bnode = external_ref_node(b); - bnum = external_ref_numbers(b); - blen = external_ref_no_of_numbers(b); + bnum = internal_thing_ref_numbers(bthing); + blen = internal_thing_ref_no_of_numbers(bthing); + } else if (is_external_ref_rel(b,b_base)) { + ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + bnode = bthing->node; + bnum = external_thing_ref_numbers(bthing); + blen = external_thing_ref_no_of_numbers(bthing); } else { a_tag = EXTERNAL_REF_DEF; goto mixed_types; } - anode = external_ref_node(a); - anum = external_ref_numbers(a); - alen = external_ref_no_of_numbers(a); + { + ExternalThing* athing = external_thing_ptr_rel(a,a_base); + anode = athing->node; + anum = external_thing_ref_numbers(athing); + alen = external_thing_ref_no_of_numbers(athing); + } goto ref_common; default: /* Must be a binary */ - ASSERT(is_binary(a)); - if (is_not_binary(b)) { + ASSERT(is_binary_rel(a,a_base)); + if (!is_binary_rel(b,b_base)) { a_tag = BINARY_DEF; goto mixed_types; } else { - Uint a_size = binary_size(a); - Uint b_size = binary_size(b); + Uint a_size = binary_size_rel(a,a_base); + Uint b_size = binary_size_rel(b,b_base); Uint a_bitsize; Uint b_bitsize; Uint a_bitoffs; @@ -2548,8 +2609,8 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; - ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); - ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); + ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); + ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { min_size = (a_size < b_size) ? a_size : b_size; if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) { @@ -2576,45 +2637,56 @@ tailrecur_ne: */ mixed_types: - b_tag = tag_val_def(b); { FloatDef f1, f2; Eterm big; - Eterm big_buf[2]; +#if HEAP_ON_C_STACK + Eterm big_buf[2]; /* If HEAP_ON_C_STACK */ +#else + Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap; +#endif +#if HALFWORD_HEAP + Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base); + Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base); +#else + Eterm aw = a; + Eterm bw = b; +#endif + b_tag = tag_val_def(bw); switch(_NUMBER_CODE(a_tag, b_tag)) { case SMALL_BIG: big = small_to_big(signed_val(a), big_buf); - j = big_comp(big, b); + j = big_comp(big, bw); break; case SMALL_FLOAT: f1.fd = signed_val(a); - GET_DOUBLE(b, f2); + GET_DOUBLE(bw, f2); j = float_comp(f1.fd, f2.fd); break; case BIG_SMALL: big = small_to_big(signed_val(b), big_buf); - j = big_comp(a, big); + j = big_comp(aw, big); break; case BIG_FLOAT: - if (big_to_double(a, &f1.fd) < 0) { + if (big_to_double(aw, &f1.fd) < 0) { j = big_sign(a) ? -1 : 1; } else { - GET_DOUBLE(b, f2); + GET_DOUBLE(bw, f2); j = float_comp(f1.fd, f2.fd); } break; case FLOAT_SMALL: - GET_DOUBLE(a, f1); + GET_DOUBLE(aw, f1); f2.fd = signed_val(b); j = float_comp(f1.fd, f2.fd); break; case FLOAT_BIG: - if (big_to_double(b, &f2.fd) < 0) { + if (big_to_double(bw, &f2.fd) < 0) { j = big_sign(b) ? 1 : -1; } else { - GET_DOUBLE(a, f1); + GET_DOUBLE(aw, f1); j = float_comp(f1.fd, f2.fd); } break; @@ -2633,7 +2705,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ while (--i) { a = *aa++; b = *bb++; - if (a != b) { + if (!is_same(a,a_base, b,b_base)) { if (is_atom(a) && is_atom(b)) { if ((j = cmp_atoms(a, b)) != 0) { goto not_equal; @@ -2644,7 +2716,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ } } else { /* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */ - ESTACK_PUSH3(stack, i, (Eterm)bb, (Eterm)aa | TAG_PRIMARY_HEADER); + WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER); goto tailrecur_ne; } } @@ -2654,20 +2726,20 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ goto tailrecur; pop_next: - if (!ESTACK_ISEMPTY(stack)) { - Eterm something = ESTACK_POP(stack); - if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */ + if (!WSTACK_ISEMPTY(stack)) { + UWord something = WSTACK_POP(stack); + if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */ aa = (Eterm*) something; - bb = (Eterm*) ESTACK_POP(stack); - i = ESTACK_POP(stack); + bb = (Eterm*) WSTACK_POP(stack); + i = WSTACK_POP(stack); goto term_array; } - a = something; - b = ESTACK_POP(stack); + a = (Eterm) something; + b = (Eterm) WSTACK_POP(stack); goto tailrecur; } - DESTROY_ESTACK(stack); + DESTROY_WSTACK(stack); return 0; not_equal: @@ -2678,21 +2750,8 @@ not_equal: } -void -erts_cleanup_externals(ExternalThing *etp) -{ - ExternalThing *tetp; - - tetp = etp; - - while(tetp) { - erts_deref_node_entry(tetp->node); - tetp = tetp->next; - } -} - Eterm -store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) +store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) { Uint i; Uint size; @@ -2711,8 +2770,8 @@ store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2); - ((ExternalThing *) to_hp)->next = *etpp; - *etpp = (ExternalThing *) to_hp; + ((struct erl_off_heap_header*) to_hp)->next = oh->first; + oh->first = (struct erl_off_heap_header*) to_hp; return make_external(to_hp); } @@ -2741,7 +2800,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns) sz = NC_HEAP_SIZE(ns); ASSERT(sz > 0); hp = HAlloc(proc, sz); - return store_external_or_ref_(&hp, &MSO(proc).externals, ns); + return store_external_or_ref_(&hp, &MSO(proc), ns); } void bin_write(int to, void *to_arg, byte* buf, int sz) @@ -2962,13 +3021,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len) return -1; } -int io_list_len(Eterm obj) +/* + * Return 0 if successful, and non-zero if unsuccessful. + */ +int erts_iolist_size(Eterm obj, Uint* sizep) { Eterm* objp; - Sint len = 0; + Uint size = 0; DECLARE_ESTACK(s); goto L_again; +#define SAFE_ADD(Var, Val) \ + do { \ + Uint valvar = (Val); \ + Var += valvar; \ + if (Var < valvar) { \ + goto L_overflow_error; \ + } \ + } while (0) + while (!ESTACK_ISEMPTY(s)) { obj = ESTACK_POP(s); L_again: @@ -2978,9 +3049,12 @@ int io_list_len(Eterm obj) /* Head */ obj = CAR(objp); if (is_byte(obj)) { - len++; + size++; + if (size == 0) { + goto L_overflow_error; + } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_list(obj)) { ESTACK_PUSH(s, CDR(objp)); goto L_iter_list; /* on head */ @@ -2992,23 +3066,29 @@ int io_list_len(Eterm obj) if (is_list(obj)) goto L_iter_list; /* on tail */ else if (is_binary(obj) && binary_bitsize(obj) == 0) { - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ - len += binary_size(obj); + SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } } +#undef SAFE_ADD DESTROY_ESTACK(s); - return len; + *sizep = size; + return ERTS_IOLIST_OK; + + L_overflow_error: + DESTROY_ESTACK(s); + return ERTS_IOLIST_OVERFLOW; L_type_error: DESTROY_ESTACK(s); - return -1; + return ERTS_IOLIST_TYPE; } /* return 0 if item is not a non-empty flat list of bytes */ @@ -3150,7 +3230,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, *timer_ref = res; - erl_set_timer(&res->timer.tm, + erts_set_timer(&res->timer.tm, (ErlTimeoutProc) ptimer_timeout, (ErlCancelProc) ptimer_cancelled, (void*) res, @@ -3164,7 +3244,7 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer) ASSERT(*ptimer->timer.timer_ref == ptimer); *ptimer->timer.timer_ref = NULL; ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED; - erl_cancel_timer(&ptimer->timer.tm); + erts_cancel_timer(&ptimer->timer.tm); } } @@ -3604,19 +3684,19 @@ erts_set_activity_error(erts_activity_error_t error, char *file, int line) } -static ERTS_INLINE int +static ERTS_INLINE erts_aint32_t threads_not_under_control(void) { - int res = system_block_state.threads_to_block; + erts_aint32_t res = system_block_state.threads_to_block; /* Waiting is always an allowed activity... */ - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.wait); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.wait); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC) - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.gc); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.gc); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO) - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.io); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.io); if (res < 0) { ASSERT(0); @@ -3676,7 +3756,7 @@ erts_block_system(Uint32 allowed_activities) } else { - erts_smp_atomic_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc(&erts_system_block_state.do_block); /* Someone else might be waiting for us to block... */ if (do_block) { @@ -3728,11 +3808,11 @@ erts_emergency_block_system(long timeout, Uint32 allowed_activities) another_blocker = erts_smp_pending_system_block(); system_block_state.emergency = 1; - erts_smp_atomic_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc(&erts_system_block_state.do_block); if (another_blocker) { if (is_blocker()) { - erts_smp_atomic_dec(&erts_system_block_state.do_block); + erts_smp_atomic32_dec(&erts_system_block_state.do_block); res = 0; goto done; } @@ -3789,7 +3869,7 @@ erts_release_system(void) if (system_block_state.recursive_block) system_block_state.recursive_block--; else { - do_block = erts_smp_atomic_dectest(&erts_system_block_state.do_block); + do_block = erts_smp_atomic32_dectest(&erts_system_block_state.do_block); system_block_state.have_blocker = 0; if (is_blockable_thread()) system_block_state.threads_to_block++; @@ -3924,10 +4004,10 @@ erts_system_block_init(void) /* Global state... */ - erts_smp_atomic_init(&erts_system_block_state.do_block, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.wait, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.gc, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.io, 0L); + erts_smp_atomic32_init(&erts_system_block_state.do_block, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.wait, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.gc, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.io, 0); /* Make sure blockable threads unregister when exiting... */ erts_smp_install_exit_handler(erts_unregister_blockable_thread); |