diff options
Diffstat (limited to 'erts/emulator/beam')
88 files changed, 6406 insertions, 4058 deletions
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index cb245a87b1..6127a658bb 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -34,7 +34,7 @@ /* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an unsigned 32 bit integer. See external.c(erts_encode_ext_dist_header_setup) for more details. */ -#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (1UL << 32)) ? MAX_ATOM_INDEX + 1 : (1UL << 32)) +#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (UWORD_CONSTANT(1) << 32)) ? MAX_ATOM_INDEX + 1 : (UWORD_CONSTANT(1) << 32)) #else #define MAX_ATOM_TABLE_SIZE (MAX_ATOM_INDEX + 1) #endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 37738faae3..71454b3e57 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -158,6 +158,8 @@ atom cr atom crlf atom creation atom current_function +atom current_location +atom current_stacktrace atom data atom debug_flags atom delay_trap diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 2561d7a630..78a9d76a20 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -33,12 +33,14 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_nif.h" +#include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp); static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp); static void delete_export_references(Eterm module); static int purge_module(int module); +static void decrement_refc(BeamInstr* code); static int is_native(BeamInstr* code); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -49,11 +51,11 @@ load_module_2(BIF_ALIST_2) { Eterm reason; Eterm* hp; - int i; int sz; byte* code; Eterm res; byte* temp_alloc = NULL; + struct LoaderState* stp; if (is_not_atom(BIF_ARG_1)) { error: @@ -63,49 +65,37 @@ load_module_2(BIF_ALIST_2) if ((code = erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc)) == NULL) { goto error; } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); - - erts_export_consolidate(); - hp = HAlloc(BIF_P, 3); + + /* + * Read the BEAM file and prepare the module for loading. + */ + stp = erts_alloc_loader_state(); sz = binary_size(BIF_ARG_2); - if ((i = erts_load_module(BIF_P, 0, - BIF_P->group_leader, &BIF_ARG_1, code, sz)) < 0) { - switch (i) { - case -1: reason = am_badfile; break; - case -2: reason = am_nofile; break; - case -3: reason = am_not_purged; break; - case -4: - reason = am_atom_put("native_code", sizeof("native_code")-1); - break; - case -5: - { - /* - * The module contains an on_load function. The loader - * has loaded the module as usual, except that the - * export entries does not point into the module, so it - * is not possible to call any code in the module. - */ - - ERTS_DECL_AM(on_load); - reason = AM_on_load; - break; - } - default: reason = am_badfile; break; - } + reason = erts_prepare_loading(stp, BIF_P, BIF_P->group_leader, + &BIF_ARG_1, code, sz); + erts_free_aligned_binary_bytes(temp_alloc); + if (reason != NIL) { res = TUPLE2(hp, am_error, reason); - goto done; + BIF_RET(res); } - set_default_trace_pattern(BIF_ARG_1); - res = TUPLE2(hp, am_module, BIF_ARG_1); + /* + * Stop all other processes and finish the loading of the module. + */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); - done: - erts_free_aligned_binary_bytes(temp_alloc); - erts_smp_release_system(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + reason = erts_finish_loading(stp, BIF_P, 0, &BIF_ARG_1); + if (reason != NIL) { + res = TUPLE2(hp, am_error, reason); + } else { + set_default_trace_pattern(BIF_ARG_1); + res = TUPLE2(hp, am_module, BIF_ARG_1); + } + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } @@ -118,12 +108,12 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_export_consolidate(); purge_res = purge_module(atom_val(BIF_ARG_1)); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); if (purge_res < 0) { @@ -152,12 +142,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) Eterm res; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_export_consolidate(); res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); return res; } @@ -239,7 +229,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) goto badarg; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); { Module *modp = erts_get_module(BIF_ARG_1); @@ -260,7 +250,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); if (res == am_badarg) { @@ -352,7 +342,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (BIF_ARG_2 == am_true) { int i; @@ -391,7 +381,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) modp->catches = BEAM_CATCHES_NIL; remove_from_address_table(code); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); } @@ -564,6 +554,7 @@ check_process_code(Process* rp, Module* modp) } else { Eterm* literals; Uint lit_size; + struct erl_off_heap_header* oh; /* * Try to get rid of constants by by garbage collecting. @@ -577,7 +568,9 @@ check_process_code(Process* rp, Module* modp) (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); literals = (Eterm *) modp->old_code[MI_LITERALS_START]; lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals; - erts_garbage_collect_literals(rp, literals, lit_size); + oh = (struct erl_off_heap_header *) + modp->old_code[MI_LITERALS_OFF_HEAP]; + erts_garbage_collect_literals(rp, literals, lit_size, oh); } } return am_false; @@ -585,7 +578,7 @@ check_process_code(Process* rp, Module* modp) } #define in_area(ptr,start,nbytes) \ - ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes)) + ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) @@ -655,9 +648,6 @@ purge_module(int module) * Any code to purge? */ if (modp->old_code == 0) { - if (display_loads) { - erts_printf("No code to purge for %T\n", make_atom(module)); - } return -1; } @@ -678,6 +668,7 @@ purge_module(int module) end = (BeamInstr *)((char *)code + modp->old_code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old_catches, code, modp->old_code_length); + decrement_refc(code); erts_free(ERTS_ALC_T_CODE, (void *) code); modp->old_code = NULL; modp->old_code_length = 0; @@ -687,6 +678,23 @@ purge_module(int module) } static void +decrement_refc(BeamInstr* code) +{ + struct erl_off_heap_header* oh = + (struct erl_off_heap_header *) code[MI_LITERALS_OFF_HEAP]; + + while (oh) { + Binary* bptr; + ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); + bptr = ((ProcBin*)oh)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + oh = oh->next; + } +} + +static void remove_from_address_table(BeamInstr* code) { int i; @@ -728,10 +736,10 @@ delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp) if (modp->code != NULL && modp->code[MI_NUM_BREAKPOINTS] > 0) { if (c_p && c_p_locks) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_clear_module_break(modp); modp->code[MI_NUM_BREAKPOINTS] = 0; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); if (c_p && c_p_locks) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } @@ -773,7 +781,7 @@ delete_export_references(Eterm module) } -int +Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) { Module* modp = erts_put_module(module); @@ -784,15 +792,12 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) */ if (modp->code != NULL && modp->old_code != NULL) { - return -3; + return am_not_purged; } else if (modp->old_code == NULL) { /* Make the current version old. */ - if (display_loads) { - erts_printf("saving old code\n"); - } delete_code(c_p, c_p_locks, modp); delete_export_references(module); } - return 0; + return NIL; } static int diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 773baad01f..692fa61fe8 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -167,7 +167,7 @@ erts_bp_init(void) { int erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, match_spec, (BeamInstr) BeamOp(op_i_trace_breakpoint), 0, tracer_pid); } @@ -175,7 +175,7 @@ erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); } @@ -184,7 +184,7 @@ erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); } @@ -198,35 +198,35 @@ void erts_clear_time_trace_bif(BeamInstr *pc) { int erts_set_debug_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, NULL, (BeamInstr) BeamOp(op_i_debug_breakpoint), 0, NIL); } int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, NULL, (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL); } int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return set_break(mfa, specified, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); } int erts_clear_trace_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_trace_breakpoint)); } int erts_clear_mtrace_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); } @@ -238,41 +238,41 @@ erts_clear_mtrace_bif(BeamInstr *pc) { int erts_clear_debug_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_debug_breakpoint)); } int erts_clear_count_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_count_breakpoint)); } int erts_clear_time_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, (BeamInstr) BeamOp(op_i_time_breakpoint)); } int erts_clear_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); return clear_break(mfa, specified, 0); } int erts_clear_module_break(Module *modp) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); return clear_module_break(modp, NULL, 0, 0); } int erts_clear_function_break(Module *modp, BeamInstr *pc) { - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); return clear_function_break(modp, pc, BREAK_IS_ERL, 0); } @@ -495,16 +495,6 @@ erts_find_local_func(Eterm mfa[3]) { return NULL; } -/* bp_hash */ -ERTS_INLINE Uint bp_sched2ix() { -#ifdef ERTS_SMP - ErtsSchedulerData *esdp; - esdp = erts_get_scheduler_data(); - return esdp->no - 1; -#else - return 0; -#endif -} static void bp_hash_init(bp_time_hash_t *hash, Uint n) { Uint size = sizeof(bp_data_time_item_t)*n; Uint i; @@ -612,9 +602,13 @@ static void bp_hash_delete(bp_time_hash_t *hash) { static void bp_time_diff(bp_data_time_item_t *item, /* out */ process_breakpoint_time_t *pbt, /* in */ Uint ms, Uint s, Uint us) { - int dms,ds,dus; + int ds,dus; +#ifdef DEBUG + int dms; + dms = ms - pbt->ms; +#endif ds = s - pbt->s; dus = us - pbt->us; @@ -622,7 +616,9 @@ static void bp_time_diff(bp_data_time_item_t *item, /* out */ * this is ok. */ +#ifdef DEBUG ASSERT(dms >= 0 || ds >= 0 || dus >= 0); +#endif if (dus < 0) { dus += 1000000; @@ -975,7 +971,7 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, BpDataTime *bdt = (BpDataTime *) bd; Uint i = 0; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); if (count_op == erts_break_stop) { bdt->pause = 1; @@ -1341,7 +1337,7 @@ static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { return NULL; } - bd = ebd = rs[bp_sched2ix()]; + bd = ebd = rs[erts_bp_sched2ix()]; ASSERT(bd); if ( (break_op == 0) || (bd->this_instr == break_op)) { return bd; diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 2ec5818688..167069552f 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -144,8 +144,6 @@ extern erts_smp_spinlock_t erts_bp_lock; #define ErtsSmpBPUnlock(BDC) #endif -ERTS_INLINE Uint bp_sched2ix(void); - #ifdef ERTS_SMP #define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1) #else @@ -247,4 +245,19 @@ BpData *erts_get_time_break(Process *p, BeamInstr *pc); BeamInstr *erts_find_local_func(Eterm mfa[3]); +ERTS_GLB_INLINE Uint erts_bp_sched2ix(void); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Uint erts_bp_sched2ix(void) +{ +#ifdef ERTS_SMP + ErtsSchedulerData *esdp; + esdp = erts_get_scheduler_data(); + return esdp->no - 1; +#else + return 0; +#endif +} +#endif + #endif /* _BEAM_BP_H */ diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index e795b4efbd..a550ec5ad0 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -22,21 +22,27 @@ #endif #include "sys.h" #include "beam_catches.h" +#include "global.h" -/* XXX: should use dynamic reallocation */ -#define TABSIZ (16*1024) -static struct { +/* R14B04 has about 380 catches when starting erlang */ +#define DEFAULT_TABSIZE (1024) +typedef struct { BeamInstr *cp; unsigned cdr; -} beam_catches[TABSIZ]; +} beam_catch_t; static int free_list; static unsigned high_mark; +static unsigned tabsize; +static beam_catch_t *beam_catches; void beam_catches_init(void) { + tabsize = DEFAULT_TABSIZE; free_list = -1; high_mark = 0; + + beam_catches = erts_alloc(ERTS_ALC_T_CODE, sizeof(beam_catch_t)*DEFAULT_TABSIZE); } unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr) @@ -50,16 +56,21 @@ unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr) * This avoids the need to initialise the free list in * beam_catches_init(), which would cost O(TABSIZ) time. */ - if( (i = free_list) >= 0 ) { + if( free_list >= 0 ) { + i = free_list; free_list = beam_catches[i].cdr; - } else if( (i = high_mark) < TABSIZ ) { - high_mark = i + 1; + } else if( high_mark < tabsize ) { + i = high_mark; + high_mark++; } else { - fprintf(stderr, "beam_catches_cons: no free slots :-(\r\n"); - exit(1); + /* No free slots and table is full: realloc table */ + tabsize = 2*tabsize; + beam_catches = erts_realloc(ERTS_ALC_T_CODE, beam_catches, sizeof(beam_catch_t)*tabsize); + i = high_mark; + high_mark++; } - beam_catches[i].cp = cp; + beam_catches[i].cp = cp; beam_catches[i].cdr = cdr; return i; @@ -67,10 +78,8 @@ unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr) BeamInstr *beam_catches_car(unsigned i) { - if( i >= TABSIZ ) { - fprintf(stderr, - "beam_catches_car: index %#x is out of range\r\n", i); - abort(); + if( i >= tabsize ) { + erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); } return beam_catches[i].cp; } @@ -80,18 +89,15 @@ void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes) unsigned i, cdr; for(i = head; i != (unsigned)-1;) { - if( i >= TABSIZ ) { - fprintf(stderr, - "beam_catches_delmod: index %#x is out of range\r\n", i); - abort(); + if( i >= tabsize ) { + erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); } if( (char*)beam_catches[i].cp - (char*)code >= code_bytes ) { - fprintf(stderr, + erl_exit(1, "beam_catches_delmod: item %#x has cp %#lx which is not " "in module's range [%#lx,%#lx[\r\n", i, (long)beam_catches[i].cp, (long)code, (long)((char*)code + code_bytes)); - abort(); } beam_catches[i].cp = 0; cdr = beam_catches[i].cdr; diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index fffb172c68..8041c92162 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -37,6 +37,7 @@ #include "beam_load.h" #include "beam_bp.h" #include "erl_binary.h" +#include "erl_thr_progress.h" #ifdef ARCH_64 # define HEXF "%016bpX" @@ -49,15 +50,18 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); -Eterm -erts_debug_same_2(Process* p, Eterm term1, Eterm term2) + +BIF_RETTYPE +erts_debug_same_2(BIF_ALIST_2) { - return (term1 == term2) ? am_true : am_false; + return (BIF_ARG_1 == BIF_ARG_2) ? am_true : am_false; } -Eterm -erts_debug_flat_size_1(Process* p, Eterm term) +BIF_RETTYPE +erts_debug_flat_size_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm term = BIF_ARG_1; Uint size = size_object(term); if (IS_USMALL(0, size)) { @@ -68,9 +72,13 @@ erts_debug_flat_size_1(Process* p, Eterm term) } } -Eterm -erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) + +BIF_RETTYPE +erts_debug_breakpoint_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm MFA = BIF_ARG_1; + Eterm bool = BIF_ARG_2; Eterm* tp; Eterm mfa[3]; int i; @@ -107,7 +115,7 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (bool == am_true) { res = make_small(erts_set_debug_break(mfa, specified)); @@ -115,7 +123,7 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) res = make_small(erts_clear_debug_break(mfa, specified)); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); return res; @@ -175,9 +183,11 @@ erts_debug_instructions_0(BIF_ALIST_0) return res; } -Eterm -erts_debug_disassemble_1(Process* p, Eterm addr) +BIF_RETTYPE +erts_debug_disassemble_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm addr = BIF_ARG_1; erts_dsprintf_buf_t *dsbufp; Eterm* hp; Eterm* tp; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 937b3d9e53..c65b2be106 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -36,6 +36,7 @@ #include "dist.h" #include "beam_bp.h" #include "beam_catches.h" +#include "erl_thr_progress.h" #ifdef HIPE #include "hipe_mode_switch.h" #include "hipe_bif1.h" @@ -44,7 +45,7 @@ /* #define HARDDEBUG 1 */ #if defined(NO_JUMP_TABLE) -# define OpCase(OpCode) case op_##OpCode: lb_##OpCode +# define OpCase(OpCode) case op_##OpCode # define CountCase(OpCode) case op_count_##OpCode # define OpCode(OpCode) ((Uint*)op_##OpCode) # define Goto(Rel) {Go = (int)(Rel); goto emulator_loop;} @@ -52,7 +53,7 @@ #else # define OpCase(OpCode) lb_##OpCode # define CountCase(OpCode) lb_count_##OpCode -# define Goto(Rel) goto *(Rel) +# define Goto(Rel) goto *((void *)Rel) # define LabelAddr(Label) &&Label # define OpCode(OpCode) (&&lb_##OpCode) #endif @@ -70,7 +71,7 @@ do { \ } \ else \ erts_lc_check_exact(NULL, 0); \ - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); \ + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ } while (0) # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN) @@ -198,7 +199,7 @@ do { \ } \ } while (0) -#define ClauseFail() goto lb_jump_f +#define ClauseFail() goto jump_f #define SAVE_CP(X) \ do { \ @@ -233,6 +234,12 @@ BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ BeamInstr beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */ BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ + +/* + * We should warn only once for tuple funs. + */ +static erts_smp_atomic_t warned_for_tuple_funs; + /* * All Beam instructions in numerical order. */ @@ -303,44 +310,6 @@ extern int count_instructions; PROCESS_MAIN_CHK_LOCKS((P)); \ ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P)) -#if defined(HYBRID) -# define POST_BIF_GC_SWAPIN_0(_p, _res) \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _res = erts_gc_after_bif_call((_p), (_res), NULL, 0); \ - } \ - SWAPIN - -# define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity) \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _regs[0] = r(0); \ - _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity)); \ - r(0) = _regs[0]; \ - } \ - SWAPIN -#else -# define POST_BIF_GC_SWAPIN_0(_p, _res) \ - ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ - PROCESS_MAIN_CHK_LOCKS((_p)); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _res = erts_gc_after_bif_call((_p), (_res), NULL, 0); \ - E = (_p)->stop; \ - } \ - HTOP = HEAP_TOP((_p)) - -# define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity) \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ - ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ - PROCESS_MAIN_CHK_LOCKS((_p)); \ - if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ - _regs[0] = r(0); \ - _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity)); \ - r(0) = _regs[0]; \ - E = (_p)->stop; \ - } \ - HTOP = HEAP_TOP((_p)) -#endif - #define db(N) (N) #define tb(N) (N) #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N))) @@ -794,11 +763,11 @@ extern int count_instructions; } \ } while (0) -#define IsFunction2(F, A, Action) \ - do { \ - if (is_function_2(c_p, F, A) != am_true ) {\ - Action; \ - } \ +#define IsFunction2(F, A, Action) \ + do { \ + if (erl_is_function(c_p, F, A) != am_true ) { \ + Action; \ + } \ } while (0) #define IsTupleOfArity(Src, Arity, Fail) \ @@ -1052,6 +1021,7 @@ init_emulator(void) #if defined(VXWORKS) init_done = 0; #endif + erts_smp_atomic_init_nob(&warned_for_tuple_funs, (erts_aint_t) 0); process_main(); } @@ -1094,7 +1064,7 @@ void process_main(void) Process* c_p = NULL; int reds_used; #ifdef DEBUG - Eterm pid; + ERTS_DECLARE_DUMMY(Eterm pid); #endif /* @@ -1145,26 +1115,11 @@ void process_main(void) Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ #endif -#ifndef ERTS_SMP -#if !HALFWORD_HEAP - static Eterm save_reg[ERTS_X_REGS_ALLOCATED]; - /* X registers -- not used directly, but - * through 'reg', because using it directly - * needs two instructions on a SPARC, - * while using it through reg needs only - * one. - */ -#endif /* - * Floating point registers. - */ - static FloatDef freg[MAX_REG]; -#else - /* X regisers and floating point registers are located in + * X registers and floating point registers are located in * scheduler specific data. */ register FloatDef *freg; -#endif /* * For keeping the negative old value of 'reds' when call saving is active. @@ -1201,14 +1156,6 @@ void process_main(void) init_done = 1; goto init_emulator; } -#ifndef ERTS_SMP -#if !HALFWORD_HEAP - reg = save_reg; /* XXX: probably wastes a register on x86 */ -#else - /* Registers need to be heap allocated (correct memory range) for tracing to work */ - reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); -#endif -#endif c_p = NULL; reds_used = 0; goto do_schedule1; @@ -1225,14 +1172,12 @@ void process_main(void) c_p = schedule(c_p, reds_used); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); #ifdef DEBUG - pid = c_p->id; + pid = c_p->id; /* Save for debugging purpouses */ #endif ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); -#ifdef ERTS_SMP - reg = c_p->scheduler_data->save_reg; - freg = c_p->scheduler_data->freg; -#endif + reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; + freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; #if !HEAP_ON_C_STACK tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; #endif @@ -1566,9 +1511,17 @@ void process_main(void) PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; - result = send_2(c_p, r(0), x(1)); + reg[0] = r(0); + result = erl_send(c_p, r(0), x(1)); PreFetch(0, next); - POST_BIF_GC_SWAPIN(c_p, result, reg, 2); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { + result = erts_gc_after_bif_call(c_p, result, reg, 2); + r(0) = reg[0]; + E = c_p->stop; + } + HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { r(0) = result; @@ -1576,10 +1529,9 @@ void process_main(void) NextPF(0, next); } else if (c_p->freason == TRAP) { SET_CP(c_p, I+1); - SET_I(*((BeamInstr **) (BeamInstr) ((c_p)->def_arg_reg + 3))); + SET_I(c_p->i); SWAPIN; - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; + r(0) = reg[0]; Dispatch(); } goto find_func_info; @@ -2234,16 +2186,16 @@ void process_main(void) OpCase(bif1_fbsd): { - Eterm (*bf)(Process*, Eterm); - Eterm arg; + Eterm (*bf)(Process*, Eterm*); + Eterm tmp_reg[1]; Eterm result; - GetArg1(2, arg); + GetArg1(2, tmp_reg[0]); bf = (BifFunction) Arg(1); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, arg); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2262,17 +2214,17 @@ void process_main(void) OpCase(bif1_body_bsd): { - Eterm (*bf)(Process*, Eterm); + Eterm (*bf)(Process*, Eterm*); - Eterm arg; + Eterm tmp_reg[1]; Eterm result; - GetArg1(1, arg); + GetArg1(1, tmp_reg[0]); bf = (BifFunction) Arg(0); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, arg); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2281,7 +2233,7 @@ void process_main(void) if (is_value(result)) { StoreBifResult(2, result); } - reg[0] = arg; + reg[0] = tmp_reg[0]; SWAPOUT; I = handle_error(c_p, I, reg, bf); goto post_error_handling; @@ -2405,14 +2357,15 @@ void process_main(void) */ OpCase(i_bif2_fbd): { - Eterm (*bf)(Process*, Eterm, Eterm); + Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm (*bf)(Process*, Eterm*); Eterm result; bf = (BifFunction) Arg(1); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_arg1, tmp_arg2); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2430,13 +2383,14 @@ void process_main(void) */ OpCase(i_bif2_body_bd): { - Eterm (*bf)(Process*, Eterm, Eterm); + Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm (*bf)(Process*, Eterm*); Eterm result; bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, tmp_arg1, tmp_arg2); + result = (*bf)(c_p, tmp_reg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2456,77 +2410,9 @@ void process_main(void) * The most general BIF call. The BIF may build any amount of data * on the heap. The result is always returned in r(0). */ - OpCase(call_bif0_e): + OpCase(call_bif_e): { - Eterm (*bf)(Process*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); - - PRE_BIF_SWAPOUT(c_p); - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); - } - - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - r(0) = (*bf)(c_p, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(r(0))); - ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN_0(c_p, r(0)); - FCALLS = c_p->fcalls; - if (is_value(r(0))) { - CHECK_TERM(r(0)); - Next(1); - } - else if (c_p->freason == TRAP) { - goto call_bif_trap3; - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); - reg[0] = r(0); - I = handle_error(c_p, I, reg, bf); - goto post_error_handling; - } - - OpCase(call_bif1_e): - { - Eterm (*bf)(Process*, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); - Eterm result; - BeamInstr *next; - - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); - } - PreFetch(1, next); - PRE_BIF_SWAPOUT(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, r(0), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN(c_p, result, reg, 1); - FCALLS = c_p->fcalls; - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(1, next); - } else if (c_p->freason == TRAP) { - goto call_bif_trap3; - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); - reg[0] = r(0); - I = handle_error(c_p, I, reg, bf); - goto post_error_handling; - } - - OpCase(call_bif2_e): - { - Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); Eterm result; BeamInstr *next; @@ -2536,61 +2422,29 @@ void process_main(void) save_calls(c_p, (Export *) Arg(0)); } PreFetch(1, next); - CHECK_TERM(r(0)); - CHECK_TERM(x(1)); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, r(0), x(1), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN(c_p, result, reg, 2); - FCALLS = c_p->fcalls; - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(1, next); - } else if (c_p->freason == TRAP) { - goto call_bif_trap3; - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); reg[0] = r(0); - I = handle_error(c_p, I, reg, bf); - goto post_error_handling; - } - - OpCase(call_bif3_e): - { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); - Eterm result; - BeamInstr *next; - - PRE_BIF_SWAPOUT(c_p); - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); - } - PreFetch(1, next); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - result = (*bf)(c_p, r(0), x(1), x(2), I); + result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_HOLE_CHECK(c_p); - POST_BIF_GC_SWAPIN(c_p, result, reg, 3); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { + Uint arity = ((Export *)Arg(0))->code[2]; + result = erts_gc_after_bif_call(c_p, result, reg, arity); + E = c_p->stop; + } + HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { r(0) = result; CHECK_TERM(r(0)); NextPF(1, next); } else if (c_p->freason == TRAP) { - call_bif_trap3: SET_CP(c_p, I+2); - SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); + SET_I(c_p->i); SWAPIN; - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; - x(2) = c_p->def_arg_reg[2]; + r(0) = reg[0]; Dispatch(); } @@ -2598,7 +2452,6 @@ void process_main(void) * Error handling. SWAPOUT is not needed because it was done above. */ ASSERT(c_p->stop == E); - reg[0] = r(0); I = handle_error(c_p, I, reg, bf); goto post_error_handling; } @@ -2694,6 +2547,7 @@ void process_main(void) lb_Cl_error: { if (Arg(0) != 0) { OpCase(jump_f): { + jump_f: SET_I((BeamInstr *) Arg(0)); Goto(*I); } @@ -3267,7 +3121,7 @@ void process_main(void) /* Fall through */ OpCase(error_action_code): { - no_error_handler: + handle_error: reg[0] = r(0); SWAPOUT; I = handle_error(c_p, NULL, reg, NULL); @@ -3351,64 +3205,23 @@ void process_main(void) ASSERT(bif_nif_arity <= 3); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - switch (bif_nif_arity) { - case 3: - { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - break; - case 2: - { - Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, r(0), x(1), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - break; - case 1: - { - Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, r(0), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - break; - case 0: - { - Eterm (*bf)(Process*, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - nif_bif_result = (*bf)(c_p, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - break; - } - default: - erl_exit(1, "apply_bif: invalid arity: %u\n", - bif_nif_arity); + reg[0] = r(0); + { + Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, reg, I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); } apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); if (c_p->mbuf) { - reg[0] = r(0); nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, reg, bif_nif_arity); - r(0) = reg[0]; } SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; @@ -3419,17 +3232,14 @@ void process_main(void) c_p->cp = 0; Goto(*I); } else if (c_p->freason == TRAP) { - SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; - x(2) = c_p->def_arg_reg[2]; + SET_I(c_p->i); + r(0) = reg[0]; if (c_p->flags & F_HIBERNATE_SCHED) { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } Dispatch(); } - reg[0] = r(0); I = handle_error(c_p, c_p->cp, reg, vbf); goto post_error_handling; } @@ -3472,7 +3282,7 @@ void process_main(void) OpCase(i_func_info_IaaI): { c_p->freason = EXC_FUNCTION_CLAUSE; c_p->current = I + 2; - goto lb_error_action_code; + goto handle_error; } OpCase(try_case_end_s): @@ -3992,8 +3802,7 @@ void process_main(void) * too big numbers). */ if (is_not_small(val) || val > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL)) || - val == make_small(0xFFFEUL) || val == make_small(0xFFFFUL)) { + (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) { goto badarg; } Next(2); @@ -4012,8 +3821,8 @@ void process_main(void) * the valid range). */ if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= tmp_arg1 && tmp_arg1 <= make_small(0xDFFFUL)) || - tmp_arg1 == make_small(0xFFFEUL) || tmp_arg1 == make_small(0xFFFFUL)) { + (make_small(0xD800UL) <= tmp_arg1 && + tmp_arg1 <= make_small(0xDFFFUL))) { ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2); mb->offset -= 32; @@ -4888,92 +4697,6 @@ void process_main(void) } /* - * Instructions for allocating on the message area. - */ - - OpCase(i_global_cons): - { - BeamInstr *next; -#ifdef HYBRID - Eterm *hp; - - PreFetch(0,next); - TestGlobalHeap(2,2,hp); - hp[0] = r(0); - hp[1] = x(1); - r(0) = make_list(hp); -#ifndef INCREMENTAL - global_htop += 2; -#endif - NextPF(0,next); -#else - PreFetch(0,next); - c_p->freason = EXC_INTERNAL_ERROR; - goto find_func_info; -#endif - } - - OpCase(i_global_tuple): - { - BeamInstr *next; - int len; -#ifdef HYBRID - Eterm list; - Eterm *hp; -#endif - - if ((len = list_length(r(0))) < 0) { - goto badarg; - } - - PreFetch(0,next); -#ifdef HYBRID - TestGlobalHeap(len + 1,1,hp); - list = r(0); - r(0) = make_tuple(hp); - *hp++ = make_arityval(len); - while(is_list(list)) - { - Eterm* cons = list_val(list); - *hp++ = CAR(cons); - list = CDR(cons); - } -#ifndef INCREMENTAL - global_htop += len + 1; -#endif - NextPF(0,next); -#else - c_p->freason = EXC_INTERNAL_ERROR; - goto find_func_info; -#endif - } - - OpCase(i_global_copy): - { - BeamInstr *next; - PreFetch(0,next); -#ifdef HYBRID - if (!IS_CONST(r(0))) - { - BM_SWAP_TIMER(system,copy); - SWAPOUT; - reg[0] = r(0); - reg[1] = NIL; - r(0) = copy_struct_lazy(c_p,r(0),0); - ASSERT(ma_src_top == 0); - ASSERT(ma_dst_top == 0); - ASSERT(ma_offset_top == 0); - SWAPIN; - BM_SWAP_TIMER(copy,system); - } - NextPF(0,next); -#else - c_p->freason = EXC_INTERNAL_ERROR; - goto find_func_info; -#endif - } - - /* * New floating point instructions. */ @@ -5032,7 +4755,12 @@ void process_main(void) OpCase(fclearerror): OpCase(i_fcheckerror): erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); +# define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT +# define ERTS_NO_FPE_ERROR ERTS_FP_ERROR #else +# define ERTS_NO_FPE_CHECK_INIT(p) +# define ERTS_NO_FPE_ERROR(p, a, b) + OpCase(fclearerror): { BeamInstr *next; @@ -5048,10 +4776,6 @@ void process_main(void) ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith); NextPF(0, next); } -# undef ERTS_FP_CHECK_INIT -# undef ERTS_FP_ERROR -# define ERTS_FP_CHECK_INIT(p) -# define ERTS_FP_ERROR(p, a, b) #endif @@ -5059,45 +4783,45 @@ void process_main(void) BeamInstr *next; PreFetch(3, next); - ERTS_FP_CHECK_INIT(c_p); + ERTS_NO_FPE_CHECK_INIT(c_p); fb(Arg(2)) = fb(Arg(0)) + fb(Arg(1)); - ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith); + ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); NextPF(3, next); } OpCase(i_fsub_lll): { BeamInstr *next; PreFetch(3, next); - ERTS_FP_CHECK_INIT(c_p); + ERTS_NO_FPE_CHECK_INIT(c_p); fb(Arg(2)) = fb(Arg(0)) - fb(Arg(1)); - ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith); + ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); NextPF(3, next); } OpCase(i_fmul_lll): { BeamInstr *next; PreFetch(3, next); - ERTS_FP_CHECK_INIT(c_p); + ERTS_NO_FPE_CHECK_INIT(c_p); fb(Arg(2)) = fb(Arg(0)) * fb(Arg(1)); - ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith); + ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); NextPF(3, next); } OpCase(i_fdiv_lll): { BeamInstr *next; PreFetch(3, next); - ERTS_FP_CHECK_INIT(c_p); + ERTS_NO_FPE_CHECK_INIT(c_p); fb(Arg(2)) = fb(Arg(0)) / fb(Arg(1)); - ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith); + ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); NextPF(3, next); } OpCase(i_fnegate_ll): { BeamInstr *next; PreFetch(2, next); - ERTS_FP_CHECK_INIT(c_p); + ERTS_NO_FPE_CHECK_INIT(c_p); fb(Arg(1)) = -fb(Arg(0)); - ERTS_FP_ERROR(c_p, fb(Arg(1)), goto fbadarith); + ERTS_NO_FPE_ERROR(c_p, fb(Arg(1)), goto fbadarith); NextPF(2, next); fbadarith: @@ -5151,10 +4875,8 @@ void process_main(void) c_p->def_arg_reg[4] = -neg_o_reds; reg[0] = r(0); c_p = hipe_mode_switch(c_p, cmd, reg); -#ifdef ERTS_SMP - reg = c_p->scheduler_data->save_reg; - freg = c_p->scheduler_data->freg; -#endif + reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; + freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; @@ -5246,7 +4968,7 @@ void process_main(void) if (I) { Goto(*I); } - goto no_error_handler; + goto handle_error; } @@ -5268,8 +4990,8 @@ void process_main(void) OpCase(int_code_end): OpCase(label_L): - OpCase(too_old_compiler): OpCase(on_load): + OpCase(line_I): erl_exit(1, "meta op\n"); /* @@ -5686,6 +5408,25 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { * that c_p->ftrace will point to a cons cell which holds the given args * and the saved data (encoded as a bignum). * + * There is an issue with line number information. Line number + * information is associated with the address *before* an operation + * that may fail or be stored stored on the stack. But continuation + * pointers point after its call instruction, not before. To avoid + * finding the wrong line number, we'll need to adjust them so that + * they point at the beginning of the call instruction or inside the + * call instruction. Since its impractical to point at the beginning, + * we'll do the simplest thing and decrement the continuation pointers + * by one. + * + * Here is an example of what can go wrong. Without the adjustment + * of continuation pointers, the call at line 42 below would seem to + * be at line 43: + * + * line 42 + * call ... + * line 43 + * gc_bif ... + * * (It would be much better to put the arglist - when it exists - in the * error value instead of in the actual trace; e.g. '{badarg, Args}' * instead of using 'badarg' with Args in the trace. The arglist may @@ -5752,7 +5493,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, } /* Save second stack entry if CP is valid and different from pc */ if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp; + s->trace[s->depth++] = c_p->cp - 1; depth--; } s->pc = NULL; @@ -5772,13 +5513,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, /* Save first stack entry */ ASSERT(c_p->cp); if (depth > 0) { - s->trace[s->depth++] = c_p->cp; + s->trace[s->depth++] = c_p->cp - 1; depth--; } s->pc = NULL; /* Ignore pc */ } else { if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp; + s->trace[s->depth++] = c_p->cp - 1; depth--; } s->pc = pc; @@ -5793,24 +5534,31 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, } /* Save the actual stack trace */ + erts_save_stacktrace(c_p, s, depth); +} + +void +erts_save_stacktrace(Process* p, struct StackTrace* s, int depth) +{ if (depth > 0) { Eterm *ptr; BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL; BeamInstr i_return_trace = beam_return_trace[0]; BeamInstr i_return_to_trace = beam_return_to_trace[0]; + /* * Traverse the stack backwards and add all unique continuation * pointers to the buffer, up to the maximum stack trace size. * * Skip trace stack frames. */ - ptr = c_p->stop; - if (ptr < STACK_START(c_p) - && (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace)) - && c_p->cp) { - /* Can not follow cp here - code may be unloaded */ - BeamInstr *cpp = c_p->cp; + ptr = p->stop; + if (ptr < STACK_START(p) && + (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace && + *cp_val(*ptr) != i_return_to_trace)) && + p->cp) { + /* Cannot follow cp here - code may be unloaded */ + BeamInstr *cpp = p->cp; if (cpp == beam_exception_trace || cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; @@ -5819,7 +5567,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, ptr += 1; } } - while (ptr < STACK_START(c_p) && depth > 0) { + while (ptr < STACK_START(p) && depth > 0) { if (is_CP(*ptr)) { if (*cp_val(*ptr) == i_return_trace) { /* Skip stack frame variables */ @@ -5834,7 +5582,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, if (cp != prev) { /* Record non-duplicates only */ prev = cp; - s->trace[s->depth++] = cp; + s->trace[s->depth++] = cp - 1; depth--; } ptr++; @@ -5902,9 +5650,14 @@ build_stacktrace(Process* c_p, Eterm exc) { struct StackTrace* s; Eterm args; int depth; - BeamInstr* current; - Eterm Where = NIL; - Eterm *next_p = &Where; + FunctionInfo fi; + FunctionInfo* stk; + FunctionInfo* stkp; + Eterm res = NIL; + Uint heap_size; + Eterm* hp; + Eterm mfa; + int i; if (! (s = get_trace_from_exc(exc))) { return NIL; @@ -5923,64 +5676,56 @@ build_stacktrace(Process* c_p, Eterm exc) { * saved s->current should already contain the proper value. */ if (s->pc != NULL) { - current = find_function_from_pc(s->pc); + erts_lookup_function_info(&fi, s->pc, 1); + } else if (GET_EXC_INDEX(s->freason) == + GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) { + erts_lookup_function_info(&fi, s->current, 1); } else { - current = s->current; + erts_set_current_function(&fi, s->current); } + /* - * If current is still NULL, default to the initial function + * If fi.current is still NULL, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (current == NULL) { - current = c_p->initial; + if (fi.current == NULL) { + erts_set_current_function(&fi, c_p->initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); } - depth = s->depth; - /* - * Add the {M,F,A} for the current function - * (where A is arity or [Argument]). + * Look up all saved continuation pointers and calculate + * needed heap space. */ - { - int i; - Eterm mfa; - Uint heap_size = 6*(depth+1); - Eterm* hp = HAlloc(c_p, heap_size); - Eterm* hp_end = hp + heap_size; - - if (args != am_true) { - /* We have an arglist - use it */ - mfa = TUPLE3(hp, current[0], current[1], args); - } else { - Eterm arity = make_small(current[2]); - mfa = TUPLE3(hp, current[0], current[1], arity); + depth = s->depth; + stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, + depth*sizeof(FunctionInfo)); + heap_size = fi.needed + 2; + for (i = 0; i < depth; i++) { + erts_lookup_function_info(stkp, s->trace[i], 1); + if (stkp->current) { + heap_size += stkp->needed + 2; + stkp++; } - hp += 4; - ASSERT(*next_p == NIL); - *next_p = CONS(hp, mfa, NIL); - next_p = &CDR(list_val(*next_p)); - hp += 2; + } - /* - * Finally, we go through the saved continuation pointers. - */ - for (i = 0; i < depth; i++) { - BeamInstr *fi = find_function_from_pc((BeamInstr *) s->trace[i]); - if (fi == NULL) continue; - mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2])); - hp += 4; - ASSERT(*next_p == NIL); - *next_p = CONS(hp, mfa, NIL); - next_p = &CDR(list_val(*next_p)); - hp += 2; - } - ASSERT(hp <= hp_end); - HRelease(c_p, hp_end, hp); + /* + * Allocate heap space and build the stacktrace. + */ + hp = HAlloc(c_p, heap_size); + while (stkp > stk) { + stkp--; + hp = erts_build_mfa_item(stkp, hp, am_true, &mfa); + res = CONS(hp, mfa, res); + hp += 2; } - return Where; + hp = erts_build_mfa_item(&fi, hp, args, &mfa); + res = CONS(hp, mfa, res); + + erts_free(ERTS_ALC_T_TMP, (void *) stk); + return res; } @@ -6450,6 +6195,26 @@ call_fun(Process* p, /* Current process. */ if (!is_atom(module) || !is_atom(function)) { goto badfun; } + + /* + * If this is the first time a tuple fun is used, + * send a warning to the logger. + */ + if (erts_smp_atomic_xchg_nob(&warned_for_tuple_funs, + (erts_aint_t) 1) == 0) { + erts_dsprintf_buf_t* dsbufp; + + dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Call to tuple fun {%T,%T}.\n\n" + "Tuple funs are deprecated and will be removed " + "in R16. Use \"fun M:F/A\" instead, for example " + "\"fun %T:%T/%d\".\n\n" + "(This warning will only be shown the first time " + "a tuple fun is called.)\n", + module, function, module, function, arity); + erts_send_warning_to_logger(p->group_leader, dsbufp); + } + if ((ep = erts_find_export_entry(module, function, arity)) == NULL) { ep = erts_find_export_entry(erts_proc_get_error_handler(p), am_undefined_function, 3); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index fceb352cf3..dd788df6e4 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -158,6 +158,7 @@ typedef struct { #define LITERAL_CHUNK 6 #define ATTR_CHUNK 7 #define COMPILE_CHUNK 8 +#define LINE_CHUNK 9 #define NUM_CHUNK_TYPES (sizeof(chunk_types)/sizeof(chunk_types[0])) @@ -182,6 +183,7 @@ static Uint chunk_types[] = { MakeIffId('L', 'i', 't', 'T'), /* 6 */ MakeIffId('A', 't', 't', 'r'), /* 7 */ MakeIffId('C', 'I', 'n', 'f'), /* 8 */ + MakeIffId('L', 'i', 'n', 'e'), /* 9 */ }; /* @@ -204,6 +206,7 @@ typedef struct { Eterm term; /* The tagged term (in the heap). */ Uint heap_size; /* (Exact) size on the heap. */ Uint offset; /* Offset from temporary location to final. */ + ErlOffHeap off_heap; /* Start of linked list of ProcBins. */ Eterm* heap; /* Heap for term. */ } Literal; @@ -231,10 +234,19 @@ struct string_patch { }; /* + * This structure associates a code offset with a source code location. + */ + +typedef struct { + int pos; /* Position in code */ + Uint32 loc; /* Location in source code */ +} LineInstr; + +/* * This structure contains all information about the module being loaded. */ -typedef struct { +typedef struct LoaderState { /* * The current logical file within the binary. */ @@ -242,6 +254,7 @@ typedef struct { char* file_name; /* Name of file we are reading (usually chunk name). */ byte* file_p; /* Current pointer within file. */ unsigned file_left; /* Number of bytes left in file. */ + ErlDrvBinary* bin; /* Binary holding BEAM file (or NULL) */ /* * The following are used mainly for diagnostics. @@ -276,7 +289,6 @@ typedef struct { BeamInstr* code; /* Loaded code. */ int ci; /* Current index into loaded code. */ Label* labels; - BeamInstr new_bs_put_strings; /* Linked list of i_new_bs_put_string instructions. */ StringPatch* string_patches; /* Linked list of position into string table to patch. */ BeamInstr catches; /* Linked list of catch_yf instructions. */ unsigned loaded_size; /* Final size of code when loaded. */ @@ -325,12 +337,41 @@ typedef struct { Literal* literals; /* Array of literals. */ LiteralPatch* literal_patches; /* Operands that need to be patched. */ Uint total_literal_size; /* Total heap size for all literals. */ + + /* + * Line table. + */ + BeamInstr* line_item; /* Line items from the BEAM file. */ + int num_line_items; /* Number of line items. */ + LineInstr* line_instr; /* Line instructions */ + int num_line_instrs; /* Maximum number of line instructions */ + int current_li; /* Current line instruction */ + int* func_line; /* Mapping from function to first line instr */ + Eterm* fname; /* List of file names */ + int num_fnames; /* Number of filenames in fname table */ + int loc_size; /* Size of location info in bytes (2/4) */ } LoaderState; -typedef struct { - unsigned num_functions; /* Number of functions. */ - Eterm* func_tab[1]; /* Pointers to each function. */ -} LoadedCode; +/* + * Layout of the line table. + */ + +#define MI_LINE_FNAME_PTR 0 +#define MI_LINE_LOC_TAB 1 +#define MI_LINE_LOC_SIZE 2 +#define MI_LINE_FUNC_TAB 3 + +#define LINE_INVALID_LOCATION (0) + +/* + * Macros for manipulating locations. + */ + +#define IS_VALID_LOCATION(File, Line) \ + ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1)) +#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line)) +#define LOC_FILE(Loc) ((Loc) >> 24) +#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) #define GetTagAndValue(Stp, Tag, Val) \ do { \ @@ -455,19 +496,20 @@ typedef struct { } while (0) -static int bin_load(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm* modp, byte* bytes, int unloaded_size); -static void init_state(LoaderState* stp); -static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, - BeamInstr* code, Uint size, BeamInstr catches); +static void free_state(LoaderState* stp); +static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm module, + BeamInstr* code, Uint size); +static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); +static int verify_chunks(LoaderState* stp); static int load_atom_table(LoaderState* stp); static int load_import_table(LoaderState* stp); static int read_export_table(LoaderState* stp); static int read_lambda_table(LoaderState* stp); static int read_literal_table(LoaderState* stp); +static int read_line_table(LoaderState* stp); static int read_code_header(LoaderState* stp); static int load_code(LoaderState* stp); static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, @@ -506,6 +548,8 @@ static Eterm native_addresses(Process* p, Eterm mod); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); +static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, + BeamInstr* modp, int idx); static int must_swap_floats; @@ -550,7 +594,7 @@ define_file(LoaderState* stp, char* name, int idx) stp->file_left = stp->chunks[idx].size; } -int +Eterm erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, /* Group leader or NIL if none. */ @@ -559,29 +603,17 @@ erts_load_module(Process *c_p, * On return, contains the actual module name. */ byte* code, /* Points to the code to load */ - int size) /* Size of code to load. */ + Uint size) /* Size of code to load. */ { - ErlDrvBinary* bin; - int result; + LoaderState* stp = erts_alloc_loader_state(); + Eterm retval; - if (size >= 4 && code[0] == 'F' && code[1] == 'O' && - code[2] == 'R' && code[3] == '1') { - /* - * The BEAM module is not compressed. - */ - result = bin_load(c_p, c_p_locks, group_leader, modp, code, size); - } else { - /* - * The BEAM module is compressed (or possibly invalid/corrupted). - */ - if ((bin = (ErlDrvBinary *) erts_gzinflate_buffer((char*)code, size)) == NULL) { - return -1; - } - result = bin_load(c_p, c_p_locks, group_leader, modp, - (byte*)bin->orig_bytes, bin->orig_size); - driver_free_binary(bin); + retval = erts_prepare_loading(stp, c_p, group_leader, modp, + code, size); + if (retval != NIL) { + return retval; } - return result; + return erts_finish_loading(stp, c_p, c_p_locks, modp); } /* #define LOAD_MEMORY_HARD_DEBUG 1*/ @@ -596,31 +628,28 @@ extern void check_allocated_block(Uint type, void *blk); #define CHKBLK(TYPE,BLK) /* nothing */ #endif -static int -bin_load(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm* modp, byte* bytes, int unloaded_size) +Eterm +erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, + Eterm* modp, byte* code, Uint unloaded_size) { - LoaderState state; - int rval = -1; - - init_state(&state); - state.module = *modp; - state.group_leader = group_leader; + Eterm retval = am_badfile; - /* - * Scan the IFF file. - */ + stp->module = *modp; + stp->group_leader = group_leader; #if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) erts_fprintf(stderr,"Loading a module\n"); #endif + /* + * Scan the IFF file. + */ + CHKALLOC(); - CHKBLK(ERTS_ALC_T_CODE,state.code); - state.file_name = "IFF header for Beam file"; - state.file_p = bytes; - state.file_left = unloaded_size; - if (!scan_iff_file(&state, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + if (!init_iff_file(stp, code, unloaded_size) || + !scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) || + !verify_chunks(stp)) { goto load_error; } @@ -628,38 +657,38 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the header for the code chunk. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - define_file(&state, "code chunk header", CODE_CHUNK); - if (!read_code_header(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + define_file(stp, "code chunk header", CODE_CHUNK); + if (!read_code_header(stp)) { goto load_error; } /* * Initialize code area. */ - state.code_buffer_size = erts_next_heap_size(2048 + state.num_functions, 0); - state.code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, - sizeof(BeamInstr) * state.code_buffer_size); + stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0); + stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, + sizeof(BeamInstr) * stp->code_buffer_size); - state.code[MI_NUM_FUNCTIONS] = state.num_functions; - state.ci = MI_FUNCTIONS + state.num_functions + 1; + stp->code[MI_NUM_FUNCTIONS] = stp->num_functions; + stp->ci = MI_FUNCTIONS + stp->num_functions + 1; - state.code[MI_ATTR_PTR] = 0; - state.code[MI_ATTR_SIZE] = 0; - state.code[MI_ATTR_SIZE_ON_HEAP] = 0; - state.code[MI_COMPILE_PTR] = 0; - state.code[MI_COMPILE_SIZE] = 0; - state.code[MI_COMPILE_SIZE_ON_HEAP] = 0; - state.code[MI_NUM_BREAKPOINTS] = 0; + stp->code[MI_ATTR_PTR] = 0; + stp->code[MI_ATTR_SIZE] = 0; + stp->code[MI_ATTR_SIZE_ON_HEAP] = 0; + stp->code[MI_COMPILE_PTR] = 0; + stp->code[MI_COMPILE_SIZE] = 0; + stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; + stp->code[MI_NUM_BREAKPOINTS] = 0; /* * Read the atom table. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - define_file(&state, "atom table", ATOM_CHUNK); - if (!load_atom_table(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + define_file(stp, "atom table", ATOM_CHUNK); + if (!load_atom_table(stp)) { goto load_error; } @@ -667,9 +696,9 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the import table. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - define_file(&state, "import table", IMP_CHUNK); - if (!load_import_table(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + define_file(stp, "import table", IMP_CHUNK); + if (!load_import_table(stp)) { goto load_error; } @@ -677,10 +706,10 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the lambda (fun) table. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - if (state.chunks[LAMBDA_CHUNK].size > 0) { - define_file(&state, "lambda (fun) table", LAMBDA_CHUNK); - if (!read_lambda_table(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + if (stp->chunks[LAMBDA_CHUNK].size > 0) { + define_file(stp, "lambda (fun) table", LAMBDA_CHUNK); + if (!read_lambda_table(stp)) { goto load_error; } } @@ -689,10 +718,22 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Read the literal table. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - if (state.chunks[LITERAL_CHUNK].size > 0) { - define_file(&state, "literals table (constant pool)", LITERAL_CHUNK); - if (!read_literal_table(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + if (stp->chunks[LITERAL_CHUNK].size > 0) { + define_file(stp, "literals table (constant pool)", LITERAL_CHUNK); + if (!read_literal_table(stp)) { + goto load_error; + } + } + + /* + * Read the line table (if present). + */ + + CHKBLK(ERTS_ALC_T_CODE,stp->code); + if (stp->chunks[LINE_CHUNK].size > 0) { + define_file(stp, "line table", LINE_CHUNK); + if (!read_line_table(stp)) { goto load_error; } } @@ -701,15 +742,15 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * Load the code chunk. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - state.file_name = "code chunk"; - state.file_p = state.code_start; - state.file_left = state.code_size; - if (!load_code(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + stp->file_name = "code chunk"; + stp->file_p = stp->code_start; + stp->file_left = stp->code_size; + if (!load_code(stp)) { goto load_error; } - CHKBLK(ERTS_ALC_T_CODE,state.code); - if (!freeze_code(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + if (!freeze_code(stp)) { goto load_error; } @@ -719,9 +760,49 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * loading the code, because it contains labels.) */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - define_file(&state, "export table", EXP_CHUNK); - if (!read_export_table(&state)) { + CHKBLK(ERTS_ALC_T_CODE,stp->code); + define_file(stp, "export table", EXP_CHUNK); + if (!read_export_table(stp)) { + goto load_error; + } + + /* + * Good so far. + */ + + retval = NIL; + + load_error: + if (retval != NIL) { + free_state(stp); + } + return retval; +} + +Eterm +erts_finish_loading(LoaderState* stp, Process* c_p, + ErtsProcLocks c_p_locks, Eterm* modp) +{ + Eterm retval; + + /* + * No other process may run since we will update the export + * table which is not protected by any locks. + */ + + ERTS_SMP_LC_ASSERT(erts_initialized == 0 || + erts_smp_thr_progress_is_blocking()); + + /* + * Make current code for the module old and insert the new code + * as current. This will fail if there already exists old code + * for the module. + */ + + CHKBLK(ERTS_ALC_T_CODE,stp->code); + retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module, + stp->code, stp->loaded_size); + if (retval != NIL) { goto load_error; } @@ -730,88 +811,43 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks, * exported and imported functions. This can't fail. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); - rval = insert_new_code(c_p, c_p_locks, state.group_leader, state.module, - state.code, state.loaded_size, state.catches); - if (rval < 0) { - goto load_error; - } - CHKBLK(ERTS_ALC_T_CODE,state.code); - final_touch(&state); + erts_export_consolidate(); + CHKBLK(ERTS_ALC_T_CODE,stp->code); + final_touch(stp); /* * Loading succeded. */ - CHKBLK(ERTS_ALC_T_CODE,state.code); + CHKBLK(ERTS_ALC_T_CODE,stp->code); #if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG) erts_fprintf(stderr,"Loaded %T\n",*modp); #if 0 - debug_dump_code(state.code,state.ci); + debug_dump_code(stp->code,stp->ci); #endif #endif - rval = 0; - state.code = NULL; /* Prevent code from being freed. */ - *modp = state.module; + stp->code = NULL; /* Prevent code from being freed. */ + *modp = stp->module; /* * If there is an on_load function, signal an error to * indicate that the on_load function must be run. */ - if (state.on_load) { - rval = -5; + if (stp->on_load) { + retval = am_on_load; } load_error: - if (state.code != 0) { - erts_free(ERTS_ALC_T_CODE, state.code); - } - if (state.labels != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); - } - if (state.atom != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); - } - if (state.import != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.import); - } - if (state.export != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); - } - if (state.lambdas != state.def_lambdas) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); - } - if (state.literals != NULL) { - int i; - for (i = 0; i < state.num_literals; i++) { - if (state.literals[i].heap != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.literals[i].heap); - } - } - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.literals); - } - while (state.literal_patches != NULL) { - LiteralPatch* next = state.literal_patches->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.literal_patches); - state.literal_patches = next; - } - while (state.string_patches != NULL) { - StringPatch* next = state.string_patches->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.string_patches); - state.string_patches = next; - } - while (state.genop_blocks) { - GenOpBlock* next = state.genop_blocks->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.genop_blocks); - state.genop_blocks = next; - } - - return rval; + free_state(stp); + return retval; } - -static void -init_state(LoaderState* stp) +LoaderState* +erts_alloc_loader_state(void) { + LoaderState* stp; + + stp = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LoaderState)); + stp->bin = NULL; stp->function = THE_NON_VALUE; /* Function not known yet */ stp->arity = 0; stp->specific_op = -1; @@ -835,23 +871,98 @@ init_state(LoaderState* stp) stp->string_patches = 0; stp->may_load_nif = 0; stp->on_load = 0; + stp->line_item = 0; + stp->line_instr = 0; + stp->func_line = 0; + stp->fname = 0; + return stp; } -static int +static void +free_state(LoaderState* stp) +{ + if (stp->bin != 0) { + driver_free_binary(stp->bin); + } + if (stp->code != 0) { + erts_free(ERTS_ALC_T_CODE, stp->code); + } + if (stp->labels != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->labels); + } + if (stp->atom != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->atom); + } + if (stp->import != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->import); + } + if (stp->export != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->export); + } + if (stp->lambdas != stp->def_lambdas) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->lambdas); + } + if (stp->literals != NULL) { + int i; + for (i = 0; i < stp->num_literals; i++) { + if (stp->literals[i].heap != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, + (void *) stp->literals[i].heap); + } + } + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literals); + } + while (stp->literal_patches != NULL) { + LiteralPatch* next = stp->literal_patches->next; + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literal_patches); + stp->literal_patches = next; + } + while (stp->string_patches != NULL) { + StringPatch* next = stp->string_patches->next; + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->string_patches); + stp->string_patches = next; + } + while (stp->genop_blocks) { + GenOpBlock* next = stp->genop_blocks->next; + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks); + stp->genop_blocks = next; + } + + if (stp->line_item != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_item); + } + + if (stp->line_instr != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_instr); + } + + if (stp->func_line != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, stp->func_line); + } + + if (stp->fname != 0) { + erts_free(ERTS_ALC_T_LOADER_TMP, stp->fname); + } + + erts_free(ERTS_ALC_T_LOADER_TMP, stp); +} + +static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, BeamInstr* code, Uint size, BeamInstr catches) + Eterm group_leader, Eterm module, BeamInstr* code, + Uint size) { Module* modp; - int rval; + Eterm retval; int i; - if ((rval = beam_make_current_old(c_p, c_p_locks, module)) < 0) { + if ((retval = beam_make_current_old(c_p, c_p_locks, module)) != NIL) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", module); erts_send_error_to_logger(group_leader, dsbufp); - return rval; + return retval; } /* @@ -862,7 +973,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp = erts_put_module(module); modp->code = code; modp->code_length = size; - modp->catches = catches; + modp->catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ /* * Update address table (used for finding a function from a PC value). @@ -884,27 +995,51 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modules[i].end = (BeamInstr *) (((byte *)code) + size); num_loaded_modules++; mid_module = &modules[num_loaded_modules/2]; - return 0; + return NIL; } static int -scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory) +init_iff_file(LoaderState* stp, byte* code, Uint size) { - MD5_CTX context; + Uint form_id = MakeIffId('F', 'O', 'R', '1'); Uint id; Uint count; - int i; + + if (size < 4) { + goto load_error; + } /* - * The binary must start with an IFF 'FOR1' chunk. + * Check if the module is compressed (or possibly invalid/corrupted). */ + if (MakeIffId(code[0], code[1], code[2], code[3]) != form_id) { + stp->bin = (ErlDrvBinary *) erts_gzinflate_buffer((char*)code, size); + if (stp->bin == NULL) { + goto load_error; + } + code = (byte*)stp->bin->orig_bytes; + size = stp->bin->orig_size; + if (size < 4) { + goto load_error; + } + } - GetInt(stp, 4, id); - if (id != MakeIffId('F', 'O', 'R', '1')) { + /* + * The binary must start with an IFF 'FOR1' chunk. + */ + if (MakeIffId(code[0], code[1], code[2], code[3]) != form_id) { LoadError0(stp, "not a BEAM file: no IFF 'FOR1' chunk"); } /* + * Initialize our "virtual file system". + */ + + stp->file_name = "IFF header for Beam file"; + stp->file_p = code + 4; + stp->file_left = size - 4; + + /* * Retrieve the chunk size and verify it. If the size is equal to * or less than the size of the binary, it is ok and we will use it * as the limit for the logical file size. @@ -925,6 +1060,21 @@ scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mand if (id != MakeIffId('B', 'E', 'A', 'M')) { LoadError0(stp, "not a BEAM file: IFF form type is not 'BEAM'"); } + return 1; + + load_error: + return 0; +} + +/* + * Scan the IFF file. The header should have been verified by init_iff_file(). + */ +static int +scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory) +{ + Uint count; + Uint id; + int i; /* * Initialize the chunks[] array in the state. @@ -981,17 +1131,25 @@ scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mand stp->file_p += count; stp->file_left -= count; } + return 1; - /* - * At this point, we have read the entire IFF file, and we - * know that it is syntactically correct. - * - * Now check that it contains all mandatory chunks. At the - * same time calculate the MD5 for the module. - */ + load_error: + return 0; +} + +/* + * Verify that all mandatory chunks are present and calculate + * MD5 for the module. + */ + +static int +verify_chunks(LoaderState* stp) +{ + int i; + MD5_CTX context; MD5Init(&context); - for (i = 0; i < num_mandatory; i++) { + for (i = 0; i < NUM_MANDATORY; i++) { if (stp->chunks[i].start != NULL) { MD5Update(&context, stp->chunks[i].start, stp->chunks[i].size); } else { @@ -1001,41 +1159,49 @@ scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mand LoadError1(stp, "mandatory chunk of type '%s' not found\n", sbuf); } } - if (LITERAL_CHUNK < num_types) { - if (stp->chunks[LAMBDA_CHUNK].start != 0) { - byte* start = stp->chunks[LAMBDA_CHUNK].start; - Uint left = stp->chunks[LAMBDA_CHUNK].size; - /* - * The idea here is to ignore the OldUniq field for the fun; it is - * based on the old broken hash function, which can be different - * on little endian and big endian machines. - */ - if (left >= 4) { - static byte zero[4]; - MD5Update(&context, start, 4); - start += 4; - left -= 4; + /* + * If there is a lambda chunk, include parts of it in the MD5. + */ + if (stp->chunks[LAMBDA_CHUNK].start != 0) { + byte* start = stp->chunks[LAMBDA_CHUNK].start; + Uint left = stp->chunks[LAMBDA_CHUNK].size; + + /* + * The idea here is to ignore the OldUniq field for the fun; it is + * based on the old broken hash function, which can be different + * on little endian and big endian machines. + */ + if (left >= 4) { + static byte zero[4]; + MD5Update(&context, start, 4); + start += 4; + left -= 4; - while (left >= 24) { - /* Include: Function Arity Index NumFree */ - MD5Update(&context, start, 20); - /* Set to zero: OldUniq */ - MD5Update(&context, zero, 4); - start += 24; - left -= 24; - } - } - /* Can't happen for a correct 'FunT' chunk */ - if (left > 0) { - MD5Update(&context, start, left); + while (left >= 24) { + /* Include: Function Arity Index NumFree */ + MD5Update(&context, start, 20); + /* Set to zero: OldUniq */ + MD5Update(&context, zero, 4); + start += 24; + left -= 24; } } - if (stp->chunks[LITERAL_CHUNK].start != 0) { - MD5Update(&context, stp->chunks[LITERAL_CHUNK].start, - stp->chunks[LITERAL_CHUNK].size); + /* Can't happen for a correct 'FunT' chunk */ + if (left > 0) { + MD5Update(&context, start, left); } } + + + /* + * If there is a literal chunk, include it in the MD5. + */ + if (stp->chunks[LITERAL_CHUNK].start != 0) { + MD5Update(&context, stp->chunks[LITERAL_CHUNK].start, + stp->chunks[LITERAL_CHUNK].size); + } + MD5Final(stp->mod_md5, &context); return 1; @@ -1268,7 +1434,7 @@ static int read_literal_table(LoaderState* stp) { int i; - BeamInstr uncompressed_sz; + uLongf uncompressed_sz; byte* uncompressed = 0; GetInt(stp, 4, uncompressed_sz); @@ -1278,7 +1444,7 @@ read_literal_table(LoaderState* stp) LoadError0(stp, "failed to uncompress literal table (constant pool)"); } stp->file_p = uncompressed; - stp->file_left = uncompressed_sz; + stp->file_left = (unsigned) uncompressed_sz; GetInt(stp, 4, stp->num_literals); stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP, stp->num_literals * sizeof(Literal)); @@ -1297,12 +1463,14 @@ read_literal_table(LoaderState* stp) GetInt(stp, 4, sz); /* Size of external term format. */ GetString(stp, p, sz); - if ((heap_size = erts_decode_ext_size(p, sz, 1)) < 0) { + if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { LoadError1(stp, "literal %d: bad external format", i); } hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm)); - val = erts_decode_ext(&hp, NULL, &p); + stp->literals[i].off_heap.first = 0; + stp->literals[i].off_heap.overhead = 0; + val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p); stp->literals[i].heap_size = hp - stp->literals[i].heap; if (stp->literals[i].heap_size > heap_size) { erl_exit(1, "overrun by %d word(s) for literal heap, term %d", @@ -1324,6 +1492,138 @@ read_literal_table(LoaderState* stp) return 0; } +static int +read_line_table(LoaderState* stp) +{ + unsigned version; + ERTS_DECLARE_DUMMY(unsigned flags); + int num_line_items; + BeamInstr* lp; + int i; + BeamInstr fname_index; + BeamInstr tag; + + /* + * If the emulator flag ignoring the line information was given, + * return immediately. + */ + + if (erts_no_line_info) { + return 1; + } + + /* + * Check version of line table. + */ + + GetInt(stp, 4, version); + if (version != 0) { + /* + * Wrong version. Silently ignore the line number chunk. + */ + return 1; + } + + /* + * Read the remaining header words. The flag word is reserved + * for possible future use; for the moment we ignore it. + */ + GetInt(stp, 4, flags); + GetInt(stp, 4, stp->num_line_instrs); + GetInt(stp, 4, num_line_items); + GetInt(stp, 4, stp->num_fnames); + + /* + * Calculate space and allocate memory for the line item table. + */ + + num_line_items++; + lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + num_line_items * sizeof(BeamInstr)); + stp->line_item = lp; + stp->num_line_items = num_line_items; + + /* + * The zeroth entry in the line item table is special. + * It contains the undefined location. + */ + + *lp++ = LINE_INVALID_LOCATION; + num_line_items--; + + /* + * Read all the line items. + */ + + stp->loc_size = stp->num_fnames ? 4 : 2; + fname_index = 0; + while (num_line_items-- > 0) { + BeamInstr val; + BeamInstr loc; + + GetTagAndValue(stp, tag, val); + if (tag == TAG_i) { + if (IS_VALID_LOCATION(fname_index, val)) { + loc = MAKE_LOCATION(fname_index, val); + } else { + /* + * Too many files or huge line number. Silently invalidate + * the location. + */ + loc = LINE_INVALID_LOCATION; + } + *lp++ = loc; + if (val > 0xFFFF) { + stp->loc_size = 4; + } + } else if (tag == TAG_a) { + if (val > stp->num_fnames) { + LoadError2(stp, "file index overflow (%d/%d)", + val, stp->num_fnames); + } + fname_index = val; + num_line_items++; + } else { + LoadError1(stp, "bad tag '%c' (expected 'a' or 'i')", + tag_to_letter[tag]); + } + } + + /* + * Read all filenames. + */ + + if (stp->num_fnames != 0) { + stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->num_fnames * + sizeof(Eterm)); + for (i = 0; i < stp->num_fnames; i++) { + byte* fname; + Uint n; + + GetInt(stp, 2, n); + GetString(stp, fname, n); + stp->fname[i] = am_atom_put((char*)fname, n); + } + } + + /* + * Allocate the arrays to be filled while code is being loaded. + */ + stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->num_line_instrs * + sizeof(LineInstr)); + stp->current_li = 0; + stp->func_line = (int *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->num_functions * + sizeof(int)); + + return 1; + + load_error: + return 0; +} + static int read_code_header(LoaderState* stp) @@ -1358,10 +1658,15 @@ read_code_header(LoaderState* stp) /* * Verify the number of the highest opcode used. */ - GetInt(stp, 4, opcode_max); if (opcode_max > MAX_GENERIC_OPCODE) { - LoadError2(stp, "use of opcode %d; this emulator supports only up to %d", + LoadError2(stp, + "This BEAM file was compiled for a later version" + " of the run-time system than " ERLANG_OTP_RELEASE ".\n" + " To fix this, please recompile this module with an " + ERLANG_OTP_RELEASE " compiler.\n" + " (Use of opcode %d; this emulator supports " + "only up to %d.)", opcode_max, MAX_GENERIC_OPCODE); } @@ -1382,7 +1687,6 @@ read_code_header(LoaderState* stp) #endif } - stp->new_bs_put_strings = 0; stp->catches = 0; return 1; @@ -1415,7 +1719,7 @@ load_code(LoaderState* stp) { int i; int ci; - int last_func_start = 0; + int last_func_start = 0; /* Needed by nif loading and line instructions */ char* sign; int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ @@ -1428,6 +1732,14 @@ load_code(LoaderState* stp) GenOp** last_op_next = NULL; int arity; + /* + * The size of the loaded func_info instruction is needed + * by both the nif functionality and line instructions. + */ + enum { + FUNC_INFO_SZ = 5 + }; + code = stp->code; code_buffer_size = stp->code_buffer_size; ci = stp->ci; @@ -1667,14 +1979,6 @@ load_code(LoaderState* stp) } /* - * Special error message instruction. - */ - if (stp->genop->op == genop_too_old_compiler_0) { - LoadError0(stp, "please re-compile this module with an " - ERLANG_OTP_RELEASE " compiler"); - } - - /* * From the collected generic instruction, find the specific * instruction. */ @@ -1725,7 +2029,27 @@ load_code(LoaderState* stp) ERLANG_OTP_RELEASE " compiler "); } - LoadError0(stp, "no specific operation found"); + /* + * Some generic instructions should have a special + * error message. + */ + switch (stp->genop->op) { + case genop_too_old_compiler_0: + LoadError0(stp, "please re-compile this module with an " + ERLANG_OTP_RELEASE " compiler"); + case genop_unsupported_guard_bif_3: + { + Eterm Mod = (Eterm) stp->genop->a[0].val; + Eterm Name = (Eterm) stp->genop->a[1].val; + Uint arity = (Uint) stp->genop->a[2].val; + FREE_GENOP(stp, stp->genop); + stp->genop = 0; + LoadError3(stp, "unsupported guard BIF: %T:%T/%d\n", + Mod, Name, arity); + } + default: + LoadError0(stp, "no specific operation found"); + } } stp->specific_op = specific; @@ -2014,7 +2338,6 @@ load_code(LoaderState* stp) case op_i_func_info_IaaI: { Uint offset; - enum { FINFO_SZ = 5 }; if (function_number >= stp->num_functions) { LoadError1(stp, "too many functions in module (header said %d)", @@ -2022,27 +2345,37 @@ load_code(LoaderState* stp) } if (stp->may_load_nif) { - const int finfo_ix = ci - FINFO_SZ; + const int finfo_ix = ci - FUNC_INFO_SZ; enum { MIN_FUNC_SZ = 3 }; if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) { /* Must make room for call_nif op */ int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); ASSERT(pad > 0 && pad < MIN_FUNC_SZ); CodeNeed(pad); - sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(BeamInstr)); + sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], + FUNC_INFO_SZ*sizeof(BeamInstr)); sys_memset(&code[finfo_ix], 0, pad*sizeof(BeamInstr)); ci += pad; stp->labels[last_label].value += pad; } } last_func_start = ci; + + /* + * Save current offset of into the line instruction array. + */ + + if (stp->func_line) { + stp->func_line[function_number] = stp->current_li; + } + /* * Save context for error messages. */ stp->function = code[ci-2]; stp->arity = code[ci-1]; - ASSERT(stp->labels[last_label].value == ci - FINFO_SZ); + ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); offset = MI_FUNCTIONS + function_number; code[offset] = stp->labels[last_label].patches; stp->labels[last_label].patches = offset; @@ -2065,32 +2398,6 @@ load_code(LoaderState* stp) stp->on_load = ci; break; case op_bs_put_string_II: - { - /* - * At entry: - * - * code[ci-3] &&lb_i_new_bs_put_string_II - * code[ci-2] length of string - * code[ci-1] offset into string table - * - * Since we don't know the address of the string table yet, - * just check the offset and length for validity, and use - * the instruction field as a link field to link all put_string - * instructions into a single linked list. At exit: - * - * code[ci-3] pointer to next i_new_bs_put_string instruction (or 0 - * if this is the last) - */ - Uint offset = code[ci-1]; - Uint len = code[ci-2]; - unsigned strtab_size = stp->chunks[STR_CHUNK].size; - if (offset > strtab_size || offset + len > strtab_size) { - LoadError2(stp, "invalid string reference %d, size %d", offset, len); - } - code[ci-3] = stp->new_bs_put_strings; - stp->new_bs_put_strings = ci - 3; - } - break; case op_i_bs_match_string_rfII: case op_i_bs_match_string_xfII: new_string_patch(stp, ci-1); @@ -2105,6 +2412,45 @@ load_code(LoaderState* stp) stp->catches = ci-3; break; + case op_line_I: + if (stp->line_item) { + BeamInstr item = code[ci-1]; + BeamInstr loc; + int li; + if (item >= stp->num_line_items) { + LoadError2(stp, "line instruction index overflow (%d/%d)", + item, stp->num_line_items); + } + li = stp->current_li; + if (li >= stp->num_line_instrs) { + LoadError2(stp, "line instruction table overflow (%d/%d)", + li, stp->num_line_instrs); + } + loc = stp->line_item[item]; + + if (ci - 2 == last_func_start) { + /* + * This line instruction directly follows the func_info + * instruction. Its address must be adjusted to point to + * func_info instruction. + */ + stp->line_instr[li].pos = last_func_start - FUNC_INFO_SZ; + stp->line_instr[li].loc = stp->line_item[item]; + stp->current_li++; + } else if (li <= stp->func_line[function_number-1] || + stp->line_instr[li-1].loc != loc) { + /* + * Only store the location if it is different + * from the previous location in the same function. + */ + stp->line_instr[li].pos = ci - 2; + stp->line_instr[li].loc = stp->line_item[item]; + stp->current_li++; + } + } + ci -= 2; /* Get rid of the instruction */ + break; + /* * End of code found. */ @@ -2141,6 +2487,8 @@ load_code(LoaderState* stp) #define no_fpe_signals(St) 0 #endif +#define never(St) 0 + /* * Predicate that tests whether a jump table can be used. */ @@ -3152,7 +3500,6 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr } size = max - min + 1; - /* * Allocate structure and fill in the fixed fields. */ @@ -3184,7 +3531,7 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr op->a[i] = Fail; } for (i = 0; i < Size.val; i += 2) { - int index; + Sint index; index = fixed_args+Rest[i].val-min; ASSERT(fixed_args <= index && index < arity); op->a[index] = Rest[i+1]; @@ -3396,10 +3743,7 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, BifFunction bf; NEW_GENOP(stp, op); - op->op = genop_i_gc_bif1_5; - op->arity = 5; - op->a[0] = Fail; - op->a[1].type = TAG_u; + op->next = NULL; bf = stp->import[Bif.val].bf; /* The translations here need to have a reverse counterpart in beam_emu.c:translate_gc_bif for error handling to work properly. */ @@ -3420,19 +3764,30 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, } else if (bf == trunc_1) { op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1; } else { - abort(); + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; } + op->op = genop_i_gc_bif1_5; + op->arity = 5; + op->a[0] = Fail; + op->a[1].type = TAG_u; op->a[2] = Src; op->a[3] = Live; op->a[4] = Dst; - op->next = NULL; return op; } /* - * This is used by the ops.tab rule that rewrites gc_bifs with two parameters + * This is used by the ops.tab rule that rewrites gc_bifs with two parameters. * The instruction returned is then again rewritten to an i_load instruction - * folowed by i_gc_bif2_jIId, to handle literals properly. + * followed by i_gc_bif2_jIId, to handle literals properly. * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is * always rewritten, regardless of if there actually are any literals. */ @@ -3444,31 +3799,39 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, BifFunction bf; NEW_GENOP(stp, op); - op->op = genop_ii_gc_bif2_6; - op->arity = 6; - op->a[0] = Fail; - op->a[1].type = TAG_u; + op->next = NULL; bf = stp->import[Bif.val].bf; /* The translations here need to have a reverse counterpart in beam_emu.c:translate_gc_bif for error handling to work properly. */ if (bf == binary_part_2) { op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; } else { - abort(); + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; } + op->op = genop_ii_gc_bif2_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1].type = TAG_u; op->a[2] = S1; op->a[3] = S2; op->a[4] = Live; op->a[5] = Dst; - op->next = NULL; return op; } /* - * This is used by the ops.tab rule that rewrites gc_bifs with three parameters + * This is used by the ops.tab rule that rewrites gc_bifs with three parameters. * The instruction returned is then again rewritten to a move instruction that * uses r[0] for temp storage, followed by an i_load instruction, - * folowed by i_gc_bif3_jIsId, to handle literals properly. Rewriting + * followed by i_gc_bif3_jIsId, to handle literals properly. Rewriting * always occur, as with the gc_bif2 counterpart. */ static GenOp* @@ -3479,18 +3842,27 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, BifFunction bf; NEW_GENOP(stp, op); - op->op = genop_ii_gc_bif3_7; - op->arity = 7; - op->a[0] = Fail; - op->a[1].type = TAG_u; + op->next = NULL; bf = stp->import[Bif.val].bf; /* The translations here need to have a reverse counterpart in beam_emu.c:translate_gc_bif for error handling to work properly. */ if (bf == binary_part_3) { op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; } else { - abort(); + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; } + op->op = genop_ii_gc_bif3_7; + op->arity = 7; + op->a[0] = Fail; + op->a[1].type = TAG_u; op->a[2] = S1; op->a[3] = S2; op->a[4] = S3; @@ -3561,15 +3933,14 @@ freeze_code(LoaderState* stp) { BeamInstr* code = stp->code; Uint *literal_end = NULL; - Uint index; int i; byte* str_table; unsigned strtab_size = stp->chunks[STR_CHUNK].size; unsigned attr_size = stp->chunks[ATTR_CHUNK].size; unsigned compile_size = stp->chunks[COMPILE_CHUNK].size; Uint size; - unsigned catches; Sint decoded_size; + Uint line_size; /* * Verify that there was a correct 'FunT' chunk if there were @@ -3580,13 +3951,19 @@ freeze_code(LoaderState* stp) LoadError0(stp, stp->lambda_error); } - /* * Calculate the final size of the code. */ - - size = (stp->ci * sizeof(BeamInstr)) + (stp->total_literal_size * sizeof(Eterm)) + - strtab_size + attr_size + compile_size; + if (stp->line_instr == 0) { + line_size = 0; + } else { + line_size = (MI_LINE_FUNC_TAB + (stp->num_functions + 1) + + (stp->current_li+1) + stp->num_fnames) * + sizeof(Eterm) + (stp->current_li+1) * stp->loc_size; + } + size = (stp->ci * sizeof(BeamInstr)) + + (stp->total_literal_size * sizeof(Eterm)) + + strtab_size + attr_size + compile_size + line_size; /* * Move the code to its final location. @@ -3623,6 +4000,8 @@ freeze_code(LoaderState* stp) Uint* low; Uint* high; LiteralPatch* lp; + struct erl_off_heap_header* off_heap = 0; + struct erl_off_heap_header** off_heap_last = &off_heap; low = (Uint *) (code+stp->ci); high = low + stp->total_literal_size; @@ -3631,6 +4010,7 @@ freeze_code(LoaderState* stp) ptr = low; for (i = 0; i < stp->num_literals; i++) { Uint offset; + struct erl_off_heap_header* t_off_heap; sys_memcpy(ptr, stp->literals[i].heap, stp->literals[i].heap_size*sizeof(Eterm)); @@ -3645,9 +4025,19 @@ freeze_code(LoaderState* stp) *ptr++ = offset_ptr(val, offset); break; case TAG_PRIMARY_HEADER: - ptr++; - if (header_is_thing(val)) { - ptr += thing_arityval(val); + if (header_is_transparent(val)) { + ptr++; + } else { + if (thing_subtag(val) == REFC_BINARY_SUBTAG) { + struct erl_off_heap_header* oh; + + oh = (struct erl_off_heap_header*) ptr; + if (oh->next) { + Eterm** uptr = (Eterm **) (void *) &oh->next; + *uptr += offset; + } + } + ptr += 1 + thing_arityval(val); } break; default: @@ -3656,7 +4046,23 @@ freeze_code(LoaderState* stp) } } ASSERT(ptr == high); + + /* + * Re-link the off_heap list for this term onto the + * off_heap list for the entire module. + */ + t_off_heap = stp->literals[i].off_heap.first; + if (t_off_heap) { + t_off_heap = (struct erl_off_heap_header *) + offset_ptr((UWord) t_off_heap, offset); + while (t_off_heap) { + *off_heap_last = t_off_heap; + off_heap_last = &t_off_heap->next; + t_off_heap = t_off_heap->next; + } + } } + code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; @@ -3674,21 +4080,72 @@ freeze_code(LoaderState* stp) } literal_end += stp->total_literal_size; } - + CHKBLK(ERTS_ALC_T_CODE,code); + /* - * Place the string table and, optionally, attributes, after the literal heap. + * If there is line information, place it here. */ - CHKBLK(ERTS_ALC_T_CODE,code); + if (stp->line_instr == 0) { + code[MI_LINE_TABLE] = (BeamInstr) 0; + str_table = (byte *) literal_end; + } else { + Eterm* line_tab = (Eterm *) literal_end; + Eterm* p; + int ftab_size = stp->num_functions; + int num_instrs = stp->current_li; + Eterm* first_line_item; + + code[MI_LINE_TABLE] = (BeamInstr) line_tab; + p = line_tab + MI_LINE_FUNC_TAB; + + first_line_item = (p + ftab_size + 1); + for (i = 0; i < ftab_size; i++) { + *p++ = (Eterm) (BeamInstr) (first_line_item + stp->func_line[i]); + } + *p++ = (Eterm) (BeamInstr) (first_line_item + num_instrs); + ASSERT(p == first_line_item); + for (i = 0; i < num_instrs; i++) { + *p++ = (Eterm) (BeamInstr) (code + stp->line_instr[i].pos); + } + *p++ = (Eterm) (BeamInstr) (code + stp->ci - 1); + + line_tab[MI_LINE_FNAME_PTR] = (Eterm) (BeamInstr) p; + memcpy(p, stp->fname, stp->num_fnames*sizeof(Eterm)); + p += stp->num_fnames; + + line_tab[MI_LINE_LOC_TAB] = (Eterm) (BeamInstr) p; + line_tab[MI_LINE_LOC_SIZE] = stp->loc_size; + if (stp->loc_size == 2) { + Uint16* locp = (Uint16 *) p; + for (i = 0; i < num_instrs; i++) { + *locp++ = (Uint16) stp->line_instr[i].loc; + } + *locp++ = LINE_INVALID_LOCATION; + str_table = (byte *) locp; + } else { + Uint32* locp = (Uint32 *) p; + ASSERT(stp->loc_size == 4); + for (i = 0; i < num_instrs; i++) { + *locp++ = stp->line_instr[i].loc; + } + *locp++ = LINE_INVALID_LOCATION; + str_table = (byte *) locp; + } - sys_memcpy(literal_end, stp->chunks[STR_CHUNK].start, strtab_size); + CHKBLK(ERTS_ALC_T_CODE,code); + } + + /* + * Place the string table and, optionally, attributes here. + */ + sys_memcpy(str_table, stp->chunks[STR_CHUNK].start, strtab_size); CHKBLK(ERTS_ALC_T_CODE,code); - str_table = (byte *) literal_end; if (attr_size) { byte* attr = str_table + strtab_size; sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size); code[MI_ATTR_PTR] = (BeamInstr) attr; code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size; - decoded_size = erts_decode_ext_size(attr, attr_size, 0); + decoded_size = erts_decode_ext_size(attr, attr_size); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of module attributes"); } @@ -3706,7 +4163,7 @@ freeze_code(LoaderState* stp) CHKBLK(ERTS_ALC_T_CODE,code); code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; CHKBLK(ERTS_ALC_T_CODE,code); - decoded_size = erts_decode_ext_size(compile_info, compile_size, 0); + decoded_size = erts_decode_ext_size(compile_info, compile_size); CHKBLK(ERTS_ALC_T_CODE,code); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of compilation information"); @@ -3723,20 +4180,8 @@ freeze_code(LoaderState* stp) ((byte *) code) + size); /* - * Go through all i_new_bs_put_strings instructions, restore the pointer to - * the instruction and convert string offsets to pointers (to the - * FIRST character). + * Patch all instructions that refer to the string table. */ - - index = stp->new_bs_put_strings; - while (index != 0) { - Uint next = code[index]; - code[index] = BeamOpCode(op_bs_put_string_II); - code[index+2] = (BeamInstr) (str_table + code[index+2]); - index = next; - } - CHKBLK(ERTS_ALC_T_CODE,code); - { StringPatch* sp = stp->string_patches; @@ -3777,21 +4222,6 @@ freeze_code(LoaderState* stp) CHKBLK(ERTS_ALC_T_CODE,code); /* - * Fix all catch_yf instructions. - */ - index = stp->catches; - catches = BEAM_CATCHES_NIL; - while (index != 0) { - BeamInstr next = code[index]; - code[index] = BeamOpCode(op_catch_yf); - catches = beam_catches_cons((BeamInstr *)code[index+2], catches); - code[index+2] = make_catch(catches); - index = next; - } - stp->catches = catches; - CHKBLK(ERTS_ALC_T_CODE,code); - - /* * Save the updated code pointer and code size. */ @@ -3816,6 +4246,26 @@ final_touch(LoaderState* stp) { int i; int on_load = stp->on_load; + unsigned catches; + Uint index; + BeamInstr* code = stp->code; + Module* modp; + + /* + * Allocate catch indices and fix up all catch_yf instructions. + */ + + index = stp->catches; + catches = BEAM_CATCHES_NIL; + while (index != 0) { + BeamInstr next = code[index]; + code[index] = BeamOpCode(op_catch_yf); + catches = beam_catches_cons((BeamInstr *)code[index+2], catches); + code[index+2] = make_catch(catches); + index = next; + } + modp = erts_put_module(stp->module); + modp->catches = catches; /* * Export functions. @@ -3899,6 +4349,7 @@ transform_engine(LoaderState* st) GenOp* instr; Uint* pc; int rval; + static Uint restart_fail[1] = {TOP_fail}; ASSERT(gen_opc[st->genop->op].transform != -1); pc = op_transform + gen_opc[st->genop->op].transform; @@ -3912,7 +4363,6 @@ transform_engine(LoaderState* st) ASSERT(restart != NULL); pc = restart; ASSERT(*pc < NUM_TOPS); /* Valid instruction? */ - ASSERT(*pc == TOP_try_me_else || *pc == TOP_fail); instr = st->genop; #define RETURN(r) rval = (r); goto do_return; @@ -3925,7 +4375,9 @@ transform_engine(LoaderState* st) op = *pc++; switch (op) { - case TOP_is_op: + case TOP_next_instr: + instr = instr->next; + ap = 0; if (instr == NULL) { /* * We'll need at least one more instruction to decide whether @@ -4112,10 +4564,6 @@ transform_engine(LoaderState* st) case TOP_next_arg: ap++; break; - case TOP_next_instr: - instr = instr->next; - ap = 0; - break; case TOP_commit: instr = instr->next; /* The next_instr was optimized away. */ @@ -4133,8 +4581,8 @@ transform_engine(LoaderState* st) #endif break; -#if defined(TOP_call) - case TOP_call: +#if defined(TOP_call_end) + case TOP_call_end: { GenOp** lastp; GenOp* new_instr; @@ -4171,7 +4619,7 @@ transform_engine(LoaderState* st) *lastp = st->genop; st->genop = new_instr; } - break; + RETURN(TE_OK); #endif case TOP_new_instr: /* @@ -4180,12 +4628,10 @@ transform_engine(LoaderState* st) NEW_GENOP(st, instr); instr->next = st->genop; st->genop = instr; + instr->op = op = *pc++; + instr->arity = gen_opc[op].arity; ap = 0; break; - case TOP_store_op: - instr->op = *pc++; - instr->arity = *pc++; - break; case TOP_store_type: i = *pc++; instr->a[ap].type = i; @@ -4195,21 +4641,25 @@ transform_engine(LoaderState* st) i = *pc++; instr->a[ap].val = i; break; - case TOP_store_var: + case TOP_store_var_next_arg: i = *pc++; ASSERT(i < TE_MAX_VARS); instr->a[ap].type = var[i].type; instr->a[ap].val = var[i].val; + ap++; break; case TOP_try_me_else: restart = pc + 1; restart += *pc++; ASSERT(*pc < NUM_TOPS); /* Valid instruction? */ break; + case TOP_try_me_else_fail: + restart = restart_fail; + break; case TOP_end: RETURN(TE_OK); case TOP_fail: - RETURN(TE_FAIL) + RETURN(TE_FAIL); default: ASSERT(0); } @@ -4532,6 +4982,8 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) lit->heap_size = heap_size; lit->heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm)); lit->term = make_boxed(lit->heap); + lit->off_heap.first = 0; + lit->off_heap.overhead = 0; *hpp = lit->heap; return stp->num_literals++; } @@ -4810,17 +5262,24 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ return result; } - /* - * Returns a pointer to {module, function, arity}, or NULL if not found. + * Find a function from the given pc and fill information in + * the FunctionInfo struct. If the full_info is non-zero, fill + * in all available information (including location in the + * source code). If no function is found, the 'current' field + * will be set to NULL. */ -BeamInstr * -find_function_from_pc(BeamInstr* pc) + +void +erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) { Range* low = modules; Range* high = low + num_loaded_modules; Range* mid = mid_module; + fi->current = NULL; + fi->needed = 5; + fi->loc = LINE_INVALID_LOCATION; while (low < high) { if (pc < mid->start) { high = mid; @@ -4837,26 +5296,160 @@ find_function_from_pc(BeamInstr* pc) high1 = mid1; } else if (pc < mid1[1]) { mid_module = mid; - return mid1[0]+2; + fi->current = mid1[0]+2; + if (full_info) { + BeamInstr** fp = (BeamInstr **) (mid->start + + MI_FUNCTIONS); + int idx = mid1 - fp; + lookup_loc(fi, pc, mid->start, idx); + } + return; } else { low1 = mid1 + 1; } } - return NULL; + return; } mid = low + (high-low) / 2; } - return NULL; +} + +static void +lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) +{ + Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; + Eterm* low; + Eterm* high; + Eterm* mid; + Eterm pc; + + if (line == 0) { + return; + } + + pc = (Eterm) (BeamInstr) orig_pc; + fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; + low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; + high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; + while (high > low) { + mid = low + (high-low) / 2; + if (pc < mid[0]) { + high = mid; + } else if (pc < mid[1]) { + int file; + int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; + + if (line[MI_LINE_LOC_SIZE] == 2) { + Uint16* loc_table = + (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + fi->loc = loc_table[index]; + } else { + Uint32* loc_table = + (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + ASSERT(line[MI_LINE_LOC_SIZE] == 4); + fi->loc = loc_table[index]; + } + if (fi->loc == LINE_INVALID_LOCATION) { + return; + } + fi->needed += 3+2+3+2; + file = LOC_FILE(fi->loc); + if (file == 0) { + /* Special case: Module name with ".erl" appended */ + Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + fi->needed += 2*(mod_atom->len+4); + } else { + Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); + fi->needed += 2*ap->len; + } + return; + } else { + low = mid + 1; + } + } +} + +/* + * Build a single {M,F,A,Loction} item to be part of + * a stack trace. + */ +Eterm* +erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) +{ + BeamInstr* current = fi->current; + Eterm loc = NIL; + + if (fi->loc != LINE_INVALID_LOCATION) { + Eterm tuple; + int line = LOC_LINE(fi->loc); + int file = LOC_FILE(fi->loc); + Eterm file_term = NIL; + + if (file == 0) { + Atom* ap = atom_tab(atom_val(fi->current[0])); + file_term = buf_to_intlist(&hp, ".erl", 4, NIL); + file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); + } else { + Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); + file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, NIL); + } + + tuple = TUPLE2(hp, am_line, make_small(line)); + hp += 3; + loc = CONS(hp, tuple, loc); + hp += 2; + tuple = TUPLE2(hp, am_file, file_term); + hp += 3; + loc = CONS(hp, tuple, loc); + hp += 2; + } + + if (is_list(args) || is_nil(args)) { + *mfa_p = TUPLE4(hp, current[0], current[1], args, loc); + } else { + Eterm arity = make_small(current[2]); + *mfa_p = TUPLE4(hp, current[0], current[1], arity, loc); + } + return hp + 5; +} + +/* + * Force setting of the current function in a FunctionInfo + * structure. No source code location will be associated with + * the function. + */ +void +erts_set_current_function(FunctionInfo* fi, BeamInstr* current) +{ + fi->current = current; + fi->needed = 5; + fi->loc = LINE_INVALID_LOCATION; +} + + +/* + * Returns a pointer to {module, function, arity}, or NULL if not found. + */ +BeamInstr* +find_function_from_pc(BeamInstr* pc) +{ + FunctionInfo fi; + + erts_lookup_function_info(&fi, pc, 0); + return fi.current; } /* * Read a specific chunk from a Beam binary. */ -Eterm -code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk) +BIF_RETTYPE +code_get_chunk_2(BIF_ALIST_2) { - LoaderState state; + Process* p = BIF_P; + Eterm Bin = BIF_ARG_1; + Eterm Chunk = BIF_ARG_2; + LoaderState* stp; Uint chunk = 0; ErlSubBin* sb; Uint offset; @@ -4868,15 +5461,16 @@ code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk) Eterm real_bin; byte* temp_alloc = NULL; + stp = erts_alloc_loader_state(); if ((start = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) { error: erts_free_aligned_binary_bytes(temp_alloc); + if (stp) { + free_state(stp); + } BIF_ERROR(p, BADARG); } - state.module = THE_NON_VALUE; /* Suppress diagnostiscs */ - state.file_name = "IFF header for Beam file"; - state.file_p = start; - state.file_left = binary_size(Bin); + stp->module = THE_NON_VALUE; /* Suppress diagnostics */ for (i = 0; i < 4; i++) { Eterm* chunkp; Eterm num; @@ -4894,25 +5488,30 @@ code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk) if (is_not_nil(Chunk)) { goto error; } - if (!scan_iff_file(&state, &chunk, 1, 1)) { - erts_free_aligned_binary_bytes(temp_alloc); - return am_undefined; + if (!init_iff_file(stp, start, binary_size(Bin)) || + !scan_iff_file(stp, &chunk, 1, 1) || + stp->chunks[0].start == NULL) { + res = am_undefined; + goto done; } ERTS_GET_REAL_BIN(Bin, real_bin, offset, bitoffs, bitsize); if (bitoffs) { - res = new_binary(p, state.chunks[0].start, state.chunks[0].size); + res = new_binary(p, stp->chunks[0].start, stp->chunks[0].size); } else { sb = (ErlSubBin *) HAlloc(p, ERL_SUB_BIN_SIZE); sb->thing_word = HEADER_SUB_BIN; sb->orig = real_bin; - sb->size = state.chunks[0].size; + sb->size = stp->chunks[0].size; sb->bitsize = 0; sb->bitoffs = 0; - sb->offs = offset + (state.chunks[0].start - start); + sb->offs = offset + (stp->chunks[0].start - start); sb->is_writable = 0; res = make_binary(sb); } + + done: erts_free_aligned_binary_bytes(temp_alloc); + free_state(stp); return res; } @@ -4920,24 +5519,34 @@ code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk) * Calculate the MD5 for a module. */ -Eterm -code_module_md5_1(Process* p, Eterm Bin) +BIF_RETTYPE +code_module_md5_1(BIF_ALIST_1) { - LoaderState state; + Process* p = BIF_P; + Eterm Bin = BIF_ARG_1; + LoaderState* stp; + byte* bytes; byte* temp_alloc = NULL; + Eterm res; - if ((state.file_p = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) { + stp = erts_alloc_loader_state(); + if ((bytes = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) { + free_state(stp); BIF_ERROR(p, BADARG); } - state.module = THE_NON_VALUE; /* Suppress diagnostiscs */ - state.file_name = "IFF header for Beam file"; - state.file_left = binary_size(Bin); - - if (!scan_iff_file(&state, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY)) { - return am_undefined; + stp->module = THE_NON_VALUE; /* Suppress diagnostiscs */ + if (!init_iff_file(stp, bytes, binary_size(Bin)) || + !scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) || + !verify_chunks(stp)) { + res = am_undefined; + goto done; } + res = new_binary(p, stp->mod_md5, sizeof(stp->mod_md5)); + + done: erts_free_aligned_binary_bytes(temp_alloc); - return new_binary(p, state.mod_md5, sizeof(state.mod_md5)); + free_state(stp); + return res; } #define WORDS_PER_FUNCTION 6 @@ -4972,7 +5581,7 @@ stub_copy_info(LoaderState* stp, if (size != 0) { memcpy(info, stp->chunks[chunk].start, size); *ptr_word = (BeamInstr) info; - decoded_size = erts_decode_ext_size(info, size, 0); + decoded_size = erts_decode_ext_size(info, size); if (decoded_size < 0) { return 0; } @@ -5179,7 +5788,17 @@ patch_funentries(Eterm Patchlist) fe = erts_get_fun_entry(Mod, uniq, index); fe->native_address = (Uint *)native_address; - erts_refc_dec(&fe->refc, 1); + + /* Deliberate MEMORY LEAK of native fun entries!!! + * + * Uncomment line below when hipe code upgrade and purging works correctly. + * Today we may get cases when old (leaked) native code of a purged module + * gets called and tries to create instances of a deleted fun entry. + * + * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge + * + * erts_refc_dec(&fe->refc, 1); + */ if (!patch(Addresses, (Uint) fe)) return 0; @@ -5200,7 +5819,7 @@ patch_funentries(Eterm Patchlist) Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) { - LoaderState state; + LoaderState* stp; BeamInstr Funcs; BeamInstr Patchlist; Eterm* tp; @@ -5213,16 +5832,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) int code_size; int rval; int i; - ErlDrvBinary* bin = NULL; byte* temp_alloc = NULL; byte* bytes; Uint size; /* - * Must initialize state.lambdas here because the error handling code + * Must initialize stp->lambdas here because the error handling code * at label 'error' uses it. */ - init_state(&state); + stp = erts_alloc_loader_state(); if (is_not_atom(Mod)) { goto error; @@ -5246,47 +5864,35 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) size = binary_size(Beam); /* - * Uncompressed if needed. - */ - if (!(size >= 4 && bytes[0] == 'F' && bytes[1] == 'O' && - bytes[2] == 'R' && bytes[3] == '1')) { - bin = (ErlDrvBinary *) erts_gzinflate_buffer((char*)bytes, size); - if (bin == NULL) { - goto error; - } - bytes = (byte*)bin->orig_bytes; - size = bin->orig_size; - } - - /* * Scan the Beam binary and read the interesting sections. */ - state.file_name = "IFF header for Beam file"; - state.file_p = bytes; - state.file_left = size; - state.module = Mod; - state.group_leader = p->group_leader; - state.num_functions = n; - if (!scan_iff_file(&state, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY)) { + stp->module = Mod; + stp->group_leader = p->group_leader; + stp->num_functions = n; + if (!init_iff_file(stp, bytes, size)) { + goto error; + } + if (!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) || + !verify_chunks(stp)) { goto error; } - define_file(&state, "code chunk header", CODE_CHUNK); - if (!read_code_header(&state)) { + define_file(stp, "code chunk header", CODE_CHUNK); + if (!read_code_header(stp)) { goto error; } - define_file(&state, "atom table", ATOM_CHUNK); - if (!load_atom_table(&state)) { + define_file(stp, "atom table", ATOM_CHUNK); + if (!load_atom_table(stp)) { goto error; } - define_file(&state, "export table", EXP_CHUNK); - if (!stub_read_export_table(&state)) { + define_file(stp, "export table", EXP_CHUNK); + if (!stub_read_export_table(stp)) { goto error; } - if (state.chunks[LAMBDA_CHUNK].size > 0) { - define_file(&state, "lambda (fun) table", LAMBDA_CHUNK); - if (!read_lambda_table(&state)) { + if (stp->chunks[LAMBDA_CHUNK].size > 0) { + define_file(stp, "lambda (fun) table", LAMBDA_CHUNK); + if (!read_lambda_table(stp)) { goto error; } } @@ -5296,8 +5902,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr); - code_size += state.chunks[ATTR_CHUNK].size; - code_size += state.chunks[COMPILE_CHUNK].size; + code_size += stp->chunks[ATTR_CHUNK].size; + code_size += stp->chunks[COMPILE_CHUNK].size; code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); if (!code) { goto error; @@ -5315,6 +5921,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_COMPILE_SIZE] = 0; code[MI_COMPILE_SIZE_ON_HEAP] = 0; code[MI_NUM_BREAKPOINTS] = 0; + code[MI_LITERALS_START] = 0; + code[MI_LITERALS_END] = 0; + code[MI_LITERALS_OFF_HEAP] = 0; code[MI_ON_LOAD_FUNCTION_PTR] = 0; ci = MI_FUNCTIONS + n + 1; @@ -5387,12 +5996,12 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ info = (byte *) fp; - info = stub_copy_info(&state, ATTR_CHUNK, info, + info = stub_copy_info(stp, ATTR_CHUNK, info, code+MI_ATTR_PTR, code+MI_ATTR_SIZE_ON_HEAP); if (info == NULL) { goto error; } - info = stub_copy_info(&state, COMPILE_CHUNK, info, + info = stub_copy_info(stp, COMPILE_CHUNK, info, code+MI_COMPILE_PTR, code+MI_COMPILE_SIZE_ON_HEAP); if (info == NULL) { goto error; @@ -5402,9 +6011,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the module in the module table. */ - rval = insert_new_code(p, 0, p->group_leader, Mod, code, code_size, - BEAM_CATCHES_NIL); - if (rval < 0) { + rval = insert_new_code(p, 0, p->group_leader, Mod, code, code_size); + if (rval != NIL) { goto error; } @@ -5414,46 +6022,19 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) fp = code + ci; for (i = 0; i < n; i++) { - stub_final_touch(&state, fp); + stub_final_touch(stp, fp); fp += WORDS_PER_FUNCTION; } if (patch_funentries(Patchlist)) { erts_free_aligned_binary_bytes(temp_alloc); - if (state.lambdas != state.def_lambdas) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); - } - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); - if (bin != NULL) { - driver_free_binary(bin); - } + free_state(stp); return Mod; } error: erts_free_aligned_binary_bytes(temp_alloc); - if (code != NULL) { - erts_free(ERTS_ALC_T_CODE, code); - } - if (state.labels != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); - } - if (state.lambdas != state.def_lambdas) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); - } - if (state.atom != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); - } - if (state.export != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); - } - if (bin != NULL) { - driver_free_binary(bin); - } - - + free_state(stp); BIF_ERROR(p, BADARG); } diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 26e3054c4b..4e22ee4d79 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -23,7 +23,9 @@ #include "beam_opcodes.h" #include "erl_process.h" -int beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module); +Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, + Eterm module); + typedef struct gen_op_entry { char* name; @@ -101,11 +103,18 @@ extern Uint erts_total_code_size; */ #define MI_LITERALS_START 8 #define MI_LITERALS_END 9 +#define MI_LITERALS_OFF_HEAP 10 + /* * Pointer to the on_load function (or NULL if none). */ -#define MI_ON_LOAD_FUNCTION_PTR 10 +#define MI_ON_LOAD_FUNCTION_PTR 11 + +/* + * Pointer to the line table (or NULL if none). + */ +#define MI_LINE_TABLE 12 /* * Start of function pointer table. This table contains pointers to @@ -116,5 +125,5 @@ extern Uint erts_total_code_size; * this table. */ -#define MI_FUNCTIONS 11 +#define MI_FUNCTIONS 13 #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 98dde066fc..55f4798892 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -36,6 +36,7 @@ #include "beam_bp.h" #include "erl_db_util.h" #include "register.h" +#include "erl_thr_progress.h" static Export* flush_monitor_message_trap = NULL; static Export* set_cpu_topology_trap = NULL; @@ -869,8 +870,6 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) } } else if (arg == am_scheduler && is_small(val)) { Sint scheduler = signed_val(val); - if (erts_common_run_queue && erts_no_schedulers > 1) - goto error; if (scheduler < 0 || erts_no_schedulers < scheduler) goto error; so.scheduler = (int) scheduler; @@ -1107,9 +1106,9 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3) /**********************************************************************/ -BIF_RETTYPE get_stacktrace_0(Process* p) +BIF_RETTYPE get_stacktrace_0(BIF_ALIST_0) { - Eterm t = build_stacktrace(p, p->ftrace); + Eterm t = build_stacktrace(BIF_P, BIF_P->ftrace); BIF_RET(t); } @@ -1119,10 +1118,10 @@ BIF_RETTYPE get_stacktrace_0(Process* p) * the process, and the final error value will be {Term,StackTrace}. */ -BIF_RETTYPE error_1(Process* p, Eterm term) +BIF_RETTYPE error_1(BIF_ALIST_1) { - p->fvalue = term; - BIF_ERROR(p, EXC_ERROR); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, EXC_ERROR); } /**********************************************************************/ @@ -1131,12 +1130,12 @@ BIF_RETTYPE error_1(Process* p, Eterm term) * in the stacktrace. */ -BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args) +BIF_RETTYPE error_2(BIF_ALIST_2) { - Eterm* hp = HAlloc(p, 3); + Eterm* hp = HAlloc(BIF_P, 3); - p->fvalue = TUPLE2(hp, value, args); - BIF_ERROR(p, EXC_ERROR_2); + BIF_P->fvalue = TUPLE2(hp, BIF_ARG_1, BIF_ARG_2); + BIF_ERROR(BIF_P, EXC_ERROR_2); } /**********************************************************************/ @@ -1146,10 +1145,10 @@ BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args) * It is useful in stub functions for NIFs. */ -BIF_RETTYPE nif_error_1(Process* p, Eterm term) +BIF_RETTYPE nif_error_1(BIF_ALIST_1) { - p->fvalue = term; - BIF_ERROR(p, EXC_ERROR); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, EXC_ERROR); } /**********************************************************************/ @@ -1159,12 +1158,12 @@ BIF_RETTYPE nif_error_1(Process* p, Eterm term) * It is useful in stub functions for NIFs. */ -BIF_RETTYPE nif_error_2(Process* p, Eterm value, Eterm args) +BIF_RETTYPE nif_error_2(BIF_ALIST_2) { - Eterm* hp = HAlloc(p, 3); + Eterm* hp = HAlloc(BIF_P, 3); - p->fvalue = TUPLE2(hp, value, args); - BIF_ERROR(p, EXC_ERROR_2); + BIF_P->fvalue = TUPLE2(hp, BIF_ARG_1, BIF_ARG_2); + BIF_ERROR(BIF_P, EXC_ERROR_2); } /**********************************************************************/ @@ -1183,14 +1182,19 @@ BIF_RETTYPE exit_1(BIF_ALIST_1) * If there is an error in the argument format, * return the atom 'badarg' instead. */ -Eterm -raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { +BIF_RETTYPE raise_3(BIF_ALIST_3) +{ + Process *c_p = BIF_P; + Eterm class = BIF_ARG_1; + Eterm value = BIF_ARG_2; + Eterm stacktrace = BIF_ARG_3; Eterm reason; Eterm l, *hp, *hp_end, *tp; int depth, cnt; size_t sz; + int must_copy = 0; struct StackTrace *s; - + if (class == am_error) { c_p->fvalue = value; reason = EXC_ERROR; @@ -1206,35 +1210,74 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { /* Check syntax of stacktrace, and count depth. * Accept anything that can be returned from erlang:get_stacktrace/0, * as well as a 2-tuple with a fun as first element that the - * error_handler may need to give us. + * error_handler may need to give us. Also allow old-style + * MFA three-tuples. */ for (l = stacktrace, depth = 0; is_list(l); l = CDR(list_val(l)), depth++) { Eterm t = CAR(list_val(l)); - int arity; + Eterm location = NIL; + if (is_not_tuple(t)) goto error; tp = tuple_val(t); - arity = arityval(tp[0]); - if ((arity == 3) && is_atom(tp[1]) && is_atom(tp[2])) continue; - if ((arity == 2) && is_fun(tp[1])) continue; - goto error; + switch (arityval(tp[0])) { + case 2: + /* {Fun,Args} */ + if (is_fun(tp[1])) { + must_copy = 1; + } else { + goto error; + } + break; + case 3: + /* + * One of: + * {Fun,Args,Location} + * {M,F,A} + */ + if (is_fun(tp[1])) { + location = tp[3]; + } else if (is_atom(tp[1]) && is_atom(tp[2])) { + must_copy = 1; + } else { + goto error; + } + break; + case 4: + if (!(is_atom(tp[1]) && is_atom(tp[2]))) { + goto error; + } + location = tp[4]; + break; + default: + goto error; + } + if (is_not_list(location) && is_not_nil(location)) { + goto error; + } } if (is_not_nil(l)) goto error; /* Create stacktrace and store */ - if (depth <= erts_backtrace_depth) { + if (erts_backtrace_depth < depth) { + depth = erts_backtrace_depth; + must_copy = 1; + } + if (must_copy) { + cnt = depth; + c_p->ftrace = NIL; + } else { + /* No need to copy the stacktrace */ cnt = 0; c_p->ftrace = stacktrace; - } else { - cnt = depth = erts_backtrace_depth; - c_p->ftrace = NIL; } + tp = &c_p->ftrace; sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm) - 1) / sizeof(Eterm); - hp = HAlloc(c_p, sz + 2*(cnt + 1)); - hp_end = hp + sz + 2*(cnt + 1); + hp = HAlloc(c_p, sz + (2+6)*(cnt + 1)); + hp_end = hp + sz + (2+6)*(cnt + 1); s = (struct StackTrace *) hp; s->header = make_neg_bignum_header(sz - 1); s->freason = reason; @@ -1242,13 +1285,29 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { s->current = NULL; s->depth = 0; hp += sz; - if (cnt > 0) { + if (must_copy) { + int cnt; + /* Copy list up to depth */ for (cnt = 0, l = stacktrace; cnt < depth; cnt++, l = CDR(list_val(l))) { + Eterm t; + Eterm *tpp; + int arity; + ASSERT(*tp == NIL); - *tp = CONS(hp, CAR(list_val(l)), *tp); + t = CAR(list_val(l)); + tpp = tuple_val(t); + arity = arityval(tpp[0]); + if (arity == 2) { + t = TUPLE3(hp, tpp[1], tpp[2], NIL); + hp += 4; + } else if (arity == 3 && is_atom(tpp[1])) { + t = TUPLE4(hp, tpp[1], tpp[2], tpp[3], NIL); + hp += 5; + } + *tp = CONS(hp, t, *tp); tp = &CDR(list_val(*tp)); hp += 2; } @@ -1256,7 +1315,7 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) { c_p->ftrace = CONS(hp, c_p->ftrace, make_big((Eterm *) s)); hp += 2; ASSERT(hp <= hp_end); - + HRelease(c_p, hp_end, hp); BIF_ERROR(c_p, reason); error: @@ -1474,8 +1533,6 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) ErtsRunQueue *old; ErtsRunQueue *new; Sint sched; - if (erts_common_run_queue && erts_no_schedulers > 1) - goto error; if (!is_small(BIF_ARG_2)) goto error; sched = signed_val(BIF_ARG_2); @@ -1674,10 +1731,10 @@ BIF_RETTYPE whereis_1(BIF_ALIST_1) * erlang:'!'/2 */ -Eterm -ebif_bang_2(Process* p, Eterm To, Eterm Message) +BIF_RETTYPE +ebif_bang_2(BIF_ALIST_2) { - return send_2(p, To, Message); + return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2); } @@ -2014,8 +2071,13 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { } -Eterm -send_3(Process *p, Eterm to, Eterm msg, Eterm opts) { +BIF_RETTYPE send_3(BIF_ALIST_3) +{ + Process *p = BIF_P; + Eterm to = BIF_ARG_1; + Eterm msg = BIF_ARG_2; + Eterm opts = BIF_ARG_3; + int connect = !0; int suspend = !0; Eterm l = opts; @@ -2079,8 +2141,13 @@ send_3(Process *p, Eterm to, Eterm msg, Eterm opts) { BIF_ERROR(p, BADARG); } -Eterm -send_2(Process *p, Eterm to, Eterm msg) { +BIF_RETTYPE send_2(BIF_ALIST_2) +{ + return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +Eterm erl_send(Process *p, Eterm to, Eterm msg) +{ Sint result = do_send(p, to, msg, !0); if (result > 0) { @@ -3256,8 +3323,11 @@ time_to_parts(Eterm date, Sint* year, Sint* month, Sint* day, /* return the universal time */ BIF_RETTYPE -localtime_to_universaltime_2(Process *p, Eterm localtime, Eterm dst) +localtime_to_universaltime_2(BIF_ALIST_2) { + Process *p = BIF_P; + Eterm localtime = BIF_ARG_1; + Eterm dst = BIF_ARG_2; Sint year, month, day; Sint hour, minute, second; int isdst; @@ -3315,6 +3385,61 @@ BIF_RETTYPE universaltime_to_localtime_1(BIF_ALIST_1) BIF_RET(TUPLE2(hp, res1, res2)); } +/* convert calendar:universaltime_to_seconds/1 */ + +BIF_RETTYPE universaltime_to_posixtime_1(BIF_ALIST_1) +{ + Sint year, month, day; + Sint hour, minute, second; + + Sint64 seconds = 0; + Eterm *hp; + Uint hsz = 0; + + if (!time_to_parts(BIF_ARG_1, &year, &month, &day, + &hour, &minute, &second)) + BIF_ERROR(BIF_P, BADARG); + + if (!univ_to_seconds(year, month, day, hour, minute, second, &seconds)) { + BIF_ERROR(BIF_P, BADARG); + } + + erts_bld_sint64(NULL, &hsz, seconds); + hp = HAlloc(BIF_P, hsz); + BIF_RET(erts_bld_sint64(&hp, NULL, seconds)); +} + +/* convert calendar:seconds_to_universaltime/1 */ + +BIF_RETTYPE posixtime_to_universaltime_1(BIF_ALIST_1) +{ + Sint year, month, day; + Sint hour, minute, second; + Eterm res1, res2; + Eterm* hp; + + Sint64 time = 0; + + if (!term_to_Sint64(BIF_ARG_1, &time)) { + BIF_ERROR(BIF_P, BADARG); + } + + if (!seconds_to_univ(time, &year, &month, &day, + &hour, &minute, &second)) { + BIF_ERROR(BIF_P, BADARG); + } + + hp = HAlloc(BIF_P, 4+4+3); + res1 = TUPLE3(hp,make_small(year),make_small(month), + make_small(day)); + hp += 4; + res2 = TUPLE3(hp,make_small(hour),make_small(minute), + make_small(second)); + hp += 4; + BIF_RET(TUPLE2(hp, res1, res2)); +} + + /**********************************************************************/ @@ -3506,9 +3631,10 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) } -Eterm -display_string_1(Process* p, Eterm string) +BIF_RETTYPE display_string_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm string = BIF_ARG_1; int len = is_string(string); char *str; @@ -3524,8 +3650,7 @@ display_string_1(Process* p, Eterm string) BIF_RET(am_true); } -Eterm -display_nl_0(Process* p) +BIF_RETTYPE display_nl_0(BIF_ALIST_0) { erts_fprintf(stderr, "\n"); BIF_RET(am_true); @@ -3589,8 +3714,13 @@ BIF_RETTYPE function_exported_3(BIF_ALIST_3) /**********************************************************************/ -BIF_RETTYPE is_builtin_3(Process* p, Eterm Mod, Eterm Name, Eterm Arity) +BIF_RETTYPE is_builtin_3(BIF_ALIST_3) { + Process* p = BIF_P; + Eterm Mod = BIF_ARG_1; + Eterm Name = BIF_ARG_2; + Eterm Arity = BIF_ARG_3; + if (is_not_atom(Mod) || is_not_atom(Name) || is_not_small(Arity)) { BIF_ERROR(p, BADARG); } @@ -3655,9 +3785,11 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3) BIF_RET(make_export(hp)); } -Eterm -fun_to_list_1(Process* p, Eterm fun) +BIF_RETTYPE fun_to_list_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm fun = BIF_ARG_1; + if (is_not_any_fun(fun)) BIF_ERROR(p, BADARG); BIF_RET(term2list_dsprintf(p, fun)); @@ -3953,11 +4085,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); H_MIN_SIZE = erts_next_heap_size(n, 0); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); @@ -3969,11 +4101,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); BIN_VH_MIN_SIZE = erts_next_heap_size(n, 0); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); @@ -3995,7 +4127,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_backtrace_depth = n; BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_trace_control_word) { - BIF_RET(db_set_trace_control_word_1(BIF_P, BIF_ARG_2)); + BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_2)); } else if (BIF_ARG_1 == am_sequential_tracer) { Eterm old_value = erts_set_system_seq_tracer(BIF_P, ERTS_PROC_LOCK_MAIN, @@ -4007,7 +4139,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) Uint i; ErlMessage* mp; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); for (i = 0; i < erts_max_processes; i++) { if (process_tab[i] != (Process*) 0) { @@ -4024,7 +4156,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); @@ -4047,8 +4179,20 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) if (is_value(res)) BIF_RET(res); } else if (ERTS_IS_ATOM_STR("cpu_topology", BIF_ARG_1)) { + erts_send_warning_to_logger_str( + BIF_P->group_leader, + "A call to erlang:system_flag(cpu_topology, _) was made.\n" + "The cpu_topology argument is deprecated and scheduled\n" + "for removal in erts-5.10/OTP-R16. For more information\n" + "see the erlang:system_flag/2 documentation.\n"); BIF_TRAP1(set_cpu_topology_trap, BIF_P, BIF_ARG_2); } else if (ERTS_IS_ATOM_STR("scheduler_bind_type", BIF_ARG_1)) { + erts_send_warning_to_logger_str( + BIF_P->group_leader, + "A call to erlang:system_flag(scheduler_bind_type, _) was\n" + "made. The scheduler_bind_type argument is deprecated and\n" + "scheduled for removal in erts-5.10/OTP-R16. For more\n" + "information see the erlang:system_flag/2 documentation.\n"); return erts_bind_schedulers(BIF_P, BIF_ARG_2); } error: @@ -4235,8 +4379,7 @@ void erts_bif_prep_await_proc_exit_data_trap(Process *c_p, Eterm pid, Eterm ret) { if (skip_current_msgq(c_p)) { - Eterm unused; - ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, pid, am_data, ret); + ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_data, ret); } } @@ -4244,8 +4387,7 @@ void erts_bif_prep_await_proc_exit_reason_trap(Process *c_p, Eterm pid) { if (skip_current_msgq(c_p)) { - Eterm unused; - ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, + ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_reason, am_undefined); } } @@ -4260,7 +4402,6 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, { ASSERT(is_atom(module) && is_atom(function)); if (skip_current_msgq(c_p)) { - Eterm unused; Eterm term; Eterm *hp; int i; @@ -4272,7 +4413,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, hp += 2; } term = TUPLE3(hp, module, function, term); - ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, pid, am_apply, term); + ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_apply, term); } } diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 8faa09feb8..d20089a9fb 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -26,14 +26,14 @@ extern Export* erts_format_cpu_topology_trap; #define BIF_P A__p -#define BIF_ALIST_0 Process* A__p -#define BIF_ALIST_1 Process* A__p, Eterm A_1 -#define BIF_ALIST_2 Process* A__p, Eterm A_1, Eterm A_2 -#define BIF_ALIST_3 Process* A__p, Eterm A_1, Eterm A_2, Eterm A_3 +#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS -#define BIF_ARG_1 A_1 -#define BIF_ARG_2 A_2 -#define BIF_ARG_3 A_3 +#define BIF_ARG_1 (BIF__ARGS[0]) +#define BIF_ARG_2 (BIF__ARGS[1]) +#define BIF_ARG_3 (BIF__ARGS[2]) #define BUMP_ALL_REDS(p) do { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) \ @@ -122,89 +122,106 @@ do { \ } while (0) -#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ -do { \ - (Proc)->arity = 0; \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ +#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ +do { \ + (Proc)->arity = 0; \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ } while (0) -#define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0) \ -do { \ - (Proc)->arity = 1; \ - (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ +#define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->arity = 1; \ + reg[0] = (Eterm) (A0); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ } while (0) -#define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1) \ -do { \ - (Proc)->arity = 2; \ - (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - (Proc)->def_arg_reg[1] = (Eterm) (A1); \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ +#define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->arity = 2; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->arity = 3; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->freason = TRAP; \ + (Ret) = THE_NON_VALUE; \ } while (0) -#define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2)\ +#define ERTS_BIF_PREP_TRAP3_NO_RET(Trap, Proc, A0, A1, A2)\ do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ (Proc)->arity = 3; \ - (Proc)->def_arg_reg[0] = (Eterm) (A0); \ - (Proc)->def_arg_reg[1] = (Eterm) (A1); \ - (Proc)->def_arg_reg[2] = (Eterm) (A2); \ - *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + (Proc)->i = (BeamInstr*) ((Trap)->address); \ (Proc)->freason = TRAP; \ - (Ret) = THE_NON_VALUE; \ } while (0) -#define BIF_TRAP0(p, Trap_) do { \ - (p)->arity = 0; \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP0(p, Trap_) do { \ + (p)->arity = 0; \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP1(Trap_, p, A0) do { \ - (p)->arity = 1; \ - (p)->def_arg_reg[0] = (A0); \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP1(Trap_, p, A0) do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + (p)->arity = 1; \ + reg[0] = (A0); \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP2(Trap_, p, A0, A1) do { \ - (p)->arity = 2; \ - (p)->def_arg_reg[0] = (A0); \ - (p)->def_arg_reg[1] = (A1); \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP2(Trap_, p, A0, A1) do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + (p)->arity = 2; \ + reg[0] = (A0); \ + reg[1] = (A1); \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP3(Trap_, p, A0, A1, A2) do { \ - (p)->arity = 3; \ - (p)->def_arg_reg[0] = (A0); \ - (p)->def_arg_reg[1] = (A1); \ - (p)->def_arg_reg[2] = (A2); \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) ((Trap_)->address); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP3(Trap_, p, A0, A1, A2) do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + (p)->arity = 3; \ + reg[0] = (A0); \ + reg[1] = (A1); \ + reg[2] = (A2); \ + (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP_CODE_PTR_0(p, Code_) do { \ - (p)->arity = 0; \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP_CODE_PTR_0(p, Code_) do { \ + (p)->arity = 0; \ + (p)->i = (BeamInstr*) (Code_); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) -#define BIF_TRAP_CODE_PTR_(p, Code_) do { \ - *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ - (p)->freason = TRAP; \ - return THE_NON_VALUE; \ +#define BIF_TRAP_CODE_PTR_(p, Code_) do { \ + (p)-> i = (BeamInstr*) (Code_); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ } while(0) extern Export bif_return_trap_export; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 987008c937..8cc568b16c 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -806,6 +806,12 @@ bif file:native_name_encoding/0 # bif erlang:check_old_code/1 + +# +# New in R15B +# +bif erlang:universaltime_to_posixtime/1 +bif erlang:posixtime_to_universaltime/1 # # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index d18de9ae5d..976f05c990 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -310,12 +310,12 @@ #define DREM(a1,a0,b,r) do { \ ErtsDigit __a1 = (a1); \ ErtsDigit __b = (b); \ - ErtsDigit __q0; \ + ERTS_DECLARE_DUMMY(ErtsDigit __q0); \ DDIVREM((__a1 % __b), (a0), __b, __q0, r); \ } while(0) #define DDIV(a1,a0,b,q) do { \ - ErtsDigit _tmp; \ + ERTS_DECLARE_DUMMY(ErtsDigit _tmp); \ DDIVREM(a1,a0,b,q,_tmp); \ } while(0) @@ -413,8 +413,8 @@ } while(0) #define DDIV2(a1,a0,b1,b0,q) do { \ - ErtsDigit _tmp_r1; \ - ErtsDigit _tmp_r0; \ + ERTS_DECLARE_DUMMY(ErtsDigit _tmp_r1); \ + ERTS_DECLARE_DUMMY(ErtsDigit _tmp_r0); \ D2DIVREM(a1,a0,b1,b0,q,_tmp_r1,_tmp_r0); \ } while(0) @@ -810,7 +810,9 @@ static dsize_t D_div(ErtsDigit* x, dsize_t xl, ErtsDigit d, ErtsDigit* q, ErtsDi } do { - ErtsDigit q0, a0, b1, b0, b; + ErtsDigit q0, a0, b0; + ERTS_DECLARE_DUMMY(ErtsDigit b); + ERTS_DECLARE_DUMMY(ErtsDigit b1); if (d > a1) { a0 = *xp; @@ -1323,7 +1325,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, return 1; } else { - long ay = (y < 0) ? -y : y; + SWord ay = (y < 0) ? -y : y; int bw = ay / D_EXP; int sw = ay % D_EXP; dsize_t rl; @@ -1448,6 +1450,20 @@ erts_make_integer(Uint x, Process *p) return uint_to_big(x,hp); } } +/* + * As erts_make_integer, but from a whole UWord. + */ +Eterm +erts_make_integer_from_uword(UWord x, Process *p) +{ + Eterm* hp; + if (IS_USMALL(0,x)) + return make_small(x); + else { + hp = HAlloc(p, BIG_UWORD_HEAP_SIZE(x)); + return uword_to_big(x,hp); + } +} /* ** convert Uint to bigint @@ -1584,6 +1600,62 @@ big_to_double(Wterm x, double* resp) return 0; } +/* + * Logic has been copied from erl_bif_guard.c and slightly + * modified to use a static instead of dynamic heap + */ +Eterm +double_to_big(double x, Eterm *heap) +{ + int is_negative; + int ds; + ErtsDigit* xp; + Eterm res; + int i; + size_t sz; + Eterm* hp; + double dbase; + + if (x >= 0) { + is_negative = 0; + } else { + is_negative = 1; + x = -x; + } + + /* Unscale & (calculate exponent) */ + ds = 0; + dbase = ((double) (D_MASK) + 1); + while (x >= 1.0) { + x /= dbase; /* "shift" right */ + ds++; + } + sz = BIG_NEED_SIZE(ds); /* number of words including arity */ + + hp = heap; + res = make_big(hp); + xp = (ErtsDigit*) (hp + 1); + + for (i = ds - 1; i >= 0; i--) { + ErtsDigit d; + + x *= dbase; /* "shift" left */ + d = x; /* trunc */ + xp[i] = d; /* store digit */ + x -= d; /* remove integer part */ + } + while ((ds & (BIG_DIGITS_PER_WORD - 1)) != 0) { + xp[ds++] = 0; + } + + if (is_negative) { + *hp = make_neg_bignum_header(sz-1); + } else { + *hp = make_pos_bignum_header(sz-1); + } + return res; +} + /* ** Estimate the number of decimal digits (include sign) diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 2afc37004f..7eb1e5afe2 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -140,10 +140,12 @@ Eterm big_lshift(Eterm, Sint, Eterm*); int big_comp (Wterm, Wterm); int big_ucomp (Eterm, Eterm); int big_to_double(Wterm x, double* resp); +Eterm double_to_big(double, Eterm*); Eterm small_to_big(Sint, Eterm*); Eterm uint_to_big(Uint, Eterm*); Eterm uword_to_big(UWord, Eterm*); Eterm erts_make_integer(Uint, Process *); +Eterm erts_make_integer_from_uword(UWord x, Process *p); dsize_t big_bytes(Eterm); Eterm bytes_to_big(byte*, dsize_t, int, Eterm*); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 1fb39c6c67..3d2725e239 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -47,7 +47,7 @@ erts_init_binary(void) away. If not, this test is not very expensive... */ erl_exit(ERTS_ABORT_EXIT, "Internal error: Address of orig_bytes[0] of a Binary" - "is *not* 8-byte aligned\n"); + " is *not* 8-byte aligned\n"); } } @@ -356,8 +356,10 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; Uint size; - int offset; byte* bytes; +#ifdef DEBUG + int offset; +#endif if (is_nil(arg)) { BIF_RET(new_binary(p,(byte*)"",0)); @@ -372,7 +374,11 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) } bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); - offset = io_list_to_buf(arg, (char*) bytes, size); +#ifdef DEBUG + offset = +#endif + io_list_to_buf(arg, (char*) bytes, size); + ASSERT(offset == 0); BIF_RET(bin); diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 432b3d0780..0d3b6a4dba 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -37,6 +37,7 @@ #include "beam_load.h" #include "erl_instrument.h" #include "erl_bif_timer.h" +#include "erl_thr_progress.h" /* Forward declarations -- should really appear somewhere else */ static void process_killer(void); @@ -94,7 +95,7 @@ process_killer(void) erts_printf("(k)ill (n)ext (r)eturn:\n"); while(1) { if ((j = sys_get_key(0)) <= 0) - halt_0(0); + erl_exit(0, ""); switch(j) { case 'k': if (rp->status == P_WAITING) { @@ -181,6 +182,7 @@ print_process_info(int to, void *to_arg, Process *p) { int garbing = 0; int running = 0; + time_t tmp_t; struct saved_calls *scb; /* display the PID */ @@ -243,8 +245,8 @@ print_process_info(int to, void *to_arg, Process *p) } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); - - erts_print(to, to_arg, "Started: %s", ctime((time_t*)&p->started.tv_sec)); + tmp_t = p->started.tv_sec; + erts_print(to, to_arg, "Started: %s", ctime(&tmp_t)); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len); @@ -653,20 +655,18 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; - /* Wait for all threads to block. If all threads haven't blocked +#ifdef ERTS_SMP + /* + * Wait for all managed threads to block. If all threads haven't blocked * after a minute, we go anyway and hope for the best... * * We do not release system again. We expect an exit() or abort() after * dump has been written. - * - * NOTE: We allow gc therefore it is important not to lock *any* - * process locks. */ - erts_smp_emergency_block_system(60000, ERTS_BS_FLG_ALLOW_GC); + erts_thr_progress_fatal_error_block(60000); /* Either worked or not... */ /* Allow us to pass certain places without locking... */ -#ifdef ERTS_SMP erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1); erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1); #else diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 90201f3a90..1d968fb147 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -134,7 +134,7 @@ Uint size_object(Eterm obj) case SUB_BINARY_SUBTAG: { Eterm real_bin; - Uint offset; /* Not used. */ + ERTS_DECLARE_DUMMY(Uint offset); /* Not used. */ Uint bitsize; Uint bitoffs; Uint extra_bytes; diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index ad042ec088..cfcdb72636 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -41,6 +41,7 @@ #include "bif.h" #include "external.h" #include "erl_binary.h" +#include "erl_thr_progress.h" /* Turn this on to get printouts of all distribution messages * which go on the line @@ -430,11 +431,11 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); nodename = erts_this_dist_entry->sysname; - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); + erts_smp_thr_progress_block(); erts_set_this_node(am_Noname, 0); erts_is_alive = 0; send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nd_reason); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); } else { /* recursive call via erts_do_exit_port() will end up here */ @@ -535,7 +536,7 @@ alloc_dist_obuf(Uint size) Binary *bin = erts_bin_drv_alloc(obuf_size); bin->flags = BIN_FLAG_DRV; erts_refc_init(&bin->refc, 1); - bin->orig_size = (long) obuf_size; + bin->orig_size = (SWord) obuf_size; obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0]; #ifdef DEBUG obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; @@ -967,7 +968,7 @@ int erts_net_message(Port *prt, res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache); if (res >= 0) - res = ctl_len = erts_decode_dist_ext_size(&ede, 0); + res = ctl_len = erts_decode_dist_ext_size(&ede); else { #ifdef ERTS_DIST_MSG_DBG erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n"); @@ -2330,11 +2331,11 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) #endif erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); + erts_smp_thr_progress_block(); erts_set_this_node(BIF_ARG_1, (Uint32) creation); erts_is_alive = 1; send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); @@ -2730,85 +2731,92 @@ BIF_RETTYPE is_alive_0(BIF_ALIST_0) /**********************************************************************/ /* erlang:monitor_node(Node, Bool, Options) -> Bool */ -BIF_RETTYPE monitor_node_3(BIF_ALIST_3) +static BIF_RETTYPE +monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) { DistEntry *dep; ErtsLink *lnk; Eterm l; - for (l = BIF_ARG_3; l != NIL && is_list(l); l = CDR(list_val(l))) { + for (l = Options; l != NIL && is_list(l); l = CDR(list_val(l))) { Eterm t = CAR(list_val(l)); /* allow_passive_connect the only available option right now */ if (t != am_allow_passive_connect) { - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } } if (l != NIL) { - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - if (is_not_atom(BIF_ARG_1) || - ((BIF_ARG_2 != am_true) && (BIF_ARG_2 != am_false)) || + if (is_not_atom(Node) || + ((Bool != am_true) && (Bool != am_false)) || ((erts_this_node->sysname == am_Noname) - && (BIF_ARG_1 != erts_this_node->sysname))) { - BIF_ERROR(BIF_P, BADARG); + && (Node != erts_this_node->sysname))) { + BIF_ERROR(p, BADARG); } - dep = erts_sysname_to_connected_dist_entry(BIF_ARG_1); + dep = erts_sysname_to_connected_dist_entry(Node); if (!dep) { do_trap: - BIF_TRAP3(dmonitor_node_trap, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options); } if (dep == erts_this_dist_entry) goto done; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK); erts_smp_de_rlock(dep); if (ERTS_DE_IS_NOT_CONNECTED(dep)) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); erts_smp_de_runlock(dep); goto do_trap; } erts_smp_de_links_lock(dep); erts_smp_de_runlock(dep); - if (BIF_ARG_2 == am_true) { + if (Bool == am_true) { ASSERT(dep->cid != NIL); lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, - BIF_P->id); + p->id); ++ERTS_LINK_REFC(lnk); - lnk = erts_add_or_lookup_link(&(BIF_P->nlinks), LINK_NODE, BIF_ARG_1); + lnk = erts_add_or_lookup_link(&(p->nlinks), LINK_NODE, Node); ++ERTS_LINK_REFC(lnk); } else { - lnk = erts_lookup_link(dep->node_links, BIF_P->id); + lnk = erts_lookup_link(dep->node_links, p->id); if (lnk != NULL) { if ((--ERTS_LINK_REFC(lnk)) == 0) { erts_destroy_link(erts_remove_link(&(dep->node_links), - BIF_P->id)); + p->id)); } } - lnk = erts_lookup_link(BIF_P->nlinks, BIF_ARG_1); + lnk = erts_lookup_link(p->nlinks, Node); if (lnk != NULL) { if ((--ERTS_LINK_REFC(lnk)) == 0) { - erts_destroy_link(erts_remove_link(&(BIF_P->nlinks), - BIF_ARG_1)); + erts_destroy_link(erts_remove_link(&(p->nlinks), + Node)); } } } erts_smp_de_links_unlock(dep); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); done: erts_deref_dist_entry(dep); BIF_RET(am_true); } +BIF_RETTYPE monitor_node_3(BIF_ALIST_3) +{ + BIF_RET(monitor_node(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3)); +} + + /* monitor_node(Node, Bool) -> Bool */ BIF_RETTYPE monitor_node_2(BIF_ALIST_2) { - BIF_RET(monitor_node_3(BIF_P,BIF_ARG_1,BIF_ARG_2,NIL)); + BIF_RET(monitor_node(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL)); } BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 80951c9b50..1e75afe6f6 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -41,6 +41,7 @@ #include "erl_monitors.h" #include "erl_bif_timer.h" #include "erl_cpu_topology.h" +#include "erl_thr_queue.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) #include "erl_check_io.h" #endif @@ -524,6 +525,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = sizeof(ErtsDrvSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] = sizeof(ErlMessage); +#ifdef ERTS_SMP + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)] + = sizeof(ErtsThrQElement_t); +#endif #ifdef HARD_DEBUG hdbg_init(); #endif @@ -1866,9 +1871,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) int only_one_value = 0; ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}}; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_smp_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); /* Figure out whats wanted... */ @@ -2506,9 +2509,7 @@ erts_allocator_info(int to, void *arg) { ErtsAlcType_t a; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_smp_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { int ai; @@ -2757,16 +2758,18 @@ erts_allocator_options(void *proc) void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size) { - UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)); + UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1) +#ifdef VALGRIND + + sizeof(UWord) +#endif + ); #ifdef VALGRIND - { /* Avoid Leak_PossiblyLost */ - static UWord vg_root_set[10]; - static unsigned ix = 0; - if (ix >= sizeof(vg_root_set) / sizeof(*vg_root_set)) { - erl_exit(ERTS_ABORT_EXIT, "Too many erts_alloc_permanent_cache_aligned's\n"); - } - vg_root_set[ix++] = v; /* not thread safe */ + { /* Link them to avoid Leak_PossiblyLost */ + static UWord* first_in_list = NULL; + *(UWord**)v = first_in_list; + first_in_list = (UWord*) v; + v += sizeof(UWord); } #endif @@ -2874,8 +2877,9 @@ reply_alloc_info(void *vair) ainfo); ainfo = erts_bld_tuple(hpp, szp, 2, erts_bld_atom(hpp, szp, - "otps"), + "options"), ainfo); + ainfo = erts_bld_cons(hpp, szp,ainfo,NIL); } ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom, @@ -3074,10 +3078,10 @@ erts_request_alloc_info(struct process *c_p, #ifdef ERTS_SMP if (erts_no_schedulers > 1) - erts_smp_schedule_misc_aux_work(1, - erts_no_schedulers, - reply_alloc_info, - (void *) air); + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_alloc_info, + (void *) air); #endif reply_alloc_info((void *) air); @@ -3114,10 +3118,10 @@ void *safe_realloc(void *ptr, Uint sz) \* */ #define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") -unsigned long erts_alc_test(unsigned long op, - unsigned long a1, - unsigned long a2, - unsigned long a3) +UWord erts_alc_test(UWord op, + UWord a1, + UWord a2, + UWord a3) { switch (op >> 8) { case 0x0: return erts_alcu_test(op, a1, a2); @@ -3131,24 +3135,24 @@ unsigned long erts_alc_test(unsigned long op, case 0xf00: #ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) - return (unsigned long) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF, (void *) a1, (Uint) a2); else #endif - return (unsigned long) erts_alcu_alloc(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_alloc(ERTS_ALC_T_UNDEF, (void *) a1, (Uint) a2); case 0xf01: #ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) - return (unsigned long) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2, (Uint) a3); else #endif - return (unsigned long) erts_alcu_realloc(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_realloc(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2, (Uint) a3); @@ -3178,7 +3182,7 @@ unsigned long erts_alc_test(unsigned long op, if (argv[i][0] == '-' && argv[i][1] == 't') handle_au_arg(&init, &argv[i][2], argv, &i); else - return (unsigned long) NULL; + return (UWord) NULL; i++; } } @@ -3219,25 +3223,25 @@ unsigned long erts_alc_test(unsigned long op, break; } - return (unsigned long) allctr; + return (UWord) allctr; } case 0xf04: erts_alcu_stop((Allctr_t *) a1); erts_free(ERTS_ALC_T_UNDEF, (void *) a1); break; #ifdef USE_THREADS - case 0xf05: return (unsigned long) 1; - case 0xf06: return (unsigned long) ((Allctr_t *) a1)->thread_safe; + case 0xf05: return (UWord) 1; + case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe; #ifdef ETHR_NO_FORKSAFETY - case 0xf07: return (unsigned long) 0; + case 0xf07: return (UWord) 0; #else - case 0xf07: return (unsigned long) ((Allctr_t *) a1)->thread_safe; + case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe; #endif case 0xf08: { ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_mutex)); if (ethr_mutex_init(mtx) != 0) ERTS_ALC_TEST_ABORT; - return (unsigned long) mtx; + return (UWord) mtx; } case 0xf09: { ethr_mutex *mtx = (ethr_mutex *) a1; @@ -3256,7 +3260,7 @@ unsigned long erts_alc_test(unsigned long op, ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); if (ethr_cond_init(cnd) != 0) ERTS_ALC_TEST_ABORT; - return (unsigned long) cnd; + return (UWord) cnd; } case 0xf0d: { ethr_cond *cnd = (ethr_cond *) a1; @@ -3282,7 +3286,7 @@ unsigned long erts_alc_test(unsigned long op, (void *) a2, NULL) != 0) ERTS_ALC_TEST_ABORT; - return (unsigned long) tid; + return (UWord) tid; } case 0xf11: { ethr_tid *tid = (ethr_tid *) a1; @@ -3299,13 +3303,13 @@ unsigned long erts_alc_test(unsigned long op, default: break; } - return (unsigned long) 0; + return (UWord) 0; default: break; } ASSERT(0); - return ~((unsigned long) 0); + return ~((UWord) 0); } #ifdef DEBUG @@ -3541,7 +3545,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) erl_exit(ERTS_ABORT_EXIT, "ERROR: Fence at beginning of memory block (p=0x%u) " "clobbered.\n", - (unsigned long) ptr); + (UWord) ptr); } memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); @@ -3558,12 +3562,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) erl_exit(ERTS_ABORT_EXIT, "ERROR: Fence at end of memory block (p=0x%u, sz=%u) " "clobbered.\n", - (unsigned long) ptr, (unsigned long) sz); + (UWord) ptr, (UWord) sz); if (found_type != GET_TYPE_OF_PATTERN(post_pattern)) erl_exit(ERTS_ABORT_EXIT, "ERROR: Fence around memory block (p=0x%u, sz=%u) " "clobbered.\n", - (unsigned long) ptr, (unsigned long) sz); + (UWord) ptr, (UWord) sz); ftype = type_no_str(found_type); if (!ftype) { @@ -3586,7 +3590,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) erl_exit(ERTS_ABORT_EXIT, "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\"," " but %s as type \"%s\".\n", - (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype); + (UWord) ptr, (UWord) sz, ftype, op_str, otype); } #ifdef HARD_DEBUG diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index f4133cdb1a..991061c48e 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -80,10 +80,10 @@ void erts_alloc_late_init(void); #if defined(GET_ERTS_ALC_TEST) || defined(ERTS_ALC_INTERNAL__) /* Only for testing */ -unsigned long erts_alc_test(unsigned long, - unsigned long, - unsigned long, - unsigned long); +UWord erts_alc_test(UWord, + UWord, + UWord, + UWord); #endif #define ERTS_ALC_O_ALLOC 0 diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 7a27ee28ec..962db8b831 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -50,6 +50,15 @@ # command line argument to make_alloc_types. The variable X is false # after a "+disable X" statement or if it has never been mentioned. ++if smp ++disable threads_no_smp ++else ++if threads ++enable threads_no_smp ++else ++disable threads_no_smp ++endif ++endif # --- Allocator declarations ------------------------------------------------- # @@ -192,7 +201,7 @@ type LINEBUF STANDARD SYSTEM line_buf type IOQ STANDARD SYSTEM io_queue type BITS_BUF STANDARD SYSTEM bits_buf type TMP_DIST_BUF TEMPORARY SYSTEM tmp_dist_buf -type ASYNC_Q LONG_LIVED SYSTEM async_queue +type ASYNC_DATA LONG_LIVED SYSTEM internal_async_data type ESTACK TEMPORARY SYSTEM estack type PORT_CALL_BUF TEMPORARY SYSTEM port_call_buf type DB_TABLE ETS ETS db_tab @@ -253,6 +262,22 @@ type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts +type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q + ++if threads_no_smp +# Need thread safe allocs, but std_alloc and fix_alloc are not; +# use driver_alloc which is... +type THR_Q_EL DRIVER SYSTEM thr_q_element +type THR_Q_EL_SL DRIVER SYSTEM sl_thr_q_element +type MISC_AUX_WORK DRIVER SYSTEM misc_aux_work ++else +type THR_Q_EL STANDARD SYSTEM thr_q_element +type THR_Q_EL_SL FIXED_SIZE SYSTEM sl_thr_q_element +type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work ++endif +type THR_Q STANDARD SYSTEM thr_queue +type THR_Q_SL SHORT_LIVED SYSTEM short_lived_thr_queue +type THR_Q_LL LONG_LIVED SYSTEM long_lived_thr_queue +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -268,10 +293,9 @@ type XPORTS_LIST SHORT_LIVED SYSTEM extra_port_list type PROC_LCK_WTR LONG_LIVED SYSTEM proc_lock_waiter type PROC_LCK_QS LONG_LIVED SYSTEM proc_lock_queues type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing -type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q -type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work type THR_PRGR_IDATA LONG_LIVED SYSTEM thr_prgr_internal_data type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data +type T_THR_PRGR_DATA SHORT_LIVED SYSTEM temp_thr_prgr_data +endif # @@ -284,12 +308,6 @@ type ETHR_STD STANDARD SYSTEM ethread_standard type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived -+ifnot smp - -type ARCALLBACK LONG_LIVED SYSTEM async_ready_callback - -+endif - +endif +if shared_heap diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index daf8822564..c32938bdff 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -482,9 +482,7 @@ do { \ else { \ ERTS_SMP_LC_ASSERT( \ ethr_equal_tids((A)->debug.tid, erts_thr_self()) \ - || erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC) \ - || (ERTS_IS_CRASH_DUMPING \ - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); \ + || erts_thr_progress_is_blocking()); \ } \ } \ } while (0) @@ -3016,9 +3014,7 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); #endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); - add_2tup(hpp, szp, &res, am.t, (allctr->t - ? bld_uint(hpp, szp, (Uint) allctr->t) - : am_false)); + add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); add_2tup(hpp, szp, &res, am.e, am_true); } diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index df560a0de2..fc1eddb116 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -227,7 +227,7 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); extern int erts_have_sbmbc_alloc; -typedef union {char c[8]; long l; double d;} Unit_t; +typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; typedef struct Carrier_t_ Carrier_t; struct Carrier_t_ { diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 64fad9fe0e..5150a8a507 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -164,14 +164,14 @@ BIF_RETTYPE bxor_2(BIF_ALIST_2) BIF_RET(erts_bxor(BIF_P, BIF_ARG_1, BIF_ARG_2)); } -BIF_RETTYPE bsl_2(Process* p, Eterm arg1, Eterm arg2) +BIF_RETTYPE bsl_2(BIF_ALIST_2) { - BIF_RET(shift(p, arg1, arg2, 0)); + BIF_RET(shift(BIF_P, BIF_ARG_1, BIF_ARG_2, 0)); } -BIF_RETTYPE bsr_2(Process* p, Eterm arg1, Eterm arg2) +BIF_RETTYPE bsr_2(BIF_ALIST_2) { - BIF_RET(shift(p, arg1, arg2, 1)); + BIF_RET(shift(BIF_P, BIF_ARG_1, BIF_ARG_2, 1)); } static Eterm diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 91b64411d4..8bca9ae582 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -24,10 +24,18 @@ #include "erl_sys_driver.h" #include "global.h" #include "erl_threads.h" +#include "erl_thr_queue.h" +#include "erl_async.h" + +#define ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ 20 + +#define ERTS_ASYNC_PRINT_JOB 0 + +#if !defined(ERTS_SMP) && defined(USE_THREADS) && !ERTS_USE_ASYNC_READY_Q +# error "Need async ready queue in non-smp case" +#endif typedef struct _erl_async { - struct _erl_async* next; - struct _erl_async* prev; DE_Handle* hndl; /* The DE_Handle is needed when port is gone */ Eterm port; long async_id; @@ -35,345 +43,499 @@ typedef struct _erl_async { ErlDrvPDL pdl; void (*async_invoke)(void*); void (*async_free)(void*); -} ErlAsync; +#if ERTS_USE_ASYNC_READY_Q + Uint sched_id; + union { + ErtsThrQPrepEnQ_t *prep_enq; + ErtsThrQFinDeQ_t fin_deq; + } q; +#endif +} ErtsAsync; + +#if ERTS_USE_ASYNC_READY_Q + +/* + * We can do without the enqueue mutex since it isn't needed for + * thread safety. Its only purpose is to put async threads to sleep + * during a blast of ready async jobs. This in order to reduce + * contention on the enqueue end of the async ready queues. During + * such a blast without the enqueue mutex much cpu time is consumed + * by the async threads without them doing much progress which in turn + * slow down progress of scheduler threads. + */ +#define ERTS_USE_ASYNC_READY_ENQ_MTX 1 + +#if ERTS_USE_ASYNC_READY_ENQ_MTX typedef struct { - erts_mtx_t mtx; - erts_cnd_t cv; - erts_tid_t thr; - int len; -#ifndef ERTS_SMP - int hndl; + erts_mtx_t enq_mtx; +} ErtsAsyncReadyQXData; + #endif - ErlAsync* head; - ErlAsync* tail; -#ifdef ERTS_ENABLE_LOCK_CHECK - int no; + +typedef struct { +#if ERTS_USE_ASYNC_READY_ENQ_MTX + union { + ErtsAsyncReadyQXData data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(ErtsAsyncReadyQXData))]; + } x; #endif -} AsyncQueue; + ErtsThrQ_t thr_q; + ErtsThrQFinDeQ_t fin_deq; +} ErtsAsyncReadyQ; -static erts_smp_spinlock_t async_id_lock; -static long async_id = 0; +typedef union { + ErtsAsyncReadyQ arq; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncReadyQ))]; +} ErtsAlgndAsyncReadyQ; -#ifndef ERTS_SMP +#endif /* ERTS_USE_ASYNC_READY_Q */ -erts_mtx_t async_ready_mtx; -static ErlAsync* async_ready_list = NULL; +typedef struct { + ErtsThrQ_t thr_q; + erts_tid_t thr_id; +} ErtsAsyncQ; + +typedef union { + ErtsAsyncQ aq; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncQ))]; +} ErtsAlgndAsyncQ; +typedef struct { + int no_initialized; + erts_mtx_t mtx; + erts_cnd_t cnd; + erts_atomic_t id; +} ErtsAsyncInit; + +typedef struct { + union { + ErtsAsyncInit data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncInit))]; + } init; + ErtsAlgndAsyncQ *queue; +#if ERTS_USE_ASYNC_READY_Q + ErtsAlgndAsyncReadyQ *ready_queue; #endif +} ErtsAsyncData; -/* -** Initialize worker threads (if supported) -*/ +int erts_async_max_threads; /* Initialized by erl_init.c */ +int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */ -/* Detach from driver */ -static void async_detach(DE_Handle* dh) -{ - return; -} +static ErtsAsyncData *async; +#ifndef USE_THREADS -#ifdef USE_THREADS +void +erts_init_async(void) +{ -static AsyncQueue* async_q; +} -static void* async_main(void*); -static void async_add(ErlAsync*, AsyncQueue*); +#else -#ifndef ERTS_SMP -typedef struct ErtsAsyncReadyCallback_ ErtsAsyncReadyCallback; -struct ErtsAsyncReadyCallback_ { - struct ErtsAsyncReadyCallback_ *next; - void (*callback)(void); -}; +static void *async_main(void *); -static ErtsAsyncReadyCallback *callbacks; -static int async_handle; +static ERTS_INLINE ErtsAsyncQ * +async_q(int i) +{ + return &async->queue[i].aq; +} + +#if ERTS_USE_ASYNC_READY_Q -int erts_register_async_ready_callback(void (*funcp)(void)) +static ERTS_INLINE ErtsAsyncReadyQ * +async_ready_q(Uint sched_id) { - ErtsAsyncReadyCallback *cb = erts_alloc(ERTS_ALC_T_ARCALLBACK, - sizeof(ErtsAsyncReadyCallback)); - cb->next = callbacks; - cb->callback = funcp; - erts_mtx_lock(&async_ready_mtx); - callbacks = cb; - erts_mtx_unlock(&async_ready_mtx); - return async_handle; + return &async->ready_queue[((int)sched_id)-1].arq; } + #endif -int init_async(int hndl) +void +erts_init_async(void) { - erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; - AsyncQueue* q; - int i; + async = NULL; + if (erts_async_max_threads > 0) { +#if ERTS_USE_ASYNC_READY_Q + ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; +#endif + erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; + char *ptr; + size_t tot_size = 0; + int i; + + tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); + tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; +#if ERTS_USE_ASYNC_READY_Q + tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; +#endif - thr_opts.detached = 0; - thr_opts.suggested_stack_size = erts_async_thread_suggested_stack_size; - -#ifndef ERTS_SMP - callbacks = NULL; - async_handle = hndl; - erts_mtx_init(&async_ready_mtx, "async_ready"); - async_ready_list = NULL; -#endif - - async_id = 0; - erts_smp_spinlock_init(&async_id_lock, "async_id"); - - async_q = q = (AsyncQueue*) - (erts_async_max_threads - ? erts_alloc(ERTS_ALC_T_ASYNC_Q, - erts_async_max_threads * sizeof(AsyncQueue)) - : NULL); - for (i = 0; i < erts_async_max_threads; i++) { - q->head = NULL; - q->tail = NULL; - q->len = 0; -#ifndef ERTS_SMP - q->hndl = hndl; -#endif -#ifdef ERTS_ENABLE_LOCK_CHECK - q->no = i; -#endif - erts_mtx_init(&q->mtx, "asyncq"); - erts_cnd_init(&q->cv); - erts_thr_create(&q->thr, async_main, (void*)q, &thr_opts); - q++; - } - return 0; -} + ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA, + tot_size); + async = (ErtsAsyncData *) ptr; + ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); -int exit_async() -{ - int i; + async->init.data.no_initialized = 0; + erts_mtx_init(&async->init.data.mtx, "async_init_mtx"); + erts_cnd_init(&async->init.data.cnd); + erts_atomic_init_nob(&async->init.data.id, 0); - /* terminate threads */ - for (i = 0; i < erts_async_max_threads; i++) { - ErlAsync* a = (ErlAsync*) erts_alloc(ERTS_ALC_T_ASYNC, - sizeof(ErlAsync)); - a->port = NIL; - async_add(a, &async_q[i]); - } + async->queue = (ErtsAlgndAsyncQ *) ptr; + ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; - for (i = 0; i < erts_async_max_threads; i++) { - erts_thr_join(async_q[i].thr, NULL); - erts_mtx_destroy(&async_q[i].mtx); - erts_cnd_destroy(&async_q[i].cv); - } -#ifndef ERTS_SMP - erts_mtx_destroy(&async_ready_mtx); +#if ERTS_USE_ASYNC_READY_Q + + qinit.live.queue = ERTS_THR_Q_LIVE_LONG; + qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; + qinit.notify = erts_notify_check_async_ready_queue; + + async->ready_queue = (ErtsAlgndAsyncReadyQ *) ptr; + ptr += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; + + for (i = 1; i <= erts_no_schedulers; i++) { + ErtsAsyncReadyQ *arq = async_ready_q(i); +#if ERTS_USE_ASYNC_READY_ENQ_MTX + erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx"); #endif - if (async_q) - erts_free(ERTS_ALC_T_ASYNC_Q, (void *) async_q); - return 0; + erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq); + qinit.arg = (void *) (SWord) i; + erts_thr_q_initialize(&arq->thr_q, &qinit); + } + +#endif + + /* Create async threads... */ + + thr_opts.detached = 0; + thr_opts.suggested_stack_size + = erts_async_thread_suggested_stack_size; + + for (i = 0; i < erts_async_max_threads; i++) { + ErtsAsyncQ *aq = async_q(i); + erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts); + } + + /* Wait for async threads to initialize... */ + + erts_mtx_lock(&async->init.data.mtx); + while (async->init.data.no_initialized != erts_async_max_threads) + erts_cnd_wait(&async->init.data.cnd, &async->init.data.mtx); + erts_mtx_unlock(&async->init.data.mtx); + + erts_mtx_destroy(&async->init.data.mtx); + erts_cnd_destroy(&async->init.data.cnd); + + } } +#if ERTS_USE_ASYNC_READY_Q -static void async_add(ErlAsync* a, AsyncQueue* q) +void * +erts_get_async_ready_queue(Uint sched_id) +{ + return (void *) async ? async_ready_q(sched_id) : NULL; +} + +#endif + +static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) { if (is_internal_port(a->port)) { - ERTS_LC_ASSERT(erts_drvportid2port(a->port)); +#if ERTS_USE_ASYNC_READY_Q + ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id); + a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q); +#endif /* make sure the driver will stay around */ - driver_lock_driver(internal_port_index(a->port)); + if (a->hndl) + erts_ddll_reference_referenced_driver(a->hndl); } - erts_mtx_lock(&q->mtx); +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "-> %ld\n", a->async_id); +#endif - if (q->len == 0) { - q->head = a; - q->tail = a; - q->len = 1; - erts_cnd_signal(&q->cv); - } - else { /* no need to signal (since the worker is working) */ - a->next = q->head; - q->head->prev = a; - q->head = a; - q->len++; - } - erts_mtx_unlock(&q->mtx); + erts_thr_q_enqueue(&q->thr_q, a); } -static ErlAsync* async_get(AsyncQueue* q) +static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, + erts_tse_t *tse, + ErtsThrQPrepEnQ_t **prep_enq) { - ErlAsync* a; +#if ERTS_USE_ASYNC_READY_Q + int saved_fin_deq = 0; + ErtsThrQFinDeQ_t fin_deq; +#endif - erts_mtx_lock(&q->mtx); - while((a = q->tail) == NULL) { - erts_cnd_wait(&q->cv, &q->mtx); - } + while (1) { + ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q); + if (a) { + +#if ERTS_USE_ASYNC_READY_Q + *prep_enq = a->q.prep_enq; + erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq); + if (saved_fin_deq) + erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq); +#endif + + return a; + } + + if (ERTS_THR_Q_DIRTY != erts_thr_q_clean(q)) { + ErtsThrQFinDeQ_t tmp_fin_deq; + + erts_tse_reset(tse); + +#if ERTS_USE_ASYNC_READY_Q + chk_fin_deq: + if (erts_thr_q_get_finalize_dequeue_data(q, &tmp_fin_deq)) { + if (!saved_fin_deq) { + erts_thr_q_finalize_dequeue_state_init(&fin_deq); + saved_fin_deq = 1; + } + erts_thr_q_append_finalize_dequeue_data(&fin_deq, + &tmp_fin_deq); + } +#endif + + switch (erts_thr_q_inspect(q, 1)) { + case ERTS_THR_Q_DIRTY: + break; + case ERTS_THR_Q_NEED_THR_PRGR: #ifdef ERTS_SMP - ASSERT(a && q->tail == a); + { + ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q); + erts_thr_progress_wakeup(NULL, prgr); + /* + * We do no dequeue finalizing in hope that a new async + * job will arrive before we are woken due to thread + * progress... + */ + erts_tse_wait(tse); + break; + } #endif - if (q->head == q->tail) { - q->head = q->tail = NULL; - q->len = 0; - } - else { - q->tail->prev->next = NULL; - q->tail = q->tail->prev; - q->len--; + case ERTS_THR_Q_CLEAN: + +#if ERTS_USE_ASYNC_READY_Q + if (saved_fin_deq) { + if (erts_thr_q_finalize_dequeue(&fin_deq)) + goto chk_fin_deq; + else + saved_fin_deq = 0; + } +#endif + + erts_tse_wait(tse); + break; + + default: + ASSERT(0); + break; + } + + } } - erts_mtx_unlock(&q->mtx); - return a; } - -static int async_del(long id) +static ERTS_INLINE void call_async_ready(ErtsAsync *a) { - int i; - /* scan all queue for an entry with async_id == 'id' */ - - for (i = 0; i < erts_async_max_threads; i++) { - ErlAsync* a; - erts_mtx_lock(&async_q[i].mtx); - - a = async_q[i].head; - while(a != NULL) { - if (a->async_id == id) { - if (a->prev != NULL) - a->prev->next = a->next; - else - async_q[i].head = a->next; - if (a->next != NULL) - a->next->prev = a->prev; - else - async_q[i].tail = a->prev; - async_q[i].len--; - erts_mtx_unlock(&async_q[i].mtx); - if (a->async_free != NULL) - a->async_free(a->async_data); - async_detach(a->hndl); - erts_free(ERTS_ALC_T_ASYNC, a); - return 1; - } - a = a->next; + Port *p = erts_id2port_sflgs(a->port, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); + if (!p) { + if (a->async_free) + a->async_free(a->async_data); + } + else { + if (async_ready(p, a->async_data)) { + if (a->async_free) + a->async_free(a->async_data); } - erts_mtx_unlock(&async_q[i].mtx); + erts_port_release(p); } - return 0; + if (a->hndl) + erts_ddll_dereference_driver(a->hndl); } -static void* async_main(void* arg) +static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq) { - AsyncQueue* q = (AsyncQueue*) arg; +#if ERTS_USE_ASYNC_READY_Q + ErtsAsyncReadyQ *arq; -#ifdef ERTS_ENABLE_LOCK_CHECK - { - char buf[27]; - erts_snprintf(&buf[0], 27, "async %d", q->no); - erts_lc_set_thread_name(&buf[0]); - } + if (a->pdl) + driver_pdl_dec_refc(a->pdl); + +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "=>> %ld\n", a->async_id); #endif - while(1) { - ErlAsync* a = async_get(q); + arq = async_ready_q(a->sched_id); - if (a->port == NIL) { /* TIME TO DIE SIGNAL */ - erts_free(ERTS_ALC_T_ASYNC, (void *) a); - break; - } - else { - (*a->async_invoke)(a->async_data); - /* Major problem if the code for async_invoke - or async_free is removed during a blocking operation */ +#if ERTS_USE_ASYNC_READY_ENQ_MTX + erts_mtx_lock(&arq->x.data.enq_mtx); +#endif + + erts_thr_q_enqueue_prepared(&arq->thr_q, (void *) a, prep_enq); + +#if ERTS_USE_ASYNC_READY_ENQ_MTX + erts_mtx_unlock(&arq->x.data.enq_mtx); +#endif + +#else /* ERTS_USE_ASYNC_READY_Q */ + + call_async_ready(a); + if (a->pdl) + driver_pdl_dec_refc(a->pdl); + erts_free(ERTS_ALC_T_ASYNC, (void *) a); + +#endif /* ERTS_USE_ASYNC_READY_Q */ +} + + +static void +async_wakeup(void *vtse) +{ + erts_tse_set((erts_tse_t *) vtse); +} + +static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) +{ + ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; + erts_tse_t *tse = erts_tse_fetch(); #ifdef ERTS_SMP - { - Port *p; - p = erts_id2port_sflgs(a->port, - NULL, - 0, - ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); - if (!p) { - if (a->async_free) - (*a->async_free)(a->async_data); - } - else { - if (async_ready(p, a->async_data)) { - if (a->async_free) - (*a->async_free)(a->async_data); - } - async_detach(a->hndl); - erts_port_release(p); - } - if (a->pdl) { - driver_pdl_dec_refc(a->pdl); - } - erts_free(ERTS_ALC_T_ASYNC, (void *) a); - } -#else - if (a->pdl) { - driver_pdl_dec_refc(a->pdl); - } - erts_mtx_lock(&async_ready_mtx); - a->next = async_ready_list; - async_ready_list = a; - erts_mtx_unlock(&async_ready_mtx); - sys_async_ready(q->hndl); + ErtsThrPrgrCallbacks callbacks; + + callbacks.arg = (void *) tse; + callbacks.wakeup = async_wakeup; + callbacks.prepare_wait = NULL; + callbacks.wait = NULL; + + erts_thr_progress_register_unmanaged_thread(&callbacks); #endif - } - } - return NULL; + qinit.live.queue = ERTS_THR_Q_LIVE_LONG; + qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; + qinit.arg = (void *) tse; + qinit.notify = async_wakeup; +#if ERTS_USE_ASYNC_READY_Q + qinit.auto_finalize_dequeue = 0; +#endif + + erts_thr_q_initialize(&aq->thr_q, &qinit); + + /* Inform main thread that we are done initializing... */ + erts_mtx_lock(&async->init.data.mtx); + async->init.data.no_initialized++; + erts_cnd_signal(&async->init.data.cnd); + erts_mtx_unlock(&async->init.data.mtx); + + return tse; } +static void *async_main(void* arg) +{ + ErtsAsyncQ *aq = (ErtsAsyncQ *) arg; + erts_tse_t *tse = async_thread_init(aq); + + while (1) { + ErtsThrQPrepEnQ_t *prep_enq; + ErtsAsync *a = async_get(&aq->thr_q, tse, &prep_enq); + if (is_nil(a->port)) + break; /* Time to die */ +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "<- %ld\n", a->async_id); #endif -#ifndef ERTS_SMP + a->async_invoke(a->async_data); + + async_reply(a, prep_enq); + } + + return NULL; +} + +#endif /* USE_THREADS */ -int check_async_ready(void) +void +erts_exit_flush_async(void) { #ifdef USE_THREADS - ErtsAsyncReadyCallback *cbs; + int i; + ErtsAsync a; + a.port = NIL; + /* + * Terminate threads in order to flush queues. We do not + * bother to clean everything up since we are about to + * terminate the runtime system and a cleanup would only + * delay the termination. + */ + for (i = 0; i < erts_async_max_threads; i++) + async_add(&a, async_q(i)); + for (i = 0; i < erts_async_max_threads; i++) + erts_thr_join(async->queue[i].aq.thr_id, NULL); #endif - ErlAsync* a; - int count = 0; +} - erts_mtx_lock(&async_ready_mtx); - a = async_ready_list; - async_ready_list = NULL; -#ifdef USE_THREADS - cbs = callbacks; -#endif - erts_mtx_unlock(&async_ready_mtx); - - while(a != NULL) { - ErlAsync* a_next = a->next; - /* Every port not dead */ - Port *p = erts_id2port_sflgs(a->port, - NULL, - 0, - ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); - if (!p) { - if (a->async_free) - (*a->async_free)(a->async_data); - } - else { - count++; - if (async_ready(p, a->async_data)) { - if (a->async_free != NULL) - (*a->async_free)(a->async_data); - } - async_detach(a->hndl); - erts_port_release(p); +#if defined(USE_THREADS) && ERTS_USE_ASYNC_READY_Q + +int erts_check_async_ready(void *varq) +{ + ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq; + int res = 1; + int i; + + for (i = 0; i < ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ; i++) { + ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(&arq->thr_q); + if (!a) { + res = 0; + break; } + +#if ERTS_ASYNC_PRINT_JOB + erts_fprintf(stderr, "<<= %ld\n", a->async_id); +#endif + erts_thr_q_append_finalize_dequeue_data(&arq->fin_deq, &a->q.fin_deq); + call_async_ready(a); erts_free(ERTS_ALC_T_ASYNC, (void *) a); - a = a_next; } -#ifdef USE_THREADS - for (; cbs; cbs = cbs->next) - (*cbs->callback)(); -#endif - return count; + + erts_thr_q_finalize_dequeue(&arq->fin_deq); + + return res; } +int erts_async_ready_clean(void *varq, void *val) +{ + ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq; + ErtsThrQCleanState_t cstate; + + cstate = erts_thr_q_clean(&arq->thr_q); + + if (erts_thr_q_finalize_dequeue(&arq->fin_deq)) + return ERTS_ASYNC_READY_DIRTY; + + switch (cstate) { + case ERTS_THR_Q_DIRTY: + return ERTS_ASYNC_READY_DIRTY; + case ERTS_THR_Q_NEED_THR_PRGR: +#ifdef ERTS_SMP + *((ErtsThrPrgrVal *) val) + = erts_thr_q_need_thr_progress(&arq->thr_q); + return ERTS_ASYNC_READY_NEED_THR_PRGR; #endif + case ERTS_THR_Q_CLEAN: + break; + } + return ERTS_ASYNC_READY_CLEAN; +} +#endif /* ** Schedule async_invoke on a worker thread @@ -393,19 +555,29 @@ long driver_async(ErlDrvPort ix, unsigned int* key, void (*async_invoke)(void*), void* async_data, void (*async_free)(void*)) { - ErlAsync* a = (ErlAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErlAsync)); - Port* prt = erts_drvport2port(ix); + ErtsAsync* a; + Port* prt; long id; unsigned int qix; +#if ERTS_USE_ASYNC_READY_Q + Uint sched_id; + sched_id = erts_get_scheduler_id(); + if (!sched_id) + sched_id = 1; +#endif + prt = erts_drvport2port(ix); if (!prt) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - a->next = NULL; - a->prev = NULL; + a = (ErtsAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErtsAsync)); + +#if ERTS_USE_ASYNC_READY_Q + a->sched_id = sched_id; +#endif a->hndl = (DE_Handle*)prt->drv_ptr->handle; a->port = prt->id; a->pdl = NULL; @@ -413,12 +585,16 @@ long driver_async(ErlDrvPort ix, unsigned int* key, a->async_invoke = async_invoke; a->async_free = async_free; - erts_smp_spin_lock(&async_id_lock); - async_id = (async_id + 1) & 0x7fffffff; - if (async_id == 0) - async_id++; - id = async_id; - erts_smp_spin_unlock(&async_id_lock); + if (!async) + id = 0; + else { + do { + id = erts_atomic_inc_read_nob(&async->init.data.id); + } while (id == 0); + if (id < 0) + id *= -1; + ASSERT(id > 0); + } a->async_id = id; @@ -437,7 +613,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key, driver_pdl_inc_refc(prt->port_data_lock); a->pdl = prt->port_data_lock; } - async_add(a, &async_q[qix]); + async_add(a, async_q(qix)); return id; } #endif @@ -455,10 +631,16 @@ long driver_async(ErlDrvPort ix, unsigned int* key, int driver_async_cancel(unsigned int id) { -#ifdef USE_THREADS - if (erts_async_max_threads > 0) - return async_del(id); -#endif + /* + * Not supported anymore. Always fail (which is backward + * compatible). + * + * This functionality could be implemented again. However, + * it is (and always has been) completely useless since + * it doesn't give you any guarantees whatsoever. The user + * needs to (and always have had to) synchronize in his/her + * own code in order to get any guarantees. + */ return 0; } diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h new file mode 100644 index 0000000000..95374a8fc9 --- /dev/null +++ b/erts/emulator/beam/erl_async.h @@ -0,0 +1,66 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_ASYNC_H__ +#define ERL_ASYNC_H__ + +#define ERTS_MAX_NO_OF_ASYNC_THREADS 1024 +extern int erts_async_max_threads; +#define ERTS_ASYNC_THREAD_MIN_STACK_SIZE 16 /* Kilo words */ +#define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ +extern int erts_async_thread_suggested_stack_size; + +#ifdef USE_THREADS + +#ifdef ERTS_SMP +/* + * With smp support we can choose to have, or not to + * have an async ready queue. + */ +#define ERTS_USE_ASYNC_READY_Q 1 +#endif + +#ifndef ERTS_SMP +/* In non-smp case we *need* the async ready queue */ +# undef ERTS_USE_ASYNC_READY_Q +# define ERTS_USE_ASYNC_READY_Q 1 +#endif + +#ifndef ERTS_USE_ASYNC_READY_Q +# define ERTS_USE_ASYNC_READY_Q 0 +#endif + +#if ERTS_USE_ASYNC_READY_Q +int erts_check_async_ready(void *); +int erts_async_ready_clean(void *, void *); +void *erts_get_async_ready_queue(Uint sched_id); +#define ERTS_ASYNC_READY_CLEAN 0 +#define ERTS_ASYNC_READY_DIRTY 1 +#ifdef ERTS_SMP +#define ERTS_ASYNC_READY_NEED_THR_PRGR 2 +#endif +#endif /* ERTS_USE_ASYNC_READY_Q */ + +#endif /* USE_THREADS */ + +void erts_init_async(void); +void erts_exit_flush_async(void); + + +#endif /* ERL_ASYNC_H__ */ diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 684fa5d12f..7e7bec9b87 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -65,6 +65,10 @@ static Export binary_copy_trap_export; static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; +static BIF_RETTYPE +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +static BIF_RETTYPE +binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); void erts_init_bif_binary(void) { @@ -1148,7 +1152,7 @@ static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend, erts_free_aligned_binary_bytes(temp_alloc); return DO_BIN_MATCH_RESTART; } else { - Eterm epos = erts_make_integer(pos+hsstart,p); + Eterm epos = erts_make_integer(pos,p); Eterm erlen = erts_make_integer(rlen,p); hp = HAlloc(p,3); ret = TUPLE2(hp, epos, erlen); @@ -1399,6 +1403,12 @@ static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3) BIF_RETTYPE binary_match_3(BIF_ALIST_3) { + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +{ Uint hsstart; Uint hsend; Eterm *tp; @@ -1408,17 +1418,17 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) int runres; Eterm result; - if (is_not_binary(BIF_ARG_1)) { + if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { goto badarg; } if (hsend == 0) { BIF_RET(am_nomatch); } - if (is_tuple(BIF_ARG_2)) { - tp = tuple_val(BIF_ARG_2); + if (is_tuple(arg2)) { + tp = tuple_val(arg2); if (arityval(*tp) != 2 || is_not_atom(tp[1])) { goto badarg; } @@ -1437,13 +1447,13 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + } else if (do_binary_match_compile(arg2,&type,&bin)) { goto badarg; } - runres = do_binary_match(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin,NIL,&result); + runres = do_binary_match(p,arg1,hsstart,hsend,type,bin,NIL,&result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); - bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + Eterm *hp = HAlloc(p, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } @@ -1451,17 +1461,23 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) case DO_BIN_MATCH_OK: BIF_RET(result); case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, bin_term); + BUMP_ALL_REDS(p); + BIF_TRAP3(&binary_match_trap_export, p, arg1, result, bin_term); default: goto badarg; } badarg: - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } BIF_RETTYPE binary_matches_3(BIF_ALIST_3) { + return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +{ Uint hsstart, hsend; Eterm *tp; Eterm type; @@ -1470,17 +1486,17 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3) int runres; Eterm result; - if (is_not_binary(BIF_ARG_1)) { + if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { goto badarg; } if (hsend == 0) { BIF_RET(NIL); } - if (is_tuple(BIF_ARG_2)) { - tp = tuple_val(BIF_ARG_2); + if (is_tuple(arg2)) { + tp = tuple_val(arg2); if (arityval(*tp) != 2 || is_not_atom(tp[1])) { goto badarg; } @@ -1499,14 +1515,14 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3) goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + } else if (do_binary_match_compile(arg2,&type,&bin)) { goto badarg; } - runres = do_binary_matches(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin, + runres = do_binary_matches(p,arg1,hsstart,hsend,type,bin, NIL,&result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); - bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + Eterm *hp = HAlloc(p, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } @@ -1514,26 +1530,26 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3) case DO_BIN_MATCH_OK: BIF_RET(result); case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + BUMP_ALL_REDS(p); + BIF_TRAP3(&binary_matches_trap_export, p, arg1, result, bin_term); default: goto badarg; } badarg: - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } BIF_RETTYPE binary_match_2(BIF_ALIST_2) { - return binary_match_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + return binary_match(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); } BIF_RETTYPE binary_matches_2(BIF_ALIST_2) { - return binary_matches_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + return binary_matches(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); } @@ -1882,9 +1898,9 @@ static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) cd = (CommonData *) ERTS_MAGIC_BIN_DATA(mb); l = list; while (is_list(l)) { - Uint bitoffs; + ERTS_DECLARE_DUMMY(Uint bitoffs); Uint bitsize; - Uint offset; + ERTS_DECLARE_DUMMY(Uint offset); Eterm real_bin; ProcBin* pb; @@ -2361,7 +2377,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) { Uint n; byte *bytes; - Uint bit_offs; + ERTS_DECLARE_DUMMY(Uint bit_offs); Uint bit_size; size_t size; Uint reds = get_reds(p, BINARY_COPY_LOOP_FACTOR); @@ -2390,9 +2406,9 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) if ((target_size - size) >= reds) { Eterm orig; - Uint offset; - Uint bit_offset; - Uint bit_size; + ERTS_DECLARE_DUMMY(Uint offset); + ERTS_DECLARE_DUMMY(Uint bit_offset); + ERTS_DECLARE_DUMMY(Uint bit_size); CopyBinState *cbs; Eterm *hp; Eterm trap_term; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index d714eacd06..b2d5722e9b 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -142,9 +142,11 @@ static void ddll_no_more_references(void *vdh); * really load and add as LOADED {ok,loaded} {ok,pending_driver} * {error, permanent} {error,load_error()} */ -BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, - Eterm name_term, Eterm options) +BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) { + Eterm path_term = BIF_ARG_1; + Eterm name_term = BIF_ARG_2; + Eterm options = BIF_ARG_3; char *path = NULL; Uint path_len; char *name = NULL; @@ -236,7 +238,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, sys_strcpy(path+path_len,name); #if DDLL_SMP - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); #endif if ((drv = lookup_driver(name)) != NULL) { @@ -247,7 +249,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, } else { dh = drv->handle; if (dh->status == ERL_DE_OK) { - int is_last = is_last_user(dh,p); + int is_last = is_last_user(dh, BIF_P); if (reload == 1 && !is_last) { /*Want reload if no other users, but there are others...*/ @@ -261,7 +263,8 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, soft_error_term = am_inconsistent; goto soft_error; } - if ((old = find_proc_entry(dh, p, ERL_DE_PROC_LOADED)) == + if ((old = find_proc_entry(dh, BIF_P, + ERL_DE_PROC_LOADED)) == NULL) { soft_error_term = am_not_loaded_by_this_process; goto soft_error; @@ -272,7 +275,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, } /* Reload requested and granted */ dereference_all_processes(dh); - set_driver_reloading(dh, p, path, name, flags); + set_driver_reloading(dh, BIF_P, path, name, flags); if (dh->flags & ERL_DE_FL_KILL_PORTS) { kill_ports = 1; } @@ -286,7 +289,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, soft_error_term = am_inconsistent; goto soft_error; } - add_proc_loaded(dh,p); + add_proc_loaded(dh, BIF_P); erts_ddll_reference_driver(dh); monitor = 0; ok_term = mkatom("already_loaded"); @@ -308,7 +311,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, notify_all(dh, drv->name, ERL_DE_PROC_AWAIT_UNLOAD, am_UP, am_unload_cancelled); - add_proc_loaded(dh,p); + add_proc_loaded(dh, BIF_P); erts_ddll_reference_driver(dh); monitor = 0; ok_term = mkatom("already_loaded"); @@ -325,7 +328,8 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto soft_error; } /* Load of granted unload... */ - add_proc_loaded_deref(dh,p); /* Dont reference, will happen after reload */ + /* Don't reference, will happen after reload */ + add_proc_loaded_deref(dh, BIF_P); ++monitor; ok_term = am_pending_driver; } else { /* ERL_DE_PERMANENT */ @@ -345,7 +349,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, goto soft_error; } else { dh->flags = flags; - add_proc_loaded(dh,p); + add_proc_loaded(dh, BIF_P); first_ddll_reference(dh); monitor = 0; ok_term = mkatom("loaded"); @@ -397,18 +401,18 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, #if DDLL_SMP erts_ddll_reference_driver(dh); unlock_drv_list(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); erts_ddll_dereference_driver(dh); #endif - p->flags |= F_USING_DDLL; + BIF_P->flags |= F_USING_DDLL; if (monitor) { - Eterm mref = add_monitor(p, dh, ERL_DE_PROC_AWAIT_LOAD); - hp = HAlloc(p,4); + Eterm mref = add_monitor(BIF_P, dh, ERL_DE_PROC_AWAIT_LOAD); + hp = HAlloc(BIF_P, 4); t = TUPLE3(hp, am_ok, ok_term, mref); } else { - hp = HAlloc(p,3); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_ok, ok_term); } #if DDLL_SMP @@ -416,33 +420,33 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, #endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); BIF_RET(t); soft_error: #if DDLL_SMP unlock_drv_list(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); #endif if (do_build_load_error) { - soft_error_term = build_load_error(p, build_this_load_error); + soft_error_term = build_load_error(BIF_P, build_this_load_error); } - hp = HAlloc(p,3); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_error, soft_error_term); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); BIF_RET(t); error: assert_drv_list_not_locked(); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); if (path != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); } if (name != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); } - BIF_ERROR(p,BADARG); + BIF_ERROR(BIF_P, BADARG); } /* @@ -481,8 +485,10 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, any AWAIT_LOAD-waiters with {'DOWN', ref(), driver, name(), load_cancelled} If the driver made itself permanent, {'UP', ref(), driver, name(), permanent} */ -Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options) +Eterm erl_ddll_try_unload_2(BIF_ALIST_2) { + Eterm name_term = BIF_ARG_1; + Eterm options = BIF_ARG_2; char *name = NULL; Eterm ok_term = NIL; Eterm soft_error_term = NIL; @@ -495,7 +501,7 @@ Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options) Eterm l; int kill_ports = 0; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); for(l = options; is_list(l); l = CDR(list_val(l))) { Eterm opt = CAR(list_val(l)); @@ -548,7 +554,7 @@ Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options) if (dh->flags & ERL_DE_FL_KILL_PORTS) { kill_ports = 1; } - if ((pe = find_proc_entry(dh, p, ERL_DE_PROC_LOADED)) == NULL) { + if ((pe = find_proc_entry(dh, BIF_P, ERL_DE_PROC_LOADED)) == NULL) { if (num_procs(dh, ERL_DE_PROC_LOADED) > 0) { soft_error_term = am_not_loaded_by_this_process; goto soft_error; @@ -624,22 +630,22 @@ done: #if DDLL_SMP erts_ddll_reference_driver(dh); unlock_drv_list(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); erts_ddll_dereference_driver(dh); #endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - p->flags |= F_USING_DDLL; + BIF_P->flags |= F_USING_DDLL; if (monitor > 0) { - Eterm mref = add_monitor(p, dh, ERL_DE_PROC_AWAIT_UNLOAD); - hp = HAlloc(p,4); + Eterm mref = add_monitor(BIF_P, dh, ERL_DE_PROC_AWAIT_UNLOAD); + hp = HAlloc(BIF_P, 4); t = TUPLE3(hp, am_ok, ok_term, mref); } else { - hp = HAlloc(p,3); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_ok, ok_term); } if (kill_ports > 1) { - ERTS_BIF_CHK_EXITED(p); /* May be exited by port killing */ + ERTS_BIF_CHK_EXITED(BIF_P); /* May be exited by port killing */ } #if DDLL_SMP unlock_drv_list(); @@ -651,8 +657,8 @@ soft_error: unlock_drv_list(); #endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - hp = HAlloc(p,3); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_error, soft_error_term); BIF_RET(t); @@ -661,21 +667,21 @@ soft_error: if (name != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - BIF_ERROR(p,BADARG); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_ERROR(BIF_P, BADARG); } /* * A shadow of the "real" demonitor BIF */ -BIF_RETTYPE erl_ddll_demonitor_1(Process *p, Eterm ref) +BIF_RETTYPE erl_ddll_demonitor_1(BIF_ALIST_1) { - if (is_not_internal_ref(ref)) { - BIF_ERROR(p, BADARG); + if (is_not_internal_ref(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); } - if (p->flags & F_USING_DDLL) { - erts_ddll_remove_monitor(p, ref, ERTS_PROC_LOCK_MAIN); + if (BIF_P->flags & F_USING_DDLL) { + erts_ddll_remove_monitor(BIF_P, BIF_ARG_1, ERTS_PROC_LOCK_MAIN); } BIF_RET(am_true); } @@ -683,18 +689,18 @@ BIF_RETTYPE erl_ddll_demonitor_1(Process *p, Eterm ref) /* * A shadow of the "real" monitor BIF */ -BIF_RETTYPE erl_ddll_monitor_2(Process *p, Eterm dr, Eterm what) +BIF_RETTYPE erl_ddll_monitor_2(BIF_ALIST_2) { - if (dr != am_driver) { - BIF_ERROR(p,BADARG); + if (BIF_ARG_1 != am_driver) { + BIF_ERROR(BIF_P, BADARG); } - return erts_ddll_monitor_driver(p, what, ERTS_PROC_LOCK_MAIN); + return erts_ddll_monitor_driver(BIF_P, BIF_ARG_2, ERTS_PROC_LOCK_MAIN); } /* * Return list of loaded drivers {ok,[string()]} */ -Eterm erl_ddll_loaded_drivers_0(Process *p) +BIF_RETTYPE erl_ddll_loaded_drivers_0(BIF_ALIST_0) { Eterm *hp; int need = 3; @@ -706,7 +712,7 @@ Eterm erl_ddll_loaded_drivers_0(Process *p) for (drv = driver_list; drv; drv = drv->next) { need += sys_strlen(drv->name)*2+2; } - hp = HAlloc(p,need); + hp = HAlloc(BIF_P, need); for (drv = driver_list; drv; drv = drv->next) { Eterm l; l = buf_to_intlist(&hp, drv->name, sys_strlen(drv->name), NIL); @@ -726,8 +732,11 @@ Eterm erl_ddll_loaded_drivers_0(Process *p) * item is processes, driver_options, port_count, linked_in_driver, * permanent, awaiting_load, awaiting_unload */ -Eterm erl_ddll_info_2(Process *p, Eterm name_term, Eterm item) +BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) { + Process *p = BIF_P; + Eterm name_term = BIF_ARG_1; + Eterm item = BIF_ARG_2; char *name = NULL; Eterm res = NIL; erts_driver_t *drv; @@ -850,8 +859,10 @@ Eterm erl_ddll_info_2(Process *p, Eterm name_term, Eterm item) * Backend for erl_ddll:format_error, handles all "soft" errors returned by builtins, * possibly by calling the system specific error handler */ -Eterm erl_ddll_format_error_int_1(Process *p, Eterm code_term) +BIF_RETTYPE erl_ddll_format_error_int_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm code_term = BIF_ARG_1; char *errstring = NULL; int errint; int len; @@ -1558,14 +1569,14 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) if ((res = erts_sys_ddll_load_driver_init(dh->handle, &init_handle)) != ERL_DE_NO_ERROR) { - erts_sys_ddll_close(dh->handle); - return ERL_DE_LOAD_ERROR_NO_INIT; + res = ERL_DE_LOAD_ERROR_NO_INIT; + goto error; } dp = erts_sys_ddll_call_init(init_handle); if (dp == NULL) { - erts_sys_ddll_close(dh->handle); - return ERL_DE_LOAD_ERROR_FAILED_INIT; + res = ERL_DE_LOAD_ERROR_FAILED_INIT; + goto error; } switch (dp->extended_marker) { @@ -1583,24 +1594,27 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) || dp->handle2 != NULL || dp->process_exit != NULL) { /* Old driver; needs to be recompiled... */ - return ERL_DE_LOAD_ERROR_INCORRECT_VERSION; + res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; + goto error; } break; case ERL_DRV_EXTENDED_MARKER: if (ERL_DRV_EXTENDED_MAJOR_VERSION != dp->major_version || ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version) { /* Incompatible driver version */ - return ERL_DE_LOAD_ERROR_INCORRECT_VERSION; + res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; + goto error; } break; default: /* Old driver; needs to be recompiled... */ - return ERL_DE_LOAD_ERROR_INCORRECT_VERSION; + res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; + goto error; } if (strcmp(name, dp->driver_name) != 0) { - erts_sys_ddll_close(dh->handle); - return ERL_DE_LOAD_ERROR_BAD_NAME; + res = ERL_DE_LOAD_ERROR_BAD_NAME; + goto error; } erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); dh->port_count = 0; @@ -1615,11 +1629,14 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) */ erts_free(ERTS_ALC_T_DDLL_HANDLE, dh->full_path); dh->full_path = NULL; - erts_sys_ddll_close(dh->handle); - return ERL_DE_LOAD_ERROR_FAILED_INIT; + res = ERL_DE_LOAD_ERROR_FAILED_INIT; + goto error; } - return ERL_DE_NO_ERROR; + +error: + erts_sys_ddll_close(dh->handle); + return res; } static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name) diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 01e6977a2c..dff59de69b 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -52,7 +52,7 @@ BIF_RETTYPE abs_1(BIF_ALIST_1) /* integer arguments */ if (is_small(BIF_ARG_1)) { i0 = signed_val(BIF_ARG_1); - i = labs(i0); + i = ERTS_SMALL_ABS(i0); if (i0 == MIN_SMALL) { hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); BIF_RET(uint_to_big(i, hp)); @@ -467,7 +467,7 @@ Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live) /* integer arguments */ if (is_small(arg)) { i0 = signed_val(arg); - i = labs(i0); + i = ERTS_SMALL_ABS(i0); if (i0 == MIN_SMALL) { if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live+1); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 70d728340a..5a806777fe 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -39,6 +39,8 @@ #include "dist.h" #include "erl_gc.h" #include "erl_cpu_topology.h" +#include "erl_async.h" +#include "erl_thr_progress.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -76,7 +78,6 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #ifdef ERTS_SMP " [smp:%beu:%beu]" #endif - " [rq:%beu]" #ifdef USE_THREADS " [async-threads:%d]" #endif @@ -122,6 +123,16 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE # define PERFMON_GETPCR _IOR('P', 2, unsigned long long) #endif +/* Cached, pre-built {OsType,OsFlavor} and {Major,Minor,Build} tuples */ +static Eterm os_type_tuple; +static Eterm os_version_tuple; + +static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item); + +static Eterm +current_function(Process* p, Process* rp, Eterm** hpp, int full_info); +static Eterm current_stacktrace(Process* p, Process* rp, Eterm** hpp); + static Eterm bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) { @@ -289,9 +300,7 @@ erts_print_system_version(int to, void *arg, Process *c_p) #endif return erts_print(to, arg, erts_system_version #ifdef ERTS_SMP - , total, online, erts_no_run_queues -#else - , 1 + , total, online #endif #ifdef USE_THREADS , erts_async_max_threads @@ -557,6 +566,8 @@ static Eterm pi_args[] = { am_suspending, am_min_heap_size, am_min_bin_vheap_size, + am_current_location, + am_current_stacktrace, #ifdef HYBRID am_message_binary #endif @@ -605,8 +616,10 @@ pi_arg2ix(Eterm arg) case am_suspending: return 26; case am_min_heap_size: return 27; case am_min_bin_vheap_size: return 28; + case am_current_location: return 29; + case am_current_stacktrace: return 30; #ifdef HYBRID - case am_message_binary: return 29; + case am_message_binary: return 31; #endif default: return -1; } @@ -1009,35 +1022,15 @@ process_info_aux(Process *BIF_P, break; case am_current_function: - if (rp->current == NULL) { - rp->current = find_function_from_pc(rp->i); - } - if (rp->current == NULL) { - hp = HAlloc(BIF_P, 3); - res = am_undefined; - } else { - BeamInstr* current; - - if (rp->current[0] == am_erlang && - rp->current[1] == am_process_info && - (rp->current[2] == 1 || rp->current[2] == 2) && - (current = find_function_from_pc(rp->cp)) != NULL) { - - /* - * The current function is erlang:process_info/2, - * which is not the answer that the application want. - * We will use the function pointed into by rp->cp - * instead. - */ + res = current_function(BIF_P, rp, &hp, 0); + break; - rp->current = current; - } + case am_current_location: + res = current_function(BIF_P, rp, &hp, 1); + break; - hp = HAlloc(BIF_P, 3+4); - res = TUPLE3(hp, rp->current[0], - rp->current[1], make_small(rp->current[2])); - hp += 4; - } + case am_current_stacktrace: + res = current_stacktrace(BIF_P, rp, &hp); break; case am_initial_call: @@ -1611,6 +1604,113 @@ process_info_aux(Process *BIF_P, } #undef MI_INC +static Eterm +current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) +{ + Eterm* hp; + Eterm res; + FunctionInfo fi; + + if (rp->current == NULL) { + erts_lookup_function_info(&fi, rp->i, full_info); + rp->current = fi.current; + } else if (full_info) { + erts_lookup_function_info(&fi, rp->i, full_info); + if (fi.current == NULL) { + /* Use the current function without location info */ + erts_set_current_function(&fi, rp->current); + } + } + + if (BIF_P->id == rp->id) { + FunctionInfo fi2; + + /* + * The current function is erlang:process_info/{1,2}, + * which is not the answer that the application want. + * We will use the function pointed into by rp->cp + * instead if it can be looked up. + */ + erts_lookup_function_info(&fi2, rp->cp, full_info); + if (fi2.current) { + fi = fi2; + rp->current = fi2.current; + } + } + + /* + * Return the result. + */ + if (rp->current == NULL) { + hp = HAlloc(BIF_P, 3); + res = am_undefined; + } else if (full_info) { + hp = HAlloc(BIF_P, 3+fi.needed); + hp = erts_build_mfa_item(&fi, hp, am_true, &res); + } else { + hp = HAlloc(BIF_P, 3+4); + res = TUPLE3(hp, rp->current[0], + rp->current[1], make_small(rp->current[2])); + hp += 4; + } + *hpp = hp; + return res; +} + +static Eterm +current_stacktrace(Process* p, Process* rp, Eterm** hpp) +{ + Uint sz; + struct StackTrace* s; + int depth; + FunctionInfo* stk; + FunctionInfo* stkp; + Uint heap_size; + int i; + Eterm* hp = *hpp; + Eterm mfa; + Eterm res = NIL; + + depth = 8; + sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth; + s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz); + s->depth = 0; + if (rp->i) { + s->trace[s->depth++] = rp->i; + depth--; + } + if (depth > 0 && rp->cp != 0) { + s->trace[s->depth++] = rp->cp - 1; + depth--; + } + erts_save_stacktrace(rp, s, depth); + + depth = s->depth; + stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, + depth*sizeof(FunctionInfo)); + heap_size = 3; + for (i = 0; i < depth; i++) { + erts_lookup_function_info(stkp, s->trace[i], 1); + if (stkp->current) { + heap_size += stkp->needed + 2; + stkp++; + } + } + + hp = HAlloc(p, heap_size); + while (stkp > stk) { + stkp--; + hp = erts_build_mfa_item(stkp, hp, am_true, &mfa); + res = CONS(hp, mfa, res); + hp += 2; + } + + erts_free(ERTS_ALC_T_TMP, stk); + erts_free(ERTS_ALC_T_TMP, s); + *hpp = hp; + return res; +} + #if defined(VALGRIND) static int check_if_xml(void) { @@ -2027,7 +2127,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_undefined); #endif } else if (BIF_ARG_1 == am_trace_control_word) { - BIF_RET(db_get_trace_control_word_0(BIF_P)); + BIF_RET(db_get_trace_control_word(BIF_P)); } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) { BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false); } else if (ERTS_IS_ATOM_STR("ets_always_compress", BIF_ARG_1)) { @@ -2087,7 +2187,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) /* Need to be the only thread running... */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (BIF_ARG_1 == am_info) info(ERTS_PRINT_DSBUF, (void *) dsbufp); @@ -2098,7 +2198,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else distribution_info(ERTS_PRINT_DSBUF, (void *) dsbufp); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); ASSERT(dsbufp && dsbufp->str); @@ -2110,7 +2210,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) i = 0; /* Need to be the only thread running... */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); for (dep = erts_visible_dist_entries; dep; dep = dep->next) ++i; for (dep = erts_hidden_dist_entries; dep; dep = dep->next) @@ -2133,7 +2233,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = CONS(hp, tpl, res); hp += 2; } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if (BIF_ARG_1 == am_system_version) { @@ -2154,16 +2254,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) return erts_instr_get_type_info(BIF_P); } else if (BIF_ARG_1 == am_os_type) { - Eterm type = am_atom_put(os_type, strlen(os_type)); - Eterm flav, tup; - char *buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */ - - os_flavor(buf, 1024); - flav = am_atom_put(buf, strlen(buf)); - hp = HAlloc(BIF_P, 3); - tup = TUPLE2(hp, type, flav); - erts_free(ERTS_ALC_T_TMP, (void *) buf); - BIF_RET(tup); + BIF_RET(os_type_tuple); } else if (BIF_ARG_1 == am_allocator) { BIF_RET(erts_allocator_options((void *) BIF_P)); @@ -2189,16 +2280,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_false); } else if (BIF_ARG_1 == am_os_version) { - int major, minor, build; - Eterm tup; - - os_version(&major, &minor, &build); - hp = HAlloc(BIF_P, 4); - tup = TUPLE3(hp, - make_small(major), - make_small(minor), - make_small(build)); - BIF_RET(tup); + BIF_RET(os_version_tuple); } else if (BIF_ARG_1 == am_version) { int n = strlen(ERLANG_VERSION); @@ -2646,9 +2728,11 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } -Eterm -port_info_1(Process* p, Eterm pid) +BIF_RETTYPE +port_info_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm pid = BIF_ARG_1; static Eterm keys[] = { am_name, am_links, @@ -2671,7 +2755,7 @@ port_info_1(Process* p, Eterm pid) for (i = 0; i < ASIZE(keys); i++) { Eterm item; - item = port_info_2(p, pid, keys[i]); + item = port_info(p, pid, keys[i]); if (is_non_value(item)) { return THE_NON_VALUE; } @@ -2680,7 +2764,7 @@ port_info_1(Process* p, Eterm pid) } items[i] = item; } - reg_name = port_info_2(p, pid, am_registered_name); + reg_name = port_info(p, pid, am_registered_name); /* * Build the resulting list. @@ -2716,24 +2800,27 @@ port_info_1(Process* p, Eterm pid) BIF_RETTYPE port_info_2(BIF_ALIST_2) { + return port_info(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item) +{ BIF_RETTYPE ret; - Eterm portid = BIF_ARG_1; Port *prt; - Eterm item = BIF_ARG_2; Eterm res; Eterm* hp; int count; if (is_internal_port(portid)) - prt = erts_id2port(portid, BIF_P, ERTS_PROC_LOCK_MAIN); + prt = erts_id2port(portid, p, ERTS_PROC_LOCK_MAIN); else if (is_atom(portid)) - erts_whereis_name(BIF_P, ERTS_PROC_LOCK_MAIN, + erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, portid, NULL, 0, 0, &prt); else if (is_external_port(portid) && external_port_dist_entry(portid) == erts_this_dist_entry) BIF_RET(am_undefined); else { - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } if (!prt) { @@ -2741,7 +2828,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) } if (item == am_id) { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = make_small(internal_port_number(portid)); } else if (item == am_links) { @@ -2753,10 +2840,10 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) erts_doforall_links(prt->nlinks, &collect_one_link, &mic); - hp = HAlloc(BIF_P, 3 + mic.sz); + hp = HAlloc(p, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -2772,11 +2859,11 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) erts_doforall_monitors(prt->monitors, &collect_one_origin_monitor, &mic); - hp = HAlloc(BIF_P, 3 + mic.sz); + hp = HAlloc(p, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { Eterm t; - item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity); t = TUPLE2(hp, am_process, item); hp += 3; res = CONS(hp, t, res); @@ -2788,25 +2875,25 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) else if (item == am_name) { count = sys_strlen(prt->name); - hp = HAlloc(BIF_P, 3 + 2*count); + hp = HAlloc(p, 3 + 2*count); res = buf_to_intlist(&hp, prt->name, count, NIL); } else if (item == am_connected) { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = prt->connected; /* internal pid */ } else if (item == am_input) { Uint hsz = 3; Uint n = prt->bytes_in; (void) erts_bld_uint(NULL, &hsz, n); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, n); } else if (item == am_output) { Uint hsz = 3; Uint n = prt->bytes_out; (void) erts_bld_uint(NULL, &hsz, n); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, n); } else if (item == am_registered_name) { @@ -2816,7 +2903,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) ERTS_BIF_PREP_RET(ret, NIL); goto done; } else { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = reg->name; } } @@ -2828,7 +2915,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) Uint size = 0; ErlHeapFragment* bp; - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); erts_doforall_links(prt->nlinks, &one_link_size, &size); @@ -2845,18 +2932,18 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) hard to retrieve... */ (void) erts_bld_uint(NULL, &hsz, size); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, size); } else if (item == am_queue_size) { Uint ioq_size = erts_port_ioq_size(prt); Uint hsz = 3; (void) erts_bld_uint(NULL, &hsz, ioq_size); - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(p, hsz); res = erts_bld_uint(&hp, NULL, ioq_size); } else if (ERTS_IS_ATOM_STR("locking", item)) { - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); #ifndef ERTS_SMP res = am_false; #else @@ -2875,7 +2962,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) #endif } else { - ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(ret, p, BADARG); goto done; } @@ -2889,9 +2976,12 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) } -Eterm -fun_info_2(Process* p, Eterm fun, Eterm what) +BIF_RETTYPE +fun_info_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm fun = BIF_ARG_1; + Eterm what = BIF_ARG_2; Eterm* hp; Eterm val; @@ -3137,7 +3227,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_runtime) { - unsigned long u1, u2, dummy; + UWord u1, u2, dummy; Eterm b1, b2; elapsed_time_both(&u1,&dummy,&u2,&dummy); b1 = erts_make_integer(u1,BIF_P); @@ -3306,10 +3396,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("memory", BIF_ARG_1)) { Eterm res; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); BIF_RET(res); } } @@ -3565,10 +3655,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) if (ms > 0) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); if (block) - erts_smp_block_system(0); + erts_smp_thr_progress_block(); while (erts_milli_sleep((long) ms) != 0); if (block) - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); } BIF_RET(am_true); @@ -3778,10 +3868,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); old_use_opt = !erts_disable_proc_not_running_opt; erts_disable_proc_not_running_opt = !use_opt; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(old_use_opt ? am_true : am_false); #else @@ -3953,7 +4043,7 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) Eterm* hp; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_SUSPEND); data = erts_lcnt_get_data(); @@ -3971,17 +4061,17 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_SUSPEND); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if (BIF_ARG_1 == am_clear) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_lcnt_clear_counters(); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_ok); @@ -3992,7 +4082,7 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) case 2: if (ERTS_IS_ATOM_STR("copy_save", tp[1])) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (tp[2] == am_true) { res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_COPYSAVE) ? am_true : am_false; @@ -4002,17 +4092,17 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) res = erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_COPYSAVE) ? am_true : am_false; } else { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_ERROR(BIF_P, BADARG); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("process_locks", tp[1])) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (tp[2] == am_true) { res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_PROCLOCK) ? am_true : am_false; @@ -4022,11 +4112,11 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_PROCLOCK) ? am_true : am_false; } else { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_ERROR(BIF_P, BADARG); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(res); } @@ -4041,6 +4131,27 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +static void os_info_init(void) +{ + Eterm type = am_atom_put(os_type, strlen(os_type)); + Eterm flav; + int major, minor, build; + char* buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */ + Eterm* hp; + + os_flavor(buf, 1024); + flav = am_atom_put(buf, strlen(buf)); + erts_free(ERTS_ALC_T_TMP, (void *) buf); + hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm)); + os_type_tuple = TUPLE2(hp, type, flav); + hp += 3; + os_version(&major, &minor, &build); + os_version_tuple = TUPLE3(hp, + make_small(major), + make_small(minor), + make_small(build)); +} + void erts_bif_info_init(void) { @@ -4050,4 +4161,5 @@ erts_bif_info_init(void) alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1); alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); process_info_init(); + os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 47c48e74d6..1805366cfe 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -34,27 +34,7 @@ static Eterm keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List); -/* - * erlang:'++'/2 - */ - -Eterm -ebif_plusplus_2(Process* p, Eterm A, Eterm B) -{ - return append_2(p, A, B); -} - -/* - * erlang:'--'/2 - */ - -Eterm -ebif_minusminus_2(Process* p, Eterm A, Eterm B) -{ - return subtract_2(p, A, B); -} - -BIF_RETTYPE append_2(BIF_ALIST_2) +static BIF_RETTYPE append(Process* p, Eterm A, Eterm B) { Eterm list; Eterm copy; @@ -63,18 +43,18 @@ BIF_RETTYPE append_2(BIF_ALIST_2) Eterm* hp; int i; - if ((i = list_length(BIF_ARG_1)) < 0) { - BIF_ERROR(BIF_P, BADARG); + if ((i = list_length(A)) < 0) { + BIF_ERROR(p, BADARG); } if (i == 0) { - BIF_RET(BIF_ARG_2); - } else if (is_nil(BIF_ARG_2)) { - BIF_RET(BIF_ARG_1); + BIF_RET(B); + } else if (is_nil(B)) { + BIF_RET(A); } need = 2*i; - hp = HAlloc(BIF_P, need); - list = BIF_ARG_1; + hp = HAlloc(p, need); + list = A; copy = last = CONS(hp, CAR(list_val(list)), make_list(hp+2)); list = CDR(list_val(list)); hp += 2; @@ -85,12 +65,31 @@ BIF_RETTYPE append_2(BIF_ALIST_2) list = CDR(listp); hp += 2; } - CDR(list_val(last)) = BIF_ARG_2; + CDR(list_val(last)) = B; BIF_RET(copy); } +/* + * erlang:'++'/2 + */ + +Eterm +ebif_plusplus_2(BIF_ALIST_2) +{ + return append(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +BIF_RETTYPE append_2(BIF_ALIST_2) +{ + return append(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +/* + * erlang:'--'/2 + */ + #define SMALL_VEC_SIZE 10 -BIF_RETTYPE subtract_2(BIF_ALIST_2) +static Eterm subtract(Process* p, Eterm A, Eterm B) { Eterm list; Eterm* hp; @@ -103,17 +102,17 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) int n; int m; - if ((n = list_length(BIF_ARG_1)) < 0) { - BIF_ERROR(BIF_P, BADARG); + if ((n = list_length(A)) < 0) { + BIF_ERROR(p, BADARG); } - if ((m = list_length(BIF_ARG_2)) < 0) { - BIF_ERROR(BIF_P, BADARG); + if ((m = list_length(B)) < 0) { + BIF_ERROR(p, BADARG); } if (n == 0) BIF_RET(NIL); if (m == 0) - BIF_RET(BIF_ARG_1); + BIF_RET(A); /* allocate element vector */ if (n <= SMALL_VEC_SIZE) @@ -123,7 +122,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) /* PUT ALL ELEMENTS IN VP */ vp = vec_p; - list = BIF_ARG_1; + list = A; i = n; while(i--) { Eterm* listp = list_val(list); @@ -132,7 +131,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) } /* UNMARK ALL DELETED CELLS */ - list = BIF_ARG_2; + list = B; m = 0; /* number of deleted elements */ while(is_list(list)) { Eterm* listp = list_val(list); @@ -153,11 +152,11 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) if (m == n) /* All deleted ? */ res = NIL; else if (m == 0) /* None deleted ? */ - res = BIF_ARG_1; + res = A; else { /* REBUILD LIST */ res = NIL; need = 2*(n - m); - hp = HAlloc(BIF_P, need); + hp = HAlloc(p, need); vp = vec_p + n - 1; while(vp >= vec_p) { if (is_value(*vp)) { @@ -172,6 +171,16 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2) BIF_RET(res); } +BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2) +{ + return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +BIF_RETTYPE subtract_2(BIF_ALIST_2) +{ + return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + BIF_RETTYPE lists_member_2(BIF_ALIST_2) { Eterm term; @@ -278,11 +287,12 @@ BIF_RETTYPE lists_reverse_2(BIF_ALIST_2) } BIF_RETTYPE -lists_keymember_3(Process* p, Eterm Key, Eterm Pos, Eterm List) +lists_keymember_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keymember_3, p, Key, Pos, List); + res = keyfind(BIF_lists_keymember_3, BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_value(res) && is_tuple(res)) { return am_true; } else { @@ -291,23 +301,25 @@ lists_keymember_3(Process* p, Eterm Key, Eterm Pos, Eterm List) } BIF_RETTYPE -lists_keysearch_3(Process* p, Eterm Key, Eterm Pos, Eterm List) +lists_keysearch_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keysearch_3, p, Key, Pos, List); + res = keyfind(BIF_lists_keysearch_3, BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_non_value(res) || is_not_tuple(res)) { return res; } else { /* Tuple */ - Eterm* hp = HAlloc(p, 3); + Eterm* hp = HAlloc(BIF_P, 3); return TUPLE2(hp, am_value, res); } } BIF_RETTYPE -lists_keyfind_3(Process* p, Eterm Key, Eterm Pos, Eterm List) +lists_keyfind_3(BIF_ALIST_3) { - return keyfind(BIF_lists_keyfind_3, p, Key, Pos, List); + return keyfind(BIF_lists_keyfind_3, BIF_P, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } static Eterm diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index deda7adc1f..13f8b1f63c 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -225,18 +225,23 @@ BIF_RETTYPE is_function_1(BIF_ALIST_1) BIF_RETTYPE is_function_2(BIF_ALIST_2) { + BIF_RET(erl_is_function(BIF_P, BIF_ARG_1, BIF_ARG_2)); +} + +Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) +{ Sint arity; /* * Verify argument 2 (arity); arity must be >= 0. */ - if (is_small(BIF_ARG_2)) { - arity = signed_val(BIF_ARG_2); + if (is_small(arg2)) { + arity = signed_val(arg2); if (arity < 0) { error: - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - } else if (is_big(BIF_ARG_2) && !bignum_header_is_neg(*big_val(BIF_ARG_2))) { + } else if (is_big(arg2) && !bignum_header_is_neg(*big_val(arg2))) { /* A positive bignum is OK, but can't possibly match. */ arity = -1; } else { @@ -244,20 +249,20 @@ BIF_RETTYPE is_function_2(BIF_ALIST_2) goto error; } - if (is_fun(BIF_ARG_1)) { - ErlFunThing* funp = (ErlFunThing *) fun_val(BIF_ARG_1); + if (is_fun(arg1)) { + ErlFunThing* funp = (ErlFunThing *) fun_val(arg1); if (funp->arity == (Uint) arity) { BIF_RET(am_true); } - } else if (is_export(BIF_ARG_1)) { - Export* exp = (Export *) EXPAND_POINTER((export_val(BIF_ARG_1))[1]); + } else if (is_export(arg1)) { + Export* exp = (Export *) EXPAND_POINTER((export_val(arg1))[1]); if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); } - } else if (is_tuple(BIF_ARG_1)) { - Eterm* tp = tuple_val(BIF_ARG_1); + } else if (is_tuple(arg1)) { + Eterm* tp = tuple_val(arg1); if (tp[0] == make_arityval(2) && is_atom(tp[1]) && is_atom(tp[2])) { BIF_RET(am_true); } diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 954b1f9729..58d48199fa 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -53,20 +53,18 @@ BIF_RETTYPE os_timestamp_0(BIF_ALIST_0) } -Eterm -os_getpid_0(Process* p) +BIF_RETTYPE os_getpid_0(BIF_ALIST_0) { char pid_string[21]; /* enough for a 64 bit number */ int n; Eterm* hp; sys_get_pid(pid_string); /* In sys.c */ n = sys_strlen(pid_string); - hp = HAlloc(p, n*2); + hp = HAlloc(BIF_P, n*2); BIF_RET(buf_to_intlist(&hp, pid_string, n, NIL)); } -Eterm -os_getenv_0(Process* p) +BIF_RETTYPE os_getenv_0(BIF_ALIST_0) { GETENV_STATE state; char *cp; @@ -80,7 +78,7 @@ os_getenv_0(Process* p) ret = NIL; while ((cp = getenv_string(&state)) != NULL) { len = strlen(cp); - hp = HAlloc(p, len*2+2); + hp = HAlloc(BIF_P, len*2+2); str = buf_to_intlist(&hp, cp, len, NIL); ret = CONS(hp, str, ret); } @@ -90,9 +88,11 @@ os_getenv_0(Process* p) return ret; } -Eterm -os_getenv_1(Process* p, Eterm key) + +BIF_RETTYPE os_getenv_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm key = BIF_ARG_1; Eterm str; int len, res; char *key_str, *val; @@ -145,9 +145,11 @@ os_getenv_1(Process* p, Eterm key) BIF_RET(str); } -Eterm -os_putenv_2(Process* p, Eterm key, Eterm value) +BIF_RETTYPE os_putenv_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm key = BIF_ARG_1; + Eterm value = BIF_ARG_2; char def_buf[1024]; char *buf = NULL; int sep_ix, i, key_len, value_len, tot_len; diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 3fd35dd963..1f6b62817d 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -48,6 +48,9 @@ static void free_args(char **); char *erts_default_arg0 = "default"; +static BIF_RETTYPE +port_call(Process* p, Eterm arg1, Eterm arg2, Eterm arg3); + BIF_RETTYPE open_port_2(BIF_ALIST_2) { int port_num; @@ -117,11 +120,9 @@ id_or_name2port(Process *c_p, Eterm id) #define ERTS_PORT_COMMAND_FLAG_FORCE (((Uint32) 1) << 0) #define ERTS_PORT_COMMAND_FLAG_NOSUSPEND (((Uint32) 1) << 1) -static BIF_RETTYPE do_port_command(Process *BIF_P, - Eterm BIF_ARG_1, - Eterm BIF_ARG_2, - Eterm BIF_ARG_3, - Uint32 flags) +static BIF_RETTYPE +do_port_command(Process *BIF_P, Eterm arg1, Eterm arg2, Eterm arg3, + Uint32 flags) { BIF_RETTYPE res; Port *p; @@ -135,7 +136,7 @@ static BIF_RETTYPE do_port_command(Process *BIF_P, profile_runnable_proc(BIF_P, am_inactive); } - p = id_or_name2port(BIF_P, BIF_ARG_1); + p = id_or_name2port(BIF_P, arg1); if (!p) { if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { trace_virtual_sched(BIF_P, am_in); @@ -172,13 +173,13 @@ static BIF_RETTYPE do_port_command(Process *BIF_P, monitor_generic(BIF_P, am_busy_port, p->id); } ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_port_command_3], BIF_P, - BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + arg1, arg2, arg3); } } else { int wres; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); ERTS_SMP_CHK_NO_PROC_LOCKS; - wres = erts_write_to_port(BIF_P->id, p, BIF_ARG_2); + wres = erts_write_to_port(BIF_P->id, p, arg2); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); if (wres != 0) { ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); @@ -237,11 +238,17 @@ BIF_RETTYPE port_command_3(BIF_ALIST_3) BIF_RETTYPE port_call_2(BIF_ALIST_2) { - return port_call_3(BIF_P,BIF_ARG_1,make_small(0),BIF_ARG_2); + return port_call(BIF_P,BIF_ARG_1, make_small(0), BIF_ARG_2); } BIF_RETTYPE port_call_3(BIF_ALIST_3) { + return port_call(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3) +{ Uint op; Port *p; Uint size; @@ -266,15 +273,15 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) /* trace of port scheduling with virtual process descheduling * lock wait */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_out); + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { + trace_virtual_sched(c_p, am_out); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_inactive); + profile_runnable_proc(c_p, am_inactive); } - p = id_or_name2port(BIF_P, BIF_ARG_1); + p = id_or_name2port(c_p, arg1); if (!p) { error: if (port_resp != port_result && @@ -286,22 +293,22 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) /* Need to virtual schedule in the process if there * was an error. */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { + trace_virtual_sched(c_p, am_in); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); + profile_runnable_proc(c_p, am_active); } if (p) erts_port_release(p); #ifdef ERTS_SMP - ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN); + ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); #else - ERTS_BIF_CHK_EXITED(BIF_P); + ERTS_BIF_CHK_EXITED(c_p); #endif - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(c_p, BADARG); } if ((drv = p->drv_ptr) == NULL) { @@ -310,10 +317,10 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) if (drv->call == NULL) { goto error; } - if (!term_to_Uint(BIF_ARG_2, &op)) { + if (!term_to_Uint(arg2, &op)) { goto error; } - p->caller = BIF_P->id; + p->caller = c_p->id; /* Lock taken, virtual schedule of port */ if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { @@ -323,19 +330,19 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { profile_runnable_port(p, am_active); } - size = erts_encode_ext_size(BIF_ARG_3); + size = erts_encode_ext_size(arg3); if (size > sizeof(port_input)) bytes = erts_alloc(ERTS_ALC_T_PORT_CALL_BUF, size); endp = bytes; - erts_encode_ext(BIF_ARG_3, &endp); + erts_encode_ext(arg3, &endp); real_size = endp - bytes; if (real_size > size) { erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); prc = (char *) port_resp; fpe_was_unmasked = erts_block_fpe(); ret = drv->call((ErlDrvData)p->drv_data, @@ -356,7 +363,7 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) port_resp = (byte *) prc; p->caller = NIL; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); #ifdef HARDDEBUG { int z; @@ -378,18 +385,18 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) /* Error or a binary without magic/ with wrong magic */ goto error; } - result_size = erts_decode_ext_size(port_resp, ret, 0); + result_size = erts_decode_ext_size(port_resp, ret); if (result_size < 0) { goto error; } - hp = HAlloc(BIF_P, result_size); + hp = HAlloc(c_p, result_size); hp_end = hp + result_size; endp = port_resp; - res = erts_decode_ext(&hp, &MSO(BIF_P), &endp); + res = erts_decode_ext(&hp, &MSO(c_p), &endp); if (res == THE_NON_VALUE) { goto error; } - HRelease(BIF_P, hp_end, hp); + HRelease(c_p, hp_end, hp); if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) { driver_free(port_resp); } @@ -398,16 +405,16 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3) if (p) erts_port_release(p); #ifdef ERTS_SMP - ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN); + ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); #else - ERTS_BIF_CHK_EXITED(BIF_P); + ERTS_BIF_CHK_EXITED(c_p); #endif - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { + trace_virtual_sched(c_p, am_in); } if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); + profile_runnable_proc(c_p, am_active); } return res; @@ -1070,7 +1077,7 @@ struct packet_callback_args }; #define in_area(ptr,start,nbytes) \ - ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes)) + ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) static Eterm http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp, diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 26891c4348..6b843d2e08 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -45,6 +45,7 @@ static Export *urun_trap_exportp = NULL; static Export *ucompile_trap_exportp = NULL; static BIF_RETTYPE re_exec_trap(BIF_ALIST_3); +static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); static void *erts_erts_pcre_malloc(size_t size) { return erts_alloc(ERTS_ALC_T_RE_HEAP,size); @@ -414,8 +415,8 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con * Compile BIFs */ -BIF_RETTYPE -re_compile_2(BIF_ALIST_2) +static BIF_RETTYPE +re_compile(Process* p, Eterm arg1, Eterm arg2) { Uint slen; char *expr; @@ -429,43 +430,49 @@ re_compile_2(BIF_ALIST_2) int unicode = 0; - if (parse_options(BIF_ARG_2,&options,NULL,&pflags,NULL,NULL) + if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL) < 0) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } if (pflags & PARSE_FLAG_UNIQUE_EXEC_OPT) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0; - if (pflags & PARSE_FLAG_UNICODE && !is_binary(BIF_ARG_1)) { - BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2); + if (pflags & PARSE_FLAG_UNICODE && !is_binary(arg1)) { + BIF_TRAP2(ucompile_trap_exportp, p, arg1, arg2); } - if (erts_iolist_size(BIF_ARG_1, &slen)) { - BIF_ERROR(BIF_P,BADARG); + if (erts_iolist_size(arg1, &slen)) { + BIF_ERROR(p,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (io_list_to_buf(BIF_ARG_1, expr, slen) != 0) { + if (io_list_to_buf(arg1, expr, slen) != 0) { erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } expr[slen]='\0'; result = erts_pcre_compile2(expr, options, &errcode, &errstr, &errofset, default_table); - ret = build_compile_result(BIF_P, am_error, result, errcode, + ret = build_compile_result(p, am_error, result, errcode, errstr, errofset, unicode, 1); erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); BIF_RET(ret); } BIF_RETTYPE +re_compile_2(BIF_ALIST_2) +{ + return re_compile(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +BIF_RETTYPE re_compile_1(BIF_ALIST_1) { - return re_compile_2(BIF_P,BIF_ARG_1,NIL); + return re_compile(BIF_P, BIF_ARG_1, NIL); } /* @@ -845,8 +852,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) /* * The actual re:run/2,3 BIFs */ -BIF_RETTYPE -re_run_3(BIF_ALIST_3) +static BIF_RETTYPE +re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) { const pcre *code_tmp; RestartContext restart; @@ -865,15 +872,15 @@ re_run_3(BIF_ALIST_3) Eterm capture[CAPSPEC_SIZE]; int is_list_cap; - if (parse_options(BIF_ARG_3,&comp_options,&options,&pflags,&startoffset,capture) + if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture) < 0) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && (capture[CAPSPEC_TYPE] == am_list)); - if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) { - if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { + if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 4)) { + if (is_binary(arg2) || is_list(arg2) || is_nil(arg2)) { /* Compile from textual RE */ Uint slen; char *expr; @@ -884,19 +891,19 @@ re_run_3(BIF_ALIST_3) int capture_count; if (pflags & PARSE_FLAG_UNICODE && - (!is_binary(BIF_ARG_2) || !is_binary(BIF_ARG_1) || + (!is_binary(arg2) || !is_binary(arg1) || (is_list_cap && !(pflags & PARSE_FLAG_GLOBAL)))) { - BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + BIF_TRAP3(urun_trap_exportp, p, arg1, arg2, arg3); } - if (erts_iolist_size(BIF_ARG_2, &slen)) { - BIF_ERROR(BIF_P,BADARG); + if (erts_iolist_size(arg2, &slen)) { + BIF_ERROR(p,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (io_list_to_buf(BIF_ARG_2, expr, slen) != 0) { + if (io_list_to_buf(arg2, expr, slen) != 0) { erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } expr[slen]='\0'; result = erts_pcre_compile2(expr, comp_options, &errcode, @@ -905,11 +912,11 @@ re_run_3(BIF_ALIST_3) erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); /* Compilation error gives badarg except in the compile function */ - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } if (pflags & PARSE_FLAG_GLOBAL) { Eterm precompiled = - build_compile_result(BIF_P, am_error, + build_compile_result(p, am_error, result, errcode, errstr, errofset, (pflags & @@ -917,13 +924,13 @@ re_run_3(BIF_ALIST_3) 0); Eterm *hp,r; erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - hp = HAlloc(BIF_P,4); - /* BIF_ARG_2 is in the tuple just to make exceptions right */ - r = TUPLE3(hp,BIF_ARG_3, + hp = HAlloc(p,4); + /* arg2 is in the tuple just to make exceptions right */ + r = TUPLE3(hp,arg3, ((pflags & PARSE_FLAG_UNIQUE_COMPILE_OPT) ? am_true : - am_false), BIF_ARG_2); - BIF_TRAP3(grun_trap_exportp, BIF_P, BIF_ARG_1, precompiled, r); + am_false), arg2); + BIF_TRAP3(grun_trap_exportp, p, arg1, precompiled, r); } erts_pcre_fullinfo(result, NULL, PCRE_INFO_SIZE, &code_size); @@ -935,31 +942,31 @@ re_run_3(BIF_ALIST_3) erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); /*unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0;*/ } else { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } } else { if (pflags & PARSE_FLAG_UNIQUE_COMPILE_OPT) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } - tp = tuple_val(BIF_ARG_2); + tp = tuple_val(arg2); if (tp[1] != am_re_pattern || is_not_small(tp[2]) || is_not_small(tp[3]) || is_not_binary(tp[4])) { - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } if (unsigned_val(tp[3]) && - (!is_binary(BIF_ARG_1) || + (!is_binary(arg1) || (is_list_cap && !(pflags & PARSE_FLAG_GLOBAL)))) { /* unicode */ - BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, - BIF_ARG_3); + BIF_TRAP3(urun_trap_exportp, p, arg1, arg2, + arg3); } if (pflags & PARSE_FLAG_GLOBAL) { Eterm *hp,r; - hp = HAlloc(BIF_P,3); - r = TUPLE2(hp,BIF_ARG_3,am_false); - BIF_TRAP3(grun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, + hp = HAlloc(p,3); + r = TUPLE2(hp,arg3,am_false); + BIF_TRAP3(grun_trap_exportp, p, arg1, arg2, r); } @@ -968,7 +975,7 @@ re_run_3(BIF_ALIST_3) if ((code_tmp = (const pcre *) erts_get_aligned_binary_bytes(tp[4], &temp_alloc)) == NULL) { erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size); memcpy(restart.code, code_tmp, code_size); @@ -980,7 +987,7 @@ re_run_3(BIF_ALIST_3) restart.ovector = erts_alloc(ERTS_ALC_T_RE_SUBJECT, ovsize * sizeof(int)); restart.extra.flags = PCRE_EXTRA_TABLES | PCRE_EXTRA_LOOP_LIMIT; restart.extra.tables = default_table; - restart.extra.loop_limit = ERTS_BIF_REDS_LEFT(BIF_P) * LOOP_FACTOR; + restart.extra.loop_limit = ERTS_BIF_REDS_LEFT(p) * LOOP_FACTOR; loop_limit_tmp = max_loop_limit; /* To lesser probability of race in debug situation (erts_debug) */ if (restart.extra.loop_limit > loop_limit_tmp) { @@ -996,7 +1003,7 @@ re_run_3(BIF_ALIST_3) if ((restart.ret_info = build_capture(capture,restart.code)) == NULL) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } } @@ -1004,7 +1011,7 @@ re_run_3(BIF_ALIST_3) copying, also binary returns can be sub binaries in that case */ restart.flags = 0; - if (is_binary(BIF_ARG_1)) { + if (is_binary(arg1)) { Eterm real_bin; Uint offset; Eterm* bptr; @@ -1012,9 +1019,9 @@ re_run_3(BIF_ALIST_3) int bitsize; ProcBin* pb; - ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + ERTS_GET_REAL_BIN(arg1, real_bin, offset, bitoffs, bitsize); - slength = binary_size(BIF_ARG_1); + slength = binary_size(arg1); bptr = binary_val(real_bin); if (bitsize != 0 || bitoffs != 0 || (*bptr != HEADER_PROC_BIN)) { goto handle_iolist; @@ -1027,24 +1034,24 @@ re_run_3(BIF_ALIST_3) restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { handle_iolist: - if (erts_iolist_size(BIF_ARG_1, &slength)) { + if (erts_iolist_size(arg1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); if (restart.ret_info != NULL) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info); } - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength); - if (io_list_to_buf(BIF_ARG_1, restart.subject, slength) != 0) { + if (io_list_to_buf(arg1, restart.subject, slength) != 0) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.subject); if (restart.ret_info != NULL) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info); } - BIF_ERROR(BIF_P,BADARG); + BIF_ERROR(p,BADARG); } } @@ -1056,7 +1063,7 @@ handle_iolist: rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, slength, startoffset, options, restart.ovector, ovsize); ASSERT(loop_count != 0xFFFFFFFF); - BUMP_REDS(BIF_P, loop_count / LOOP_FACTOR); + BUMP_REDS(p, loop_count / LOOP_FACTOR); if (rc == PCRE_ERROR_LOOP_LIMIT) { /* Trap */ Binary *mbp = erts_create_magic_binary(sizeof(RestartContext), @@ -1065,17 +1072,17 @@ handle_iolist: Eterm magic_bin; Eterm *hp; memcpy(restartp,&restart,sizeof(RestartContext)); - BUMP_ALL_REDS(BIF_P); - hp = HAlloc(BIF_P, PROC_BIN_SIZE); - magic_bin = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mbp); + BUMP_ALL_REDS(p); + hp = HAlloc(p, PROC_BIN_SIZE); + magic_bin = erts_mk_magic_binary_term(&hp, &MSO(p), mbp); BIF_TRAP3(&re_exec_trap_export, - BIF_P, - BIF_ARG_1, - BIF_ARG_2 /* To avoid GC of precompiled code, XXX: not utilized yet */, + p, + arg1, + arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */, magic_bin); } - res = build_exec_return(BIF_P, rc, &restart, BIF_ARG_1); + res = build_exec_return(p, rc, &restart, arg1); cleanup_restart_context(&restart); @@ -1083,9 +1090,15 @@ handle_iolist: } BIF_RETTYPE +re_run_3(BIF_ALIST_3) +{ + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE re_run_2(BIF_ALIST_2) { - return re_run_3(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); } /* diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index db771bd216..a922a33da3 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -26,6 +26,7 @@ #include "bif.h" #include "error.h" #include "big.h" +#include "erl_thr_progress.h" /**************************************************************************** ** BIF Timer support @@ -686,7 +687,7 @@ erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *), { int i; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); for (i = 0; i < TIMER_HASH_VEC_SZ; i++) { ErtsBifTimer *btm; diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 7a08182e18..b0a58c80ea 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -37,6 +37,7 @@ #include "erl_version.h" #include "beam_bp.h" #include "erl_binary.h" +#include "erl_thr_progress.h" #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -47,6 +48,11 @@ static Binary *erts_default_meta_match_spec; static struct trace_pattern_flags erts_default_trace_pattern_flags; static Eterm erts_default_meta_tracer_pid; +static Eterm +trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist); +static BIF_RETTYPE +system_monitor(Process *p, Eterm monitor_pid, Eterm list); + static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/ static int already_traced(Process *p, Process *tracee_p, Eterm tracer); static int port_already_traced(Process *p, Port *tracee_port, Eterm tracer); @@ -76,13 +82,19 @@ erts_bif_trace_init(void) */ Eterm -trace_pattern_2(Process* p, Eterm MFA, Eterm Pattern) +trace_pattern_2(BIF_ALIST_2) { - return trace_pattern_3(p,MFA,Pattern,NIL); + return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL); } Eterm -trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) +trace_pattern_3(BIF_ALIST_3) +{ + return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static Eterm +trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; @@ -97,7 +109,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) Eterm meta_tracer_pid = p->id; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); UseTmpHeap(3,p); /* @@ -326,7 +338,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) done: UnUseTmpHeap(3,p); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); return make_small(matches); @@ -336,7 +348,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetUnref(match_prog_set); UnUseTmpHeap(3,p); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_ERROR(p, BADARG); } @@ -435,9 +447,12 @@ erts_trace_flags(Eterm List, return 0; } -Eterm -trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) +Eterm trace_3(BIF_ALIST_3) { + Process* p = BIF_P; + Eterm pid_spec = BIF_ARG_1; + Eterm how = BIF_ARG_2; + Eterm list = BIF_ARG_3; int on; Eterm tracer = NIL; int matches = 0; @@ -630,7 +645,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) #ifdef ERTS_SMP erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); system_blocked = 1; #endif @@ -711,7 +726,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) #ifdef ERTS_SMP if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif @@ -726,7 +741,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list) #ifdef ERTS_SMP if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif @@ -820,9 +835,11 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer) * Return information about a process or an external function being traced. */ -Eterm -trace_info_2(Process* p, Eterm What, Eterm Key) +Eterm trace_info_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm What = BIF_ARG_1; + Eterm Key = BIF_ARG_2; Eterm res; if (What == am_on_load) { res = trace_info_on_load(p, Key); @@ -1060,7 +1077,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) #ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); } #endif @@ -1068,7 +1085,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) #ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif @@ -1752,23 +1769,20 @@ new_seq_trace_token(Process* p) } } -BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) +BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) { - Eterm item; Eterm res; Eterm* hp; Uint current_flag; - if (is_not_atom(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + if (is_not_atom(item)) { + BIF_ERROR(p, BADARG); } - item = BIF_ARG_1; - - if (SEQ_TRACE_TOKEN(BIF_P) == NIL) { + if (SEQ_TRACE_TOKEN(p) == NIL) { if ((item == am_send) || (item == am_receive) || (item == am_print) || (item == am_timestamp)) { - hp = HAlloc(BIF_P,3); + hp = HAlloc(p,3); res = TUPLE2(hp, item, am_false); BIF_RET(res); } else if ((item == am_label) || (item == am_serial)) { @@ -1778,35 +1792,40 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) } } - if (BIF_ARG_1 == am_send) { + if (item == am_send) { current_flag = SEQ_TRACE_SEND; - } else if (BIF_ARG_1 == am_receive) { + } else if (item == am_receive) { current_flag = SEQ_TRACE_RECEIVE; - } else if (BIF_ARG_1 == am_print) { + } else if (item == am_print) { current_flag = SEQ_TRACE_PRINT; - } else if (BIF_ARG_1 == am_timestamp) { + } else if (item == am_timestamp) { current_flag = SEQ_TRACE_TIMESTAMP; } else { current_flag = 0; } if (current_flag) { - res = unsigned_val(SEQ_TRACE_TOKEN_FLAGS(BIF_P)) & current_flag ? + res = unsigned_val(SEQ_TRACE_TOKEN_FLAGS(p)) & current_flag ? am_true : am_false; } else if (item == am_label) { - res = SEQ_TRACE_TOKEN_LABEL(BIF_P); + res = SEQ_TRACE_TOKEN_LABEL(p); } else if (item == am_serial) { - hp = HAlloc(BIF_P, 3); - res = TUPLE2(hp, SEQ_TRACE_TOKEN_LASTCNT(BIF_P), SEQ_TRACE_TOKEN_SERIAL(BIF_P)); + hp = HAlloc(p, 3); + res = TUPLE2(hp, SEQ_TRACE_TOKEN_LASTCNT(p), SEQ_TRACE_TOKEN_SERIAL(p)); } else { error: - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - hp = HAlloc(BIF_P, 3); + hp = HAlloc(p, 3); res = TUPLE2(hp, item, res); BIF_RET(res); } +BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) +{ + BIF_RET(erl_seq_trace_info(BIF_P, BIF_ARG_1)); +} + /* seq_trace_print(Message) -> true | false This function passes Message to the system_tracer @@ -1852,7 +1871,7 @@ void erts_system_monitor_clear(Process *c_p) { #ifdef ERTS_SMP if (c_p) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); } #endif erts_set_system_monitor(NIL); @@ -1862,7 +1881,7 @@ void erts_system_monitor_clear(Process *c_p) { erts_system_monitor_flags.busy_dist_port = 0; #ifdef ERTS_SMP if (c_p) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } #endif @@ -1919,23 +1938,35 @@ static Eterm system_monitor_get(Process *p) } -BIF_RETTYPE system_monitor_0(Process *p) { - BIF_RET(system_monitor_get(p)); +BIF_RETTYPE system_monitor_0(BIF_ALIST_0) +{ + BIF_RET(system_monitor_get(BIF_P)); } -BIF_RETTYPE system_monitor_1(Process *p, Eterm spec) { +BIF_RETTYPE system_monitor_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm spec = BIF_ARG_1; + if (spec == am_undefined) { - BIF_RET(system_monitor_2(p, spec, NIL)); + BIF_RET(system_monitor(p, spec, NIL)); } else if (is_tuple(spec)) { Eterm *tp = tuple_val(spec); if (tp[0] != make_arityval(2)) goto error; - BIF_RET(system_monitor_2(p, tp[1], tp[2])); + BIF_RET(system_monitor(p, tp[1], tp[2])); } error: BIF_ERROR(p, BADARG); } -BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { +BIF_RETTYPE system_monitor_2(BIF_ALIST_2) +{ + return system_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +static BIF_RETTYPE +system_monitor(Process *p, Eterm monitor_pid, Eterm list) +{ Eterm prev; int system_blocked = 0; @@ -1951,7 +1982,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { system_blocked = 1; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, monitor_pid, 0)) goto error; @@ -1985,7 +2016,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { erts_system_monitor_flags.busy_port = !!busy_port; erts_system_monitor_flags.busy_dist_port = !!busy_dist_port; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(prev); } @@ -1993,7 +2024,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) { error: if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } @@ -2006,7 +2037,7 @@ void erts_system_profile_clear(Process *c_p) { #ifdef ERTS_SMP if (c_p) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); } #endif erts_set_system_profile(NIL); @@ -2016,7 +2047,7 @@ void erts_system_profile_clear(Process *c_p) { erts_system_profile_flags.exclusive = 0; #ifdef ERTS_SMP if (c_p) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } #endif @@ -2053,11 +2084,16 @@ static Eterm system_profile_get(Process *p) { } } -BIF_RETTYPE system_profile_0(Process *p) { - BIF_RET(system_profile_get(p)); +BIF_RETTYPE system_profile_0(BIF_ALIST_0) +{ + BIF_RET(system_profile_get(BIF_P)); } -BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { +BIF_RETTYPE system_profile_2(BIF_ALIST_2) +{ + Process *p = BIF_P; + Eterm profiler = BIF_ARG_1; + Eterm list = BIF_ARG_2; Eterm prev; int system_blocked = 0; Process *profiler_p = NULL; @@ -2075,7 +2111,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { system_blocked = 1; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); /* Check if valid process, no locks are taken */ @@ -2117,7 +2153,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { erts_system_profile_flags.runnable_procs = !!runnable_procs; erts_system_profile_flags.exclusive = !!exclusive; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(prev); @@ -2126,7 +2162,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) { error: if (system_blocked) { - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 326a5c136b..6f7309f493 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -845,8 +845,7 @@ erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm arg)) dst[1] = 0x80 | (val & 0x3F); num_bits = 16; } else if (val < 0x10000UL) { - if ((0xD800 <= val && val <= 0xDFFF) || - val == 0xFFFE || val == 0xFFFF) { + if (0xD800 <= val && val <= 0xDFFF) { return 0; } dst[0] = 0xE0 | (val >> 12); @@ -886,8 +885,7 @@ erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm arg, Uint flags)) return 0; } val = unsigned_val(arg); - if (val > 0x10FFFF || (0xD800 <= val && val <= 0xDFFF) || - val == 0xFFFE || val == 0xFFFF) { + if (val > 0x10FFFF || (0xD800 <= val && val <= 0xDFFF)) { return 0; } @@ -1652,8 +1650,7 @@ erts_bs_get_utf8(ErlBinMatchBuffer* mb) return THE_NON_VALUE; } result = (((result << 6) + a) << 6) + b - (Eterm) 0x000E2080UL; - if ((0xD800 <= result && result <= 0xDFFF) || - result == 0xFFFE || result == 0xFFFF) { + if (0xD800 <= result && result <= 0xDFFF) { return THE_NON_VALUE; } mb->offset += 24; @@ -1723,9 +1720,6 @@ erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags) w1 = (src[0] << 8) | src[1]; } if (w1 < 0xD800 || w1 > 0xDFFF) { - if (w1 == 0xFFFE || w1 == 0xFFFF) { - return THE_NON_VALUE; - } mb->offset += 16; return make_small(w1); } else if (w1 > 0xDBFF) { diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 03c0ef904a..fe3693d0ca 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -486,10 +486,7 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) erts_thr_set_main_status(1, (int) esdp->no); /* Make sure we check if we should bind to a cpu or not... */ - if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 1); - else - esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; } #endif @@ -502,11 +499,7 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) erts_cpu_groups_callback_list_t *cgcl; erts_cpu_groups_callback_call_t *cgcc; #ifdef ERTS_SMP - if (erts_common_run_queue) - erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 0); - else { - esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; - } + esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; #endif erts_smp_runq_unlock(esdp->run_queue); erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); @@ -1729,16 +1722,8 @@ erts_init_cpu_topology(void) scheduler2cpu_map[ix].bound_id = -1; } - if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) { - int ncpus = erts_get_cpu_configured(cpuinfo); - if (ncpus < 1 || erts_no_schedulers < ncpus) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else - cpu_bind_order = ((system_cpudata || user_cpudata) - && (erts_bind_to_cpu(cpuinfo, -1) != -ENOTSUP) - ? ERTS_CPU_BIND_DEFAULT_BIND - : ERTS_CPU_BIND_NONE); - } + if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) + cpu_bind_order = ERTS_CPU_BIND_NONE; reader_groups_map = add_cpu_groups(reader_groups, reader_groups_callback, diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 0327850cb9..eb89baf1c9 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -129,8 +129,6 @@ static Uint meta_main_tab_slot_mask; /* The slot index part of an unnamed tab static Uint meta_main_tab_seq_incr; static Uint meta_main_tab_seq_cnt = 0; /* To give unique(-ish) table identifiers */ - - /* ** The meta hash table of all NAMED ets tables */ @@ -202,12 +200,17 @@ static int free_table_cont(Process *p, int first, int clean_meta_tab); static void print_table(int to, void *to_arg, int show, DbTable* tb); -static BIF_RETTYPE ets_select_delete_1(Process *p, Eterm a1); -static BIF_RETTYPE ets_select_count_1(Process *p, Eterm a1); -static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1); -static BIF_RETTYPE ets_delete_trap(Process *p, Eterm a1); +static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1); +static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1); +static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1); +static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1); static Eterm table_info(Process* p, DbTable* tb, Eterm What); +static BIF_RETTYPE ets_select1(Process* p, Eterm arg1); +static BIF_RETTYPE ets_select2(Process* p, Eterm arg1, Eterm arg2); +static BIF_RETTYPE ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3); + + /* * Exported global */ @@ -277,8 +280,7 @@ static void schedule_free_dbtable(DbTable* tb) ASSERT(scheds >= 1); ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); erts_refc_init(&tb->common.ref, scheds); - ERTS_THR_MEMORY_BARRIER; - erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb); + erts_schedule_multi_misc_aux_work(0, scheds, chk_free_dbtable, tb); #else free_dbtable(tb); #endif @@ -1296,8 +1298,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) UWord heir_data; Uint32 status; Sint keypos; - int is_named, is_fine_locked, frequent_read, is_compressed; + int is_named, is_compressed; +#ifdef ERTS_SMP + int is_fine_locked, frequent_read; +#endif +#ifdef DEBUG int cret; +#endif DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; erts_smp_rwmtx_t *mmtl; @@ -1312,8 +1319,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) status = DB_NORMAL | DB_SET | DB_PROTECTED; keypos = 1; is_named = 0; +#ifdef ERTS_SMP is_fine_locked = 0; frequent_read = 0; +#endif heir = am_none; heir_data = (UWord) am_undefined; is_compressed = erts_ets_always_compress; @@ -1342,18 +1351,31 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) keypos = signed_val(tp[2]); } else if (tp[1] == am_write_concurrency) { +#ifdef ERTS_SMP if (tp[2] == am_true) { is_fine_locked = 1; } else if (tp[2] == am_false) { is_fine_locked = 0; } else break; +#else + if ((tp[2] != am_true) && (tp[2] != am_false)) { + break; + } +#endif } else if (tp[1] == am_read_concurrency) { +#ifdef ERTS_SMP if (tp[2] == am_true) { frequent_read = 1; } else if (tp[2] == am_false) { frequent_read = 0; } else break; +#else + if ((tp[2] != am_true) && (tp[2] != am_false)) { + break; + } +#endif + } else if (tp[1] == am_heir && tp[2] == am_none) { heir = am_none; @@ -1393,11 +1415,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) } if (IS_HASH_TABLE(status)) { meth = &db_hash; - #ifdef ERTS_SMP +#ifdef ERTS_SMP if (is_fine_locked && !(status & DB_PRIVATE)) { status |= DB_FINE_LOCKED; } - #endif +#endif } else if (IS_TREE_TABLE(status)) { meth = &db_tree; @@ -1445,7 +1467,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.fixations = NULL; tb->common.compress = is_compressed; - cret = meth->db_create(BIF_P, tb); +#ifdef DEBUG + cret = +#endif + meth->db_create(BIF_P, tb); ASSERT(cret == DB_ERROR_NONE); erts_smp_spin_lock(&meta_main_tab_main_lock); @@ -1942,8 +1967,10 @@ BIF_RETTYPE ets_delete_object_2(BIF_ALIST_2) /* ** This is for trapping, cannot be called directly. */ -static BIF_RETTYPE ets_select_delete_1(Process *p, Eterm a1) +static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; BIF_RETTYPE result; DbTable* tb; int cret; @@ -2109,7 +2136,7 @@ BIF_RETTYPE ets_slot_2(BIF_ALIST_2) BIF_RETTYPE ets_match_1(BIF_ALIST_1) { - return ets_select_1(BIF_P, BIF_ARG_1); + return ets_select1(BIF_P, BIF_ARG_1); } BIF_RETTYPE ets_match_2(BIF_ALIST_2) @@ -2125,7 +2152,7 @@ BIF_RETTYPE ets_match_2(BIF_ALIST_2) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select2(BIF_P, BIF_ARG_1, ms); UnUseTmpHeap(8,BIF_P); return res; } @@ -2143,7 +2170,7 @@ BIF_RETTYPE ets_match_3(BIF_ALIST_3) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); UnUseTmpHeap(8,BIF_P); return res; } @@ -2151,6 +2178,12 @@ BIF_RETTYPE ets_match_3(BIF_ALIST_3) BIF_RETTYPE ets_select_3(BIF_ALIST_3) { + return ets_select3(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +static BIF_RETTYPE +ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3) +{ BIF_RETTYPE result; DbTable* tb; int cret; @@ -2161,22 +2194,22 @@ BIF_RETTYPE ets_select_3(BIF_ALIST_3) CHECK_TABLES(); /* Chunk size strictly greater than 0 */ - if (is_not_small(BIF_ARG_3) || (chunk_size = signed_val(BIF_ARG_3)) <= 0) { - BIF_ERROR(BIF_P, BADARG); + if (is_not_small(arg3) || (chunk_size = signed_val(arg3)) <= 0) { + BIF_ERROR(p, BADARG); } - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) { + BIF_ERROR(p, BADARG); } - safety = ITERATION_SAFETY(BIF_P,tb); + safety = ITERATION_SAFETY(p,tb); if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_chunk(BIF_P, tb, - BIF_ARG_2, chunk_size, + cret = tb->common.meth->db_select_chunk(p, tb, + arg2, chunk_size, 0 /* not reversed */, &ret); - if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { - fix_table_locked(BIF_P, tb); + if (DID_TRAP(p,ret) && safety != ITER_SAFE) { + fix_table_locked(p, tb); } if (safety == ITER_UNSAFE) { local_unfix_table(tb); @@ -2188,22 +2221,24 @@ BIF_RETTYPE ets_select_3(BIF_ALIST_3) ERTS_BIF_PREP_RET(result, ret); break; case DB_ERROR_SYSRES: - ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT); + ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT); break; default: - ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(result, p, BADARG); break; } - erts_match_set_release_result(BIF_P); + erts_match_set_release_result(p); return result; } /* We get here instead of in the real BIF when trapping */ -static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1) +static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; BIF_RETTYPE result; DbTable* tb; int cret; @@ -2248,6 +2283,11 @@ static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1) BIF_RETTYPE ets_select_1(BIF_ALIST_1) { + return ets_select1(BIF_P, BIF_ARG_1); +} + +static BIF_RETTYPE ets_select1(Process *p, Eterm arg1) +{ BIF_RETTYPE result; DbTable* tb; int cret; @@ -2261,28 +2301,27 @@ BIF_RETTYPE ets_select_1(BIF_ALIST_1) * Make sure that the table exists. */ - if (!is_tuple(BIF_ARG_1)) { - if (BIF_ARG_1 == am_EOT) { + if (!is_tuple(arg1)) { + if (arg1 == am_EOT) { BIF_RET(am_EOT); } - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); } - tptr = tuple_val(BIF_ARG_1); + tptr = tuple_val(arg1); if (arityval(*tptr) < 1 || - (tb = db_get_table(BIF_P, tptr[1], DB_READ, LCK_READ)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + (tb = db_get_table(p, tptr[1], DB_READ, LCK_READ)) == NULL) { + BIF_ERROR(p, BADARG); } - safety = ITERATION_SAFETY(BIF_P,tb); + safety = ITERATION_SAFETY(p,tb); if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select_continue(BIF_P,tb, - BIF_ARG_1, &ret); + cret = tb->common.meth->db_select_continue(p,tb, arg1, &ret); - if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { - fix_table_locked(BIF_P, tb); + if (DID_TRAP(p,ret) && safety != ITER_SAFE) { + fix_table_locked(p, tb); } if (safety == ITER_UNSAFE) { local_unfix_table(tb); @@ -2294,20 +2333,26 @@ BIF_RETTYPE ets_select_1(BIF_ALIST_1) ERTS_BIF_PREP_RET(result, ret); break; case DB_ERROR_SYSRES: - ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT); + ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT); break; default: - ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(result, p, BADARG); break; } - erts_match_set_release_result(BIF_P); + erts_match_set_release_result(p); return result; } BIF_RETTYPE ets_select_2(BIF_ALIST_2) { + return ets_select2(BIF_P, BIF_ARG_1, BIF_ARG_2); +} + +static BIF_RETTYPE +ets_select2(Process* p, Eterm arg1, Eterm arg2) +{ BIF_RETTYPE result; DbTable* tb; int cret; @@ -2320,19 +2365,19 @@ BIF_RETTYPE ets_select_2(BIF_ALIST_2) * Make sure that the table exists. */ - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) { + BIF_ERROR(p, BADARG); } - safety = ITERATION_SAFETY(BIF_P,tb); + safety = ITERATION_SAFETY(p,tb); if (safety == ITER_UNSAFE) { local_fix_table(tb); } - cret = tb->common.meth->db_select(BIF_P, tb, BIF_ARG_2, + cret = tb->common.meth->db_select(p, tb, arg2, 0, &ret); - if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) { - fix_table_locked(BIF_P, tb); + if (DID_TRAP(p,ret) && safety != ITER_SAFE) { + fix_table_locked(p, tb); } if (safety == ITER_UNSAFE) { local_unfix_table(tb); @@ -2344,21 +2389,23 @@ BIF_RETTYPE ets_select_2(BIF_ALIST_2) ERTS_BIF_PREP_RET(result, ret); break; case DB_ERROR_SYSRES: - ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT); + ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT); break; default: - ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG); + ERTS_BIF_PREP_ERROR(result, p, BADARG); break; } - erts_match_set_release_result(BIF_P); + erts_match_set_release_result(p); return result; } /* We get here instead of in the real BIF when trapping */ -static BIF_RETTYPE ets_select_count_1(Process *p, Eterm a1) +static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; BIF_RETTYPE result; DbTable* tb; int cret; @@ -2499,7 +2546,7 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3) BIF_RETTYPE ets_select_reverse_1(BIF_ALIST_1) { - return ets_select_1(BIF_P, BIF_ARG_1); + return ets_select1(BIF_P, BIF_ARG_1); } BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2) @@ -2553,7 +2600,7 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2) */ BIF_RETTYPE ets_match_object_1(BIF_ALIST_1) { - return ets_select_1(BIF_P, BIF_ARG_1); + return ets_select1(BIF_P, BIF_ARG_1); } BIF_RETTYPE ets_match_object_2(BIF_ALIST_2) @@ -2569,7 +2616,7 @@ BIF_RETTYPE ets_match_object_2(BIF_ALIST_2) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_2(BIF_P, BIF_ARG_1, ms); + res = ets_select2(BIF_P, BIF_ARG_1, ms); UnUseTmpHeap(8,BIF_P); return res; } @@ -2587,7 +2634,7 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); hp += 4; ms = CONS(hp, ms, NIL); - res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); + res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3); UnUseTmpHeap(8,BIF_P); return res; } @@ -2606,7 +2653,9 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) int i; Eterm* hp; /*Process* rp = NULL;*/ + /* If/when we implement lockless private tables: Eterm owner; + */ if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) { @@ -2615,7 +2664,9 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } + /* If/when we implement lockless private tables: owner = tb->common.owner; + */ /* If/when we implement lockless private tables: if ((tb->common.status & DB_PRIVATE) && owner != BIF_P->id) { @@ -2802,10 +2853,10 @@ void init_db(void) bits = erts_fit_in_bits(db_max_tabs-1); if (bits > SMALL_BITS) { erl_exit(1,"Max limit for ets tabled too high %u (max %u).", - db_max_tabs, 1L<<SMALL_BITS); + db_max_tabs, ((Uint)1)<<SMALL_BITS); } - meta_main_tab_slot_mask = (1L<<bits) - 1; - meta_main_tab_seq_incr = (1L<<bits); + meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1; + meta_main_tab_seq_incr = (((Uint)1)<<bits); size = sizeof(*meta_main_tab)*db_max_tabs; meta_main_tab = erts_db_alloc_nt(ERTS_ALC_T_DB_TABLES, size); @@ -2818,7 +2869,7 @@ void init_db(void) SET_NEXT_FREE_SLOT(db_max_tabs-1, (Uint)-1); meta_main_tab_first_free = 0; - meta_name_tab_mask = (1L<<(bits-1)) - 1; /* At least half the size of main tab */ + meta_name_tab_mask = (((Uint) 1)<<(bits-1)) - 1; /* At least half the size of main tab */ size = sizeof(struct meta_name_tab_entry)*(meta_name_tab_mask+1); meta_name_tab = erts_db_alloc_nt(ERTS_ALC_T_DB_TABLES, size); ERTS_ETS_MISC_MEM_ADD(size); @@ -3521,8 +3572,10 @@ static void free_heir_data(DbTable* tb) #endif } -static BIF_RETTYPE ets_delete_trap(Process *p, Eterm cont) +static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm cont = BIF_ARG_1; int trap; Eterm* ptr = big_val(cont); DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index e3380a57b2..038a667b06 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -312,15 +312,24 @@ struct ext_segment { struct segment* segtab[1]; /* The segment table */ }; #define SIZEOF_EXTSEG(NSEGS) \ - (sizeof(struct ext_segment) - sizeof(struct segment*) + sizeof(struct segment*)*(NSEGS)) + (offsetof(struct ext_segment,segtab) + sizeof(struct segment*)*(NSEGS)) -#ifdef DEBUG -# include <stddef.h> /* offsetof */ +#if defined(DEBUG) || defined(VALGRIND) # define EXTSEG(SEGTAB_PTR) \ ((struct ext_segment*) (((char*)(SEGTAB_PTR)) - offsetof(struct ext_segment,segtab))) #endif +static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, + struct segment** segtab) +{ + erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab); +#ifdef VALGRIND + tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab); +#endif +} + + /* How the table segments relate to each other: ext_segment: ext_segment: "plain" segment @@ -649,7 +658,8 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK); erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); - erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); + erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); + SET_SEGTAB(tb, alloc_ext_seg(tb,0,NULL)->segtab); tb->nsegs = NSEG_1; tb->nslots = SEGSZ; @@ -2357,7 +2367,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) eseg->segtab); + SET_SEGTAB(tb, eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2429,7 +2439,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)newtop->prev_segtab); + SET_SEGTAB(tb, newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2446,7 +2456,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)NULL); + SET_SEGTAB(tb, NULL); } #endif tb->nslots -= SEGSZ; diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e0285fa5ed..23ac493118 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -58,6 +58,9 @@ typedef struct db_table_hash { #ifdef ERTS_SMP DbTableHashFineLocks* locks; #endif +#ifdef VALGRIND + struct ext_segment* top_ptr_to_segment_with_active_segtab; +#endif } DbTableHash; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index c6f0d80e32..312050b931 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -344,8 +344,8 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b, int *done); static int do_partly_bound_can_match_greater(Eterm a, Eterm b, int *done); -static BIF_RETTYPE ets_select_reverse(Process *p, Eterm a1, - Eterm a2, Eterm a3); +static BIF_RETTYPE ets_select_reverse(BIF_ALIST_3); + /* Method interface functions */ static int db_first_tree(Process *p, DbTable *tbl, @@ -844,8 +844,12 @@ static int db_slot_tree(Process *p, DbTable *tbl, -static BIF_RETTYPE ets_select_reverse(Process *p, Eterm a1, Eterm a2, Eterm a3) +static BIF_RETTYPE ets_select_reverse(BIF_ALIST_3) { + Process *p = BIF_P; + Eterm a1 = BIF_ARG_1; + Eterm a2 = BIF_ARG_2; + Eterm a3 = BIF_ARG_3; Eterm list; Eterm result; Eterm* hp; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 7dfbb2ed02..4821a7d9fb 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -35,6 +35,7 @@ #include "bif.h" #include "big.h" #include "erl_binary.h" +#include "erl_thr_progress.h" #include "erl_db_util.h" @@ -495,7 +496,7 @@ static erts_smp_atomic32_t trace_control_word; /* This needs to be here, before the bif table... */ -static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val); +static Eterm db_set_trace_control_word_fake_1(BIF_ALIST_1); /* ** The table of callable bif's, i e guard bif's and @@ -908,14 +909,18 @@ static void db_free_tmp_uncompressed(DbTerm* obj); /* ** Pseudo BIF:s to be callable from the PAM VM. */ - -BIF_RETTYPE db_get_trace_control_word_0(Process *p) +BIF_RETTYPE db_get_trace_control_word(Process *p) { Uint32 tcw = (Uint32) erts_smp_atomic32_read_acqb(&trace_control_word); BIF_RET(erts_make_integer((Uint) tcw, p)); } -BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) +BIF_RETTYPE db_get_trace_control_word_0(BIF_ALIST_0) +{ + BIF_RET(db_get_trace_control_word(BIF_P)); +} + +BIF_RETTYPE db_set_trace_control_word(Process *p, Eterm new) { Uint val; Uint32 old_tcw; @@ -923,20 +928,27 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) BIF_ERROR(p, BADARG); if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - + old_tcw = (Uint32) erts_smp_atomic32_xchg_relb(&trace_control_word, (erts_aint32_t) val); BIF_RET(erts_make_integer((Uint) old_tcw, p)); } -static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm new) +BIF_RETTYPE db_set_trace_control_word_1(BIF_ALIST_1) { + BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_1)); +} + +static Eterm db_set_trace_control_word_fake_1(BIF_ALIST_1) +{ + Process *p = BIF_P; + Eterm new = BIF_ARG_1; Uint val; if (!term_to_Uint(new, &val)) BIF_ERROR(p, BADARG); if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - BIF_RET(db_get_trace_control_word_0(p)); + BIF_RET(db_get_trace_control_word(p)); } /* @@ -1704,6 +1716,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Process *current_scheduled; ErtsSchedulerData *esdp; Eterm (*bif)(Process*, ...); + Eterm bif_args[3]; int fail_label; int atomic_trace; #if HALFWORD_HEAP @@ -1734,14 +1747,14 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, if (! atomic_trace) { \ erts_refc_inc(&bprog->refc, 2); \ erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \ - erts_smp_block_system(0); \ + erts_smp_thr_progress_block(); \ atomic_trace = !0; \ } \ } while (0) #define END_ATOMIC_TRACE(p) \ do { \ if (atomic_trace) { \ - erts_smp_release_system(); \ + erts_smp_thr_progress_unblock(); \ erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \ if (erts_refc_dectest(&bprog->refc, 0) == 0) {\ erts_bin_free(bprog); \ @@ -1957,7 +1970,7 @@ restart: break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc); + t = (*bif)(build_proc, bif_args); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1968,7 +1981,7 @@ restart: break; case matchCall1: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc, esp[-1]); + t = (*bif)(build_proc, esp-1); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1979,7 +1992,9 @@ restart: break; case matchCall2: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc, esp[-1], esp[-2]); + bif_args[0] = esp[-1]; + bif_args[1] = esp[-2]; + t = (*bif)(build_proc, bif_args); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1991,7 +2006,10 @@ restart: break; case matchCall3: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(build_proc, esp[-1], esp[-2], esp[-3]); + bif_args[0] = esp[-1]; + bif_args[1] = esp[-2]; + bif_args[2] = esp[-3]; + t = (*bif)(build_proc, bif_args); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -2846,7 +2864,9 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) Uint new_sz = offset + db_size_dbterm_comp(tb, obj); byte* basep; DbTerm* newp; +#ifdef DEBUG byte* top; +#endif ASSERT(tb->compress); if (old != 0) { @@ -2868,7 +2888,10 @@ void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) } newp->size = size_object(obj); - top = copy_to_comp(tb, obj, newp, new_sz); +#ifdef DEBUG + top = +#endif + copy_to_comp(tb, obj, newp, new_sz); ASSERT(top <= basep + new_sz); /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */ @@ -4970,7 +4993,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) static Eterm seq_trace_fake(Process *p, Eterm arg1) { - Eterm result = seq_trace_info_1(p,arg1); + Eterm result = erl_seq_trace_info(p, arg1); if (is_tuple(result) && *tuple_val(result) == 2) { return (tuple_val(result))[2]; } diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index bb1751d309..6a96e174e1 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -326,8 +326,10 @@ ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) (T)->common.owner == (P)->id) /* Function prototypes */ -Eterm db_get_trace_control_word_0(Process *p); -Eterm db_set_trace_control_word_1(Process *p, Eterm val); +BIF_RETTYPE db_get_trace_control_word(Process* p); +BIF_RETTYPE db_set_trace_control_word(Process* p, Eterm tcw); +BIF_RETTYPE db_get_trace_control_word_0(BIF_ALIST_0); +BIF_RETTYPE db_set_trace_control_word_1(BIF_ALIST_1); void db_initialize_util(void); Eterm db_getkey(int keypos, Eterm obj); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 401967a8de..25483380ed 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -28,6 +28,14 @@ # include "config.h" #endif +#define ERL_DRV_DEPRECATED_FUNC +#ifdef __GNUC__ +# if __GNUC__ >= 3 +# undef ERL_DRV_DEPRECATED_FUNC +# define ERL_DRV_DEPRECATED_FUNC __attribute__((deprecated)) +# endif +#endif + #ifdef SIZEOF_CHAR # define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR # undef SIZEOF_CHAR @@ -152,10 +160,15 @@ typedef struct { /* * Integer types */ - +#if defined(__WIN32__) && (SIZEOF_VOID_P == 8) +typedef unsigned __int64 ErlDrvTermData; +typedef unsigned __int64 ErlDrvUInt; +typedef signed __int64 ErlDrvSInt; +#else typedef unsigned long ErlDrvTermData; typedef unsigned long ErlDrvUInt; typedef signed long ErlDrvSInt; +#endif #if defined(__WIN32__) typedef unsigned __int64 ErlDrvUInt64; @@ -176,7 +189,7 @@ typedef long long ErlDrvSInt64; */ typedef struct erl_drv_binary { - long orig_size; /* total length of binary */ + ErlDrvSInt orig_size; /* total length of binary */ char orig_bytes[1]; /* the data (char instead of byte!) */ } ErlDrvBinary; @@ -582,8 +595,11 @@ EXTERN long driver_async(ErlDrvPort ix, void* async_data, void (*async_free)(void*)); - -EXTERN int driver_async_cancel(unsigned int key); +/* + * driver_async_cancel() is deprecated. It is scheduled for removal + * in OTP-R16. For more information see the erl_driver(3) documentation. + */ +EXTERN int driver_async_cancel(unsigned int key) ERL_DRV_DEPRECATED_FUNC; /* Locks the driver in the machine "forever", there is no unlock function. Note that this is almost never useful, as an open diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index dc578f6d2a..a49a155701 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -158,7 +158,9 @@ erl_drv_mutex_create(char *name) (sizeof(ErlDrvMutex) + (name ? sys_strlen(name) + 1 : 0))); if (dmtx) { - if (ethr_mutex_init(&dmtx->mtx) != 0) { + ethr_mutex_opt opt = ETHR_MUTEX_OPT_DEFAULT_INITER; + opt.posix_compliant = 1; + if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) { erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); dmtx = NULL; } @@ -226,7 +228,9 @@ erl_drv_cond_create(char *name) (sizeof(ErlDrvCond) + (name ? sys_strlen(name) + 1 : 0))); if (dcnd) { - if (ethr_cond_init(&dcnd->cnd) != 0) { + ethr_cond_opt opt = ETHR_COND_OPT_DEFAULT_INITER; + opt.posix_compliant = 1; + if (ethr_cond_init_opt(&dcnd->cnd, &opt) != 0) { erts_free(ERTS_ALC_T_DRV_CND, (void *) dcnd); dcnd = NULL; } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index e3445bcdc5..eb2b945877 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -315,7 +315,12 @@ erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) if (is_non_value(result)) { if (p->freason == TRAP) { - cost = erts_garbage_collect(p, 0, p->def_arg_reg, p->arity); + #if HIPE + if (regs == NULL) { + regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array; + } + #endif + cost = erts_garbage_collect(p, 0, regs, p->arity); } else { cost = erts_garbage_collect(p, 0, regs, arity); } @@ -357,8 +362,6 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC); - ERTS_CHK_OFFHEAP(p); ErtsGcQuickSanityCheck(p); @@ -392,8 +395,6 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) trace_gc(p, am_gc_end); } - erts_smp_locked_activity_end(ERTS_ACTIVITY_GC); - if (erts_system_monitor_long_gc != 0) { Uint ms2, s2, us2; Sint t; @@ -477,7 +478,6 @@ erts_garbage_collect_hibernate(Process* p) p->gcstatus = p->status; p->status = P_GARBING; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC); ErtsGcQuickSanityCheck(p); ASSERT(p->mbuf_sz == 0); ASSERT(p->mbuf == 0); @@ -591,12 +591,13 @@ erts_garbage_collect_hibernate(Process* p) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); p->status = p->gcstatus; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_end(ERTS_ACTIVITY_GC); } void -erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) +erts_garbage_collect_literals(Process* p, Eterm* literals, + Uint lit_size, + struct erl_off_heap_header* oh) { Uint byte_lit_size = sizeof(Eterm)*lit_size; Uint old_heap_size; @@ -608,6 +609,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) Uint area_size; Eterm* old_htop; Uint n; + struct erl_off_heap_header** prev; /* * Set GC state. @@ -616,7 +618,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) p->gcstatus = p->status; p->status = P_GARBING; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC); /* * We assume that the caller has already done a major collection @@ -642,6 +643,9 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) offset_heap(temp_lit, lit_size, offs, (char *) literals, byte_lit_size); offset_heap(p->heap, p->htop - p->heap, offs, (char *) literals, byte_lit_size); offset_rootset(p, offs, (char *) literals, byte_lit_size, p->arg_reg, p->arity); + if (oh) { + oh = (struct erl_off_heap_header *) ((Eterm *)(void *) oh + offs); + } /* * Now the literals are placed in memory that is safe to write into, @@ -709,6 +713,45 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) p->old_htop = old_htop; /* + * Prepare to sweep binaries. Since all MSOs on the new heap + * must be come before MSOs on the old heap, find the end of + * current MSO list and use that as a starting point. + */ + + if (oh) { + prev = &MSO(p).first; + while (*prev) { + prev = &(*prev)->next; + } + } + + /* + * Sweep through all binaries in the temporary literal area. + */ + + while (oh) { + if (IS_MOVED_BOXED(oh->thing_word)) { + Binary* bptr; + struct erl_off_heap_header* ptr; + + ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word); + ASSERT(thing_subtag(ptr->thing_word) == REFC_BINARY_SUBTAG); + bptr = ((ProcBin*)ptr)->val; + + /* + * This binary has been copied to the heap. + * We must increment its reference count and + * link it into the MSO list for the process. + */ + + erts_refc_inc(&bptr->refc, 1); + *prev = ptr; + prev = &ptr->next; + } + oh = oh->next; + } + + /* * We no longer need this temporary area. */ erts_free(ERTS_ALC_T_TMP, (void *) temp_lit); @@ -719,7 +762,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); p->status = p->gcstatus; erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_locked_activity_end(ERTS_ACTIVITY_GC); } static int diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 807ef8ae8d..0ba1009bd3 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -62,7 +62,7 @@ do { \ } while(0) #define in_area(ptr,start,nbytes) \ - ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes)) + ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) extern Uint erts_test_long_gc_sleep; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index ef86e6db5e..717315d8bd 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -43,6 +43,8 @@ #include "packet_parser.h" #include "erl_cpu_topology.h" #include "erl_thr_progress.h" +#include "erl_thr_queue.h" +#include "erl_async.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -69,6 +71,8 @@ static void erl_init(int ncpu); #define ERTS_MIN_COMPAT_REL 7 +static erts_atomic_t exiting; + #ifdef ERTS_SMP erts_smp_atomic32_t erts_writing_erl_crash_dump; erts_tsd_key_t erts_is_crash_dumping_key; @@ -88,7 +92,6 @@ int erts_use_sender_punish; */ Uint display_items; /* no of items to display in traces etc */ -Uint display_loads; /* print info about loaded modules */ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ @@ -100,8 +103,6 @@ int erts_backtrace_depth; /* How many functions to show in a backtrace * in error codes. */ -int erts_async_max_threads; /* number of threads for async support */ -int erts_async_thread_suggested_stack_size; erts_smp_atomic32_t erts_max_gen_gcs; Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, @@ -110,7 +111,6 @@ Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, int erts_compat_rel; -static int use_multi_run_queue; static int no_schedulers; static int no_schedulers_online; @@ -128,6 +128,8 @@ int erts_modified_timing_level; int erts_no_crash_dump = 0; /* Use -d to suppress crash dump. */ +int erts_no_line_info = 0; /* -L: Don't load line information */ + /* * Other global variables. */ @@ -246,17 +248,12 @@ erl_init(int ncpu) { init_benchmarking(); -#ifdef ERTS_SMP - erts_system_block_init(); -#endif - erts_init_monitors(); erts_init_gc(); erts_init_time(); erts_init_sys_common_misc(); erts_init_process(ncpu); - erts_init_scheduling(use_multi_run_queue, - no_schedulers, + erts_init_scheduling(no_schedulers, no_schedulers_online); erts_init_cpu_topology(); /* Must be after init_scheduling */ erts_alloc_late_init(); @@ -282,6 +279,7 @@ erl_init(int ncpu) erts_init_node_tables(); init_dist(); erl_drv_thr_init(); + erts_init_async(); init_io(); init_copy(); init_load(); @@ -436,7 +434,7 @@ static void load_preloaded(void) { int i; - int res; + Eterm res; Preload* preload_p; Eterm module_name; byte* code; @@ -455,8 +453,9 @@ load_preloaded(void) name); res = erts_load_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); - if (res < 0) - erl_exit(1,"Failed loading preloaded module %s\n", name); + if (res != NIL) + erl_exit(1,"Failed loading preloaded module %s (%T)\n", + name, res); i++; } } @@ -498,8 +497,6 @@ void erts_usage(void) erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); - erts_fprintf(stderr, "-l turn on auto load tracing\n"); - erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n"); @@ -514,6 +511,8 @@ void erts_usage(void) erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); + erts_fprintf(stderr, "-scl bool enable/disable compaction of scheduler load,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); @@ -608,12 +607,13 @@ early_init(int *argc, char **argv) /* int max_main_threads; int max_reader_groups; int reader_groups; + char envbuf[21]; /* enough for any 64-bit integer */ + size_t envbufsz; - use_multi_run_queue = 1; + erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; display_items = 200; - display_loads = 0; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = 0; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; @@ -645,6 +645,7 @@ early_init(int *argc, char **argv) /* erts_use_r9_pids_ports = 0; erts_sys_pre_init(); + erts_atomic_init_nob(&exiting, 0); #ifdef ERTS_SMP erts_thr_progress_pre_init(); #endif @@ -679,6 +680,16 @@ early_init(int *argc, char **argv) /* schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; + envbufsz = sizeof(envbuf); + + /* erts_sys_getenv() not initialized yet; need erts_sys_getenv__() */ + if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) + erts_async_max_threads = atoi(envbuf); + else + erts_async_max_threads = 0; + if (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS) + erts_async_max_threads = ERTS_MAX_NO_OF_ASYNC_THREADS; + if (argc && argv) { int i = 1; while (i < *argc) { @@ -706,6 +717,20 @@ early_init(int *argc, char **argv) /* } break; } + case 'A': { + /* set number of threads in thread pool */ + char *arg = get_arg(argv[i]+2, argv[i+1], &i); + if (((erts_async_max_threads = atoi(arg)) < 0) || + (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { + erts_fprintf(stderr, + "bad number of async threads %s\n", + arg); + erts_usage(); + VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n", + erts_async_max_threads)); + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -778,8 +803,22 @@ early_init(int *argc, char **argv) /* -M flags. */ /* Require allocators */ #ifdef ERTS_SMP - erts_thr_progress_init(no_schedulers, no_schedulers+1, 0); + /* + * Thread progress management: + * + * * Managed threads: + * ** Scheduler threads (see erl_process.c) + * ** Aux thread (see erl_process.c) + * ** Sys message dispatcher thread (see erl_trace.c) + * + * * Unmanaged threads that need to register: + * ** Async threads (see erl_async.c) + */ + erts_thr_progress_init(no_schedulers, + no_schedulers+2, + erts_async_max_threads); #endif + erts_thr_q_init(); erts_init_utils(); erts_early_init_cpu_topology(no_schedulers, &max_main_threads, @@ -859,7 +898,6 @@ erl_start(int argc, char **argv) int have_break_handler = 1; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; - int async_max_threads = erts_async_max_threads; int ncpu = early_init(&argc, argv); envbufsz = sizeof(envbuf); @@ -875,11 +913,6 @@ erl_start(int argc, char **argv) (erts_aint32_t) max_gen_gcs); } - envbufsz = sizeof(envbuf); - if (erts_sys_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) { - async_max_threads = atoi(envbuf); - } - #if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) /* * The default stack size on MacOS X is too small for pcre. @@ -945,10 +978,9 @@ erl_start(int argc, char **argv) erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); erts_usage(); } - case 'l': - display_loads++; + case 'L': + erts_no_line_info = 1; break; - case 'v': #ifdef DEBUG if (argv[i][2] == '\0') { @@ -1166,6 +1198,19 @@ erl_start(int argc, char **argv) erts_usage(); } } + else if (has_prefix("cl", sub_param)) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if (sys_strcmp("true", arg) == 0) + erts_sched_compact_load = 1; + else if (sys_strcmp("false", arg) == 0) + erts_sched_compact_load = 0; + else { + erts_fprintf(stderr, + "bad scheduler compact load value '%s'\n", + arg); + erts_usage(); + } + } else if (has_prefix("ct", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); res = erts_init_cpu_topology_string(arg); @@ -1209,12 +1254,8 @@ erl_start(int argc, char **argv) erts_usage(); } } - else if (sys_strcmp("mrq", sub_param) == 0) - use_multi_run_queue = 1; else if (sys_strcmp("nsp", sub_param) == 0) erts_use_sender_punish = 0; - else if (sys_strcmp("srq", sub_param) == 0) - use_multi_run_queue = 0; else if (sys_strcmp("wt", sub_param) == 0) { arg = get_arg(sub_param+2, argv[i+1], &i); if (erts_sched_set_wakeup_limit(arg) != 0) { @@ -1307,17 +1348,8 @@ erl_start(int argc, char **argv) break; } - case 'A': - /* set number of threads in thread pool */ - arg = get_arg(argv[i]+2, argv[i+1], &i); - if (((async_max_threads = atoi(arg)) < 0) || - (async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { - erts_fprintf(stderr, "bad number of async threads %s\n", arg); - erts_usage(); - } - - VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n", - async_max_threads)); + case 'A': /* Was handled in early init just read past it */ + (void) get_arg(argv[i]+2, argv[i+1], &i); break; case 'a': @@ -1406,10 +1438,6 @@ erl_start(int argc, char **argv) i++; } -#ifdef USE_THREADS - erts_async_max_threads = async_max_threads; -#endif - /* Delayed check of +P flag */ if (erts_max_processes < ERTS_MIN_PROCESSES || erts_max_processes > ERTS_MAX_PROCESSES @@ -1455,6 +1483,10 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else erts_thr_set_main_status(1, 1); +#if ERTS_USE_ASYNC_READY_Q + erts_get_scheduler_data()->aux_work_data.async_ready.queue + = erts_get_async_ready_queue(1); +#endif set_main_stack_size(); process_main(); #endif @@ -1480,6 +1512,29 @@ __decl_noreturn void erts_thr_fatal_error(int err, char *what) static void system_cleanup(int exit_code) { + /* + * Make sure only one thread exits the runtime system. + */ + if (erts_atomic_inc_read_nob(&exiting) != 1) { + /* + * Another thread is currently exiting the system; + * wait for it to do its job. + */ +#ifdef ERTS_SMP + if (erts_thr_progress_is_managed_thread()) { + /* + * The exiting thread might be waiting for + * us to block; need to update status... + */ + erts_thr_progress_active(NULL, 0); + erts_thr_progress_prepare_wait(NULL); + } +#endif + /* Wait forever... */ + while (1) + erts_milli_sleep(10000000); + } + /* No cleanup wanted if ... * 1. we are about to do an abnormal exit * 2. we haven't finished initializing, or @@ -1499,7 +1554,6 @@ system_cleanup(int exit_code) #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); /* We never release it... */ #endif #ifdef HYBRID @@ -1528,14 +1582,7 @@ system_cleanup(int exit_code) erts_cleanup_incgc(); #endif -#if defined(USE_THREADS) - exit_async(); -#endif - - /* - * A lot more cleaning could/should have been done... - */ - + erts_exit_flush_async(); } /* @@ -1552,10 +1599,10 @@ __decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...) va_start(args, fmt); - save_statistics(); - system_cleanup(n); + save_statistics(); + an = abs(n); if (erts_mtrace_enabled) @@ -1592,10 +1639,10 @@ __decl_noreturn void erl_exit(int n, char *fmt,...) va_start(args, fmt); - save_statistics(); - system_cleanup(n); + save_statistics(); + an = abs(n); if (erts_mtrace_enabled) diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 02d1407a2d..09e85893c3 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -110,10 +110,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "fun_tab", NULL }, { "environ", NULL }, #endif - { "asyncq", "address" }, -#ifndef ERTS_SMP - { "async_ready", NULL }, -#endif { "efile_drv", "address" }, #if defined(ENABLE_CHILD_WAITER_THREAD) || defined(ERTS_SMP) { "child_status", NULL }, @@ -138,6 +134,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "alcu_init_atoms", NULL }, { "mseg_init_atoms", NULL }, { "drv_tsd", NULL }, + { "async_enq_mtx", NULL }, #ifdef ERTS_SMP { "sys_msg_q", NULL }, { "atom_tab", NULL }, @@ -173,14 +170,11 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "timeofday", NULL }, { "breakpoints", NULL }, { "pollsets_lock", NULL }, - { "async_id", NULL }, { "pix_lock", "address" }, { "run_queues_lists", NULL }, - { "misc_aux_work_queue", "index" }, - { "misc_aux_work_pre_alloc_lock", "address" }, { "sched_stat", NULL }, - { "run_queue_sleep_list", "address" }, #endif + { "async_init_mtx", NULL }, #ifdef ERTS_SMP { "proc_lck_qs_alloc", NULL }, #endif @@ -1258,7 +1252,7 @@ erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags) { lck->id = erts_lc_get_lock_order_id(name); - lck->extra = &lck->extra; + lck->extra = (UWord) &lck->extra; ASSERT(is_not_immed(lck->extra)); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 82f272d28a..16be47d540 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -240,7 +240,7 @@ erts_msg_distext2heap(Process *pp, Sint sz; *bpp = NULL; - sz = erts_decode_dist_ext_size(dist_extp, 0); + sz = erts_decode_dist_ext_size(dist_extp); if (sz < 0) goto decode_error; if (is_not_nil(*tokenp)) { @@ -713,7 +713,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) ASSERT(msg->data.dist_ext); ASSERT(msg->data.dist_ext->heap_size < 0); - sz = erts_decode_dist_ext_size(msg->data.dist_ext, 0); + sz = erts_decode_dist_ext_size(msg->data.dist_ext); if (sz < 0) { /* Bad external; remove it */ if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 47597f302b..1a84950120 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -948,8 +948,10 @@ static void erts_dump_links(ErtsLink *root, int indent) erts_destroy_tmp_dsbuf(dsbufp); } -Eterm erts_debug_dump_monitors_1(Process *p, Eterm pid) +Eterm erts_debug_dump_monitors_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm pid = BIF_ARG_1; Process *rp; DistEntry *dep; rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK); @@ -976,8 +978,10 @@ Eterm erts_debug_dump_monitors_1(Process *p, Eterm pid) } } -Eterm erts_debug_dump_links_1(Process *p, Eterm pid) +Eterm erts_debug_dump_links_1(BIF_ALIST_1) { + Process *p = BIF_P; + Eterm pid = BIF_ARG_1; Process *rp; DistEntry *dep; if (is_internal_port(pid)) { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6e7ac43676..58a09986d2 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -32,6 +32,7 @@ #include "error.h" #include "big.h" #include "beam_bp.h" +#include "erl_thr_progress.h" #include <limits.h> #include <stddef.h> /* offsetof */ @@ -130,10 +131,13 @@ static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) env->tmp_obj_list = NULL; } -/* Temporary object header, auto-deallocated when NIF returns. */ +/* Temporary object header, auto-deallocated when NIF returns + * or when independent environment is cleared. + */ struct enif_tmp_obj_t { struct enif_tmp_obj_t* next; void (*dtor)(struct enif_tmp_obj_t*); + ErtsAlcType_t allocator; /*char data[];*/ }; @@ -244,7 +248,7 @@ ErlNifEnv* enif_alloc_env(void) msg_env->env.hp_end = phony_heap; msg_env->env.heap_frag = NULL; msg_env->env.mod_nif = NULL; - msg_env->env.tmp_obj_list = (struct enif_tmp_obj_t*) 1; /* invalid non-NULL */ + msg_env->env.tmp_obj_list = NULL; msg_env->env.proc = &msg_env->phony_proc; memset(&msg_env->phony_proc, 0, sizeof(Process)); HEAP_START(&msg_env->phony_proc) = phony_heap; @@ -289,6 +293,7 @@ void enif_clear_env(ErlNifEnv* env) menv->env.hp = menv->env.hp_end = HEAP_TOP(p); ASSERT(!is_offheap(&MSO(p))); + free_tmp_objs(env); } int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg) @@ -435,24 +440,36 @@ int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term) return term == THE_NON_VALUE; } +int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_number(term); +} + +static ERTS_INLINE int is_proc_bound(ErlNifEnv* env) +{ + return env->mod_nif != NULL; +} + static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { - erts_free_aligned_binary_bytes_extra((byte*)obj,ERTS_ALC_T_TMP); + erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); } int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) { + ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; union { struct enif_tmp_obj_t* tmp; byte* raw_ptr; }u; u.tmp = NULL; - bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, ERTS_ALC_T_TMP, + bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator, sizeof(struct enif_tmp_obj_t)); if (bin->data == NULL) { return 0; } if (u.tmp != NULL) { + u.tmp->allocator = allocator; u.tmp->next = env->tmp_obj_list; u.tmp->dtor = &aligned_binary_dtor; env->tmp_obj_list = u.tmp; @@ -466,12 +483,13 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) { - erts_free(ERTS_ALC_T_TMP, obj); + erts_free(obj->allocator, obj); } int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { struct enif_tmp_obj_t* tobj; + ErtsAlcType_t allocator; Uint sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); @@ -486,8 +504,10 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) if (erts_iolist_size(term, &sz)) { return 0; } - - tobj = erts_alloc(ERTS_ALC_T_TMP, sz + sizeof(struct enif_tmp_obj_t)); + + allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t)); + tobj->allocator = allocator; tobj->next = env->tmp_obj_list; tobj->dtor = &tmp_alloc_dtor; env->tmp_obj_list = tobj; @@ -511,7 +531,7 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) } refbin->flags = BIN_FLAG_DRV; /* BUGBUG: Flag? */ erts_refc_init(&refbin->refc, 1); - refbin->orig_size = (long) size; + refbin->orig_size = (SWord) size; bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; @@ -676,6 +696,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlSubBin* sb; Eterm orig; Uint offset, bit_offset, bit_size; +#ifdef DEBUG unsigned src_size; ASSERT(is_binary(bin_term)); @@ -683,6 +704,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ASSERT(pos <= src_size); ASSERT(size <= src_size); ASSERT(pos + size <= src_size); +#endif sb = (ErlSubBin*) alloc_heap(env, ERL_SUB_BIN_SIZE); ERTS_GET_REAL_BIN(bin_term, orig, offset, bit_offset, bit_size); sb->thing_word = HEADER_SUB_BIN; @@ -722,7 +744,8 @@ int enif_get_int(ErlNifEnv* env, Eterm term, int* ip) { #if SIZEOF_INT == ERTS_SIZEOF_ETERM return term_to_Sint(term, (Sint*)ip); -#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM +#elif (SIZEOF_LONG == ERTS_SIZEOF_ETERM) || \ + (SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM) Sint i; if (!term_to_Sint(term, &i) || i < INT_MIN || i > INT_MAX) { return 0; @@ -738,7 +761,8 @@ int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip) { #if SIZEOF_INT == ERTS_SIZEOF_ETERM return term_to_Uint(term, (Uint*)ip); -#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM +#elif (SIZEOF_LONG == ERTS_SIZEOF_ETERM) || \ + (SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM) Uint i; if (!term_to_Uint(term, &i) || i > UINT_MAX) { return 0; @@ -754,6 +778,13 @@ int enif_get_long(ErlNifEnv* env, Eterm term, long* ip) return term_to_Sint(term, ip); #elif SIZEOF_LONG == 8 return term_to_Sint64(term, ip); +#elif SIZEOF_LONG == SIZEOF_INT + int tmp,ret; + ret = enif_get_int(env,term,&tmp); + if (ret) { + *ip = (long) tmp; + } + return ret; #else # error Unknown long word size #endif @@ -765,6 +796,14 @@ int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip) return term_to_Uint(term, ip); #elif SIZEOF_LONG == 8 return term_to_Uint64(term, ip); +#elif SIZEOF_LONG == SIZEOF_INT + int ret; + unsigned int tmp; + ret = enif_get_uint(env,term,&tmp); + if (ret) { + *ip = (unsigned long) tmp; + } + return ret; #else # error Unknown long word size #endif @@ -825,7 +864,8 @@ ERL_NIF_TERM enif_make_int(ErlNifEnv* env, int i) { #if SIZEOF_INT == ERTS_SIZEOF_ETERM return IS_SSMALL(i) ? make_small(i) : small_to_big(i,alloc_heap(env,2)); -#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM +#elif (SIZEOF_LONG == ERTS_SIZEOF_ETERM) || \ + (SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM) return make_small(i); #endif } @@ -834,7 +874,8 @@ ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i) { #if SIZEOF_INT == ERTS_SIZEOF_ETERM return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); -#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM +#elif (SIZEOF_LONG == ERTS_SIZEOF_ETERM) || \ + (SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM) return make_small(i); #endif } @@ -846,6 +887,8 @@ ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) } #if SIZEOF_LONG == ERTS_SIZEOF_ETERM return small_to_big(i, alloc_heap(env,2)); +#elif SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM + return make_small(i); #elif SIZEOF_LONG == 8 ensure_heap(env,3); return erts_sint64_to_big(i, &env->hp); @@ -859,6 +902,8 @@ ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) } #if SIZEOF_LONG == ERTS_SIZEOF_ETERM return uint_to_big(i,alloc_heap(env,2)); +#elif SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM + return make_small(i); #elif SIZEOF_LONG == 8 ensure_heap(env,3); return erts_uint64_to_big(i, &env->hp); @@ -1021,6 +1066,29 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size) driver_system_info(sip, si_size); } +int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) { + Eterm *listptr, ret = NIL, *hp; + + if (is_nil(term)) { + *list = term; + return 1; + } + + ret = NIL; + + while (is_not_nil(term)) { + if (is_not_list(term)) { + return 0; + } + hp = alloc_heap(env, 2); + listptr = list_val(term); + ret = CONS(hp, CAR(listptr), ret); + term = CDR(listptr); + } + *list = ret; + return 1; +} + ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); } void enif_mutex_destroy(ErlNifMutex *mtx) { erl_drv_mutex_destroy(mtx); } @@ -1112,7 +1180,7 @@ static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) } #define in_area(ptr,start,nbytes) \ - ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes)) + ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) static void close_lib(struct erl_module_nif* lib) @@ -1160,7 +1228,7 @@ enif_open_resource_type(ErlNifEnv* env, ErlNifResourceFlags op = flags; Eterm module_am, name_am; - ASSERT(erts_smp_is_system_blocked(0)); + ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(module_str == NULL); /* for now... */ module_am = make_atom(env->mod_nif->mod->module); name_am = enif_make_atom(env, name_str); @@ -1439,6 +1507,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm ret = am_ok; int veto; struct erl_module_nif* lib = NULL; + int reload_warning = 0; len = list_length(BIF_ARG_1); if (len < 0) { @@ -1454,7 +1523,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Block system (is this the right place to do it?) */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); /* Find calling module */ ASSERT(BIF_P->current != NULL); @@ -1578,6 +1647,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) else { mod->nif->entry = NULL; /* to prevent 'unload' callback */ erts_unload_nif(mod->nif); + reload_warning = 1; } } else { @@ -1624,7 +1694,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } else { /* Function traced, patch the original instruction word */ BpData** bps = (BpData**) code_ptr[1]; - BpData* bp = (BpData*) bps[bp_sched2ix()]; + BpData* bp = (BpData*) bps[erts_bp_sched2ix()]; bp->orig_instr = (BeamInstr) BeamOp(op_call_nif); } code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr; @@ -1643,9 +1713,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_sys_ddll_free_error(&errdesc); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_free(ERTS_ALC_T_TMP, lib_name); + + if (reload_warning) { + erts_dsprintf_buf_t* dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, + "Repeated calls to erlang:load_nif from module '%T'.\n\n" + "The NIF reload mechanism is deprecated and must not " + "be used in production systems.\n", mod_atom); + erts_send_warning_to_logger(BIF_P->group_leader, dsbufp); + } BIF_RET(ret); } @@ -1655,7 +1734,7 @@ erts_unload_nif(struct erl_module_nif* lib) { ErlNifResourceType* rt; ErlNifResourceType* next; - ASSERT(erts_smp_is_system_blocked(0)); + ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(lib != NULL); ASSERT(lib->mod != NULL); for (rt = resource_type_list.next; @@ -1715,8 +1794,10 @@ struct readonly_check_t }; static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) { - struct readonly_check_t* obj = erts_alloc(ERTS_ALC_T_TMP, + ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + struct readonly_check_t* obj = erts_alloc(allocator, sizeof(struct readonly_check_t)); + obj->hdr.allocator = allocator; obj->hdr.next = env->tmp_obj_list; env->tmp_obj_list = &obj->hdr; obj->hdr.dtor = &readonly_check_dtor; @@ -1733,7 +1814,7 @@ static void readonly_check_dtor(struct enif_tmp_obj_t* o) " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); abort(); } - erts_free(ERTS_ALC_T_TMP, obj); + erts_free(obj->hdr.allocator, obj); } static unsigned calc_checksum(unsigned char* ptr, unsigned size) { diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index d028567faf..e5d99dc4f1 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -32,9 +32,10 @@ ** 2.0: R14A ** 2.1: R14B02 "vm_variant" ** 2.2: R14B03 enif_is_exception +** 2.3: R15 enif_make_reverse_list */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 2 +#define ERL_NIF_MINOR_VERSION 3 #include <stdlib.h> @@ -86,7 +87,11 @@ typedef long long ErlNifSInt64; typedef unsigned int ERL_NIF_TERM; #else # define ERL_NIF_VM_VARIANT "beam.vanilla" +# if SIZEOF_LONG == SIZEOF_VOID_P typedef unsigned long ERL_NIF_TERM; +# elif SIZEOF_LONG_LONG == SIZEOF_VOID_P +typedef unsigned long long ERL_NIF_TERM; +# endif #endif struct enif_environment_t; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index c991b61abe..6396af09d0 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -136,6 +136,8 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); #endif ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list)); +ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term)); /* ** Add new entries here to keep compatibility on Windows!!! @@ -256,12 +258,207 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); #endif # define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) +# define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list) +# define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number) /* ** Add new entries here */ #endif + +#if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) + +/* Inline functions for compile time type checking of arguments to + variadic functions. +*/ + +# define ERL_NIF_INLINE __inline__ + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple1(ErlNifEnv* env, + ERL_NIF_TERM e1) +{ + return enif_make_tuple(env, 1, e1); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple2(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2) +{ + return enif_make_tuple(env, 2, e1, e2); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple3(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3) +{ + return enif_make_tuple(env, 3, e1, e2, e3); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple4(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4) +{ + return enif_make_tuple(env, 4, e1, e2, e3, e4); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple5(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5) +{ + return enif_make_tuple(env, 5, e1, e2, e3, e4, e5); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple6(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6) +{ + return enif_make_tuple(env, 6, e1, e2, e3, e4, e5, e6); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple7(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7) +{ + return enif_make_tuple(env, 7, e1, e2, e3, e4, e5, e6, e7); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple8(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8) +{ + return enif_make_tuple(env, 8, e1, e2, e3, e4, e5, e6, e7, e8); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple9(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8, + ERL_NIF_TERM e9) +{ + return enif_make_tuple(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list1(ErlNifEnv* env, + ERL_NIF_TERM e1) +{ + return enif_make_list(env, 1, e1); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list2(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2) +{ + return enif_make_list(env, 2, e1, e2); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list3(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3) +{ + return enif_make_list(env, 3, e1, e2, e3); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list4(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4) +{ + return enif_make_list(env, 4, e1, e2, e3, e4); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list5(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5) +{ + return enif_make_list(env, 5, e1, e2, e3, e4, e5); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list6(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6) +{ + return enif_make_list(env, 6, e1, e2, e3, e4, e5, e6); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list7(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7) +{ + return enif_make_list(env, 7, e1, e2, e3, e4, e5, e6, e7); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list8(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8) +{ + return enif_make_list(env, 8, e1, e2, e3, e4, e5, e6, e7, e8); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8, + ERL_NIF_TERM e9) +{ + return enif_make_list(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9); +} + +# undef ERL_NIF_INLINE + +#else /* fallback with macros */ + #ifndef enif_make_list1 # define enif_make_list1(ENV,E1) enif_make_list(ENV,1,E1) # define enif_make_list2(ENV,E1,E2) enif_make_list(ENV,2,E1,E2) @@ -281,6 +478,11 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); # define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7) # define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) # define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) +#endif + +#endif /* __GNUC__ && !WIN32 */ + +#ifndef enif_make_pid # define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) diff --git a/erts/emulator/beam/erl_nmgc.c b/erts/emulator/beam/erl_nmgc.c index d7bfb2ab12..2a8c819360 100644 --- a/erts/emulator/beam/erl_nmgc.c +++ b/erts/emulator/beam/erl_nmgc.c @@ -1391,7 +1391,7 @@ Eterm *erts_inc_alloc(int need) if (ma_gc_flags & GC_MAJOR) { if (need > 254) { blackmap[(Eterm*)this - global_old_heap] = 255; - *(int*)((long)(&blackmap[(Eterm*)this - global_old_heap]+4) & ~3) = + *(int*)((UWord)(&blackmap[(Eterm*)this - global_old_heap]+4) & ~3) = need; } else blackmap[(Eterm*)this - global_old_heap] = need; diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 2c67e781e0..329a2204cc 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -176,7 +176,7 @@ extern int erts_use_r9_pids_ports; * 32-bit CPU. */ -#define ERTS_MAX_PROCESSES ((1L << 27)-1) +#define ERTS_MAX_PROCESSES ((SWORD_CONSTANT(1) << 27)-1) #if (ERTS_MAX_PROCESSES > MAX_SMALL) # error "The maximum number of processes must fit in a SMALL." #endif diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index af3873995e..908ba755ed 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -26,6 +26,7 @@ #include "dist.h" #include "big.h" #include "error.h" +#include "erl_thr_progress.h" Hash erts_dist_table; Hash erts_node_table; @@ -907,7 +908,7 @@ erts_get_node_and_dist_references(struct process *proc) #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - erts_smp_block_system(0); + erts_smp_thr_progress_block(); /* No need to lock any thing since we are alone... */ if (references_atoms_need_init) { @@ -951,7 +952,7 @@ erts_get_node_and_dist_references(struct process *proc) delete_reference_table(); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); return res; } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 6aa5161b08..2b5e65b11a 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -711,23 +711,6 @@ typedef struct { int *resp; } ErtsPortTaskExeBlockData; -static void -prepare_for_block(void *vd) -{ - ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; - erts_smp_runq_unlock(d->runq); -} - -static void -resume_after_block(void *vd) -{ - ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; - erts_smp_runq_lock(d->runq); - if (d->resp) - *d->resp = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) - != (erts_aint_t) 0); -} - /* * Run all scheduled tasks for the first port in run queue. If * new tasks appear while running reschedule port (free task is @@ -752,11 +735,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - erts_smp_activity_begin(ERTS_ACTIVITY_IO, - prepare_for_block, - resume_after_block, - (void *) &blk_data); - ERTS_PT_CHK_PORTQ(runq); pp = pop_port(runq); @@ -988,10 +966,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) done: blk_data.resp = &res; - erts_smp_activity_end(ERTS_ACTIVITY_IO, - prepare_for_block, - resume_after_block, - (void *) &blk_data); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -1074,8 +1048,6 @@ erts_port_migrate(Port *prt, int *prt_locked, ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - ASSERT(!erts_common_run_queue); - if (!*from_locked || !*to_locked) { if (from_rq < to_rq) { if (!*to_locked) { diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4dda17f559..b8c6b64fc0 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -40,6 +40,8 @@ #include "beam_bp.h" #include "erl_cpu_topology.h" #include "erl_thr_progress.h" +#include "erl_thr_queue.h" +#include "erl_async.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -112,6 +114,7 @@ static Sint p_serial; static Uint p_serial_mask; static Uint p_serial_shift; +int erts_sched_compact_load; Uint erts_no_schedulers; Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; Uint erts_process_tab_index_mask; @@ -125,7 +128,6 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif #ifdef ERTS_SMP - int erts_disable_proc_not_running_opt; static ErtsAuxWorkData *aux_thread_aux_work_data; @@ -155,7 +157,6 @@ do { \ static struct { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - int aux_thread; int online; int curr_online; int wait_curr_online; @@ -195,8 +196,6 @@ do { \ erts_sched_stat_t erts_sched_stat; -ErtsRunQueue *erts_common_run_queue; - #ifdef USE_THREADS static erts_tsd_key_t sched_data_key; #endif @@ -224,10 +223,6 @@ typedef union { static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -#ifndef BM_COUNTERS -static int processes_busy; -#endif - Process** process_tab; static Uint last_reductions; static Uint last_exact_reductions; @@ -362,6 +357,15 @@ dbg_chk_aux_work_val(erts_aint32_t value) #ifdef ERTS_SSI_AUX_WORK_MISC valid |= ERTS_SSI_AUX_WORK_MISC; #endif +#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR + valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; +#endif +#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; +#endif +#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#endif #ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; @@ -487,9 +491,6 @@ erts_init_process(int ncpu) p_serial_shift = erts_fit_in_bits(erts_max_processes - 1); p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift); erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift); -#ifndef BM_COUNTERS - processes_busy = 0; -#endif last_reductions = 0; last_exact_reductions = 0; erts_default_process_flags = 0; @@ -622,18 +623,6 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP -static void -prepare_for_block(void *vrq) -{ - erts_smp_runq_unlock((ErtsRunQueue *) vrq); -} - -static void -resume_after_block(void *vrq) -{ - erts_smp_runq_lock((ErtsRunQueue *) vrq); -} - void erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) { @@ -641,6 +630,13 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case ERTS_SSI_FLG_POLL_SLEEPING: erts_sys_schedule_interrupt(1); break; + case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING: + /* + * Thread progress blocking while poll sleeping; need + * to signal on both... + */ + erts_sys_schedule_interrupt(1); + /* fall through */ case ERTS_SSI_FLG_TSE_SLEEPING: erts_tse_set(ssi->event); break; @@ -713,37 +709,37 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); } -#ifdef ERTS_SMP - typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { - erts_misc_aux_work_t *next; void (*func)(void *); void *arg; }; -typedef struct { - erts_smp_mtx_t mtx; - erts_misc_aux_work_t *first; - erts_misc_aux_work_t *last; -} erts_misc_aux_work_q_t; +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, + erts_misc_aux_work_t, + 200, + ERTS_ALC_T_MISC_AUX_WORK) typedef union { - erts_misc_aux_work_q_t data; - char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))]; + ErtsThrQ_t q; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))]; } erts_algnd_misc_aux_work_q_t; static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues; -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, - erts_misc_aux_work_t, - 200, - ERTS_ALC_T_MISC_AUX_WORK) +static void +notify_aux_work(void *vssi) +{ + set_aux_work_flags_wakeup_nob((ErtsSchedulerSleepInfo *) vssi, + ERTS_SSI_AUX_WORK_MISC); +} static void init_misc_aux_work(void) { int ix; + ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; + qinit.notify = notify_aux_work; init_misc_aux_work_alloc(); @@ -752,88 +748,190 @@ init_misc_aux_work(void) sizeof(erts_algnd_misc_aux_work_q_t) * (erts_no_schedulers+1)); - for (ix = 0; ix <= erts_no_schedulers; ix++) { - erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, - "misc_aux_work_queue", - make_small(ix)); - misc_aux_work_queues[ix].data.first = NULL; - misc_aux_work_queues[ix].data.last = NULL; +#ifdef ERTS_SMP + ix = 0; /* aux_thread + schedulers */ +#else + ix = 1; /* scheduler only */ +#endif + + for (; ix <= erts_no_schedulers; ix++) { + qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1); + erts_thr_q_initialize(&misc_aux_work_queues[ix].q, &qinit); } } static erts_aint32_t +misc_aux_work_clean(ErtsThrQ_t *q, + ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + switch (erts_thr_q_clean(q)) { + case ERTS_THR_Q_DIRTY: + set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); + return aux_work | ERTS_SSI_AUX_WORK_MISC; + case ERTS_THR_Q_NEED_THR_PRGR: +#ifdef ERTS_SMP + set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); + erts_thr_progress_wakeup(awdp->esdp, + erts_thr_q_need_thr_progress(q)); +#endif + case ERTS_THR_Q_CLEAN: + break; + } + return aux_work; +} + +static erts_aint32_t handle_misc_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - int ix = (int) awdp->sched_id; - erts_misc_aux_work_t *mawp; + ErtsThrQ_t *q = &misc_aux_work_queues[awdp->sched_id].q; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); - - erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); - mawp = misc_aux_work_queues[ix].data.first; - misc_aux_work_queues[ix].data.first = NULL; - misc_aux_work_queues[ix].data.last = NULL; - erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); - - while (mawp) { - erts_misc_aux_work_t *free_mawp; + while (1) { + erts_misc_aux_work_t *mawp = erts_thr_q_dequeue(q); + if (!mawp) + break; mawp->func(mawp->arg); - free_mawp = mawp; - mawp = mawp->next; - misc_aux_work_free(free_mawp); + misc_aux_work_free(mawp); } - return aux_work & ~ERTS_SSI_AUX_WORK_MISC; + return misc_aux_work_clean(q, awdp, aux_work & ~ERTS_SSI_AUX_WORK_MISC); } -static void -smp_schedule_misc_aux_work(int ix, - void (*func)(void *), - void *arg) +#ifdef ERTS_SMP + +static erts_aint32_t +handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) { - erts_aint32_t aux_work; + if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr)) + return aux_work; + + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); + + return misc_aux_work_clean(&misc_aux_work_queues[awdp->sched_id].q, + awdp, + aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR); +} + +#endif + +static ERTS_INLINE void +schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg) +{ + ErtsThrQ_t *q; erts_misc_aux_work_t *mawp; - ErtsSchedulerSleepInfo *ssi; - mawp = misc_aux_work_alloc(); +#ifdef ERTS_SMP + ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers); +#else + ASSERT(sched_id == 1); +#endif + q = &misc_aux_work_queues[sched_id].q; + mawp = misc_aux_work_alloc(); mawp->func = func; mawp->arg = arg; - mawp->next = NULL; - - erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); - if (!misc_aux_work_queues[ix].data.last) - misc_aux_work_queues[ix].data.first = mawp; - else - misc_aux_work_queues[ix].data.last->next = mawp; - misc_aux_work_queues[ix].data.last = mawp; - erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + erts_thr_q_enqueue(q, mawp); +} - set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), - ERTS_SSI_AUX_WORK_MISC); +void +erts_schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg) +{ + schedule_misc_aux_work(sched_id, func, arg); } void -erts_smp_schedule_misc_aux_work(int ignore_self, - int max_sched, - void (*func)(void *), - void *arg) +erts_schedule_multi_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg) { - int ix, ignore_ix = -1; + int id, self = 0; if (ignore_self) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); if (esdp) - ignore_ix = (int) esdp->no; + self = (int) esdp->no; } ASSERT(0 < max_sched && max_sched <= erts_no_schedulers); - for (ix = 1; ix <= max_sched; ix++) { - if (ix == ignore_ix) + for (id = 1; id <= max_sched; id++) { + if (id == self) continue; - smp_schedule_misc_aux_work(ix, func, arg); + schedule_misc_aux_work(id, func, arg); + } +} + +#if ERTS_USE_ASYNC_READY_Q + +void +erts_notify_check_async_ready_queue(void *vno) +{ + int ix = ((int) (SWord) vno) -1; + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix), + ERTS_SSI_AUX_WORK_ASYNC_READY); +} + +static erts_aint32_t +handle_async_ready(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY); + if (erts_check_async_ready(awdp->async_ready.queue)) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY) + & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + aux_work &= ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; + } + return aux_work; + } +#ifdef ERTS_SMP + awdp->async_ready.need_thr_prgr = 0; +#endif + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + return ((aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY) + | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); +} + +static erts_aint32_t +handle_async_ready_clean(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + void *thr_prgr_p; + +#ifdef ERTS_SMP + if (awdp->async_ready.need_thr_prgr + && !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) { + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; + } + + awdp->async_ready.need_thr_prgr = 0; + thr_prgr_p = (void *) &awdp->async_ready.thr_prgr; +#else + thr_prgr_p = NULL; +#endif + + switch (erts_async_ready_clean(awdp->async_ready.queue, thr_prgr_p)) { + case ERTS_ASYNC_READY_CLEAN: + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#ifdef ERTS_SMP + case ERTS_ASYNC_READY_NEED_THR_PRGR: + erts_thr_progress_wakeup(awdp->esdp, + awdp->async_ready.thr_prgr); + awdp->async_ready.need_thr_prgr = 1; + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#endif + default: + return aux_work; } } @@ -970,14 +1068,14 @@ prep_setup_completed_dealloc(void *vproc) erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { /* scheduler threads */ - erts_smp_schedule_misc_aux_work(0, - erts_no_schedulers, - setup_completed_dealloc, - vproc); + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + setup_completed_dealloc, + vproc); /* aux_thread */ - smp_schedule_misc_aux_work(0, - setup_completed_dealloc, - vproc); + erts_schedule_misc_aux_work(0, + setup_completed_dealloc, + vproc); } } @@ -998,14 +1096,14 @@ erts_debug_wait_deallocations(Process *c_p) erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); erts_smp_proc_inc_refc(c_p); /* scheduler threads */ - erts_smp_schedule_misc_aux_work(0, - erts_no_schedulers, - prep_setup_completed_dealloc, - (void *) c_p); + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + prep_setup_completed_dealloc, + (void *) c_p); /* aux_thread */ - smp_schedule_misc_aux_work(0, - prep_setup_completed_dealloc, - (void *) c_p); + erts_schedule_misc_aux_work(0, + prep_setup_completed_dealloc, + (void *) c_p); return 1; } return 0; @@ -1068,10 +1166,24 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); } #ifdef ERTS_SMP + if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) { + aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } +#endif if (aux_work & ERTS_SSI_AUX_WORK_MISC) { aux_work = handle_misc_aux_work(awdp, aux_work); ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); } +#if ERTS_USE_ASYNC_READY_Q + if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) { + aux_work = handle_async_ready(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } + if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { + aux_work = handle_async_ready_clean(awdp, aux_work); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + } #endif #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { @@ -1482,11 +1594,50 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) static void -poke_ssi(void *vssi) +thr_prgr_wakeup(void *vssi) { erts_sched_poke((ErtsSchedulerSleepInfo *) vssi); } +static void +thr_prgr_prep_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_smp_atomic32_read_bor_acqb(&ssi->flags, + ERTS_SSI_FLG_SLEEPING); +} + +static void +thr_prgr_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING; + + erts_tse_reset(ssi->event); + + while (1) { + erts_aint32_t aflgs, nflgs; + nflgs = xflgs | ERTS_SSI_FLG_TSE_SLEEPING; + aflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + if (aflgs == xflgs) { + erts_tse_wait(ssi->event); + break; + } + if ((aflgs & ERTS_SSI_FLG_SLEEPING) == 0) + break; + xflgs = aflgs; + } +} + +static void +thr_prgr_fin_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_smp_atomic32_read_band_nob(&ssi->flags, + ~(ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING)); +} + static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp); static void * @@ -1495,25 +1646,21 @@ aux_thread(void *unused) ErtsAuxWorkData *awdp = aux_thread_aux_work_data; ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1); erts_aint32_t aux_work; - ErtsThrPrgrWakeupCallback wake_me; + ErtsThrPrgrCallbacks callbacks; int thr_prgr_active = 1; - wake_me.wakeup = poke_ssi; - wake_me.arg = (void *) ssi; + ssi->event = erts_tse_fetch(); - erts_thr_progress_register_managed_thread(NULL, &wake_me, 1); + callbacks.arg = (void *) ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + + erts_thr_progress_register_managed_thread(NULL, &callbacks, 1); init_aux_work_data(awdp, NULL); awdp->ssi = ssi; - erts_register_blockable_thread(); - - ssi->event = erts_tse_fetch(); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - schdlr_sspnd.aux_thread = 1; - erts_smp_cnd_signal(&schdlr_sspnd.cnd); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - sched_prep_spin_wait(ssi); while (1) { @@ -1528,8 +1675,6 @@ aux_thread(void *unused) erts_thr_progress_leader_update(NULL); } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (!aux_work) { if (thr_prgr_active) erts_thr_progress_active(NULL, thr_prgr_active = 0); @@ -1552,8 +1697,6 @@ aux_thread(void *unused) erts_thr_progress_finalize_wait(NULL); } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - flgs = sched_prep_spin_wait(ssi); } return NULL; @@ -1573,21 +1716,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - erts_smp_spin_lock(&rq->sleepers.lock); flgs = sched_prep_spin_wait(ssi); if (flgs & ERTS_SSI_FLG_SUSPENDED) { /* Go suspend instead... */ - erts_smp_spin_unlock(&rq->sleepers.lock); return; } - ssi->prev = NULL; - ssi->next = rq->sleepers.list; - if (rq->sleepers.list) - rq->sleepers.list->prev = ssi; - rq->sleepers.list = ssi; - erts_smp_spin_unlock(&rq->sleepers.lock); - /* * If all schedulers are waiting, one of them *should* * be waiting in erl_sys_schedule() @@ -1614,8 +1748,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_leader_update(esdp); } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (aux_work) flgs = erts_smp_atomic32_read_acqb(&ssi->flags); else { @@ -1639,8 +1771,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_finalize_wait(esdp); } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); break; @@ -1659,6 +1789,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + erts_smp_runq_lock(rq); sched_active(esdp->no, rq); @@ -1765,12 +1898,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto sys_poll_aux_work; } #ifdef ERTS_SMP - if (thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 0); flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { - if (!(flgs & ERTS_SSI_FLG_WAITING)) + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); goto sys_locked_woken; + } erts_smp_runq_unlock(rq); flgs = sched_prep_cont_spin_wait(ssi); if (!(flgs & ERTS_SSI_FLG_WAITING)) { @@ -1787,6 +1920,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_unlock(rq); +#ifdef ERTS_SMP + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); +#endif + ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(0); @@ -1804,8 +1942,15 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto sys_aux_work; sys_woken: + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); erts_smp_runq_lock(rq); sys_locked_woken: + if (!thr_prgr_active) { + erts_smp_runq_unlock(rq); + erts_thr_progress_active(esdp, thr_prgr_active = 1); + erts_smp_runq_lock(rq); + } clear_sys_scheduling(); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); @@ -1813,12 +1958,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_active_sys(esdp->no, rq); } -#ifdef ERTS_SMP - if (!thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 1); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif } #ifdef ERTS_SMP @@ -1840,10 +1980,10 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) } static void -wake_scheduler(ErtsRunQueue *rq, int incq, int one) +wake_scheduler(ErtsRunQueue *rq, int incq) { ErtsSchedulerSleepInfo *ssi; - ErtsSchedulerSleepList *sl; + erts_aint32_t flgs; /* * The unlocked run queue is not strictly necessary @@ -1855,56 +1995,13 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) */ ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); - sl = &rq->sleepers; - - erts_smp_spin_lock(&sl->lock); - ssi = sl->list; - if (!ssi) - erts_smp_spin_unlock(&sl->lock); - else if (one) { - erts_aint32_t flgs; - if (ssi->prev) - ssi->prev->next = ssi->next; - else { - ASSERT(sl->list == ssi); - sl->list = ssi->next; - } - if (ssi->next) - ssi->next->prev = ssi->prev; - - erts_smp_spin_unlock(&sl->lock); + ssi = rq->scheduler->ssi; - flgs = ssi_flags_set_wake(ssi); - erts_sched_finish_poke(ssi, flgs); + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); - if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING)) - non_empty_runq(rq); - } - else { - sl->list = NULL; - erts_smp_spin_unlock(&sl->lock); - - ERTS_THR_MEMORY_BARRIER; - do { - ErtsSchedulerSleepInfo *wake_ssi = ssi; - ssi = ssi->next; - erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi)); - } while (ssi); - } -} - -static void -wake_all_schedulers(void) -{ - if (erts_common_run_queue) - wake_scheduler(erts_common_run_queue, 0, 0); - else { - int ix; - for (ix = 0; ix < erts_no_run_queues; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq, 0, 1); - } - } + if (incq && (flgs & ERTS_SSI_FLG_WAITING)) + non_empty_runq(rq); } #define ERTS_NO_USED_RUNQS_SHIFT 16 @@ -1997,7 +2094,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) erts_smp_xrunq_unlock(crq, wrq); } } - wake_scheduler(wrq, 0, 1); + wake_scheduler(wrq, 0); return 1; } return 0; @@ -2045,7 +2142,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP if (runq) - wake_scheduler(runq, 1, 1); + wake_scheduler(runq, 1); #endif } @@ -2060,19 +2157,12 @@ erts_sched_notify_check_cpu_bind(void) { #ifdef ERTS_SMP int ix; - if (erts_common_run_queue) { - for (ix = 0; ix < erts_no_schedulers; ix++) - erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); - wake_all_schedulers(); - } - else { - for (ix = 0; ix < erts_no_run_queues; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - erts_smp_runq_unlock(rq); - wake_scheduler(rq, 0, 1); - }; + for (ix = 0; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0); } #else erts_sched_check_cpu_bind(erts_get_scheduler_data()); @@ -2341,7 +2431,7 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) if (notify_to_rq) smp_notify_inc_runq(rq); - wake_scheduler(evac_rq, 0, 1); + wake_scheduler(evac_rq, 0); } static int @@ -2499,9 +2589,6 @@ static int try_steal_task(ErtsRunQueue *rq) { int res, rq_locked, vix, active_rqs, blnc_rqs; - - if (erts_common_run_queue) - return 0; /* * We are not allowed to steal jobs to this run queue @@ -2793,6 +2880,9 @@ check_balance(ErtsRunQueue *c_rq) mmax_len = run_queue_info[qix].max_len; } + if (!erts_sched_compact_load) + goto all_active; + if (!forced && half_full_scheds != blnc_no_rqs) { int min = 1; if (min < half_full_scheds) @@ -3160,21 +3250,25 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp) awdp->esdp = esdp; awdp->ssi = esdp ? esdp->ssi : NULL; #ifdef ERTS_SMP + awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.completed_callback = NULL; awdp->dd.completed_arg = NULL; #endif +#ifdef ERTS_USE_ASYNC_READY_Q +#ifdef ERTS_SMP + awdp->async_ready.need_thr_prgr = 0; + awdp->async_ready.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; +#endif + awdp->async_ready.queue = NULL; +#endif } void -erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) +erts_init_scheduling(int no_schedulers, int no_schedulers_online) { int ix, n, no_ssi; -#ifndef ERTS_SMP - mrq = 0; -#endif - init_misc_op_list_alloc(); ASSERT(no_schedulers_online <= no_schedulers); @@ -3183,7 +3277,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) /* Create and initialize run queues */ - n = (int) (mrq ? no_schedulers : 1); + n = no_schedulers; erts_aligned_run_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, @@ -3208,14 +3302,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); -#ifdef ERTS_SMP - erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list"); - rq->sleepers.list = NULL; -#endif - rq->waiting = 0; rq->woken = 0; - rq->flags = !mrq ? ERTS_RUNQ_FLG_SHARED_RUNQ : 0; + rq->flags = 0; rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; rq->full_reds_history_sum = 0; for (rix = 0; rix < ERTS_FULL_REDS_HISTORY_SIZE; rix++) { @@ -3261,8 +3350,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) rq->ports.end = NULL; } - erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; - #ifdef ERTS_SMP if (erts_no_run_queues != 1) { @@ -3318,11 +3405,14 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; esdp->free_process = NULL; -#if HALFWORD_HEAP - /* Registers need to be heap allocated (correct memory range) for tracing to work */ - esdp->save_reg = erts_alloc(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * sizeof(Eterm)); -#endif #endif + esdp->x_reg_array = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, + ERTS_X_REGS_ALLOCATED * + sizeof(Eterm)); + esdp->f_reg_array = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, + MAX_REG * sizeof(FloatDef)); #if !HEAP_ON_C_STACK esdp->num_tmp_heap_used = 0; #endif @@ -3336,24 +3426,16 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_init_atom_cache_map(&esdp->atom_cache_map); - if (erts_common_run_queue) { - esdp->run_queue = erts_common_run_queue; - esdp->run_queue->scheduler = NULL; - } - else { - esdp->run_queue = ERTS_RUNQ_IX(ix); - esdp->run_queue->scheduler = esdp; - } + esdp->run_queue = ERTS_RUNQ_IX(ix); + esdp->run_queue->scheduler = esdp; -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&esdp->chk_cpu_bind, 0); -#endif init_aux_work_data(&esdp->aux_work_data, esdp); } -#ifdef ERTS_SMP init_misc_aux_work(); +#ifdef ERTS_SMP + erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ aux_thread_aux_work_data = @@ -3364,14 +3446,12 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_cnd_init(&schdlr_sspnd.cnd); erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); - schdlr_sspnd.aux_thread = 0; schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; schdlr_sspnd.msb.ongoing = 0; erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - init_no_runqs(no_schedulers, - erts_common_run_queue ? 1 : no_schedulers_online); + init_no_runqs(no_schedulers, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; @@ -3384,16 +3464,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) balance_info.n = 0; if (no_schedulers_online < no_schedulers) { - if (erts_common_run_queue) { - for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); - } - else { - for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % no_schedulers_online)); - } + for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) + evacuate_run_queue(ERTS_RUNQ_IX(ix), + ERTS_RUNQ_IX(ix % no_schedulers_online)); } schdlr_sspnd.wait_curr_online = no_schedulers_online; @@ -3438,8 +3511,6 @@ ErtsRunQueue * erts_schedid2runq(Uint id) { int ix; - if (erts_common_run_queue) - return erts_common_run_queue; ix = (int) id - 1; ASSERT(0 <= ix && ix < erts_no_run_queues); return ERTS_RUNQ_IX(ix); @@ -3558,18 +3629,6 @@ erts_get_max_no_executing_schedulers(void) #ifdef ERTS_SMP static void -susp_sched_prep_block(void *unused) -{ - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); -} - -static void -susp_sched_resume_block(void *unused) -{ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); -} - -static void scheduler_ix_resume_wake(Uint ix) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); @@ -3742,9 +3801,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) wake = 0; } - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; + if (curr_online && !ongoing_multi_scheduling_block()) { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); while (1) { @@ -3759,8 +3820,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_thr_progress_leader_update(esdp); } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (!aux_work) { if (thr_prgr_active) erts_thr_progress_active(esdp, thr_prgr_active = 0); @@ -3785,8 +3844,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_thr_progress_finalize_wait(esdp); } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) @@ -3796,7 +3853,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) break; } - erts_smp_mtx_lock(&schdlr_sspnd.mtx); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); } @@ -3906,12 +3962,16 @@ erts_set_schedulers_online(Process *p, Sint new_no, Sint *old_no) { - int ix, res, no, have_unlocked_plocks; + ErtsSchedulerData *esdp; + int ix, res, no, have_unlocked_plocks, end_wait; erts_aint32_t changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; + esdp = ERTS_PROC_GET_SCHDATA(p); + end_wait = 0; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); have_unlocked_plocks = 0; @@ -3937,10 +3997,6 @@ erts_set_schedulers_online(Process *p, for (ix = online; ix < no; ix++) erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - else if (erts_common_run_queue) { - for (ix = online; ix < no; ix++) - scheduler_ix_resume_wake(ix); - } else { if (plocks) { have_unlocked_plocks = 1; @@ -3988,15 +4044,6 @@ erts_set_schedulers_online(Process *p, for (ix = no; ix < online; ix++) erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - else if (erts_common_run_queue) { - for (ix = no; ix < online; ix++) { - ErtsSchedulerSleepInfo *ssi; - ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_all_schedulers(); - } else { if (plocks) { have_unlocked_plocks = 1; @@ -4023,21 +4070,26 @@ erts_set_schedulers_online(Process *p, erts_smp_mtx_lock(&schdlr_sspnd.mtx); for (ix = no; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq, 0, 1); + wake_scheduler(rq, 0); } } } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (plocks && !have_unlocked_plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + end_wait = 1; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE ? (ERTS_SCHDLR_SSPND_CHNG_WAITER & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) @@ -4045,10 +4097,15 @@ erts_set_schedulers_online(Process *p, == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + } } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (end_wait) { + erts_thr_progress_finalize_wait(esdp); + erts_thr_progress_active(esdp, 1); + } if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); @@ -4106,44 +4163,58 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; schdlr_sspnd.msb.wait_active = 2; } - if (erts_common_run_queue) { - for (ix = 1; ix < online; ix++) - erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); - wake_all_schedulers(); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_smp_mtx_lock(&balance_info.update_mtx); + set_no_used_runqs(1); + for (ix = 0; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); + erts_smp_runq_unlock(rq); } - else { + /* + * Evacuate all activities in all other run queues + * into the first run queue. Note order is important, + * online run queues has to be evacuated last. + */ + for (ix = erts_no_run_queues-1; ix >= 1; ix--) + evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0)); + erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) + != schdlr_sspnd.msb.wait_active) { + ErtsSchedulerData *esdp; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - set_no_used_runqs(1); - for (ix = 0; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); - ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); - erts_smp_runq_unlock(rq); + + if (plocks && !have_unlocked_plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); } - /* - * Evacuate all activities in all other run queues - * into the first run queue. Note order is important, - * online run queues has to be evacuated last. - */ - for (ix = erts_no_run_queues-1; ix >= 1; ix--) - evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0)); - erts_smp_mtx_unlock(&balance_info.update_mtx); + + esdp = ERTS_PROC_GET_SCHDATA(p); + + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) + != schdlr_sspnd.msb.wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, + &schdlr_sspnd.mtx); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + erts_thr_progress_active(esdp, 1); + erts_thr_progress_finalize_wait(esdp); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED ? (ERTS_SCHDLR_SSPND_CHNG_WAITER & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) @@ -4223,12 +4294,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } - else if (erts_common_run_queue) { - for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic32_read_band_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ~ERTS_SSI_FLG_SUSPENDED); - wake_all_schedulers(); - } else { int online = schdlr_sspnd.online; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -4331,13 +4396,18 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { - ErtsThrPrgrWakeupCallback wake_me; + ErtsThrPrgrCallbacks callbacks; ErtsSchedulerData *esdp = vesdp; Uint no = esdp->no; #ifdef ERTS_SMP - wake_me.wakeup = poke_ssi; - wake_me.arg = (void *) esdp->ssi; - erts_thr_progress_register_managed_thread(esdp, &wake_me, 0); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + callbacks.arg = (void *) esdp->ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + + erts_thr_progress_register_managed_thread(esdp, &callbacks, 0); erts_alloc_register_scheduler(vesdp); #endif #ifdef ERTS_ENABLE_LOCK_CHECK @@ -4352,19 +4422,25 @@ sched_thread_func(void *vesdp) #if HAVE_ERTS_MSEG erts_mseg_late_init(); #endif +#if ERTS_USE_ASYNC_READY_Q + esdp->aux_work_data.async_ready.queue = erts_get_async_ready_queue(no); +#endif erts_sched_init_check_cpu_bind(esdp); erts_proc_lock_prepare_proc_lock_waiter(); - ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); - - #endif - erts_register_blockable_thread(); + #ifdef HIPE hipe_thread_signal_init(); #endif erts_thread_init_float(); + + if (no == 1) { + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + } + erts_smp_mtx_lock(&schdlr_sspnd.mtx); ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing) @@ -4378,23 +4454,17 @@ sched_thread_func(void *vesdp) } if (no == 1) { - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online - || !schdlr_sspnd.aux_thread) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - } + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (no == 1) { + erts_thr_progress_finalize_wait(esdp); + erts_thr_progress_active(esdp, 1); + } + #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC esdp->verify_unused_temp_alloc = erts_alloc_get_verify_unused_temp_alloc( @@ -4431,12 +4501,6 @@ erts_start_schedulers(void) res = ENOTSUP; } - erts_block_system(0); - - res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); - if (res != 0) - erl_exit(1, "Failed to create aux thread\n"); - while (actual < wanted) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); actual++; @@ -4449,7 +4513,12 @@ erts_start_schedulers(void) } erts_no_schedulers = actual; - erts_release_system(); + + ERTS_THR_MEMORY_BARRIER; + + res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); + if (res != 0) + erl_exit(1, "Failed to create aux thread\n"); if (actual < 1) erl_exit(1, @@ -4903,7 +4972,7 @@ suspend_process_2(BIF_ALIST_2) /* This is really a piece of cake without SMP support... */ if (!smon->active) { - suspend_process(erts_common_run_queue, suspendee); + suspend_process(ERTS_RUNQ_IX(0), suspendee); smon->active++; res = am_true; } @@ -5473,8 +5542,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, || from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - ASSERT(!erts_common_run_queue); /* * If we have the lock on the run queue to migrate to, @@ -5625,25 +5692,17 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, int i; ErtsSchedulerData *esdp; - if (erts_common_run_queue) - erts_smp_runq_lock(erts_common_run_queue); - for (i = 0; i < erts_no_schedulers; i++) { esdp = ERTS_SCHEDULER_IX(i); - if (!erts_common_run_queue) - erts_smp_runq_lock(esdp->run_queue); + erts_smp_runq_lock(esdp->run_queue); if (esdp->free_process && esdp->free_process->id == rpid) { res = am_free; - if (!erts_common_run_queue) - erts_smp_runq_unlock(esdp->run_queue); + erts_smp_runq_unlock(esdp->run_queue); break; } - if (!erts_common_run_queue) - erts_smp_runq_unlock(esdp->run_queue); + erts_smp_runq_unlock(esdp->run_queue); } - if (erts_common_run_queue) - erts_smp_runq_unlock(erts_common_run_queue); #endif } @@ -5810,7 +5869,7 @@ Process *schedule(Process *p, int calls) input_reductions = INPUT_REDUCTIONS; } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); /* * Clean up after the process being scheduled out. @@ -5948,17 +6007,16 @@ Process *schedule(Process *p, int calls) } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + check_activities_to_run: { #ifdef ERTS_SMP - if (!(rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && rq->check_balance_reds <= 0) { + if (rq->check_balance_reds <= 0) check_balance(rq); - } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) @@ -5966,20 +6024,15 @@ Process *schedule(Process *p, int calls) continue_check_activities_to_run: - if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_CHK_CPU_BIND + if (rq->flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { - if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED)) { + if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } - if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) { + if (rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) erts_sched_check_cpu_bind(esdp); - } } { @@ -5996,11 +6049,7 @@ Process *schedule(Process *p, int calls) } } - erts_smp_chk_system_block(prepare_for_block, - resume_after_block, - (void *) rq); - - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); #else /* ERTS_SMP */ @@ -6025,16 +6074,11 @@ Process *schedule(Process *p, int calls) empty_runq(rq); - if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_SUSPENDED)) { - if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED); - non_empty_runq(rq); - goto continue_check_activities_to_run; - } + if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { + ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); + non_empty_runq(rq); + goto continue_check_activities_to_run; } else if (!(rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { /* @@ -6099,11 +6143,7 @@ Process *schedule(Process *p, int calls) else if (rq->wakeup_other < wakeup_other_limit) rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC; else { - if (erts_common_run_queue) { - if (erts_common_run_queue->waiting) - wake_scheduler(erts_common_run_queue, 0, 1); - } - else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { + if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -6327,14 +6367,14 @@ erts_sched_stat_modify(int what) int ix; switch (what) { case ERTS_SCHED_STAT_MODIFY_ENABLE: - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_sched_stat.enabled = 1; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_DISABLE: - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_sched_stat.enabled = 1; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_CLEAR: erts_smp_spin_lock(&erts_sched_stat.lock); @@ -6489,7 +6529,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) * Wait for other schedulers to schedule out their processes * and update 'reductions'. */ - erts_smp_block_system(0); + erts_smp_thr_progress_block(); for (reds = 0, ix = 0; ix < erts_no_run_queues; ix++) reds += ERTS_RUNQ_IX(ix)->procs.reductions; if (redsp) @@ -6497,7 +6537,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) if (diffp) *diffp = reds - last_exact_reductions; last_exact_reductions = reds; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } @@ -6702,7 +6742,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } +#ifdef BM_COUNTERS processes_busy++; +#endif BM_COUNT(processes_spawned); #ifndef HYBRID @@ -6950,7 +6992,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->pending_exit.bp = NULL; #endif -#if !defined(NO_FPE_SIGNALS) +#if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; #endif @@ -7124,7 +7166,7 @@ void erts_init_empty_process(Process *p) p->run_queue = ERTS_RUNQ_IX(0); #endif -#if !defined(NO_FPE_SIGNALS) +#if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; #endif @@ -8225,7 +8267,9 @@ continue_exit_process(Process *p pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); +#ifdef BM_COUNTERS processes_busy--; +#endif if (dep) { erts_do_net_exits(dep, reason); @@ -9302,6 +9346,22 @@ init_processes_bif(void) * Debug stuff */ +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int +erts_dbg_check_halloc_lock(Process *p) +{ + if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) + return 1; + if (p->id == ERTS_INVALID_PID) + return 1; + if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) + return 1; + if (erts_thr_progress_is_blocking()) + return 1; + return 0; +} +#endif + Eterm erts_debug_processes(Process *c_p) { diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 43593f32d9..a51b380bb0 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -54,6 +54,7 @@ typedef struct process Process; #include "erl_atom_table.h" #include "external.h" #include "erl_mseg.h" +#include "erl_async.h" #ifdef HIPE #include "hipe_process.h" @@ -94,6 +95,7 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; +extern int erts_sched_compact_load; extern Uint erts_no_schedulers; extern Uint erts_no_run_queues; extern int erts_sched_thread_suggested_stack_size; @@ -142,12 +144,10 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1)) #define ERTS_RUNQ_FLG_SUSPENDED \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2)) -#define ERTS_RUNQ_FLG_SHARED_RUNQ \ - (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3)) #define ERTS_RUNQ_FLG_CHK_CPU_BIND \ - (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4)) + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3)) #define ERTS_RUNQ_FLG_INACTIVE \ - (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5)) + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4)) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ @@ -251,13 +251,18 @@ typedef enum { #define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0) #define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1) #define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 3) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 4) #ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 3) #endif -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 7) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) +#ifdef ERTS_SMP +#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 8) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 9) +#endif +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) #if !HAVE_ERTS_MSEG # undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK @@ -265,11 +270,6 @@ typedef enum { typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; -typedef struct { - erts_smp_spinlock_t lock; - ErtsSchedulerSleepInfo *list; -} ErtsSchedulerSleepList; - struct ErtsSchedulerSleepInfo_ { #ifdef ERTS_SMP ErtsSchedulerSleepInfo *next; @@ -332,10 +332,6 @@ struct ErtsRunQueue_ { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; -#ifdef ERTS_SMP - ErtsSchedulerSleepList sleepers; -#endif - ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ int woken; @@ -381,7 +377,6 @@ typedef union { } ErtsAlignedRunQueue; extern ErtsAlignedRunQueue *erts_aligned_run_queues; -extern ErtsRunQueue *erts_common_run_queue; #define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS) \ do { \ @@ -404,6 +399,9 @@ typedef struct { ErtsSchedulerSleepInfo *ssi; struct { int ix; +#ifdef ERTS_SMP + ErtsThrPrgrVal thr_prgr; +#endif } misc; #ifdef ERTS_SMP struct { @@ -412,22 +410,27 @@ typedef struct { void (*completed_arg)(void *); } dd; #endif +#ifdef ERTS_USE_ASYNC_READY_Q + struct { +#ifdef ERTS_SMP + int need_thr_prgr; + ErtsThrPrgrVal thr_prgr; +#endif + void *queue; + } async_ready; +#endif } ErtsAuxWorkData; struct ErtsSchedulerData_ { - -#ifdef ERTS_SMP /* * Keep X registers first (so we get as many low * numbered registers as possible in the same cache * line). */ -#if !HALFWORD_HEAP - Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers */ -#else - Eterm *save_reg; -#endif - FloatDef freg[MAX_REG]; /* Floating point registers. */ + Eterm* x_reg_array; /* X registers */ + FloatDef* f_reg_array; /* Floating point registers. */ + +#ifdef ERTS_SMP ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ @@ -454,11 +457,6 @@ struct ErtsSchedulerData_ { ErtsSchedAllocData alloc_data; -#ifdef ERTS_SMP - /* NOTE: These fields are modified under held mutexes by other threads */ - erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */ -#endif - #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; Allctr_t *verify_unused_temp_alloc_data; @@ -603,7 +601,7 @@ struct process { Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ -#if !defined(NO_FPE_SIGNALS) +#if !defined(NO_FPE_SIGNALS) || defined(HIPE) volatile unsigned long fp_exception; #endif @@ -1064,7 +1062,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; void erts_pre_init_process(void); void erts_late_init_process(void); void erts_early_init_scheduling(int); -void erts_init_scheduling(int, int, int); +void erts_init_scheduling(int, int); ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); @@ -1072,6 +1070,9 @@ int erts_proclist_same(ErtsProcList *, Process *); int erts_sched_set_wakeup_limit(char *str); +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int erts_dbg_check_halloc_lock(Process *p); +#endif #ifdef DEBUG void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); #endif @@ -1091,12 +1092,17 @@ Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); void erts_smp_notify_check_children_needed(void); -void -erts_smp_schedule_misc_aux_work(int ignore_self, - int max_sched, - void (*func)(void *), - void *arg); #endif +#if ERTS_USE_ASYNC_READY_Q +void erts_notify_check_async_ready_queue(void *); +#endif +void erts_schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg); +void erts_schedule_multi_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg); erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); @@ -1252,16 +1258,11 @@ erts_psd_get(Process *p, int ix) #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks) - ERTS_SMP_LC_ASSERT(locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); else { locks &= erts_psd_required_locks[ix].get_locks; ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + || erts_thr_progress_is_blocking()); } #endif ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); @@ -1278,16 +1279,11 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks) - ERTS_SMP_LC_ASSERT(locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); else { locks &= erts_psd_required_locks[ix].set_locks; ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks - || erts_is_system_blocked(0) - || (ERTS_IS_CRASH_DUMPING - && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC))); + || erts_thr_progress_is_blocking()); } #endif ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); @@ -1451,8 +1447,7 @@ erts_get_runq_proc(Process *p) ASSERT(p->run_queue); return p->run_queue; #else - ASSERT(erts_common_run_queue); - return erts_common_run_queue; + return ERTS_RUNQ_IX(0); #endif } @@ -1465,8 +1460,7 @@ erts_get_runq_current(ErtsSchedulerData *esdp) esdp = erts_get_scheduler_data(); return esdp->run_queue; #else - ASSERT(erts_common_run_queue); - return erts_common_run_queue; + return ERTS_RUNQ_IX(0); #endif } @@ -1634,8 +1628,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) erts_aint32_t flags; ERTS_THR_MEMORY_BARRIER; flags = erts_smp_atomic32_read_nob(&ssi->flags); - ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) - || (flags & ERTS_SSI_FLG_WAITING)); if (flags & ERTS_SSI_FLG_SLEEPING) { flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); erts_sched_finish_poke(ssi, flags); diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 83379d7352..a5a753b798 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -123,10 +123,10 @@ erts_init_proc_lock(int cpus) erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { #ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, - "pix_lock", make_small(i)); + erts_mtx_init_x(&erts_pix_locks[i].u.mtx, + "pix_lock", make_small(i)); #else - erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock"); + erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); #endif } queue_free_list = NULL; @@ -669,7 +669,9 @@ proc_safelock(Process *a_proc, ErtsProcLocks b_need_locks) { Process *p1, *p2; +#ifdef ERTS_ENABLE_LOCK_CHECK Eterm pid1, pid2; +#endif erts_pix_lock_t *pix_lck1, *pix_lck2; ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2; ErtsProcLocks unlock_mask; @@ -684,24 +686,32 @@ proc_safelock(Process *a_proc, if (a_proc) { if (a_proc->id < b_proc->id) { p1 = a_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; +#endif pix_lck1 = a_pix_lck; need_locks1 = a_need_locks; have_locks1 = a_have_locks; p2 = b_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = b_proc->id; +#endif pix_lck2 = b_pix_lck; need_locks2 = b_need_locks; have_locks2 = b_have_locks; } else if (a_proc->id > b_proc->id) { p1 = b_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; +#endif pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = a_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = a_proc->id; +#endif pix_lck2 = a_pix_lck; need_locks2 = a_need_locks; have_locks2 = a_have_locks; @@ -710,12 +720,16 @@ proc_safelock(Process *a_proc, ERTS_LC_ASSERT(a_proc == b_proc); ERTS_LC_ASSERT(a_proc->id == b_proc->id); p1 = a_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = a_proc->id; +#endif pix_lck1 = a_pix_lck; need_locks1 = a_need_locks | b_need_locks; have_locks1 = a_have_locks | b_have_locks; p2 = NULL; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; +#endif pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; @@ -723,12 +737,16 @@ proc_safelock(Process *a_proc, } else { p1 = b_proc; +#ifdef ERTS_ENABLE_LOCK_CHECK pid1 = b_proc->id; +#endif pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = NULL; +#ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; +#endif pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 97f250138e..97e554914e 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -255,8 +255,8 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); typedef struct { union { - erts_smp_spinlock_t spnlck; - char buf[64]; /* Try to get locks in different cache lines */ + erts_mtx_t mtx; + char buf[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))]; } u; } erts_pix_lock_t; @@ -380,18 +380,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int); ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); - erts_smp_spin_lock(&pixlck->u.spnlck); + erts_mtx_lock(&pixlck->u.mtx); } ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); - erts_smp_spin_unlock(&pixlck->u.spnlck); + erts_mtx_unlock(&pixlck->u.mtx); } ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) { - return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck); + return erts_lc_mtx_is_locked(&pixlck->u.mtx); } /* diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index a89ddfbcc1..63179dfad4 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -822,6 +822,16 @@ erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx) #endif } +/* + * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast() + * + * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast' + * even though the associated mutex/mutexes isn't/aren't locked by the + * caller. Our implementation do not allow that in order to avoid a + * performance penalty. That is, all associated mutexes *need* to be + * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()! + */ + ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd) { diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 1d75fa313c..c270d13365 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -253,15 +253,15 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define SMALL_BITS (28) #define SMALL_DIGITS (8) #endif -#define MAX_SMALL ((1L << (SMALL_BITS-1))-1) -#define MIN_SMALL (-(1L << (SMALL_BITS-1))) +#define MAX_SMALL ((SWORD_CONSTANT(1) << (SMALL_BITS-1))-1) +#define MIN_SMALL (-(SWORD_CONSTANT(1) << (SMALL_BITS-1))) #define make_small(x) (((Uint)(x) << _TAG_IMMED1_SIZE) + _TAG_IMMED1_SMALL) #define is_small(x) (((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_SMALL) #define is_not_small(x) (!is_small((x))) #define is_byte(x) (((x) & ((~(Uint)0 << (_TAG_IMMED1_SIZE+8)) + _TAG_IMMED1_MASK)) == _TAG_IMMED1_SMALL) #define is_valid_bit_size(x) (((Sint)(x)) >= 0 && ((x) & 0x7F) == _TAG_IMMED1_SMALL) #define is_not_valid_bit_size(x) (!is_valid_bit_size((x))) -#define MY_IS_SSMALL(x) (((Uint) (((x) >> (SMALL_BITS-1)) + 1)) < 2) +#define MY_IS_SSMALL(x) (((Uint) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2) #define _unchecked_unsigned_val(x) ((x) >> _TAG_IMMED1_SIZE) _ET_DECLARE_CHECKED(Uint,unsigned_val,Eterm) #define unsigned_val(x) _ET_APPLY(unsigned_val,(x)) @@ -331,7 +331,13 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) * we now use a non-zero bit-pattern in debug mode. */ #if ET_DEBUG -#define THE_NON_VALUE _make_header(0,_TAG_HEADER_FLOAT) +# ifdef HIPE + /* A very large (or negative) value as work-around for ugly hipe-bifs + that return untagged integers (eg hipe_bs_put_utf8) */ +# define THE_NON_VALUE _make_header((Uint)~0,_TAG_HEADER_FLOAT) +# else +# define THE_NON_VALUE _make_header(0,_TAG_HEADER_FLOAT) +# endif #else #define THE_NON_VALUE (0) #endif diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 826570b928..02f36fc75e 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -76,6 +76,7 @@ #include <stddef.h> /* offsetof() */ #include "erl_thr_progress.h" +#include "global.h" #ifdef ERTS_SMP @@ -88,9 +89,14 @@ #define ERTS_THR_PRGR_PRINT_LEADER 0 #define ERTS_THR_PRGR_PRINT_VAL 0 +#define ERTS_THR_PRGR_PRINT_BLOCKERS 0 -#define ERTS_THR_PRGR_LFLG_NO_LEADER (((erts_aint32_t) 1) << 31) -#define ERTS_THR_PRGR_LFLG_ACTIVE_MASK (~ERTS_THR_PRGR_LFLG_NO_LEADER) +#define ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL 100 + +#define ERTS_THR_PRGR_LFLG_BLOCK (((erts_aint32_t) 1) << 31) +#define ERTS_THR_PRGR_LFLG_NO_LEADER (((erts_aint32_t) 1) << 30) +#define ERTS_THR_PRGR_LFLG_ACTIVE_MASK (~(ERTS_THR_PRGR_LFLG_NO_LEADER \ + | ERTS_THR_PRGR_LFLG_BLOCK)) #define ERTS_THR_PRGR_LFLGS_ACTIVE(LFLGS) \ ((LFLGS) & ERTS_THR_PRGR_LFLG_ACTIVE_MASK) @@ -232,6 +238,11 @@ do { \ #endif /* ERTS_THR_PROGRESS_STATE_DEBUG */ +#define ERTS_THR_PRGR_BLCKR_INVALID (~((erts_aint32_t) 0)) +#define ERTS_THR_PRGR_BLCKR_UNMANAGED (((erts_aint32_t) 1) << 31) + +#define ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING (((erts_aint32_t) 1) << 31) + #define ERTS_THR_PRGR_BM_BITS 32 #define ERTS_THR_PRGR_BM_SHIFT 5 #define ERTS_THR_PRGR_BM_MASK 0x1f @@ -256,11 +267,13 @@ typedef struct { typedef struct { erts_atomic32_t lflgs; - + erts_atomic32_t block_count; + erts_atomic_t blocker_event; erts_atomic32_t pref_wakeup_used; + erts_atomic32_t managed_count; erts_atomic32_t managed_id; erts_atomic32_t unmanaged_id; -} ErtsThrPrgrMiscVolatile; +} ErtsThrPrgrMiscData; typedef struct { ERTS_THR_PRGR_ATOMIC current; @@ -276,19 +289,19 @@ typedef union { typedef struct { union { - ErtsThrPrgrMiscVolatile tile; + ErtsThrPrgrMiscData data; char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( - sizeof(ErtsThrPrgrMiscVolatile))]; - } vola; + sizeof(ErtsThrPrgrMiscData))]; + } misc; ErtsThrPrgrArray *thr; struct { int no; - ErtsThrPrgrWakeupCallback *callback; + ErtsThrPrgrCallbacks *callbacks; ErtsThrPrgrManagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; } managed; struct { int no; - ErtsThrPrgrWakeupCallback *callback; + ErtsThrPrgrCallbacks *callbacks; ErtsThrPrgrUnmanagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; } unmanaged; } ErtsThrPrgrInternalData; @@ -301,36 +314,106 @@ erts_tsd_key_t erts_thr_prgr_data_key__; static void handle_wakeup_requests(ErtsThrPrgrVal current); static int got_sched_wakeups(void); +static erts_aint32_t block_thread(ErtsThrPrgrData *tpd); static ERTS_INLINE void wakeup_managed(int id) { - ErtsThrPrgrWakeupCallback *wdp = &intrnl->managed.callback[id]; + ErtsThrPrgrCallbacks *cbp = &intrnl->managed.callbacks[id]; ASSERT(0 <= id && id < intrnl->managed.no); - wdp->wakeup(wdp->arg); + cbp->wakeup(cbp->arg); } static ERTS_INLINE void wakeup_unmanaged(int id) { - ErtsThrPrgrWakeupCallback *wdp = &intrnl->unmanaged.callback[id]; + ErtsThrPrgrCallbacks *cbp = &intrnl->unmanaged.callbacks[id]; ASSERT(0 <= id && id < intrnl->unmanaged.no); - wdp->wakeup(wdp->arg); + cbp->wakeup(cbp->arg); } static ERTS_INLINE ErtsThrPrgrData * -thr_prgr_data(ErtsSchedulerData *esdp) +perhaps_thr_prgr_data(ErtsSchedulerData *esdp) { - ErtsThrPrgrData *tpd; if (esdp) - tpd = &esdp->thr_progress_data; + return &esdp->thr_progress_data; else - tpd = erts_tsd_get(erts_thr_prgr_data_key__); + return erts_tsd_get(erts_thr_prgr_data_key__); +} + +static ERTS_INLINE ErtsThrPrgrData * +thr_prgr_data(ErtsSchedulerData *esdp) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp); ASSERT(tpd); return tpd; } +static void +init_tmp_thr_prgr_data(ErtsThrPrgrData *tpd) +{ + tpd->id = -1; + tpd->is_managed = 0; + tpd->is_blocking = 0; + tpd->is_temporary = 1; + + erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); +} + +static ERTS_INLINE ErtsThrPrgrData * +tmp_thr_prgr_data(ErtsSchedulerData *esdp) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp); + + if (!tpd) { + /* + * We only allocate the part up to the wakeup_request field + * which is the first field only used by registered threads + */ + tpd = erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA, + offsetof(ErtsThrPrgrData, wakeup_request)); + init_tmp_thr_prgr_data(tpd); + } + + return tpd; +} + +static ERTS_INLINE void +return_tmp_thr_prgr_data(ErtsThrPrgrData *tpd) +{ + if (tpd->is_temporary) { + erts_tsd_set(erts_thr_prgr_data_key__, NULL); + erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd); + } +} + +static ERTS_INLINE int +block_count_dec(void) +{ + erts_aint32_t block_count; + block_count = erts_atomic32_dec_read_mb(&intrnl->misc.data.block_count); + if (block_count == 0) { + erts_tse_t *event; + event = ((erts_tse_t*) + erts_atomic_read_nob(&intrnl->misc.data.blocker_event)); + if (event) + erts_tse_set(event); + return 1; + } + + return (block_count & ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING) == 0; +} + +static ERTS_INLINE int +block_count_inc(void) +{ + erts_aint32_t block_count; + block_count = erts_atomic32_inc_read_mb(&intrnl->misc.data.block_count); + return (block_count & ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING) == 0; +} + + void erts_thr_progress_pre_init(void) { @@ -350,7 +433,7 @@ erts_thr_progress_init(int no_schedulers, int managed, int unmanaged) intrnl_sz = sizeof(ErtsThrPrgrInternalData); intrnl_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(intrnl_sz); - cb_sz = sizeof(ErtsThrPrgrWakeupCallback)*(managed+unmanaged); + cb_sz = sizeof(ErtsThrPrgrCallbacks)*(managed+unmanaged); cb_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(cb_sz); thr_arr_sz = sizeof(ErtsThrPrgrArray)*managed; @@ -379,31 +462,36 @@ erts_thr_progress_init(int no_schedulers, int managed, int unmanaged) intrnl = (ErtsThrPrgrInternalData *) ptr; ptr += intrnl_sz; - erts_atomic32_init_nob(&intrnl->vola.tile.lflgs, + erts_atomic32_init_nob(&intrnl->misc.data.lflgs, ERTS_THR_PRGR_LFLG_NO_LEADER); - erts_atomic32_init_nob(&intrnl->vola.tile.pref_wakeup_used, 0); - erts_atomic32_init_nob(&intrnl->vola.tile.managed_id, no_schedulers); - erts_atomic32_init_nob(&intrnl->vola.tile.unmanaged_id, -1); + erts_atomic32_init_nob(&intrnl->misc.data.block_count, + (ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING + | (erts_aint32_t) managed)); + erts_atomic_init_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL); + erts_atomic32_init_nob(&intrnl->misc.data.pref_wakeup_used, 0); + erts_atomic32_init_nob(&intrnl->misc.data.managed_count, 0); + erts_atomic32_init_nob(&intrnl->misc.data.managed_id, no_schedulers); + erts_atomic32_init_nob(&intrnl->misc.data.unmanaged_id, -1); intrnl->thr = (ErtsThrPrgrArray *) ptr; ptr += thr_arr_sz; for (i = 0; i < managed; i++) init_nob(&intrnl->thr[i].data.current, 0); - intrnl->managed.callback = (ErtsThrPrgrWakeupCallback *) ptr; - intrnl->unmanaged.callback = &intrnl->managed.callback[managed]; + intrnl->managed.callbacks = (ErtsThrPrgrCallbacks *) ptr; + intrnl->unmanaged.callbacks = &intrnl->managed.callbacks[managed]; ptr += cb_sz; intrnl->managed.no = managed; for (i = 0; i < managed; i++) { - intrnl->managed.callback[i].arg = NULL; - intrnl->managed.callback[i].wakeup = NULL; + intrnl->managed.callbacks[i].arg = NULL; + intrnl->managed.callbacks[i].wakeup = NULL; } intrnl->unmanaged.no = unmanaged; for (i = 0; i < unmanaged; i++) { - intrnl->unmanaged.callback[i].arg = NULL; - intrnl->unmanaged.callback[i].wakeup = NULL; + intrnl->unmanaged.callbacks[i].arg = NULL; + intrnl->unmanaged.callbacks[i].wakeup = NULL; } for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) { @@ -446,21 +534,30 @@ init_wakeup_request_array(ErtsThrPrgrVal *w) } void -erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrWakeupCallback *callback) +erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) { - ErtsThrPrgrData *tpd; - if (erts_tsd_get(erts_thr_prgr_data_key__)) - erl_exit(ERTS_ABORT_EXIT, - "%s:%d:%s(): Double register of thread\n", - __FILE__, __LINE__, __func__); + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + int is_blocking = 0; + + if (tpd) { + if (!tpd->is_temporary) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Double register of thread\n", + __FILE__, __LINE__, __func__); + is_blocking = tpd->is_blocking; + return_tmp_thr_prgr_data(tpd); + } + /* * We only allocate the part up to the leader field * which is the first field only used by managed threads */ tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, offsetof(ErtsThrPrgrData, leader)); - tpd->id = (int) erts_atomic32_inc_read_nob(&intrnl->vola.tile.unmanaged_id); + tpd->id = (int) erts_atomic32_inc_read_nob(&intrnl->misc.data.unmanaged_id); tpd->is_managed = 0; + tpd->is_blocking = is_blocking; + tpd->is_temporary = 0; ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->unmanaged.no) erl_exit(ERTS_ABORT_EXIT, @@ -470,32 +567,41 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrWakeupCallback *callback) init_wakeup_request_array(&tpd->wakeup_request[0]); erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); - intrnl->unmanaged.callback[tpd->id] = *callback; + ASSERT(callbacks->wakeup); + + intrnl->unmanaged.callbacks[tpd->id] = *callbacks; } void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, - ErtsThrPrgrWakeupCallback *callback, + ErtsThrPrgrCallbacks *callbacks, int pref_wakeup) { - ErtsThrPrgrData *tpd; - if (erts_tsd_get(erts_thr_prgr_data_key__)) - erl_exit(ERTS_ABORT_EXIT, - "%s:%d:%s(): Double register of thread\n", - __FILE__, __LINE__, __func__); + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + int is_blocking = 0, managed; + + if (tpd) { + if (!tpd->is_temporary) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Double register of thread\n", + __FILE__, __LINE__, __func__); + is_blocking = tpd->is_blocking; + return_tmp_thr_prgr_data(tpd); + } + if (esdp) tpd = &esdp->thr_progress_data; else tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, sizeof(ErtsThrPrgrData)); if (pref_wakeup - && !erts_atomic32_xchg_nob(&intrnl->vola.tile.pref_wakeup_used, 1)) + && !erts_atomic32_xchg_nob(&intrnl->misc.data.pref_wakeup_used, 1)) tpd->id = 0; else if (esdp) tpd->id = (int) esdp->no; else - tpd->id = erts_atomic32_inc_read_nob(&intrnl->vola.tile.managed_id); + tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id); ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->managed.no) erl_exit(ERTS_ABORT_EXIT, @@ -503,6 +609,8 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, __FILE__, __LINE__, __func__); tpd->is_managed = 1; + tpd->is_blocking = is_blocking; + tpd->is_temporary = 0; init_wakeup_request_array(&tpd->wakeup_request[0]); @@ -514,14 +622,46 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); - erts_atomic32_inc_nob(&intrnl->vola.tile.lflgs); - intrnl->managed.callback[tpd->id] = *callback; + erts_atomic32_inc_nob(&intrnl->misc.data.lflgs); + + ASSERT(callbacks->wakeup); + ASSERT(callbacks->prepare_wait); + ASSERT(callbacks->wait); + ASSERT(callbacks->finalize_wait); + + intrnl->managed.callbacks[tpd->id] = *callbacks; + + callbacks->prepare_wait(callbacks->arg); + managed = erts_atomic32_inc_read_relb(&intrnl->misc.data.managed_count); + if (managed != intrnl->managed.no) { + /* Wait until all managed threads have registered... */ + do { + callbacks->wait(callbacks->arg); + callbacks->prepare_wait(callbacks->arg); + managed = erts_atomic32_read_acqb(&intrnl->misc.data.managed_count); + } while (managed != intrnl->managed.no); + } + else { + int id; + /* All managed threads have registered; lets go... */ + for (id = 0; id < managed; id++) + if (id != tpd->id) + wakeup_managed(id); + } + callbacks->finalize_wait(callbacks->arg); } static ERTS_INLINE int leader_update(ErtsThrPrgrData *tpd) { - if (tpd->leader) { +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + if (!tpd->leader) { + /* Probably need to block... */ + block_thread(tpd); + } + else { erts_aint32_t lflgs; ErtsThrPrgrVal next; int ix, sz, make_progress; @@ -577,16 +717,24 @@ leader_update(ErtsThrPrgrData *tpd) handle_wakeup_requests(current); } - if (!tpd->active) { + if (tpd->active) { + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + (void) block_thread(tpd); + } + else { tpd->leader = 0; tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; #if ERTS_THR_PRGR_PRINT_LEADER erts_fprintf(stderr, "L <- %d\n", tpd->id); #endif + ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0); - lflgs = erts_atomic32_read_bor_relb(&intrnl->vola.tile.lflgs, + lflgs = erts_atomic32_read_bor_relb(&intrnl->misc.data.lflgs, ERTS_THR_PRGR_LFLG_NO_LEADER); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + lflgs = block_thread(tpd); if (ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0 && got_sched_wakeups()) wakeup_managed(0); } @@ -598,10 +746,14 @@ leader_update(ErtsThrPrgrData *tpd) static int update(ErtsThrPrgrData *tpd) { + int res; ErtsThrPrgrVal val; - if (!tpd->leader) { + if (tpd->leader) + res = 1; + else { erts_aint32_t lflgs; + res = 0; val = read_acqb(&erts_thr_prgr__.current); if (tpd->previous.local == val) { val++; @@ -611,13 +763,16 @@ update(ErtsThrPrgrData *tpd) set_mb(&intrnl->thr[tpd->id].data.current, val); } - lflgs = erts_atomic32_read_nob(&intrnl->vola.tile.lflgs); + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + res = 1; /* Need to block in leader_update() */ + if ((lflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) && (tpd->active || ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0)) { /* Try to take over leadership... */ erts_aint32_t olflgs; olflgs = erts_atomic32_read_band_acqb( - &intrnl->vola.tile.lflgs, + &intrnl->misc.data.lflgs, ~ERTS_THR_PRGR_LFLG_NO_LEADER); if (olflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) { tpd->leader = 1; @@ -627,8 +782,9 @@ update(ErtsThrPrgrData *tpd) ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 1); } } + res |= tpd->leader; } - return tpd->leader; + return res; } int @@ -650,10 +806,16 @@ erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp) erts_aint32_t lflgs; ErtsThrPrgrData *tpd = thr_prgr_data(esdp); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + + block_count_dec(); + tpd->previous.local = ERTS_THR_PRGR_VAL_WAITING; set_mb(&intrnl->thr[tpd->id].data.current, ERTS_THR_PRGR_VAL_WAITING); - lflgs = erts_atomic32_read_nob(&intrnl->vola.tile.lflgs); + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); if (ERTS_THR_PRGR_LFLGS_ALL_WAITING(lflgs) && got_sched_wakeups()) wakeup_managed(0); /* Someone need to make progress */ } @@ -664,6 +826,10 @@ erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp) ErtsThrPrgrData *tpd = thr_prgr_data(esdp); ErtsThrPrgrVal current, val; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + /* * We aren't allowed to continue until our thread * progress is past global current. @@ -680,6 +846,8 @@ erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp) break; current = val; } + if (block_count_inc()) + block_thread(tpd); if (update(tpd)) leader_update(tpd); } @@ -689,25 +857,28 @@ erts_thr_progress_active(ErtsSchedulerData *esdp, int on) { ErtsThrPrgrData *tpd = thr_prgr_data(esdp); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_exact(NULL, 0); +#endif + ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(tpd->id, on); if (on) { ASSERT(!tpd->active); tpd->active = 1; - erts_atomic32_inc_nob(&intrnl->vola.tile.lflgs); + erts_atomic32_inc_nob(&intrnl->misc.data.lflgs); } else { ASSERT(tpd->active); tpd->active = 0; - erts_atomic32_dec_nob(&intrnl->vola.tile.lflgs); - + erts_atomic32_dec_nob(&intrnl->misc.data.lflgs); if (update(tpd)) leader_update(tpd); } #ifdef DEBUG { - erts_aint32_t n = erts_atomic32_read_nob(&intrnl->vola.tile.lflgs); + erts_aint32_t n = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); n &= ERTS_THR_PRGR_LFLG_ACTIVE_MASK; ASSERT(tpd->active <= n && n <= intrnl->managed.no); } @@ -883,7 +1054,7 @@ erts_thr_progress_wakeup(ErtsSchedulerData *esdp, ErtsThrPrgrVal value) { ErtsThrPrgrData *tpd = thr_prgr_data(esdp); - ASSERT(tpd); + ASSERT(!tpd->is_temporary); if (tpd->is_managed) request_wakeup_managed(tpd, value); else @@ -972,6 +1143,198 @@ got_sched_wakeups(void) return 0; } +static erts_aint32_t +block_thread(ErtsThrPrgrData *tpd) +{ + erts_aint32_t lflgs; + ErtsThrPrgrCallbacks *cbp = &intrnl->managed.callbacks[tpd->id]; + + do { + block_count_dec(); + + while (1) { + cbp->prepare_wait(cbp->arg); + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + cbp->wait(cbp->arg); + else + break; + } + + } while (block_count_inc()); + + cbp->finalize_wait(cbp->arg); + + return lflgs; +} + +static erts_aint32_t +thr_progress_block(ErtsThrPrgrData *tpd, int wait) +{ + erts_tse_t *event = NULL; /* Remove erroneous warning... sigh... */ + erts_aint32_t lflgs, bc; + + if (tpd->is_blocking++) + return (erts_aint32_t) 0; + + while (1) { + lflgs = erts_atomic32_read_bor_nob(&intrnl->misc.data.lflgs, + ERTS_THR_PRGR_LFLG_BLOCK); + if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) + block_thread(tpd); + else + break; + } + +#if ERTS_THR_PRGR_PRINT_BLOCKERS + erts_fprintf(stderr, "block(%d)\n", tpd->id); +#endif + + ASSERT(ERTS_AINT_NULL + == erts_atomic_read_nob(&intrnl->misc.data.blocker_event)); + + if (wait) { + event = erts_tse_fetch(); + erts_tse_reset(event); + erts_atomic_set_nob(&intrnl->misc.data.blocker_event, + (erts_aint_t) event); + } + if (tpd->is_managed) + erts_atomic32_dec_nob(&intrnl->misc.data.block_count); + bc = erts_atomic32_read_band_mb(&intrnl->misc.data.block_count, + ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING); + bc &= ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING; + if (wait) { + while (bc != 0) { + erts_tse_wait(event); + erts_tse_reset(event); + bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count); + } + } + return bc; + +} + +void +erts_thr_progress_block(void) +{ + thr_progress_block(tmp_thr_prgr_data(NULL), 1); +} + +void +erts_thr_progress_fatal_error_block(SWord timeout) +{ + ErtsThrPrgrData tpd_buf; + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + erts_aint32_t bc; + SWord time_left = timeout; + SysTimeval to; + + /* + * Counting poll intervals may give us a too long timeout + * if cpu is busy. If we got tolerant time of day we use it + * to prevent this. + */ + if (!erts_disable_tolerant_timeofday) { + erts_get_timeval(&to); + to.tv_sec += timeout / 1000; + to.tv_sec += timeout % 1000; + } + + if (!tpd) { + /* + * We stack allocate since failure to allocate memory may + * have caused the problem in the first place. This is ok + * since we never complete an unblock after a fatal error + * block. + */ + tpd = &tpd_buf; + init_tmp_thr_prgr_data(tpd); + } + + bc = thr_progress_block(tpd, 0); + if (bc == 0) + return; /* Succefully blocked all managed threads */ + + while (1) { + if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0) + time_left -= ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL; + bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count); + if (bc == 0) + break; /* Succefully blocked all managed threads */ + if (time_left <= 0) + break; /* Timeout */ + if (!erts_disable_tolerant_timeofday) { + SysTimeval now; + erts_get_timeval(&now); + if (now.tv_sec > to.tv_sec) + break; /* Timeout */ + if (now.tv_sec == to.tv_sec && now.tv_usec >= to.tv_usec) + break; /* Timeout */ + } + } +} + +void +erts_thr_progress_unblock(void) +{ + erts_tse_t *event; + int id, break_id, sz, wakeup; + ErtsThrPrgrData *tpd = thr_prgr_data(NULL); + + ASSERT(tpd->is_blocking); + if (--tpd->is_blocking) + return; + + sz = intrnl->managed.no; + + wakeup = 1; + if (!tpd->is_managed) + id = break_id = tpd->id < 0 ? 0 : tpd->id % sz; + else { + break_id = tpd->id; + id = break_id + 1; + if (id >= sz) + id = 0; + if (id == break_id) + wakeup = 0; + erts_atomic32_inc_nob(&intrnl->misc.data.block_count); + } + + event = ((erts_tse_t *) + erts_atomic_read_nob(&intrnl->misc.data.blocker_event)); + ASSERT(event); + erts_atomic_set_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL); + + erts_atomic32_read_bor_relb(&intrnl->misc.data.block_count, + ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING); +#if ERTS_THR_PRGR_PRINT_BLOCKERS + erts_fprintf(stderr, "unblock(%d)\n", tpd->id); +#endif + erts_atomic32_read_band_mb(&intrnl->misc.data.lflgs, + ~ERTS_THR_PRGR_LFLG_BLOCK); + + if (wakeup) { + do { + ErtsThrPrgrVal tmp; + tmp = read_nob(&intrnl->thr[id].data.current); + if (tmp != ERTS_THR_PRGR_VAL_WAITING) + wakeup_managed(id); + if (++id >= sz) + id = 0; + } while (id != break_id); + } + + return_tmp_thr_prgr_data(tpd); + erts_tse_return(event); +} + +int +erts_thr_progress_is_blocking(void) +{ + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + return tpd && tpd->is_blocking; +} void erts_thr_progress_dbg_print_state(void) { diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 36880c10f0..68d14174b9 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -32,7 +32,22 @@ #include "sys.h" -#ifdef ERTS_SMP +#ifndef ERTS_SMP + +#define erts_smp_thr_progress_block() ((void) 0) +#define erts_smp_thr_progress_unblock() ((void) 0) +#define erts_smp_thr_progress_is_blocking() 1 + +#else /* ERTS_SMP */ + +#define erts_smp_thr_progress_block erts_thr_progress_block +#define erts_smp_thr_progress_unblock erts_thr_progress_unblock +#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking + +void erts_thr_progress_fatal_error_block(SWord timeout); +void erts_thr_progress_block(void); +void erts_thr_progress_unblock(void); +int erts_thr_progress_is_blocking(void); typedef Uint64 ErtsThrPrgrVal; @@ -41,9 +56,14 @@ typedef Uint64 ErtsThrPrgrVal; typedef struct { int id; int is_managed; + int is_blocking; + int is_temporary; + + /* --- Part below only for registered threads --- */ + ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE]; - /* --- Part below only for managed threads */ + /* --- Part below only for managed threads --- */ int leader; /* Needs to be first in the managed threads part */ int active; @@ -77,8 +97,11 @@ extern erts_tsd_key_t erts_thr_prgr_data_key__; typedef struct { void *arg; - void (*wakeup)(void *arg); -} ErtsThrPrgrWakeupCallback; + void (*wakeup)(void *); + void (*prepare_wait)(void *); + void (*wait)(void *); + void (*finalize_wait)(void *); +} ErtsThrPrgrCallbacks; typedef struct { ERTS_THR_PRGR_ATOMIC current; @@ -89,9 +112,9 @@ extern ErtsThrPrgr erts_thr_prgr__; void erts_thr_progress_pre_init(void); void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged); void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, - ErtsThrPrgrWakeupCallback *, + ErtsThrPrgrCallbacks *, int); -void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrWakeupCallback *); +void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *); void erts_thr_progress_active(ErtsSchedulerData *esdp, int on); void erts_thr_progress_wakeup(ErtsSchedulerData *esdp, ErtsThrPrgrVal value); diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c new file mode 100644 index 0000000000..efb8c635d7 --- /dev/null +++ b/erts/emulator/beam/erl_thr_queue.c @@ -0,0 +1,763 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Lock-free queue for communication between threads. + * + * Currently only a many-to-one version has been, + * implemented, i.e., many threads can enqueue but + * only one thread can dequeue at a time. It doesn't + * have to be the same thread dequeuing every time, but + * synchronization so that only one thread dequeues + * at a time has to be provided by other means. + * + * When/If the need for a many-to-many queue arises, + * this implementation can relatively easy be extended + * to support that too. + * + * Usage instructions below. + * + * Author: Rickard Green + */ + +/* + * ------ Usage instructions ----------------------------------------------- + * + * Dequeuing generates garbage that needs to be cleaned up. + * erts_thr_q_dequeue() automatically cleans, but garbage may have to be + * cleaned up also when the queue is empty. This is done by calling + * erts_thr_q_clean(). In the SMP case thread progress may have to be made + * before cleaning can continue. If so, erts_thr_q_need_thr_progress() in + * combination with erts_thr_progress_wakeup() can be used in order to + * request a wakeup at appropriate time. + * + * Enqueuing implies memory allocation and dequeuing implies memory + * deallocation. Memory allocation can be moved to another more suitable + * thread using erts_thr_q_prepare_enqueue() together with + * erts_thr_q_enqueue_prepared() instead of using erts_thr_q_enqueue(). + * Memory deallocation can can be moved to another more suitable thread by + * disabling auto_finalize_dequeue when initializing the queue and then use + * erts_thr_q_get_finalize_dequeue_data() together + * erts_thr_q_finalize_dequeue() after dequeuing or cleaning. + * + * Ending the life of the queue using either erts_thr_q_destroy() + * or erts_thr_q_finalize() impies cleaning the queue. Both functions + * return the cleaning result and may have to be called multiple times + * until the queue is clean. Once one of these functions have been called + * enqueuing is not allowed. This has to be synchronized by the user. + * If auto_finalize_dequeue has been disabled, the finalize dequeue + * functionality has to be called after ending the life of the queue just + * as when dequeuing or cleaning on a queue that is alive. + * + * ------------------------------------------------------------------------- + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "erl_thr_queue.h" + +#if defined(DEBUG) +#define ERTS_THR_Q_DBG_CHK_DATA 1 +#else +#define ERTS_THR_Q_DBG_CHK_DATA 0 +#endif + +#define ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT 100 +#define ERTS_THR_Q_MAX_SCHED_CLEAN_OPS 50 +#define ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS 3 + +#define ERTS_THR_Q_MAX_FINI_DEQ_OPS 50 + +#ifdef ERTS_SMP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(sl_element, + ErtsThrQElement_t, + 1000, + ERTS_ALC_T_THR_Q_EL_SL) +#else + +static void +init_sl_element_alloc(void) +{ +} + +static ErtsThrQElement_t * +sl_element_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_THR_Q_EL_SL, + sizeof(ErtsThrQElement_t)); +} + +static void +sl_element_free(ErtsThrQElement_t *p) +{ + erts_free(ERTS_ALC_T_THR_Q_EL_SL, p); +} + +#endif + +typedef union { + ErtsThrQ_t q; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))]; +} ErtsAlignedThrQ_t; + +void +erts_thr_q_init(void) +{ + init_sl_element_alloc(); +} + +static void noop_callback(void *arg) { } + +void +erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) +{ +#ifndef USE_THREADS + q->init = *qi; + if (!q->init.notify) + q->init.notify = noop_callback; + q->first = NULL; + q->last = NULL; + q->q.blk = NULL; +#else + erts_atomic_init_nob(&q->tail.data.marker.next.atmc, ERTS_AINT_NULL); + q->tail.data.marker.data.ptr = NULL; + erts_atomic_init_nob(&q->tail.data.last, + (erts_aint_t) &q->tail.data.marker); + erts_atomic_init_nob(&q->tail.data.um_refc[0], 0); + erts_atomic_init_nob(&q->tail.data.um_refc[1], 0); + erts_atomic32_init_nob(&q->tail.data.um_refc_ix, 0); + q->tail.data.live = qi->live.objects; + q->tail.data.arg = qi->arg; + q->tail.data.notify = qi->notify; + if (!q->tail.data.notify) + q->tail.data.notify = noop_callback; + + q->head.head.ptr = &q->tail.data.marker; + q->head.live = qi->live.objects; + q->head.first = &q->tail.data.marker; + q->head.unref_end = &q->tail.data.marker; + q->head.clean_reached_head_count = 0; + q->head.deq_fini.automatic = qi->auto_finalize_dequeue; + q->head.deq_fini.start = NULL; + q->head.deq_fini.end = NULL; +#ifdef ERTS_SMP + q->head.next.thr_progress = erts_thr_progress_current(); + q->head.next.thr_progress_reached = 1; +#endif + q->head.next.um_refc_ix = 1; + q->head.next.unref_end = &q->tail.data.marker; + q->head.used_marker = 1; + q->head.arg = qi->arg; + q->head.notify = q->tail.data.notify; + q->q.finalizing = 0; + q->q.live = qi->live.queue; + q->q.blk = NULL; +#endif +} + +ErtsThrQCleanState_t +erts_thr_q_finalize(ErtsThrQ_t *q) +{ +#ifdef USE_THREADS + q->q.finalizing = 1; +#endif + while (erts_thr_q_dequeue(q)); + return erts_thr_q_clean(q); +} + +ErtsThrQ_t * +erts_thr_q_create(ErtsThrQInit_t *qi) +{ + ErtsAlcType_t atype; + ErtsThrQ_t *q, *qblk; + UWord qw; + + switch (qi->live.queue) { + case ERTS_THR_Q_LIVE_SHORT: + atype = ERTS_ALC_T_THR_Q_SL; + break; + case ERTS_THR_Q_LIVE_LONG: + atype = ERTS_ALC_T_THR_Q_LL; + break; + default: + atype = ERTS_ALC_T_THR_Q; + break; + } + + qw = (UWord) erts_alloc(atype, + sizeof(ErtsThrQ_t) + (ERTS_CACHE_LINE_SIZE-1)); + qblk = (ErtsThrQ_t *) qw; + if (qw & ERTS_CACHE_LINE_MASK) + qw = (qw & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + ASSERT((qw & ERTS_CACHE_LINE_MASK) == 0); + q = (ErtsThrQ_t *) qw; + erts_thr_q_initialize(q, qi); + q->q.blk = qblk; + return q; +} + +ErtsThrQCleanState_t +erts_thr_q_destroy(ErtsThrQ_t *q) +{ + if (!q->q.blk) + erl_exit(ERTS_ABORT_EXIT, + "Trying to destroy not created thread queue\n"); + return erts_thr_q_finalize(q); +} + +#ifdef USE_THREADS + +static void +destroy(ErtsThrQ_t *q) +{ + ErtsAlcType_t atype; + switch (q->q.live) { + case ERTS_THR_Q_LIVE_SHORT: + atype = ERTS_ALC_T_THR_Q_SL; + break; + case ERTS_THR_Q_LIVE_LONG: + atype = ERTS_ALC_T_THR_Q_LL; + break; + default: + atype = ERTS_ALC_T_THR_Q; + break; + } + erts_free(atype, q->q.blk); +} + +#endif + +static ERTS_INLINE ErtsThrQElement_t * +element_live_alloc(ErtsThrQLive_t live) +{ + switch (live) { + case ERTS_THR_Q_LIVE_SHORT: + return sl_element_alloc(); + default: + return (ErtsThrQElement_t *) erts_alloc(ERTS_ALC_T_THR_Q_EL, + sizeof(ErtsThrQElement_t)); + } +} + +static ERTS_INLINE ErtsThrQElement_t * +element_alloc(ErtsThrQ_t *q) +{ + ErtsThrQLive_t live; +#ifdef USE_THREADS + live = q->tail.data.live; +#else + live = q->init.live.objects; +#endif + return element_live_alloc(live); +} + +static ERTS_INLINE void +element_live_free(ErtsThrQLive_t live, ErtsThrQElement_t *el) +{ + switch (live) { + case ERTS_THR_Q_LIVE_SHORT: + sl_element_free(el); + break; + default: + erts_free(ERTS_ALC_T_THR_Q_EL, el); + } +} + +static ERTS_INLINE void +element_free(ErtsThrQ_t *q, ErtsThrQElement_t *el) +{ + ErtsThrQLive_t live; +#ifdef USE_THREADS + live = q->head.live; +#else + live = q->init.live.objects; +#endif + element_live_free(live, el); +} + +#ifdef USE_THREADS + +static ERTS_INLINE ErtsThrQElement_t * +enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this, int want_last) +{ + erts_aint_t ilast, itmp; + + erts_atomic_init_nob(&this->next.atmc, ERTS_AINT_NULL); + /* Enqueue at end of list... */ + + ilast = erts_atomic_read_nob(&q->tail.data.last); + while (1) { + ErtsThrQElement_t *last = (ErtsThrQElement_t *) ilast; + itmp = erts_atomic_cmpxchg_mb(&last->next.atmc, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) + break; + ilast = itmp; + } + + /* Move last pointer forward... */ + while (1) { + if (want_last) { + if (erts_atomic_read_rb(&this->next.atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + ilast = erts_atomic_read_rb(&q->tail.data.last); + return (ErtsThrQElement_t *) ilast; + } + } + else { + if (erts_atomic_read_nob(&this->next.atmc) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + return NULL; + } + } + itmp = erts_atomic_cmpxchg_mb(&q->tail.data.last, + (erts_aint_t) this, + ilast); + if (ilast == itmp) + return want_last ? this : NULL; + ilast = itmp; + } +} + +static ErtsThrQCleanState_t +clean(ErtsThrQ_t *q, int max_ops, int do_notify) +{ + erts_aint_t ilast; + int um_refc_ix; + int ops; + + for (ops = 0; ops < max_ops; ops++) { + ErtsThrQElement_t *tmp; + restart: + ASSERT(q->head.first); + if (q->head.first == q->head.head.ptr) { + q->head.clean_reached_head_count++; + if (q->head.clean_reached_head_count + >= ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT) { + q->head.clean_reached_head_count = 0; + break; + } + goto inspect_head; + } + if (q->head.first == q->head.unref_end) + break; + if (q->head.first == &q->tail.data.marker) { + q->head.used_marker = 0; + q->head.first = q->head.first->next.ptr; + goto restart; + } + tmp = q->head.first; + q->head.first = q->head.first->next.ptr; + if (q->head.deq_fini.automatic) + element_free(q, tmp); + else { + tmp->data.ptr = (void *) (UWord) q->head.live; + if (!q->head.deq_fini.start) + q->head.deq_fini.start = tmp; + else if (q->head.deq_fini.end->next.ptr == &q->tail.data.marker) + q->head.deq_fini.end->next.ptr = tmp; + q->head.deq_fini.end = tmp; + } + } + + ilast = erts_atomic_read_nob(&q->tail.data.last); + if (q->head.first == ((ErtsThrQElement_t *) ilast) + && ((ErtsThrQElement_t *) ilast) == &q->tail.data.marker + && q->head.first == &q->tail.data.marker) { + /* Empty and clean queue */ + if (q->q.finalizing) + destroy(q); + return ERTS_THR_Q_CLEAN; + } + +#ifdef ERTS_SMP + if (q->head.next.thr_progress_reached + || erts_thr_progress_has_reached(q->head.next.thr_progress)) { + q->head.next.thr_progress_reached = 1; +#endif + um_refc_ix = q->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) { + /* Move unreferenced end pointer forward... */ + q->head.clean_reached_head_count = 0; + q->head.unref_end = q->head.next.unref_end; + + if (!q->head.used_marker + && q->head.unref_end == (ErtsThrQElement_t *) ilast) { + q->head.used_marker = 1; + ilast = (erts_aint_t) enqueue_managed(q, + &q->tail.data.marker, + 1); + if (q->head.head.ptr == q->head.unref_end) { + ErtsThrQElement_t *next; + next = ((ErtsThrQElement_t *) + erts_atomic_read_acqb(&q->head.head.ptr->next.atmc)); + if (next == &q->tail.data.marker) { + q->head.head.ptr->next.ptr = &q->tail.data.marker; + q->head.head.ptr = &q->tail.data.marker; + } + } + } + + if (q->head.unref_end == (ErtsThrQElement_t *) ilast) + ERTS_THR_MEMORY_BARRIER; + else { + q->head.next.unref_end = (ErtsThrQElement_t *) ilast; + ERTS_THR_MEMORY_BARRIER; +#ifdef ERTS_SMP + q->head.next.thr_progress = erts_thr_progress_later(); +#endif + erts_atomic32_set_relb(&q->tail.data.um_refc_ix, + um_refc_ix); + q->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0; +#ifdef ERTS_SMP + q->head.next.thr_progress_reached = 0; +#endif + } + } +#ifdef ERTS_SMP + } +#endif + + if (q->head.first == q->head.head.ptr) { + inspect_head: + if (!q->head.used_marker) { + erts_aint_t inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) { + q->head.used_marker = 1; + (void) enqueue_managed(q, &q->tail.data.marker, 0); + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == (erts_aint_t) &q->tail.data.marker) { + q->head.head.ptr->next.ptr = &q->tail.data.marker; + q->head.head.ptr = &q->tail.data.marker; + goto check_thr_progress; + } + } + } + + if (q->q.finalizing) { + ilast = erts_atomic_read_nob(&q->tail.data.last); + if (q->head.first == ((ErtsThrQElement_t *) ilast) + && ((ErtsThrQElement_t *) ilast) == &q->tail.data.marker + && q->head.first == &q->tail.data.marker) { + destroy(q); + } + else { + goto dirty; + } + } + return ERTS_THR_Q_CLEAN; + } + + if (q->head.first != q->head.unref_end) + goto dirty; + +check_thr_progress: + +#ifdef ERTS_SMP + if (q->head.next.thr_progress_reached) +#endif + { + int um_refc_ix = q->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) { + dirty: + if (do_notify) + q->head.notify(q->head.arg); + return ERTS_THR_Q_DIRTY; + } + } + + return ERTS_THR_Q_NEED_THR_PRGR; +} + +#endif + +ErtsThrQCleanState_t +erts_thr_q_clean(ErtsThrQ_t *q) +{ +#ifdef USE_THREADS + return clean(q, ERTS_THR_Q_MAX_SCHED_CLEAN_OPS, 0); +#else + return ERTS_THR_Q_CLEAN; +#endif +} + +ErtsThrQCleanState_t +erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty) +{ +#ifndef USE_THREADS + return ERTS_THR_Q_CLEAN; +#else + if (ensure_empty) { + erts_aint_t inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext != ERTS_AINT_NULL) { + if (&q->tail.data.marker != (ErtsThrQElement_t *) inext) + return ERTS_THR_Q_DIRTY; + else { + q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; + q->head.head.ptr = (ErtsThrQElement_t *) inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext != ERTS_AINT_NULL) + return ERTS_THR_Q_DIRTY; + } + } + } + + if (q->head.first == q->head.head.ptr) { + if (!q->head.used_marker) { + erts_aint_t inext; + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) + return ERTS_THR_Q_DIRTY; + } + return ERTS_THR_Q_CLEAN; + } + + if (q->head.first != q->head.unref_end) + return ERTS_THR_Q_DIRTY; + +#ifdef ERTS_SMP + if (q->head.next.thr_progress_reached) +#endif + { + int um_refc_ix = q->head.next.um_refc_ix; + if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) + return ERTS_THR_Q_DIRTY; + } + return ERTS_THR_Q_NEED_THR_PRGR; +#endif +} + +static void +enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) +{ +#ifndef USE_THREADS + ASSERT(data); + + this->next.ptr = NULL; + this->data.ptr = data; + + if (q->last) + q->last->next.ptr = this; + else { + q->first = q->last = this; + q->init.notify(q->init.arg); + } +#else + int notify; + int um_refc_ix = 0; +#ifdef ERTS_SMP + int unmanaged_thread; +#endif + +#if ERTS_THR_Q_DBG_CHK_DATA + if (!data) + erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); +#endif + + ASSERT(!q->q.finalizing); + + this->data.ptr = data; + +#ifdef ERTS_SMP + unmanaged_thread = !erts_thr_progress_is_managed_thread(); + if (unmanaged_thread) +#endif + { + um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix); + while (1) { + int tmp_um_refc_ix; + erts_atomic_inc_acqb(&q->tail.data.um_refc[um_refc_ix]); + tmp_um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix); + if (tmp_um_refc_ix == um_refc_ix) + break; + erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]); + um_refc_ix = tmp_um_refc_ix; + } + } + + notify = this == enqueue_managed(q, this, 1); + + +#ifdef ERTS_SMP + if (unmanaged_thread) +#endif + { + if (notify) + erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]); + else if (erts_atomic_dec_read_relb(&q->tail.data.um_refc[um_refc_ix]) == 0) + notify = 1; + } + if (notify) + q->tail.data.notify(q->tail.data.arg); +#endif +} + +void +erts_thr_q_enqueue(ErtsThrQ_t *q, void *data) +{ + enqueue(q, data, element_alloc(q)); +} + +ErtsThrQPrepEnQ_t * +erts_thr_q_prepare_enqueue(ErtsThrQ_t *q) +{ + return (ErtsThrQPrepEnQ_t *) element_alloc(q); +} + +int +erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp) +{ +#ifndef USE_THREADS + return 0; +#else +#ifdef DEBUG + if (!q->head.deq_fini.start) { + ASSERT(!q->head.deq_fini.end); + } + else { + ErtsThrQElement_t *e = q->head.deq_fini.start; + ErtsThrQElement_t *end = q->head.deq_fini.end; + while (e != end) { + ASSERT(q->head.head.ptr != e); + ASSERT(q->head.first != e); + ASSERT(q->head.unref_end != e); + e = e->next.ptr; + } + } +#endif + fdp->start = q->head.deq_fini.start; + fdp->end = q->head.deq_fini.end; + if (fdp->end) + fdp->end->next.ptr = NULL; + q->head.deq_fini.start = NULL; + q->head.deq_fini.end = NULL; + return fdp->start != NULL; +#endif +} + +void +erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0, + ErtsThrQFinDeQ_t *fdp1) +{ +#ifdef USE_THREADS + if (fdp1->start) { + if (fdp0->end) + fdp0->end->next.ptr = fdp1->start; + else + fdp0->start = fdp1->start; + fdp0->end = fdp1->end; + } +#endif +} + + +int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state) +{ +#ifdef USE_THREADS + ErtsThrQElement_t *start = state->start; + if (start) { + ErtsThrQLive_t live; + int i; + for (i = 0; i < ERTS_THR_Q_MAX_FINI_DEQ_OPS; i++) { + ErtsThrQElement_t *tmp; + if (!start) + break; + tmp = start; + start = start->next.ptr; + live = (ErtsThrQLive_t) (UWord) tmp->data.ptr; + element_live_free(live, tmp); + } + state->start = start; + if (start) + return 1; /* More to do */ + state->end = NULL; + } +#endif + return 0; +} + +void +erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *state) +{ +#ifdef USE_THREADS + state->start = NULL; + state->end = NULL; +#endif +} + + +void +erts_thr_q_enqueue_prepared(ErtsThrQ_t *q, void *data, ErtsThrQPrepEnQ_t *prep) +{ + ASSERT(prep); + enqueue(q, data, (ErtsThrQElement_t *) prep); +} + +void * +erts_thr_q_dequeue(ErtsThrQ_t *q) +{ +#ifndef USE_THREADS + void *res; + ErtsThrQElement_t *tmp; + + if (!q->first) + return NULL; + tmp = q->first; + res = tmp->data.ptr; + q->first = tmp->next.ptr; + if (!q->first) + q->last = NULL; + + element_free(q, tmp); + + return res; +#else + erts_aint_t inext; + void *res; + + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) + return NULL; + q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; + q->head.head.ptr = (ErtsThrQElement_t *) inext; + if (q->head.head.ptr == &q->tail.data.marker) { + inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + if (inext == ERTS_AINT_NULL) + return NULL; + q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; + q->head.head.ptr = (ErtsThrQElement_t *) inext; + } + res = q->head.head.ptr->data.ptr; +#if ERTS_THR_Q_DBG_CHK_DATA + q->head.head.ptr->data.ptr = NULL; + if (!res) + erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); +#endif + clean(q, + (q->head.deq_fini.automatic + ? ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS + : ERTS_THR_Q_MAX_SCHED_CLEAN_OPS), 1); + return res; +#endif +} diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h new file mode 100644 index 0000000000..edcf2c3823 --- /dev/null +++ b/erts/emulator/beam/erl_thr_queue.h @@ -0,0 +1,209 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Lock-free queue for communication between threads. + * + * Currently only a many-to-one version has been, + * implemented, i.e., many threads can enqueue but + * only one thread can dequeue at a time. It doesn't + * have to be the same thread dequeuing every time, but + * synchronization so that only one thread dequeues + * at a time has to be provided by other means. + * + * When/If the need for a many-to-many queue arises, + * this implementation can relatively easy be extended + * to support that too. + * + * Usage instructions can be found in erts_thr_queue.c + * + * Author: Rickard Green + */ + +#ifndef ERL_THR_QUEUE_H__ +#define ERL_THR_QUEUE_H__ + +#include "sys.h" +#include "erl_threads.h" +#include "erl_alloc.h" +#include "erl_thr_progress.h" + +typedef enum { + ERTS_THR_Q_LIVE_UNDEF, + ERTS_THR_Q_LIVE_SHORT, + ERTS_THR_Q_LIVE_LONG +} ErtsThrQLive_t; + +#define ERTS_THR_Q_INIT_DEFAULT \ +{ \ + { \ + ERTS_THR_Q_LIVE_UNDEF, \ + ERTS_THR_Q_LIVE_SHORT \ + }, \ + NULL, \ + NULL, \ + 1 \ +} + +typedef struct ErtsThrQ_t_ ErtsThrQ_t; + +typedef struct { + struct { + ErtsThrQLive_t queue; + ErtsThrQLive_t objects; + } live; + void *arg; + void (*notify)(void *); + int auto_finalize_dequeue; +} ErtsThrQInit_t; + +typedef struct ErtsThrQElement_t_ ErtsThrQElement_t; +typedef struct ErtsThrQElement_t ErtsThrQPrepEnQ_t; + +typedef union { + erts_atomic_t atmc; + ErtsThrQElement_t *ptr; +} ErtsThrQPtr_t; + +struct ErtsThrQElement_t_ { + ErtsThrQPtr_t next; + union { + erts_atomic_t atmc; + void *ptr; + } data; +}; + +typedef struct { + ErtsThrQElement_t *start; + ErtsThrQElement_t *end; +} ErtsThrQFinDeQ_t; + +typedef enum { + ERTS_THR_Q_CLEAN, + ERTS_THR_Q_NEED_THR_PRGR, + ERTS_THR_Q_DIRTY, +} ErtsThrQCleanState_t; + +#ifdef USE_THREADS + +typedef struct { + ErtsThrQElement_t marker; + erts_atomic_t last; + erts_atomic_t um_refc[2]; + erts_atomic32_t um_refc_ix; + ErtsThrQLive_t live; +#ifdef ERTS_SMP + erts_atomic32_t thr_prgr_clean_scheduled; +#endif + void *arg; + void (*notify)(void *); +} ErtsThrQTail_t; + +struct ErtsThrQ_t_ { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* Modified by threads enqueuing */ + ErtsThrQTail_t data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQTail_t))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread dequeuing. + */ + struct { + ErtsThrQPtr_t head; + ErtsThrQLive_t live; + ErtsThrQElement_t *first; + ErtsThrQElement_t *unref_end; + int clean_reached_head_count; + struct { + int automatic; + ErtsThrQElement_t *start; + ErtsThrQElement_t *end; + } deq_fini; + struct { +#ifdef ERTS_SMP + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; +#endif + int um_refc_ix; + ErtsThrQElement_t *unref_end; + } next; + int used_marker; + void *arg; + void (*notify)(void *); + } head; + struct { + int finalizing; + ErtsThrQLive_t live; + void *blk; + } q; +}; + +#else /* !USE_THREADS */ + +struct ErtsThrQ_t_ { + ErtsThrQInit_t init; + ErtsThrQElement_t *first; + ErtsThrQElement_t *last; + struct { + void *blk; + } q; +}; + +#endif + +void erts_thr_q_init(void); +void erts_thr_q_initialize(ErtsThrQ_t *, ErtsThrQInit_t *); +ErtsThrQCleanState_t erts_thr_q_finalize(ErtsThrQ_t *); +ErtsThrQ_t *erts_thr_q_create(ErtsThrQInit_t *); +ErtsThrQCleanState_t erts_thr_q_destroy(ErtsThrQ_t *); +ErtsThrQCleanState_t erts_thr_q_clean(ErtsThrQ_t *); +ErtsThrQCleanState_t erts_thr_q_inspect(ErtsThrQ_t *, int); +ErtsThrQPrepEnQ_t *erts_thr_q_prepare_enqueue(ErtsThrQ_t *); +void erts_thr_q_enqueue_prepared(ErtsThrQ_t *, void *, ErtsThrQPrepEnQ_t *); +void erts_thr_q_enqueue(ErtsThrQ_t *, void *); +void * erts_thr_q_dequeue(ErtsThrQ_t *); +int erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *, + ErtsThrQFinDeQ_t *); +void erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *, + ErtsThrQFinDeQ_t *); +int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *); +void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *); + +#ifdef ERTS_SMP +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q); +#endif + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ERTS_SMP +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_q_need_thr_progress(ErtsThrQ_t *q) +{ + return q->head.next.thr_progress; +} +#endif + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERL_THR_QUEUE_H__ */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index b4b6d0dfd5..065e7077c0 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -92,6 +92,8 @@ typedef struct { #endif } erts_rwmtx_t; +#define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER +#define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER #define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER #define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL #define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ @@ -1130,6 +1132,16 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) #endif } +/* + * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast() + * + * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast' + * even though the associated mutex/mutexes isn't/aren't locked by the + * caller. Our implementation do not allow that in order to avoid a + * performance penalty. That is, all associated mutexes *need* to be + * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()! + */ + ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 7a09d30ff6..6c6e193818 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -20,7 +20,11 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ -extern erts_smp_atomic_t do_time; /* set at clock interrupt */ +#define ERTS_SHORT_TIME_T_MAX ERTS_AINT32_T_MAX +#define ERTS_SHORT_TIME_T_MIN ERTS_AINT32_T_MIN +typedef erts_aint32_t erts_short_time_t; + +extern erts_smp_atomic32_t do_time; /* set at clock interrupt */ extern SysTimeval erts_first_emu_time; /* @@ -71,22 +75,32 @@ void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer); void erts_init_time(void); void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); void erts_cancel_timer(ErlTimer*); -void erts_bump_timer(erts_aint_t); +void erts_bump_timer(erts_short_time_t); Uint erts_timer_wheel_memory_size(void); Uint erts_time_left(ErlTimer *); -erts_aint_t erts_next_time(void); +erts_short_time_t erts_next_time(void); #ifdef DEBUG void erts_p_slpq(void); #endif -ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void); -ERTS_GLB_INLINE void erts_do_time_add(long); +ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void); +ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg_acqb(&do_time, 0L); } -ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add_relb(&do_time, elapsed); } +ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void) +{ + erts_short_time_t time = erts_smp_atomic32_xchg_acqb(&do_time, 0); + if (time < 0) + erl_exit(ERTS_ABORT_EXIT, "Internal time management error\n"); + return time; +} + +ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) +{ + erts_smp_atomic32_add_relb(&do_time, elapsed); +} #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ @@ -105,7 +119,7 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif void erts_get_timeval(SysTimeval *tv); -long erts_get_time(void); +erts_time_t erts_get_time(void); void erts_get_emu_time(SysTimeval *); ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index ca4b54188e..f782e2f0b1 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -371,7 +371,7 @@ static void init_erts_deliver_time(const SysTimeval *inittv) static void do_erts_deliver_time(const SysTimeval *current) { SysTimeval cur_time; - long elapsed; + erts_time_t elapsed; /* calculate and deliver appropriate number of ticks */ cur_time = *current; @@ -385,7 +385,10 @@ static void do_erts_deliver_time(const SysTimeval *current) this by simply pretend as if the time stood still. :) */ if (elapsed > 0) { - erts_do_time_add(elapsed); + + ASSERT(elapsed < ((erts_time_t) ERTS_SHORT_TIME_T_MAX)); + + erts_do_time_add((erts_short_time_t) elapsed); last_delivered = cur_time; } } @@ -421,11 +424,11 @@ erts_init_time_sup(void) /* info functions */ void -elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, - unsigned long *ms_user_diff, unsigned long *ms_sys_diff) +elapsed_time_both(UWord *ms_user, UWord *ms_sys, + UWord *ms_user_diff, UWord *ms_sys_diff) { - unsigned long prev_total_user, prev_total_sys; - unsigned long total_user, total_sys; + UWord prev_total_user, prev_total_sys; + UWord total_user, total_sys; SysTimes now; sys_times(&now); @@ -456,9 +459,9 @@ elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, /* wall clock routines */ void -wall_clock_elapsed_time_both(unsigned long *ms_total, unsigned long *ms_diff) +wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff) { - unsigned long prev_total; + UWord prev_total; SysTimeval tv; erts_smp_mtx_lock(&erts_timeofday_mtx); @@ -491,7 +494,7 @@ get_time(int *hour, int *minute, int *second) the_clock = time((time_t *)0); #ifdef HAVE_LOCALTIME_R - localtime_r(&the_clock, (tm = &tmbuf)); + tm = localtime_r(&the_clock, &tmbuf); #else tm = localtime(&the_clock); #endif @@ -513,7 +516,7 @@ get_date(int *year, int *month, int *day) the_clock = time((time_t *)0); #ifdef HAVE_LOCALTIME_R - localtime_r(&the_clock, (tm = &tmbuf)); + tm = localtime_r(&the_clock, &tmbuf); #else tm = localtime(&the_clock); #endif @@ -583,7 +586,44 @@ static const int mdays[14] = {0, 31, 28, 31, 30, 31, 30, (((y) % 100) != 0)) || \ (((y) % 400) == 0)) -#define BASEYEAR 1970 +/* This is the earliest year we are sure to be able to handle + on all platforms w/o problems */ +#define BASEYEAR 1902 + +/* A more "clever" mktime + * return 1, if successful + * return -1, if not successful + */ + +static int erl_mktime(time_t *c, struct tm *tm) { + time_t clock; + + clock = mktime(tm); + + if (clock != -1) { + *c = clock; + return 1; + } + + /* in rare occasions mktime returns -1 + * when a correct value has been entered + * + * decrease seconds with one second + * if the result is -2, epochs should be -1 + */ + + tm->tm_sec = tm->tm_sec - 1; + clock = mktime(tm); + tm->tm_sec = tm->tm_sec + 1; + + *c = -1; + + if (clock == -2) { + return 1; + } + + return -1; +} /* * gregday @@ -592,10 +632,10 @@ static const int mdays[14] = {0, 31, 28, 31, 30, 31, 30, * greater of equal to 1600 , and month [1-12] and day [1-31] * are within range. Otherwise it returns -1. */ -static int long gregday(int year, int month, int day) +static time_t gregday(int year, int month, int day) { - int long ndays = 0; - int gyear, pyear, m; + Sint ndays = 0; + Sint gyear, pyear, m; /* number of days in previous years */ gyear = year - 1600; @@ -610,10 +650,72 @@ static int long gregday(int year, int month, int day) if (is_leap_year(year) && (month > 2)) ndays++; ndays += day - 1; - return ndays - 135140; /* 135140 = Jan 1, 1970 */ + return (time_t) (ndays - 135140); /* 135140 = Jan 1, 1970 */ +} + +#define SECONDS_PER_MINUTE (60) +#define SECONDS_PER_HOUR (60 * SECONDS_PER_MINUTE) +#define SECONDS_PER_DAY (24 * SECONDS_PER_HOUR) + +int seconds_to_univ(Sint64 time, Sint *year, Sint *month, Sint *day, + Sint *hour, Sint *minute, Sint *second) { + + Sint y,mi; + Sint days = time / SECONDS_PER_DAY; + Sint secs = time % SECONDS_PER_DAY; + Sint tmp; + + if (secs < 0) { + days--; + secs += SECONDS_PER_DAY; + } + + tmp = secs % SECONDS_PER_HOUR; + + *hour = secs / SECONDS_PER_HOUR; + *minute = tmp / SECONDS_PER_MINUTE; + *second = tmp % SECONDS_PER_MINUTE; + + days += 719468; + y = (10000*((Sint64)days) + 14780) / 3652425; + tmp = days - (365 * y + y/4 - y/100 + y/400); + + if (tmp < 0) { + y--; + tmp = days - (365*y + y/4 - y/100 + y/400); + } + mi = (100 * tmp + 52)/3060; + *month = (mi + 2) % 12 + 1; + *year = y + (mi + 2) / 12; + *day = tmp - (mi * 306 + 5)/10 + 1; + + return 1; } +int univ_to_seconds(Sint year, Sint month, Sint day, Sint hour, Sint minute, Sint second, Sint64 *time) { + Sint days; + + if (!(IN_RANGE(1600, year, INT_MAX - 1) && + IN_RANGE(1, month, 12) && + IN_RANGE(1, day, (mdays[month] + + (month == 2 + && (year % 4 == 0) + && (year % 100 != 0 || year % 400 == 0)))) && + IN_RANGE(0, hour, 23) && + IN_RANGE(0, minute, 59) && + IN_RANGE(0, second, 59))) { + return 0; + } + + days = gregday(year, month, day); + *time = SECONDS_PER_DAY; + *time *= days; /* don't try overflow it, it hurts */ + *time += SECONDS_PER_HOUR * hour; + *time += SECONDS_PER_MINUTE * minute; + *time += second; + return 1; +} int local_to_univ(Sint *year, Sint *month, Sint *day, @@ -644,15 +746,18 @@ local_to_univ(Sint *year, Sint *month, Sint *day, t.tm_min = *minute; t.tm_sec = *second; t.tm_isdst = isdst; - the_clock = mktime(&t); - if (the_clock == -1) { + + /* the nature of mktime makes this a bit interesting, + * up to four mktime calls could happen here + */ + + if (erl_mktime(&the_clock, &t) < 0) { if (isdst) { /* If this is a timezone without DST and the OS (correctly) refuses to give us a DST time, we simulate the Linux/Solaris behaviour of giving the same data as if is_dst was not set. */ t.tm_isdst = 0; - the_clock = mktime(&t); - if (the_clock == -1) { + if (erl_mktime(&the_clock, &t)) { /* Failed anyway, something else is bad - will be a badarg */ return 0; } @@ -662,10 +767,13 @@ local_to_univ(Sint *year, Sint *month, Sint *day, } } #ifdef HAVE_GMTIME_R - gmtime_r(&the_clock, (tm = &tmbuf)); + tm = gmtime_r(&the_clock, &tmbuf); #else tm = gmtime(&the_clock); #endif + if (!tm) { + return 0; + } *year = tm->tm_year + 1900; *month = tm->tm_mon +1; *day = tm->tm_mday; @@ -719,17 +827,20 @@ univ_to_local(Sint *year, Sint *month, Sint *day, #endif #ifdef HAVE_LOCALTIME_R - localtime_r(&the_clock, (tm = &tmbuf)); + tm = localtime_r(&the_clock, &tmbuf); #else tm = localtime(&the_clock); #endif - *year = tm->tm_year + 1900; - *month = tm->tm_mon +1; - *day = tm->tm_mday; - *hour = tm->tm_hour; - *minute = tm->tm_min; - *second = tm->tm_sec; - return 1; + if (tm) { + *year = tm->tm_year + 1900; + *month = tm->tm_mon +1; + *day = tm->tm_mday; + *hour = tm->tm_hour; + *minute = tm->tm_min; + *second = tm->tm_sec; + return 1; + } + return 0; } @@ -798,13 +909,14 @@ void erts_deliver_time(void) { void erts_time_remaining(SysTimeval *rem_time) { - int ticks; + erts_time_t ticks; SysTimeval cur_time; - long elapsed; + erts_time_t elapsed; /* erts_next_time() returns no of ticks to next timeout or -1 if none */ - if ((ticks = erts_next_time()) == -1) { + ticks = (erts_time_t) erts_next_time(); + if (ticks == (erts_time_t) -1) { /* timer queue empty */ /* this will cause at most 100000000 ticks */ rem_time->tv_sec = 100000; @@ -839,7 +951,7 @@ void erts_get_timeval(SysTimeval *tv) erts_smp_mtx_unlock(&erts_timeofday_mtx); } -long +erts_time_t erts_get_time(void) { SysTimeval sys_tv; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8833137112..b487dbf054 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -36,6 +36,7 @@ #include "error.h" #include "erl_binary.h" #include "erl_bits.h" +#include "erl_thr_progress.h" #if 0 #define DEBUG_PRINTOUTS @@ -159,7 +160,7 @@ static Uint active_sched; void erts_system_profile_setup_active_schedulers(void) { - ERTS_SMP_LC_ASSERT(erts_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); active_sched = erts_active_schedulers(); } @@ -1940,7 +1941,8 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) Eterm* hp; int need; - ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) + || erts_thr_progress_is_blocking()); if (is_internal_port(t_p->tracer_proc)) { #define LOCAL_HEAP_SIZE (5+5) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); @@ -2092,8 +2094,7 @@ void save_calls(Process *p, Export *e) * entries instead of the original BIF functions. */ Eterm -erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I) +erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) { Eterm result; int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META); @@ -2107,10 +2108,10 @@ erts_bif_trace(int bif_index, Process* p, * no tracing will occur. Doing the whole else branch will * also do nothing, only slower. */ - Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = bif_table[bif_index].f; - result = func(p, arg1, arg2, arg3, I); + Eterm (*func)(Process*, Eterm*, BeamInstr*) = bif_table[bif_index].f; + result = func(p, args, I); } else { - Eterm (*func)(Process*, Eterm, Eterm, Eterm, BeamInstr*); + Eterm (*func)(Process*, Eterm*, BeamInstr*); Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL); @@ -2122,8 +2123,6 @@ erts_bif_trace(int bif_index, Process* p, * export entry */ BeamInstr *cp = p->cp; - Eterm args[3] = {arg1, arg2, arg3}; - /* * Make continuation pointer OK, it is not during direct BIF calls, * but it is correct during apply of bif. @@ -2155,7 +2154,7 @@ erts_bif_trace(int bif_index, Process* p, func = bif_table[bif_index].f; - result = func(p, arg1, arg2, arg3, I); + result = func(p, args, I); if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { BeamInstr i_return_trace = beam_return_trace[0]; @@ -2745,7 +2744,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) { Eterm mess; Eterm* hp; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + || erts_thr_progress_is_blocking()); if (is_internal_port(t_p->tracer_proc)) { #define LOCAL_HEAP_SIZE (5+5) @@ -3021,8 +3021,6 @@ static ErtsSysMsgQ *sys_message_queue_end; static erts_tid_t sys_msg_dispatcher_tid; static erts_cnd_t smq_cnd; -static int dispatcher_waiting; - ERTS_QUALLOC_IMPL(smq_element, ErtsSysMsgQ, 20, ERTS_ALC_T_SYS_MSG_Q) static void @@ -3066,18 +3064,6 @@ enqueue_sys_msg(enum ErtsSysMsgType type, erts_smp_mtx_unlock(&smq_mtx); } -static void -prepare_for_block(void *unused) -{ - erts_smp_mtx_unlock(&smq_mtx); -} - -static void -resume_after_block(void *unused) -{ - erts_smp_mtx_lock(&smq_mtx); -} - void erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp) { @@ -3143,10 +3129,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) && !erts_system_monitor_flags.busy_port && !erts_system_monitor_flags.busy_dist_port) break; /* Everything is disabled */ - erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); + erts_smp_thr_progress_block(); if (system_monitor == receiver || receiver == NIL) erts_system_monitor_clear(NULL); - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case SYS_MSG_TYPE_SYSPROF: if (receiver == NIL @@ -3156,11 +3142,11 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) && !erts_system_profile_flags.scheduler) break; /* Block system to clear flags */ - erts_smp_block_system(0); + erts_smp_thr_progress_block(); if (system_profile == receiver || receiver == NIL) { erts_system_profile_clear(NULL); } - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case SYS_MSG_TYPE_ERRLGR: { char *no_elgger = "(no error logger present)"; @@ -3201,22 +3187,68 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) } } +static void +sys_msg_dispatcher_wakeup(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + *wait_p = 0; + erts_smp_cnd_signal(&smq_cnd); + erts_smp_mtx_unlock(&smq_mtx); +} + +static void +sys_msg_dispatcher_prep_wait(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + *wait_p = 1; + erts_smp_mtx_unlock(&smq_mtx); +} + +static void +sys_msg_dispatcher_fin_wait(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + *wait_p = 0; + erts_smp_mtx_unlock(&smq_mtx); +} + +static void +sys_msg_dispatcher_wait(void *vwait_p) +{ + int *wait_p = (int *) vwait_p; + erts_smp_mtx_lock(&smq_mtx); + while (*wait_p) + erts_smp_cnd_wait(&smq_cnd, &smq_mtx); + erts_smp_mtx_unlock(&smq_mtx); +} + static void * sys_msg_dispatcher_func(void *unused) { + ErtsThrPrgrCallbacks callbacks; ErtsSysMsgQ *local_sys_message_queue = NULL; + int wait = 0; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_set_thread_name("system message dispatcher"); #endif - erts_register_blockable_thread(); - erts_smp_activity_begin(ERTS_ACTIVITY_IO, NULL, NULL, NULL); + callbacks.arg = (void *) &wait; + callbacks.wakeup = sys_msg_dispatcher_wakeup; + callbacks.prepare_wait = sys_msg_dispatcher_prep_wait; + callbacks.wait = sys_msg_dispatcher_wait; + callbacks.finalize_wait = sys_msg_dispatcher_fin_wait; + + erts_thr_progress_register_managed_thread(NULL, &callbacks, 0); while (1) { + int end_wait = 0; ErtsSysMsgQ *smqp; - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); erts_smp_mtx_lock(&smq_mtx); @@ -3228,20 +3260,16 @@ sys_msg_dispatcher_func(void *unused) } /* Fetch current trace message queue ... */ - erts_smp_activity_change(ERTS_ACTIVITY_IO, - ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - NULL); - dispatcher_waiting = 1; + if (!sys_message_queue) { + erts_smp_mtx_unlock(&smq_mtx); + end_wait = 1; + erts_thr_progress_active(NULL, 0); + erts_thr_progress_prepare_wait(NULL); + erts_smp_mtx_lock(&smq_mtx); + } + while (!sys_message_queue) erts_smp_cnd_wait(&smq_cnd, &smq_mtx); - dispatcher_waiting = 0; - erts_smp_activity_change(ERTS_ACTIVITY_WAIT, - ERTS_ACTIVITY_IO, - prepare_for_block, - resume_after_block, - NULL); local_sys_message_queue = sys_message_queue; sys_message_queue = NULL; @@ -3249,6 +3277,11 @@ sys_msg_dispatcher_func(void *unused) erts_smp_mtx_unlock(&smq_mtx); + if (end_wait) { + erts_thr_progress_finalize_wait(NULL); + erts_thr_progress_active(NULL, 1); + } + /* Send trace messages ... */ ASSERT(local_sys_message_queue); @@ -3259,6 +3292,9 @@ sys_msg_dispatcher_func(void *unused) Process *proc = NULL; Port *port = NULL; + if (erts_thr_progress_update(NULL)) + erts_thr_progress_leader_update(NULL); + #ifdef DEBUG_PRINTOUTS print_msg_type(smqp); #endif @@ -3372,7 +3408,6 @@ sys_msg_dispatcher_func(void *unused) } } - erts_smp_activity_end(ERTS_ACTIVITY_IO, NULL, NULL, NULL); return NULL; } @@ -3422,7 +3457,6 @@ init_sys_msg_dispatcher(void) sys_message_queue_end = NULL; erts_smp_cnd_init(&smq_cnd); erts_smp_mtx_init(&smq_mtx, "sys_msg_q"); - dispatcher_waiting = 0; erts_smp_thr_create(&sys_msg_dispatcher_tid, sys_msg_dispatcher_func, NULL, diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 158eb361a4..6d5eae73b0 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -47,7 +47,7 @@ typedef struct _restart_context { static Uint max_loop_limit; -static BIF_RETTYPE utf8_to_list(BIF_ALIST_1); +static BIF_RETTYPE utf8_to_list(Process *p, Eterm arg1); static BIF_RETTYPE finalize_list_to_list(Process *p, byte *bytes, Eterm rest, @@ -227,8 +227,8 @@ static ERTS_INLINE int simple_loops_to_common(int cost) static Sint aligned_binary_size(Eterm binary) { - unsigned char *bytes; - Uint bitoffs; + ERTS_DECLARE_DUMMY(unsigned char *bytes); + ERTS_DECLARE_DUMMY(Uint bitoffs); Uint bitsize; ERTS_GET_BINARY_BYTES(binary, bytes, bitoffs, bitsize); @@ -348,12 +348,6 @@ static int copy_utf8_bin(byte *target, byte *source, Uint size, return copied; } - if (((*source) == 0xEF) && (source[1] == 0xBF) && - ((source[2] == 0xBE) || (source[2] == 0xBF))) { - *err_pos = source; - return copied; - } - *(target++) = *(source++); *(target++) = *(source++); *(target++) = *(source++); @@ -714,9 +708,8 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ target[(*pos)++] = (((byte) (x & 0x3F)) | ((byte) 0x80)); } else if (x < 0x10000) { - if ((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF)) { /* Invalid unicode range */ + if (x >= 0xD800 && x <= 0xDFFF) { + /* Invalid unicode range */ *err = 1; goto done; } @@ -901,7 +894,9 @@ static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,int pos, static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3) { +#ifdef DEBUG Eterm *real_bin; +#endif byte* bytes; Eterm rest_term; int left, sleft; @@ -915,8 +910,10 @@ static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3) /*erts_printf("Trap %T!\r\n",BIF_ARG_2);*/ ASSERT(is_binary(BIF_ARG_1)); +#ifdef DEBUG real_bin = binary_val(BIF_ARG_1); ASSERT(*real_bin == HEADER_PROC_BIN); +#endif pos = (int) binary_size(BIF_ARG_1); bytes = binary_bytes(BIF_ARG_1); sleft = left = allowed_iterations(BIF_P); @@ -1230,10 +1227,6 @@ int erts_analyze_utf8(byte *source, Uint size, ((source[1] & 0x20) != 0)) { return ERTS_UTF8_ERROR; } - if (((*source) == 0xEF) && (source[1] == 0xBF) && - ((source[2] == 0xBE) || (source[2] == 0xBF))) { - return ERTS_UTF8_ERROR; - } source += 3; size -= 3; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { @@ -1730,7 +1723,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, if (b_sz) { ErlSubBin *sb; Eterm orig; - Uint offset; + ERTS_DECLARE_DUMMY(Uint offset); ASSERT(state != ERTS_UTF8_OK); hp = HAlloc(p, ERL_SUB_BIN_SIZE); sb = (ErlSubBin *) hp; @@ -1839,13 +1832,13 @@ static BIF_RETTYPE characters_to_list_trap_4(BIF_ALIST_1) * Instead of building an utf8 buffer, we analyze the binary given and use that. */ -static BIF_RETTYPE utf8_to_list(BIF_ALIST_1) +static BIF_RETTYPE utf8_to_list(Process* p, Eterm arg) { - if (!is_binary(BIF_ARG_1) || aligned_binary_size(BIF_ARG_1) < 0) { - BIF_ERROR(BIF_P,BADARG); + if (!is_binary(arg) || aligned_binary_size(arg) < 0) { + BIF_ERROR(p, BADARG); } - return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, - ERTS_UTF8_ANALYZE_MORE,NIL); + return do_bif_utf8_to_list(p, arg, 0U, 0U, 0U, + ERTS_UTF8_ANALYZE_MORE, NIL); } @@ -2166,9 +2159,8 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ } else if (x < 0x800) { need += 2; } else if (x < 0x10000) { - if ((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF)) { /* Invalid unicode range */ + if (x >= 0xD800 && x <= 0xDFFF) { + /* Invalid unicode range */ DESTROY_ESTACK(stack); return ((Sint) -1); } @@ -2314,9 +2306,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ *p++ = (((byte) (x & 0x3F)) | ((byte) 0x80)); } else if (x < 0x10000) { - ASSERT(!((x >= 0xD800 && x <= 0xDFFF) || - (x == 0xFFFE) || - (x == 0xFFFF))); + ASSERT(!(x >= 0xD800 && x <= 0xDFFF)); *p++ = (((byte) (x >> 12)) | ((byte) 0xE0)); *p++ = ((((byte) (x >> 6)) & 0x3F) | @@ -2580,11 +2570,11 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1) BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1) { - Eterm real_bin; - Uint offset; + ERTS_DECLARE_DUMMY(Eterm real_bin); + ERTS_DECLARE_DUMMY(Uint offset); Uint size,num_chars; Uint bitsize; - Uint bitoffs; + ERTS_DECLARE_DUMMY(Uint bitoffs); Eterm ret; byte *temp_alloc = NULL; byte *bytes; diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index e7fd144ec3..5dc307e383 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -55,7 +55,7 @@ heap data on the C stack or if we use the buffers in the scheduler data. */ #define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers small heap for transient heap data */ -#define CMP_TMP_HEAP_SIZE 2 /* cmp wants its own tmp-heap... */ +#define CMP_TMP_HEAP_SIZE 32 /* cmp wants its own tmp-heap... */ #define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ #define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ @@ -83,11 +83,7 @@ #define CP_SIZE 1 #define ErtsHAllocLockCheck(P) \ - ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks((P))) \ - || ((P)->id == ERTS_INVALID_PID) \ - || ((P)->scheduler_data \ - && (P) == (P)->scheduler_data->match_pseudo_process) \ - || erts_is_system_blocked(0)) + ERTS_SMP_LC_ASSERT(erts_dbg_check_halloc_lock((P))) #ifdef DEBUG diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 5bc402fe22..18d62dac1d 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -208,7 +208,8 @@ erts_export_put(Eterm mod, Eterm func, unsigned int arity) Export e; int ix; - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_smp_thr_progress_is_blocking()); ASSERT(is_atom(mod)); ASSERT(is_atom(func)); e.code[0] = mod; @@ -265,7 +266,8 @@ erts_export_consolidate(void) HashInfo hi; #endif - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_smp_thr_progress_is_blocking()); export_write_lock(); erts_index_merge(&secondary_export_table, &export_table); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 6953e7fe7d..152dbcf085 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -46,7 +46,7 @@ #ifdef HIPE #include "hipe_mode_switch.h" #endif -#define in_area(ptr,start,nbytes) ((Uint)((char*)(ptr) - (char*)(start)) < (nbytes)) +#define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff @@ -88,7 +88,7 @@ static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); -static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins, int internal_tags); +static Sint decoded_size(byte *ep, byte* endp, int internal_tags); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); @@ -810,7 +810,7 @@ bad_dist_ext(ErtsDistExternal *edep) } Sint -erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins) +erts_decode_dist_ext_size(ErtsDistExternal *edep) { Sint res; byte *ep; @@ -829,7 +829,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins) goto fail; ep = edep->extp+1; } - res = decoded_size(ep, edep->ext_endp, no_refc_bins, 0); + res = decoded_size(ep, edep->ext_endp, 0); if (res >= 0) return res; fail: @@ -837,16 +837,16 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins) return -1; } -Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins) +Sint erts_decode_ext_size(byte *ext, Uint size) { if (size == 0 || *ext != VERSION_MAGIC) return -1; - return decoded_size(ext+1, ext+size, no_refc_bins, 0); + return decoded_size(ext+1, ext+size, 0); } Sint erts_decode_ext_size_ets(byte *ext, Uint size) { - Sint sz = decoded_size(ext, ext+size, 0, 1); + Sint sz = decoded_size(ext, ext+size, 1); ASSERT(sz >= 0); return sz; } @@ -968,7 +968,7 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) ede.extp = binary_bytes(real_bin)+offset; ede.ext_endp = ede.extp + size; - hsz = erts_decode_dist_ext_size(&ede, 0); + hsz = erts_decode_dist_ext_size(&ede); if (hsz < 0) goto badarg; @@ -988,16 +988,16 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) } -Eterm -term_to_binary_1(Process* p, Eterm Term) +BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { - return erts_term_to_binary(p, Term, 0, TERM_TO_BINARY_DFLAGS); + return erts_term_to_binary(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS); } - -Eterm -term_to_binary_2(Process* p, Eterm Term, Eterm Flags) +BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) { + Process* p = BIF_P; + Eterm Term = BIF_ARG_1; + Eterm Flags = BIF_ARG_2; int level = 0; Uint flags = TERM_TO_BINARY_DFLAGS; @@ -1106,7 +1106,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) goto error; size = (Sint) dest_len; } - res = decoded_size(state->extp, state->extp + size, 0, 0); + res = decoded_size(state->extp, state->extp + size, 0); if (res < 0) goto error; return res; @@ -1256,8 +1256,11 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) } Eterm -external_size_1(Process* p, Eterm Term) +external_size_1(BIF_ALIST_1) { + Process* p = BIF_P; + Eterm Term = BIF_ARG_1; + Uint size = erts_encode_ext_size(Term); if (IS_USMALL(0, size)) { BIF_RET(make_small(size)); @@ -1268,13 +1271,13 @@ external_size_1(Process* p, Eterm Term) } Eterm -external_size_2(Process* p, Eterm Term, Eterm Flags) +external_size_2(BIF_ALIST_2) { Uint size; Uint flags = TERM_TO_BINARY_DFLAGS; - while (is_list(Flags)) { - Eterm arg = CAR(list_val(Flags)); + while (is_list(BIF_ARG_2)) { + Eterm arg = CAR(list_val(BIF_ARG_2)); Eterm* tp; if (is_tuple(arg) && *(tp = tuple_val(arg)) == make_arityval(2)) { @@ -1293,19 +1296,19 @@ external_size_2(Process* p, Eterm Term, Eterm Flags) } } else { error: - BIF_ERROR(p, BADARG); + BIF_ERROR(BIF_P, BADARG); } - Flags = CDR(list_val(Flags)); + BIF_ARG_2 = CDR(list_val(BIF_ARG_2)); } - if (is_not_nil(Flags)) { + if (is_not_nil(BIF_ARG_2)) { goto error; } - size = erts_encode_ext_size_2(Term, flags); + size = erts_encode_ext_size_2(BIF_ARG_1, flags); if (IS_USMALL(0, size)) { BIF_RET(make_small(size)); } else { - Eterm* hp = HAlloc(p, BIG_UINT_HEAP_SIZE); + Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); BIF_RET(uint_to_big(size, hp)); } } @@ -2451,7 +2454,7 @@ dec_term_atom_common: n = get_int32(ep); ep += 4; - if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) { + if (n <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb = (ErlHeapBin *) hp; hb->thing_word = header_heap_bin(n); @@ -2489,7 +2492,7 @@ dec_term_atom_common: n = get_int32(ep); bitsize = ep[4]; ep += 5; - if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) { + if (n <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb = (ErlHeapBin *) hp; hb->thing_word = header_heap_bin(n); @@ -3058,7 +3061,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } static Sint -decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags) +decoded_size(byte *ep, byte* endp, int internal_tags) { int heap_size = 0; int terms; @@ -3220,7 +3223,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags) CHKSIZE(4); n = get_int32(ep); SKIP2(n, 4); - if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) { + if (n <= ERL_ONHEAP_BIN_LIMIT) { heap_size += heap_bin_size(n); } else { heap_size += PROC_BIN_SIZE; @@ -3231,7 +3234,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags) CHKSIZE(5); n = get_int32(ep); SKIP2(n, 5); - if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) { + if (n <= ERL_ONHEAP_BIN_LIMIT) { heap_size += heap_bin_size(n) + ERL_SUB_BIN_SIZE; } else { heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE; diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index 671b8b8781..eddd4571dd 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -175,10 +175,10 @@ void *erts_dist_ext_trailer(ErtsDistExternal *); void erts_destroy_dist_ext_copy(ErtsDistExternal *); int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, DistEntry *, ErtsAtomCache *); -Sint erts_decode_dist_ext_size(ErtsDistExternal *, int); +Sint erts_decode_dist_ext_size(ErtsDistExternal *); Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); -Sint erts_decode_ext_size(byte*, Uint, int); +Sint erts_decode_ext_size(byte*, Uint); Sint erts_decode_ext_size_ets(byte*, Uint); Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 6687e02485..f98232246b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -37,16 +37,11 @@ #include "erl_process.h" #include "erl_sys_driver.h" #include "erl_debug.h" +#include "error.h" typedef struct port Port; #include "erl_port_task.h" -#define ERTS_MAX_NO_OF_ASYNC_THREADS 1024 -extern int erts_async_max_threads; -#define ERTS_ASYNC_THREAD_MIN_STACK_SIZE 16 /* Kilo words */ -#define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ -extern int erts_async_thread_suggested_stack_size; - typedef struct erts_driver_t_ erts_driver_t; #define SMALL_IO_QUEUE 5 /* Number of fixed elements */ @@ -177,7 +172,7 @@ struct port { DistEntry *dist_entry; /* Dist entry used in DISTRIBUTION */ char *name; /* String used in the open */ erts_driver_t* drv_ptr; - long drv_data; + UWord drv_data; ErtsProcList *suspended; /* List of suspended processes. */ LineBuf *linebuf; /* Buffer to hold data not ready for process to get (line oriented I/O)*/ @@ -210,7 +205,7 @@ erts_port_runq(Port *prt) rq1 = rq2; } #else - return erts_common_run_queue; + return ERTS_RUNQ_IX(0); #endif } @@ -403,7 +398,7 @@ extern Eterm erts_ddll_monitor_driver(Process *p, typedef struct binary { ERTS_INTERNAL_BINARY_FIELDS - long orig_size; + SWord orig_size; char orig_bytes[1]; /* to be continued */ } Binary; @@ -412,7 +407,7 @@ typedef struct binary { typedef struct { ERTS_INTERNAL_BINARY_FIELDS - long orig_size; + SWord orig_size; void (*destructor)(Binary *); char magic_bin_data[1]; } ErtsMagicBinary; @@ -560,7 +555,6 @@ extern Eterm node_cookie; extern erts_smp_atomic_t erts_bytes_out; /* no bytes written out */ extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ extern Uint display_items; /* no of items to display in traces etc */ -extern Uint display_loads; /* print info about loaded modules */ extern int erts_backtrace_depth; extern erts_smp_atomic32_t erts_max_gen_gcs; @@ -851,18 +845,41 @@ void erts_queue_monitor_message(Process *, Eterm, Eterm); void erts_init_bif(void); +Eterm erl_send(Process *p, Eterm to, Eterm msg); + +/* erl_bif_op.c */ + +Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* erl_bif_port.c */ /* erl_bif_trace.c */ +Eterm erl_seq_trace_info(Process *p, Eterm arg1); void erts_system_monitor_clear(Process *c_p); void erts_system_profile_clear(Process *c_p); /* beam_load.c */ -int erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm* mod, byte* code, int size); +typedef struct { + BeamInstr* current; /* Pointer to: Mod, Name, Arity */ + Uint needed; /* Heap space needed for entire tuple */ + Uint32 loc; /* Location in source code */ + Eterm* fname_ptr; /* Pointer to fname table */ +} FunctionInfo; + +struct LoaderState* erts_alloc_loader_state(void); +Eterm erts_prepare_loading(struct LoaderState*, Process *c_p, + Eterm group_leader, Eterm* modp, + byte* code, Uint size); +Eterm erts_finish_loading(struct LoaderState* stp, Process* c_p, + ErtsProcLocks c_p_locks, Eterm* modp); +Eterm erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); BeamInstr* find_function_from_pc(BeamInstr* pc); +Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, + Eterm args, Eterm* mfa_p); +void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); +void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); @@ -1053,6 +1070,7 @@ void init_emulator(void); void process_main(void); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); +void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); /* erl_init.c */ @@ -1074,6 +1092,7 @@ extern ErtsModifiedTimings erts_modified_timings[]; #define ERTS_MODIFIED_TIMING_INPUT_REDS \ (erts_modified_timings[erts_modified_timing_level].input_reds) +extern int erts_no_line_info; extern Eterm erts_error_logger_warnings; extern int erts_initialized; extern int erts_compat_rel; @@ -1107,7 +1126,9 @@ void erts_init_gc(void); int erts_garbage_collect(Process*, int, Eterm*, int); void erts_garbage_collect_hibernate(Process* p); Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity); -void erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size); +void erts_garbage_collect_literals(Process* p, Eterm* literals, + Uint lit_size, + struct erl_off_heap_header* oh); Uint erts_next_heap_size(Uint, Uint); Eterm erts_heap_sizes(Process* p); @@ -1627,8 +1648,7 @@ void monitor_generic(Process *p, Eterm type, Eterm spec); Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, Eterm *pTracer, int *pCpuTimestamp); -Eterm erts_bif_trace(int bif_index, Process* p, - Eterm arg1, Eterm arg2, Eterm arg3, BeamInstr *I); +Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); #ifdef ERTS_SMP void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 151c776a3d..759621d3c2 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -42,6 +42,7 @@ #include "erl_bits.h" #include "erl_version.h" #include "error.h" +#include "erl_async.h" extern ErlDrvEntry fd_driver_entry; extern ErlDrvEntry vanilla_driver_entry; @@ -444,7 +445,7 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, prt->control_flags = 0; prt->connected = pid; - prt->drv_data = (long) drv_data; + prt->drv_data = (SWord) drv_data; prt->bytes_in = 0; prt->bytes_out = 0; prt->dist_entry = NULL; @@ -643,11 +644,10 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ name, opts); erts_unblock_fpe(fpe_was_unmasked); port->caller = NIL; - erts_unblock_fpe(fpe_was_unmasked); if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_out, am_start); } - if (error_number_ptr && ((long) drv_data) == (long) -2) + if (error_number_ptr && ((SWord) drv_data) == (SWord) -2) *error_number_ptr = errno; #ifdef ERTS_SMP if (port->xports) @@ -656,10 +656,10 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ #endif } - if (((long)drv_data) == -1 || - ((long)drv_data) == -2 || - ((long)drv_data) == -3) { - int res = (int) ((long) drv_data); + if (((SWord)drv_data) == -1 || + ((SWord)drv_data) == -2 || + ((SWord)drv_data) == -3) { + int res = (int) ((SWord) drv_data); if (res == -3 && error_number_ptr) { *error_number_ptr = BADARG; @@ -688,7 +688,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ erts_port_release(port); return res; } - port->drv_data = (long) drv_data; + port->drv_data = (SWord) drv_data; return port_ix; } @@ -951,7 +951,7 @@ io_list_to_vec(Eterm obj, /* io-list */ do { \ int _size = binary_size(obj); \ Eterm _real; \ - Uint _offset; \ + ERTS_DECLARE_DUMMY(Uint _offset); \ int _bitoffs; \ int _bitsize; \ ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \ @@ -2170,8 +2170,8 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) * and with its length in to_len. */ if (is_binary(iolist) && binary_bitoffset(iolist) == 0) { - Uint bitoffs; - Uint bitsize; + ERTS_DECLARE_DUMMY(Uint bitoffs); + ERTS_DECLARE_DUMMY(Uint bitsize); ERTS_GET_BINARY_BYTES(iolist, to_port, bitoffs, bitsize); to_len = binary_size(iolist); } else { @@ -3082,7 +3082,7 @@ driver_deliver_term(ErlDrvPort port, Binary* bp = erts_bin_nrml_alloc(size); ASSERT(bufp); bp->flags = 0; - bp->orig_size = (long) size; + bp->orig_size = (SWord) size; erts_refc_init(&bp->refc, 1); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); pbp = (ProcBin *) hp; @@ -3448,7 +3448,7 @@ driver_alloc_binary(int size) return NULL; /* The driver write must take action */ bin->flags = BIN_FLAG_DRV; erts_refc_init(&bin->refc, 1); - bin->orig_size = (long) size; + bin->orig_size = (SWord) size; return Binary2ErlDrvBinary(bin); } @@ -4075,7 +4075,7 @@ drv_cancel_timer(Port *prt) erts_port_task_abort(prt->id, &prt->timeout_task); } -int driver_set_timer(ErlDrvPort ix, UWord t) +int driver_set_timer(ErlDrvPort ix, unsigned long t) { Port* prt = erts_drvport2port(ix); @@ -4579,7 +4579,10 @@ int driver_lock_driver(ErlDrvPort ix) erts_smp_mtx_lock(&erts_driver_list_lock); - if (prt == NULL) return -1; + if (prt == NULL) { + erts_smp_mtx_unlock(&erts_driver_list_lock); + return -1; + } ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if ((dh = (DE_Handle*)prt->drv_ptr->handle ) == NULL) { diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 91e4ccce70..b93b1ad09a 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -108,7 +108,8 @@ erts_put_module(Eterm mod) int index; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0)); + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_smp_thr_progress_is_blocking()); e.module = atom_val(mod); index = index_put(&module_table, (void*) &e); return (Module*) erts_index_lookup(&module_table, index); diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 304ce22ef2..fc53a88a3a 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -25,30 +25,12 @@ # instruction transformations; thus, they never occur in BEAM files. # -# Special instruction used to generate an error message when -# trying to load a module compiled by the V1 compiler (R5 & R6). -# (Specially treated in beam_load.c.) +# The too_old_compiler/0 instruction is specially handled in beam_load.c +# to produce a user-friendly message informing the user that the module +# needs to be re-compiled with a modern compiler. too_old_compiler/0 -too_old_compiler - -# -# Obsolete instruction usage follow. (Nowdays we use f with -# a zero label instead of p.) -# - -is_list p S => too_old_compiler -is_nonempty_list p R => too_old_compiler -is_nil p R => too_old_compiler - -is_tuple p S => too_old_compiler -test_arity p S Arity => too_old_compiler - -is_integer p R => too_old_compiler -is_float p R => too_old_compiler -is_atom p R => too_old_compiler - -is_eq_exact p S1 S2 => too_old_compiler +too_old_compiler | never() => # In R9C and earlier, the loader used to insert special instructions inside # the module_info/0,1 functions. (In R10B and later, the compiler inserts @@ -88,12 +70,42 @@ i_time_breakpoint i_return_time_trace i_return_to_trace i_yield -i_global_cons -i_global_tuple -i_global_copy return +# +# To ensure that a "move Src x(0)" instruction can be combined +# with the following call instruction, we need to make sure that +# there is no line/1 instruction between the move and the call. +# + +move S r | line Loc | call_ext Ar Func => \ + line Loc | move S r | call_ext Ar Func +move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \ + line Loc | move S r | call_ext_last Ar Func D +move S r | line Loc | call_ext_only Ar Func=u$is_bif => \ + line Loc | move S r | call_ext_only Ar Func +move S r | line Loc | call Ar Func => \ + line Loc | move S r | call Ar Func + +# +# A tail-recursive call to an external function (non-BIF) will +# never be saved on the stack, so there is no reason to keep +# the line instruction. (The compiler did not remove the line +# instruction because it cannot tell the difference between +# BIFs and ordinary Erlang functions.) +# + +line Loc | call_ext_last Ar Func=u$is_not_bif D => \ + call_ext_last Ar Func D +line Loc | call_ext_only Ar Func=u$is_not_bif => \ + call_ext_only Ar Func + +line Loc | func_info M F A => func_info M F A | line Loc + +line I + + %macro: allocate Allocate -pack %macro: allocate_zero AllocateZero -pack %macro: allocate_heap AllocateHeap -pack @@ -277,8 +289,6 @@ raise s s badarg j system_limit j -move R R => - move C=cxy r | jump Lbl => move_jump Lbl C %macro: move_jump MoveJump -nonext @@ -585,8 +595,6 @@ get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg original_reg Reg Pos => -get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst - original_reg/2 extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ @@ -837,11 +845,11 @@ call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only # thus there is no need to generate any return instruction. # -call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif1 Bif -call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif1 Bif +call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif Bif +call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif Bif -call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif1 Bif -call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif1 Bif +call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif Bif +call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif Bif # # The error/1 and error/2 BIFs never execute the instruction following them; @@ -851,13 +859,13 @@ call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif1 Bif # the continuation pointer on the stack. # -call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif1 Bif -call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif2 Bif +call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif Bif +call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif Bif call_ext_only Ar=u==1 Bif=u$bif:erlang:error/1 => \ - allocate u Ar | call_bif1 Bif + allocate u Ar | call_bif Bif call_ext_only Ar=u==2 Bif=u$bif:erlang:error/2 => \ - allocate u Ar | call_bif2 Bif + allocate u Ar | call_bif Bif # # The yield/0 BIF is an instruction @@ -875,47 +883,18 @@ call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate # -# Hybrid memory architecture need special cons and tuple instructions -# that allocate on the message area. These looks like BIFs in the BEAM code. -# - -call_ext u==2 u$func:hybrid:cons/2 => i_global_cons -call_ext_last u==2 u$func:hybrid:cons/2 D => i_global_cons | deallocate_return D -call_ext_only Ar=u==2 u$func:hybrid:cons/2 => i_global_cons | return - -call_ext u==1 u$func:hybrid:tuple/1 => i_global_tuple -call_ext_last u==1 u$func:hybrid:tuple/1 D => i_global_tuple | deallocate_return D -call_ext_only Ar=u==1 u$func:hybrid:tuple/1 => i_global_tuple | return - -call_ext u==1 u$func:hybrid:copy/1 => i_global_copy -call_ext_last u==1 u$func:hybrid:copy/1 D => i_global_copy | deallocate_return D -call_ext_only u==1 Ar=u$func:hybrid:copy/1 => i_global_copy | return - -# # The general case for BIFs that have no special instructions. # A BIF used in the tail must be followed by a return instruction. # # To make trapping and stack backtraces work correctly, we make sure that # the continuation pointer is always stored on the stack. -call_ext u==0 Bif=u$is_bif => call_bif0 Bif -call_ext u==1 Bif=u$is_bif => call_bif1 Bif -call_ext u==2 Bif=u$is_bif => call_bif2 Bif -call_ext u==3 Bif=$is_bif => call_bif3 Bif +call_ext u Bif=u$is_bif => call_bif Bif -call_ext_last u==0 Bif=u$is_bif D => call_bif0 Bif | deallocate_return D -call_ext_last u==1 Bif=u$is_bif D => call_bif1 Bif | deallocate_return D -call_ext_last u==2 Bif=u$is_bif D => call_bif2 Bif | deallocate_return D -call_ext_last u==3 Bif=u$is_bif D => call_bif3 Bif | deallocate_return D +call_ext_last u Bif=u$is_bif D => call_bif Bif | deallocate_return D -call_ext_only Ar=u==0 Bif=u$is_bif => \ - allocate u Ar | call_bif0 Bif | deallocate_return u -call_ext_only Ar=u==1 Bif=u$is_bif => \ - allocate u Ar | call_bif1 Bif | deallocate_return u -call_ext_only Ar=u==2 Bif=u$is_bif => \ - allocate u Ar | call_bif2 Bif | deallocate_return u -call_ext_only Ar=u==3 Bif=u$is_bif => \ - allocate u Ar | call_bif3 Bif | deallocate_return u +call_ext_only Ar=u Bif=u$is_bif => \ + allocate u Ar | call_bif Bif | deallocate_return u # # Any remaining calls are calls to Erlang functions, not BIFs. @@ -928,9 +907,9 @@ move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r -call_ext Ar=u Func => i_call_ext Func -call_ext_last Ar=u Func D => i_call_ext_last Func D -call_ext_only Ar=u Func => i_call_ext_only Func +call_ext Ar Func => i_call_ext Func +call_ext_last Ar Func D => i_call_ext_last Func D +call_ext_only Ar Func => i_call_ext_only Func i_apply i_apply_last P @@ -942,10 +921,7 @@ i_apply_fun_only i_hibernate -call_bif0 e -call_bif1 e -call_bif2 e -call_bif3 e +call_bif e # # Calls to non-building and guard BIFs. @@ -964,7 +940,7 @@ bif1 p Bif S1 Dst => bif1_body Bif S1 Dst bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst -bif2 Fail=f Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst +bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst i_get s d @@ -1047,8 +1023,8 @@ i_move_call_ext_only e c r # Fun calls. -call_fun Arity=u | deallocate D | return => i_call_fun_last Arity D -call_fun Arity=u => i_call_fun Arity +call_fun Arity | deallocate D | return => i_call_fun_last Arity D +call_fun Arity => i_call_fun Arity i_call_fun I i_call_fun_last I P @@ -1304,13 +1280,13 @@ i_bs_utf16_size s d bs_put_utf8 Fail=j Flags=u Literal=q => \ move Literal x | bs_put_utf8 Fail Flags x -bs_put_utf8 Fail=j u Src=s => i_bs_put_utf8 Fail Src +bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src i_bs_put_utf8 j s bs_put_utf16 Fail=j Flags=u Literal=q => \ move Literal x | bs_put_utf16 Fail Flags x -bs_put_utf16 Fail=j Flags=u Src=s => i_bs_put_utf16 Fail Flags Src +bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src i_bs_put_utf16 j I s @@ -1475,34 +1451,13 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler # # Guard BIFs. # -gc_bif1 Fail I Bif=u$bif:erlang:length/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:size/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:bit_size/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:byte_size/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:abs/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:float/1 Src Dst=d => \ +gc_bif1 Fail I Bif Src Dst => \ gen_guard_bif1(Fail, I, Bif, Src, Dst) -gc_bif1 Fail I Bif=u$bif:erlang:round/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif1 Fail I Bif=u$bif:erlang:trunc/1 Src Dst=d => \ - gen_guard_bif1(Fail, I, Bif, Src, Dst) - -gc_bif2 Fail I Bif=u$bif:erlang:binary_part/2 S1 S2 Dst=d => \ +gc_bif2 Fail I Bif S1 S2 Dst => \ gen_guard_bif2(Fail, I, Bif, S1, S2, Dst) -gc_bif3 Fail I Bif=u$bif:erlang:binary_part/3 S1 S2 S3 Dst=d => \ +gc_bif3 Fail I Bif S1 S2 S3 Dst => \ gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D @@ -1520,6 +1475,15 @@ ii_gc_bif3/7 ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D i_gc_bif3 j I s I d + +# +# The following instruction is specially handled in beam_load.c +# to produce a user-friendly message if an unsupported guard BIF is +# encountered. +# +unsupported_guard_bif/3 +unsupported_guard_bif A B C | never() => + # # R13B03 # diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index a66d60aa22..4d4b6ea196 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -301,7 +301,11 @@ int packet_get_length(enum PacketParseType htype, /* TCP_PB_LINE_LF: [Data ... \n] */ const char* ptr2; if ((ptr2 = memchr(ptr, '\n', n)) == NULL) { - if (n >= trunc_len && trunc_len!=0) { /* buffer full */ + if (n > max_plen && max_plen != 0) { /* packet full */ + DEBUGF((" => packet full (no NL)=%d\r\n", n)); + goto error; + } + else if (n >= trunc_len && trunc_len!=0) { /* buffer full */ DEBUGF((" => line buffer full (no NL)=%d\r\n", n)); return trunc_len; } @@ -309,6 +313,10 @@ int packet_get_length(enum PacketParseType htype, } else { int len = (ptr2 - ptr) + 1; /* including newline */ + if (len > max_plen && max_plen!=0) { + DEBUGF((" => packet_size %d exceeded\r\n", max_plen)); + goto error; + } if (len > trunc_len && trunc_len!=0) { DEBUGF((" => truncated line=%d\r\n", trunc_len)); return trunc_len; @@ -397,33 +405,50 @@ int packet_get_length(enum PacketParseType htype, const char* ptr1 = ptr; int len = plen; + if (!max_plen) { + /* This is for backward compatibility with old user of decode_packet + * that might use option 'line_length' to limit accepted length of + * http lines. + */ + max_plen = trunc_len; + } + while (1) { const char* ptr2 = memchr(ptr1, '\n', len); if (ptr2 == NULL) { - if (n >= trunc_len && trunc_len!=0) { /* buffer full */ - plen = trunc_len; - goto done; + if (max_plen != 0) { + if (n >= max_plen) /* packet full */ + goto error; } goto more; } else { plen = (ptr2 - ptr) + 1; - - if (*statep == 0) + + if (*statep == 0) { + if (max_plen != 0 && plen > max_plen) + goto error; goto done; - + } + if (plen < n) { if (SP(ptr2+1) && plen>2) { /* header field value continue on next line */ ptr1 = ptr2+1; len = n - plen; } - else + else { + if (max_plen != 0 && plen > max_plen) + goto error; goto done; + } } - else + else { + if (max_plen != 0 && plen > max_plen) + goto error; goto more; + } } } } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 9f5747205d..6b4b382caa 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -103,6 +103,15 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_LIKELY(BOOL) (BOOL) # define ERTS_UNLIKELY(BOOL) (BOOL) #endif +#ifdef __GNUC__ +# if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5) +# define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused)) +# else +# define ERTS_DECLARE_DUMMY(X) X +# endif +#else +# define ERTS_DECLARE_DUMMY(X) X +#endif #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) # undef ERTS_CAN_INLINE @@ -212,7 +221,8 @@ int real_printf(const char *fmt, ...); */ #if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \ - && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG))) + && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \ + (SIZEOF_VOID_P == SIZEOF_LONG_LONG))) #error Cannot handle this combination of int/long/void*/size_t sizes #endif @@ -253,9 +263,18 @@ typedef int Sint; #if SIZEOF_VOID_P == SIZEOF_LONG typedef unsigned long UWord; typedef long SWord; +#define SWORD_CONSTANT(Const) Const##L +#define UWORD_CONSTANT(Const) Const##UL #elif SIZEOF_VOID_P == SIZEOF_INT typedef unsigned int UWord; typedef int SWord; +#define SWORD_CONSTANT(Const) Const +#define UWORD_CONSTANT(Const) Const##U +#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG +typedef unsigned long long UWord; +typedef long long SWord; +#define SWORD_CONSTANT(Const) Const##LL +#define UWORD_CONSTANT(Const) Const##ULL #else #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' #endif @@ -266,12 +285,23 @@ typedef int SWord; typedef unsigned long Eterm; typedef unsigned long Uint; typedef long Sint; +#define SWORD_CONSTANT(Const) Const##L +#define UWORD_CONSTANT(Const) Const##UL #define ERTS_SIZEOF_ETERM SIZEOF_LONG #elif SIZEOF_VOID_P == SIZEOF_INT typedef unsigned int Eterm; typedef unsigned int Uint; typedef int Sint; +#define SWORD_CONSTANT(Const) Const +#define UWORD_CONSTANT(Const) Const##U #define ERTS_SIZEOF_ETERM SIZEOF_INT +#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG +typedef unsigned long long Eterm; +typedef unsigned long long Uint; +typedef long long Sint; +#define SWORD_CONSTANT(Const) Const##LL +#define UWORD_CONSTANT(Const) Const##ULL +#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG #else #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' #endif @@ -475,15 +505,6 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); #define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ #define ERTS_DUMP_EXIT (127) /* crash dump; then exit() */ - -#ifndef ERTS_SMP -int check_async_ready(void); -#ifdef USE_THREADS -void sys_async_ready(int hndl); -int erts_register_async_ready_callback(void (*funcp)(void)); -#endif -#endif - Eterm erts_check_io_info(void *p); /* Size of misc memory allocated from system dependent code */ @@ -615,10 +636,11 @@ extern char *erts_sys_ddll_error(int code); /* * System interfaces for startup. */ +#include "erl_time.h" void erts_sys_schedule_interrupt(int set); #ifdef ERTS_SMP -void erts_sys_schedule_interrupt_timed(int set, long msec); +void erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec); void erts_sys_main_thread(void); #endif @@ -635,17 +657,24 @@ Preload* sys_preloaded(void); unsigned char* sys_preload_begin(Preload*); void sys_preload_end(Preload*); int sys_get_key(int); -void elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, - unsigned long *ms_user_diff, unsigned long *ms_sys_diff); -void wall_clock_elapsed_time_both(unsigned long *ms_total, - unsigned long *ms_diff); +void elapsed_time_both(UWord *ms_user, UWord *ms_sys, + UWord *ms_user_diff, UWord *ms_sys_diff); +void wall_clock_elapsed_time_both(UWord *ms_total, + UWord *ms_diff); void get_time(int *hour, int *minute, int *second); void get_date(int *year, int *month, int *day); void get_localtime(int *year, int *month, int *day, int *hour, int *minute, int *second); void get_universaltime(int *year, int *month, int *day, int *hour, int *minute, int *second); -int univ_to_local(Sint *year, Sint *month, Sint *day, +int seconds_to_univ(Sint64 seconds, + Sint *year, Sint *month, Sint *day, + Sint *hour, Sint *minute, Sint *second); +int univ_to_seconds(Sint year, Sint month, Sint day, + Sint hour, Sint minute, Sint second, + Sint64* seconds); +int univ_to_local( + Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second); int local_to_univ(Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second, int isdst); @@ -671,6 +700,8 @@ int erts_sys_putenv(char *key_value, int sep_ix); *size), a value > 0 if value buffer is too small (*size is set to needed size), and a value < 0 on failure. */ int erts_sys_getenv(char *key, char *value, size_t *size); +/* erts_sys_getenv__() is only allowed to be used in early init phase */ +int erts_sys_getenv__(char *key, char *value, size_t *size); /* Easier to use, but not as efficient, environment functions */ char *erts_read_env(char *key); @@ -702,283 +733,6 @@ typedef struct { void sys_alloc_stat(SysAllocStat *); -/* Block the whole system... */ - -#define ERTS_BS_FLG_ALLOW_GC (((Uint32) 1) << 0) -#define ERTS_BS_FLG_ALLOW_IO (((Uint32) 1) << 1) - -/* Activities... */ -typedef enum { - ERTS_ACTIVITY_UNDEFINED, /* Undefined activity */ - ERTS_ACTIVITY_WAIT, /* Waiting */ - ERTS_ACTIVITY_GC, /* Garbage collecting */ - ERTS_ACTIVITY_IO /* I/O including message passing to erl procs */ -} erts_activity_t; - -#ifdef ERTS_SMP - -typedef enum { - ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED, - ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, - ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY -} erts_activity_error_t; - -typedef struct { - erts_smp_atomic32_t do_block; - struct { - erts_smp_atomic32_t wait; - erts_smp_atomic32_t gc; - erts_smp_atomic32_t io; - } in_activity; -} erts_system_block_state_t; - -extern erts_system_block_state_t erts_system_block_state; - -int erts_is_system_blocked(erts_activity_t allowed_activities); -void erts_block_me(void (*prepare)(void *), void (*resume)(void *), void *arg); -void erts_register_blockable_thread(void); -void erts_unregister_blockable_thread(void); -void erts_note_activity_begin(erts_activity_t activity); -void -erts_check_block(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg); -void erts_block_system(Uint32 allowed_activities); -int erts_emergency_block_system(long timeout, Uint32 allowed_activities); -void erts_release_system(void); -void erts_system_block_init(void); -void erts_set_activity_error(erts_activity_error_t, char *, int); -#ifdef ERTS_ENABLE_LOCK_CHECK -void erts_lc_activity_change_begin(void); -void erts_lc_activity_change_end(void); -int erts_lc_is_blocking(void); -#define ERTS_LC_IS_BLOCKING \ - (erts_smp_pending_system_block() && erts_lc_is_blocking()) -#endif -#endif - -#define erts_smp_activity_begin(NACT, PRP, RSM, ARG) \ - erts_smp_set_activity(ERTS_ACTIVITY_UNDEFINED, \ - (NACT), \ - 0, \ - (PRP), \ - (RSM), \ - (ARG), \ - __FILE__, \ - __LINE__) -#define erts_smp_activity_change(OACT, NACT, PRP, RSM, ARG) \ - erts_smp_set_activity((OACT), \ - (NACT), \ - 0, \ - (PRP), \ - (RSM), \ - (ARG), \ - __FILE__, \ - __LINE__) -#define erts_smp_activity_end(OACT, PRP, RSM, ARG) \ - erts_smp_set_activity((OACT), \ - ERTS_ACTIVITY_UNDEFINED, \ - 0, \ - (PRP), \ - (RSM), \ - (ARG), \ - __FILE__, \ - __LINE__) - -#define erts_smp_locked_activity_begin(NACT) \ - erts_smp_set_activity(ERTS_ACTIVITY_UNDEFINED, \ - (NACT), \ - 1, \ - NULL, \ - NULL, \ - NULL, \ - __FILE__, \ - __LINE__) -#define erts_smp_locked_activity_change(OACT, NACT) \ - erts_smp_set_activity((OACT), \ - (NACT), \ - 1, \ - NULL, \ - NULL, \ - NULL, \ - __FILE__, \ - __LINE__) -#define erts_smp_locked_activity_end(OACT) \ - erts_smp_set_activity((OACT), \ - ERTS_ACTIVITY_UNDEFINED, \ - 1, \ - NULL, \ - NULL, \ - NULL, \ - __FILE__, \ - __LINE__) - - -ERTS_GLB_INLINE int erts_smp_is_system_blocked(erts_activity_t allowed_activities); -ERTS_GLB_INLINE void erts_smp_block_system(Uint32 allowed_activities); -ERTS_GLB_INLINE int erts_smp_emergency_block_system(long timeout, - Uint32 allowed_activities); -ERTS_GLB_INLINE void erts_smp_release_system(void); -ERTS_GLB_INLINE int erts_smp_pending_system_block(void); -ERTS_GLB_INLINE void erts_smp_chk_system_block(void (*prepare)(void *), - void (*resume)(void *), - void *arg); -ERTS_GLB_INLINE void -erts_smp_set_activity(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg, - char *file, - int line); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - - -ERTS_GLB_INLINE int -erts_smp_is_system_blocked(erts_activity_t allowed_activities) -{ -#ifdef ERTS_SMP - return erts_is_system_blocked(allowed_activities); -#else - return 1; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_block_system(Uint32 allowed_activities) -{ -#ifdef ERTS_SMP - erts_block_system(allowed_activities); -#endif -} - -ERTS_GLB_INLINE int -erts_smp_emergency_block_system(long timeout, Uint32 allowed_activities) -{ -#ifdef ERTS_SMP - return erts_emergency_block_system(timeout, allowed_activities); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_release_system(void) -{ -#ifdef ERTS_SMP - erts_release_system(); -#endif -} - -ERTS_GLB_INLINE int -erts_smp_pending_system_block(void) -{ -#ifdef ERTS_SMP - return (int) erts_smp_atomic32_read_nob(&erts_system_block_state.do_block); -#else - return 0; -#endif -} - - -ERTS_GLB_INLINE void -erts_smp_chk_system_block(void (*prepare)(void *), - void (*resume)(void *), - void *arg) -{ -#ifdef ERTS_SMP - if (erts_smp_pending_system_block()) - erts_block_me(prepare, resume, arg); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_set_activity(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg, - char *file, - int line) -{ -#ifdef ERTS_SMP -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_activity_change_begin(); -#endif - switch (old_activity) { - case ERTS_ACTIVITY_UNDEFINED: - break; - case ERTS_ACTIVITY_WAIT: - erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.wait); - if (locked) { - /* You are not allowed to leave activity waiting - * without supplying the possibility to block - * unlocked. - */ - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED, - file, line); - } - break; - case ERTS_ACTIVITY_GC: - erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.gc); - break; - case ERTS_ACTIVITY_IO: - erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.io); - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, - file, line); - break; - } - - /* We are not allowed to block when going to activity waiting... */ - if (new_activity != ERTS_ACTIVITY_WAIT && erts_smp_pending_system_block()) - erts_check_block(old_activity,new_activity,locked,prepare,resume,arg); - - switch (new_activity) { - case ERTS_ACTIVITY_UNDEFINED: - break; - case ERTS_ACTIVITY_WAIT: - erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.wait); - break; - case ERTS_ACTIVITY_GC: - erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.gc); - break; - case ERTS_ACTIVITY_IO: - erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.io); - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, - file, line); - break; - } - - switch (new_activity) { - case ERTS_ACTIVITY_WAIT: - case ERTS_ACTIVITY_GC: - case ERTS_ACTIVITY_IO: - if (erts_smp_pending_system_block()) - erts_note_activity_begin(new_activity); - break; - default: - break; - } - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_activity_change_end(); -#endif - -#endif -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) #undef ERTS_REFC_DEBUG #define ERTS_REFC_DEBUG @@ -1243,6 +997,19 @@ void erl_bin_write(unsigned char *, int, int); #endif +#ifdef __WIN32__ +#ifdef ARCH_64 +#define ERTS_ALLOC_ALIGN_BYTES 16 +#define ERTS_SMALL_ABS(Small) _abs64(Small) +#else +#define ERTS_ALLOC_ALIGN_BYTES 8 +#define ERTS_SMALL_ABS(Small) labs(Small) +#endif +#else +#define ERTS_ALLOC_ALIGN_BYTES 8 +#define ERTS_SMALL_ABS(Small) labs(Small) +#endif + #ifdef __WIN32__ diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index db9a24e0a3..932d157cd8 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -107,20 +107,31 @@ static ErlTimer *tiw_min_ptr; /* Actual interval time chosen by sys_init_time() */ static int itime; /* Constant after init */ -erts_smp_atomic_t do_time; /* set at clock interrupt */ -static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read_acqb(&do_time); } -static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); } -static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init_nob(&do_time, 0L); } +erts_smp_atomic32_t do_time; /* set at clock interrupt */ +static ERTS_INLINE erts_short_time_t do_time_read(void) +{ + return erts_smp_atomic32_read_acqb(&do_time); +} + +static ERTS_INLINE erts_short_time_t do_time_update(void) +{ + return do_time_read(); +} + +static ERTS_INLINE void do_time_init(void) +{ + erts_smp_atomic32_init_nob(&do_time, 0); +} /* get the time (in units of itime) to the next timeout, or -1 if there are no timeouts */ -static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ +static erts_short_time_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ { int i, tm, nto; - unsigned int min; + Uint32 min; ErlTimer* p; - erts_aint_t dt; + erts_short_time_t dt; if (tiw_nto == 0) return -1; /* no timeouts in wheel */ @@ -133,7 +144,7 @@ static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ /* start going through wheel to find next timeout */ tm = nto = 0; - min = (unsigned int) -1; /* max unsigned int */ + min = (Uint32) -1; /* max Uint32 */ i = tiw_pos; do { p = tiw[i]; @@ -162,7 +173,11 @@ static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ i = (i + 1) % TIW_SIZE; } while (i != tiw_pos); dt = do_time_read(); - return ((min >= dt) ? (min - dt) : 0); + if (min <= (Uint32) dt) + return 0; + if ((min - (Uint32) dt) > (Uint32) ERTS_SHORT_TIME_T_MAX) + return ERTS_SHORT_TIME_T_MAX; + return (erts_short_time_t) (min - (Uint32) dt); } static void remove_timer(ErlTimer *p) { @@ -191,9 +206,9 @@ static void remove_timer(ErlTimer *p) { } /* Private export to erl_time_sup.c */ -erts_aint_t erts_next_time(void) +erts_short_time_t erts_next_time(void) { - erts_aint_t ret; + erts_short_time_t ret; erts_smp_mtx_lock(&tiw_lock); (void)do_time_update(); @@ -202,7 +217,7 @@ erts_aint_t erts_next_time(void) return ret; } -static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is write-locked */ +static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lock is write-locked */ { Uint keep_pos; Uint count; @@ -273,7 +288,7 @@ static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is } } -void erts_bump_timer(erts_aint_t dt) /* dt is value from do_time */ +void erts_bump_timer(erts_short_time_t dt) /* dt is value from do_time */ { erts_smp_mtx_lock(&tiw_lock); bump_timer_internal(dt); @@ -378,8 +393,8 @@ erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, insert_timer(p, t); erts_smp_mtx_unlock(&tiw_lock); #if defined(ERTS_SMP) - if (t <= (Uint) LONG_MAX) - erts_sys_schedule_interrupt_timed(1, (long) t); + if (t <= (Uint) ERTS_SHORT_TIME_T_MAX) + erts_sys_schedule_interrupt_timed(1, (erts_short_time_t) t); #endif } @@ -419,7 +434,7 @@ Uint erts_time_left(ErlTimer *p) { Uint left; - erts_aint_t dt; + erts_short_time_t dt; erts_smp_mtx_lock(&tiw_lock); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 3fa84bd13c..4105f194a9 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -43,7 +43,9 @@ #include "erl_smp.h" #include "erl_time.h" #include "erl_thr_progress.h" +#include "erl_thr_queue.h" #include "erl_sched_spec_pre_alloc.h" +#include "beam_bp.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD @@ -77,6 +79,7 @@ typedef struct { #ifdef ERTS_SMP +#if 0 /* Unused */ static void dispatch_profile_msg_q(profile_sched_msg_q *psmq) { @@ -88,6 +91,7 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq) profile_scheduler_q(make_small(msg->scheduler_id), msg->state, am_undefined, msg->Ms, msg->s, msg->us); } } +#endif #endif @@ -2644,7 +2648,7 @@ tailrecur_ne: FloatDef f1, f2; Eterm big; #if HEAP_ON_C_STACK - Eterm big_buf[2]; /* If HEAP_ON_C_STACK */ + Eterm big_buf[CMP_TMP_HEAP_SIZE]; /* If HEAP_ON_C_STACK */ #else Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap; #endif @@ -2655,42 +2659,98 @@ tailrecur_ne: Eterm aw = a; Eterm bw = b; #endif +#define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2)) +#define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1)) +#define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */ b_tag = tag_val_def(bw); switch(_NUMBER_CODE(a_tag, b_tag)) { case SMALL_BIG: - big = small_to_big(signed_val(a), big_buf); - j = big_comp(big, bw); + j = big_sign(bw) ? 1 : -1; + break; + case BIG_SMALL: + j = big_sign(aw) ? -1 : 1; break; case SMALL_FLOAT: - f1.fd = signed_val(a); GET_DOUBLE(bw, f2); - j = float_comp(f1.fd, f2.fd); - break; - case BIG_SMALL: - big = small_to_big(signed_val(b), big_buf); - j = big_comp(aw, big); + if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { + // Float is within the no loss limit + f1.fd = signed_val(aw); + j = float_comp(f1.fd, f2.fd); +#if ERTS_SIZEOF_ETERM == 8 + } else if (f2.fd > (double) (MAX_SMALL + 1)) { + // Float is a positive bignum, i.e. bigger + j = -1; + } else if (f2.fd < (double) (MIN_SMALL - 1)) { + // Float is a negative bignum, i.e. smaller + j = 1; + } else { // Float is a Sint but less precise + j = signed_val(aw) - (Sint) f2.fd; + } +#else + } else { + // If float is positive it is bigger than small + j = (f2.fd > 0.0) ? -1 : 1; + } +#endif // ERTS_SIZEOF_ETERM == 8 break; + case FLOAT_BIG: + { + Wterm tmp = aw; + aw = bw; + bw = tmp; + }/* fall through */ case BIG_FLOAT: - if (big_to_double(aw, &f1.fd) < 0) { - j = big_sign(a) ? -1 : 1; + GET_DOUBLE(bw, f2); + if ((f2.fd < (double) (MAX_SMALL + 1)) + && (f2.fd > (double) (MIN_SMALL - 1))) { + // Float is a Sint + j = big_sign(aw) ? -1 : 1; + } else if (big_arity(aw) > BIG_ARITY_FLOAT_MAX + || pow(2.0,(big_arity(aw)-1)*D_EXP) > fabs(f2.fd)) { + // If bignum size shows that it is bigger than the abs float + j = big_sign(aw) ? -1 : 1; + } else if (big_arity(aw) < BIG_ARITY_FLOAT_MAX + && (pow(2.0,(big_arity(aw))*D_EXP)-1.0) < fabs(f2.fd)) { + // If bignum size shows that it is smaller than the abs float + j = f2.fd < 0 ? 1 : -1; + } else if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { + // Float is within the no loss limit + if (big_to_double(aw, &f1.fd) < 0) { + j = big_sign(aw) ? -1 : 1; + } else { + j = float_comp(f1.fd, f2.fd); + } } else { - GET_DOUBLE(bw, f2); - j = float_comp(f1.fd, f2.fd); + big = double_to_big(f2.fd, big_buf); + j = big_comp(aw, big); + } + if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) { + j = -j; } break; case FLOAT_SMALL: GET_DOUBLE(aw, f1); - f2.fd = signed_val(b); - j = float_comp(f1.fd, f2.fd); - break; - case FLOAT_BIG: - if (big_to_double(bw, &f2.fd) < 0) { - j = big_sign(b) ? 1 : -1; - } else { - GET_DOUBLE(aw, f1); + if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) { + // Float is within the no loss limit + f2.fd = signed_val(bw); j = float_comp(f1.fd, f2.fd); +#if ERTS_SIZEOF_ETERM == 8 + } else if (f1.fd > (double) (MAX_SMALL + 1)) { + // Float is a positive bignum, i.e. bigger + j = 1; + } else if (f1.fd < (double) (MIN_SMALL - 1)) { + // Float is a negative bignum, i.e. smaller + j = -1; + } else { // Float is a Sint but less precise it + j = (Sint) f1.fd - signed_val(bw); + } +#else + } else { + // If float is positive it is bigger than small + j = (f1.fd > 0.0) ? 1 : -1; } +#endif // ERTS_SIZEOF_ETERM == 8 break; default: j = b_tag - a_tag; @@ -2885,14 +2945,14 @@ Eterm buf_to_intlist(Eterm** hpp, char *buf, int len, Eterm tail) { Eterm* hp = *hpp; + int i = len - 1; - buf += (len-1); - while(len > 0) { - tail = CONS(hp, make_small((byte)*buf), tail); + while(i >= 0) { + tail = CONS(hp, make_small((Uint)(byte)buf[i]), tail); hp += 2; - buf--; - len--; + --i; } + *hpp = hp; return tail; } @@ -3338,688 +3398,6 @@ sys_alloc_stat(SysAllocStat *sasp) } -#ifdef ERTS_SMP - -/* Local system block state */ - -struct { - int emergency; - long emergency_timeout; - erts_smp_cnd_t watchdog_cnd; - erts_smp_tid_t watchdog_tid; - int threads_to_block; - int have_blocker; - erts_smp_tid_t blocker_tid; - int recursive_block; - Uint32 allowed_activities; - erts_smp_tsd_key_t blockable_key; - erts_smp_mtx_t mtx; - erts_smp_cnd_t cnd; -#ifdef ERTS_ENABLE_LOCK_CHECK - int activity_changing; - int checking; -#endif -} system_block_state; - -/* Global system block state */ -erts_system_block_state_t erts_system_block_state; - - -static ERTS_INLINE int -is_blockable_thread(void) -{ - return erts_smp_tsd_get(system_block_state.blockable_key) != NULL; -} - -static ERTS_INLINE int -is_blocker(void) -{ - return (system_block_state.have_blocker - && erts_smp_equal_tids(system_block_state.blocker_tid, - erts_smp_thr_self())); -} - -#ifdef ERTS_ENABLE_LOCK_CHECK -int -erts_lc_is_blocking(void) -{ - int res; - erts_smp_mtx_lock(&system_block_state.mtx); - res = erts_smp_pending_system_block() && is_blocker(); - erts_smp_mtx_unlock(&system_block_state.mtx); - return res; -} -#endif - -static ERTS_INLINE void -block_me(void (*prepare)(void *), - void (*resume)(void *), - void *arg, - int mtx_locked, - int want_to_block, - int update_act_changing, - profile_sched_msg_q *psmq) -{ - if (prepare) - (*prepare)(arg); - - /* Locks might be held... */ - - if (!mtx_locked) - erts_smp_mtx_lock(&system_block_state.mtx); - - if (erts_smp_pending_system_block() && !is_blocker()) { - int is_blockable = is_blockable_thread(); - ASSERT(is_blockable); - - if (is_blockable) - system_block_state.threads_to_block--; - - if (erts_system_profile_flags.scheduler && psmq) { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - if (esdp) { - profile_sched_msg *msg = NULL; - - ASSERT(psmq->n < 2); - msg = &((psmq->msg)[psmq->n]); - msg->scheduler_id = esdp->no; - get_now(&(msg->Ms), &(msg->s), &(msg->us)); - msg->no_schedulers = 0; - msg->state = am_inactive; - psmq->n++; - } - } - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (update_act_changing) - system_block_state.activity_changing--; -#endif - - erts_smp_cnd_broadcast(&system_block_state.cnd); - - do { - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - } while (erts_smp_pending_system_block() - && !(want_to_block && !system_block_state.have_blocker)); - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (update_act_changing) - system_block_state.activity_changing++; -#endif - if (erts_system_profile_flags.scheduler && psmq) { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - if (esdp) { - profile_sched_msg *msg = NULL; - - ASSERT(psmq->n < 2); - msg = &((psmq->msg)[psmq->n]); - msg->scheduler_id = esdp->no; - get_now(&(msg->Ms), &(msg->s), &(msg->us)); - msg->no_schedulers = 0; - msg->state = am_active; - psmq->n++; - } - } - - if (is_blockable) - system_block_state.threads_to_block++; - } - - if (!mtx_locked) - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (resume) - (*resume)(arg); -} - -void -erts_block_me(void (*prepare)(void *), - void (*resume)(void *), - void *arg) -{ - profile_sched_msg_q psmq; - psmq.n = 0; - if (prepare) - (*prepare)(arg); - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_check_exact(NULL, 0); /* No locks should be locked */ -#endif - - block_me(NULL, NULL, NULL, 0, 0, 0, &psmq); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); - - if (resume) - (*resume)(arg); -} - -void -erts_register_blockable_thread(void) -{ - profile_sched_msg_q psmq; - psmq.n = 0; - if (!is_blockable_thread()) { - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.threads_to_block++; - erts_smp_tsd_set(system_block_state.blockable_key, - (void *) &erts_system_block_state); - - /* Someone might be waiting for us to block... */ - if (erts_smp_pending_system_block()) - block_me(NULL, NULL, NULL, 1, 0, 0, &psmq); - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); - } -} - -void -erts_unregister_blockable_thread(void) -{ - if (is_blockable_thread()) { - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.threads_to_block--; - ASSERT(system_block_state.threads_to_block >= 0); - erts_smp_tsd_set(system_block_state.blockable_key, NULL); - - /* Someone might be waiting for us to block... */ - if (erts_smp_pending_system_block()) - erts_smp_cnd_broadcast(&system_block_state.cnd); - erts_smp_mtx_unlock(&system_block_state.mtx); - } -} - -void -erts_note_activity_begin(erts_activity_t activity) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - if (erts_smp_pending_system_block()) { - Uint32 broadcast = 0; - switch (activity) { - case ERTS_ACTIVITY_GC: - broadcast = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_GC); - break; - case ERTS_ACTIVITY_IO: - broadcast = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_IO); - break; - case ERTS_ACTIVITY_WAIT: - broadcast = 1; - break; - default: - abort(); - break; - } - if (broadcast) - erts_smp_cnd_broadcast(&system_block_state.cnd); - } - erts_smp_mtx_unlock(&system_block_state.mtx); -} - -void -erts_check_block(erts_activity_t old_activity, - erts_activity_t new_activity, - int locked, - void (*prepare)(void *), - void (*resume)(void *), - void *arg) -{ - int do_block; - profile_sched_msg_q psmq; - - psmq.n = 0; - if (!locked && prepare) - (*prepare)(arg); - - erts_smp_mtx_lock(&system_block_state.mtx); - - /* First check if it is ok to block... */ - if (!locked) - do_block = 1; - else { - switch (old_activity) { - case ERTS_ACTIVITY_UNDEFINED: - do_block = 0; - break; - case ERTS_ACTIVITY_GC: - do_block = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_GC); - break; - case ERTS_ACTIVITY_IO: - do_block = (system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_IO); - break; - case ERTS_ACTIVITY_WAIT: - /* You are not allowed to leave activity waiting - * without supplying the possibility to block - * unlocked. - */ - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED, - __FILE__, __LINE__); - do_block = 0; - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, - __FILE__, __LINE__); - do_block = 0; - break; - } - } - - if (do_block) { - /* ... then check if it is necessary to block... */ - - switch (new_activity) { - case ERTS_ACTIVITY_UNDEFINED: - do_block = 1; - break; - case ERTS_ACTIVITY_GC: - do_block = !(system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_GC); - break; - case ERTS_ACTIVITY_IO: - do_block = !(system_block_state.allowed_activities - & ERTS_BS_FLG_ALLOW_IO); - break; - case ERTS_ACTIVITY_WAIT: - /* No need to block if we are going to wait */ - do_block = 0; - break; - default: - erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, - __FILE__, __LINE__); - break; - } - } - - if (do_block) { - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (!locked) { - /* Only system_block_state.mtx should be held */ - erts_lc_check_exact(&system_block_state.mtx.lc, 1); - } -#endif - - block_me(NULL, NULL, NULL, 1, 0, 1, &psmq); - - } - - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); - - if (!locked && resume) - (*resume)(arg); -} - - - -void -erts_set_activity_error(erts_activity_error_t error, char *file, int line) -{ - switch (error) { - case ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED: - erl_exit(1, "%s:%d: Fatal error: Leaving activity waiting without " - "supplying the possibility to block unlocked.", - file, line); - break; - case ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY: - erl_exit(1, "%s:%d: Fatal error: Leaving unknown activity.", - file, line); - break; - case ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY: - erl_exit(1, "%s:%d: Fatal error: Leaving unknown activity.", - file, line); - break; - default: - erl_exit(1, "%s:%d: Internal error in erts_smp_set_activity()", - file, line); - break; - } - -} - - -static ERTS_INLINE erts_aint32_t -threads_not_under_control(void) -{ - erts_aint32_t res = system_block_state.threads_to_block; - - ERTS_THR_MEMORY_BARRIER; - - /* Waiting is always an allowed activity... */ - res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.wait); - - if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC) - res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.gc); - - if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO) - res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.io); - - if (res < 0) { - ASSERT(0); - return 0; - } - return res; -} - -/* - * erts_block_system() blocks all threads registered as blockable. - * It doesn't return until either all threads have blocked (0 is returned) - * or it has timed out (ETIMEDOUT) is returned. - * - * If allowed activities == 0, blocked threads will release all locks - * before blocking. - * - * If allowed_activities is != 0, erts_block_system() will allow blockable - * threads to continue executing as long as they are doing an allowed - * activity. When they are done with the allowed activity they will block, - * *but* they will block holding locks. Therefore, the thread calling - * erts_block_system() must *not* try to aquire any locks that might be - * held by blocked threads holding locks from allowed activities. - * - * Currently allowed_activities are: - * * ERTS_BS_FLG_ALLOW_GC Thread continues with garbage - * collection and blocks with - * main process lock on current - * process locked. - * * ERTS_BS_FLG_ALLOW_IO Thread continues with I/O - */ - -void -erts_block_system(Uint32 allowed_activities) -{ - int do_block; - profile_sched_msg_q psmq; - - psmq.n = 0; -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_check_exact(NULL, 0); /* No locks should be locked */ -#endif - - erts_smp_mtx_lock(&system_block_state.mtx); - - do_block = erts_smp_pending_system_block(); - if (do_block - && system_block_state.have_blocker - && erts_smp_equal_tids(system_block_state.blocker_tid, - erts_smp_thr_self())) { - ASSERT(system_block_state.recursive_block >= 0); - system_block_state.recursive_block++; - - /* You are not allowed to restrict allowed activites - in a recursive block! */ - ERTS_SMP_LC_ASSERT((system_block_state.allowed_activities - & ~allowed_activities) == 0); - } - else { - - erts_smp_atomic32_inc_nob(&erts_system_block_state.do_block); - - /* Someone else might be waiting for us to block... */ - if (do_block) { - do_block_me: - block_me(NULL, NULL, NULL, 1, 1, 0, &psmq); - } - - ASSERT(!system_block_state.have_blocker); - system_block_state.have_blocker = 1; - system_block_state.blocker_tid = erts_smp_thr_self(); - system_block_state.allowed_activities = allowed_activities; - - if (is_blockable_thread()) - system_block_state.threads_to_block--; - - while (threads_not_under_control() && !system_block_state.emergency) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - - if (system_block_state.emergency) { - system_block_state.have_blocker = 0; - goto do_block_me; - } - } - - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0 ) - dispatch_profile_msg_q(&psmq); -} - -/* - * erts_emergency_block_system() should only be called when we are - * about to write a crash dump... - */ - -int -erts_emergency_block_system(long timeout, Uint32 allowed_activities) -{ - int res = 0; - long another_blocker; - - erts_smp_mtx_lock(&system_block_state.mtx); - - if (system_block_state.emergency) { - /* Argh... */ - res = EINVAL; - goto done; - } - - another_blocker = erts_smp_pending_system_block(); - system_block_state.emergency = 1; - erts_smp_atomic32_inc_nob(&erts_system_block_state.do_block); - - if (another_blocker) { - if (is_blocker()) { - erts_smp_atomic32_dec_nob(&erts_system_block_state.do_block); - res = 0; - goto done; - } - /* kick the other blocker */ - erts_smp_cnd_broadcast(&system_block_state.cnd); - while (system_block_state.have_blocker) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - } - - ASSERT(!system_block_state.have_blocker); - system_block_state.have_blocker = 1; - system_block_state.blocker_tid = erts_smp_thr_self(); - system_block_state.allowed_activities = allowed_activities; - - if (is_blockable_thread()) - system_block_state.threads_to_block--; - - if (timeout < 0) { - while (threads_not_under_control()) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - } - else { - system_block_state.emergency_timeout = timeout; - erts_smp_cnd_signal(&system_block_state.watchdog_cnd); - - while (system_block_state.emergency_timeout >= 0 - && threads_not_under_control()) { - erts_smp_cnd_wait(&system_block_state.cnd, - &system_block_state.mtx); - } - } - done: - erts_smp_mtx_unlock(&system_block_state.mtx); - return res; -} - -void -erts_release_system(void) -{ - long do_block; - profile_sched_msg_q psmq; - - psmq.n = 0; - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_check_exact(NULL, 0); /* No locks should be locked */ -#endif - - erts_smp_mtx_lock(&system_block_state.mtx); - ASSERT(is_blocker()); - - ASSERT(system_block_state.recursive_block >= 0); - - if (system_block_state.recursive_block) - system_block_state.recursive_block--; - else { - do_block = erts_smp_atomic32_dec_read_nob(&erts_system_block_state.do_block); - system_block_state.have_blocker = 0; - if (is_blockable_thread()) - system_block_state.threads_to_block++; - else - do_block = 0; - - /* Someone else might be waiting for us to block... */ - if (do_block) - block_me(NULL, NULL, NULL, 1, 0, 0, &psmq); - else - erts_smp_cnd_broadcast(&system_block_state.cnd); - } - - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_system_profile_flags.scheduler && psmq.n > 0) - dispatch_profile_msg_q(&psmq); -} - -#ifdef ERTS_ENABLE_LOCK_CHECK - -void -erts_lc_activity_change_begin(void) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.activity_changing++; - erts_smp_mtx_unlock(&system_block_state.mtx); -} - -void -erts_lc_activity_change_end(void) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.activity_changing--; - if (system_block_state.checking && !system_block_state.activity_changing) - erts_smp_cnd_broadcast(&system_block_state.cnd); - erts_smp_mtx_unlock(&system_block_state.mtx); -} - -#endif - -int -erts_is_system_blocked(erts_activity_t allowed_activities) -{ - int blkd; - - erts_smp_mtx_lock(&system_block_state.mtx); - blkd = (erts_smp_pending_system_block() - && system_block_state.have_blocker - && erts_smp_equal_tids(system_block_state.blocker_tid, - erts_smp_thr_self()) - && !(system_block_state.allowed_activities & ~allowed_activities)); -#ifdef ERTS_ENABLE_LOCK_CHECK - if (blkd) { - system_block_state.checking = 1; - while (system_block_state.activity_changing) - erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx); - system_block_state.checking = 0; - blkd = !threads_not_under_control(); - } -#endif - erts_smp_mtx_unlock(&system_block_state.mtx); - return blkd; -} - -static void * -emergency_watchdog(void *unused) -{ - erts_smp_mtx_lock(&system_block_state.mtx); - while (1) { - long timeout; - while (system_block_state.emergency_timeout < 0) - erts_smp_cnd_wait(&system_block_state.watchdog_cnd, &system_block_state.mtx); - timeout = system_block_state.emergency_timeout; - erts_smp_mtx_unlock(&system_block_state.mtx); - - if (erts_disable_tolerant_timeofday) - erts_milli_sleep(timeout); - else { - SysTimeval to; - erts_get_timeval(&to); - to.tv_sec += timeout / 1000; - to.tv_usec += timeout % 1000; - - while (1) { - SysTimeval curr; - erts_milli_sleep(timeout); - erts_get_timeval(&curr); - if (curr.tv_sec > to.tv_sec - || (curr.tv_sec == to.tv_sec && curr.tv_usec >= to.tv_usec)) { - break; - } - timeout = (to.tv_sec - curr.tv_sec)*1000; - timeout += (to.tv_usec - curr.tv_usec)/1000; - } - } - - erts_smp_mtx_lock(&system_block_state.mtx); - system_block_state.emergency_timeout = -1; - erts_smp_cnd_broadcast(&system_block_state.cnd); - } - erts_smp_mtx_unlock(&system_block_state.mtx); - return NULL; -} - -void -erts_system_block_init(void) -{ - erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; - /* Local state... */ - system_block_state.emergency = 0; - system_block_state.emergency_timeout = -1; - erts_smp_cnd_init(&system_block_state.watchdog_cnd); - system_block_state.threads_to_block = 0; - system_block_state.have_blocker = 0; - /* system_block_state.block_tid */ - system_block_state.recursive_block = 0; - system_block_state.allowed_activities = 0; - erts_smp_tsd_key_create(&system_block_state.blockable_key); - erts_smp_mtx_init(&system_block_state.mtx, "system_block"); - erts_smp_cnd_init(&system_block_state.cnd); -#ifdef ERTS_ENABLE_LOCK_CHECK - system_block_state.activity_changing = 0; - system_block_state.checking = 0; -#endif - - thr_opts.suggested_stack_size = 8; - erts_smp_thr_create(&system_block_state.watchdog_tid, - emergency_watchdog, - NULL, - &thr_opts); - - /* Global state... */ - - erts_smp_atomic32_init_nob(&erts_system_block_state.do_block, 0); - erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.wait, 0); - erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.gc, 0); - erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.io, 0); - - /* Make sure blockable threads unregister when exiting... */ - erts_smp_install_exit_handler(erts_unregister_blockable_thread); -} - - -#endif /* #ifdef ERTS_SMP */ - char * erts_read_env(char *key) { |