diff options
Diffstat (limited to 'erts/emulator/beam')
68 files changed, 1712 insertions, 946 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 190e7817dc..cb6d294a41 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -71,6 +71,7 @@ atom absoluteURI atom ac atom accessor atom active +atom active_tasks atom all atom all_but_first atom all_names @@ -268,6 +269,7 @@ atom getenv atom gather_gc_info_result atom gather_io_bytes atom gather_sched_wall_time_result +atom gather_system_check_result atom getting_linked atom getting_unlinked atom global @@ -369,6 +371,7 @@ atom monitor atom monitor_nodes atom monitors atom monotonic +atom monotonic_timestamp atom more atom multi_scheduling atom multiline @@ -512,6 +515,7 @@ atom return_from atom return_to atom return_trace atom run_queue +atom run_queue_lengths atom runnable atom runnable_ports atom runnable_procs @@ -557,6 +561,7 @@ atom static atom stderr_to_stdout atom stop atom stream +atom strict_monotonic_timestamp atom sunrm atom suspend atom suspended @@ -579,7 +584,9 @@ atom timeout_value atom Times='*' atom timestamp atom total +atom total_active_tasks atom total_heap_size +atom total_run_queue_lengths atom tpkt atom trace trace_ts traced atom trace_control_word diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 016d0aaa32..9860968687 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -75,6 +75,16 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_smp_atomic32_t erts_active_bp_index; erts_smp_atomic32_t erts_staging_bp_index; +/* + * Inlined helpers + */ + +static ERTS_INLINE ErtsMonotonicTime +get_mtime(Process *c_p) +{ + return erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(c_p)); +} + /* ************************************************************************* ** Local prototypes */ @@ -97,9 +107,6 @@ static int clear_function_break(BeamInstr *pc, Uint break_flags); static BpDataTime* get_time_break(BeamInstr *pc); static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); -static void bp_time_diff(bp_data_time_item_t *item, - process_breakpoint_time_t *pbt, - Uint ms, Uint s, Uint us); static void bp_meta_unref(BpMetaPid* bmp); static void bp_count_unref(BpCount* bcp); @@ -110,13 +117,8 @@ static void uninstall_breakpoint(BeamInstr* pc); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ do { \ - Uint r; \ (pi0)->count += (pi1)->count; \ - (pi0)->s_time += (pi1)->s_time; \ - (pi0)->us_time += (pi1)->us_time; \ - r = (pi0)->us_time / 1000000; \ - (pi0)->s_time += r; \ - (pi0)->us_time = (pi0)->us_time % 1000000; \ + (pi0)->time += (pi1)->time; \ } while(0) static void bp_hash_init(bp_time_hash_t *hash, Uint n); @@ -948,7 +950,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, void erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) { - Uint ms,s,us; + ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -961,7 +963,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) * from the process psd */ pbt = ERTS_PROC_GET_CALL_TIME(c_p); - get_sys_now(&ms, &s, &us); + time = get_mtime(c_p); /* get pbt * timestamp = t0 @@ -976,7 +978,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) } else { ASSERT(pbt->pc); /* add time to previous code */ - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = time - pbt->time; sitem.pid = c_p->common.id; sitem.count = 0; @@ -1002,8 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* Add count to this code */ sitem.pid = c_p->common.id; sitem.count = 1; - sitem.s_time = 0; - sitem.us_time = 0; + sitem.time = 0; /* this breakpoint */ ASSERT(bdt); @@ -1020,15 +1021,13 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) } pbt->pc = I; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = time; } void erts_trace_time_return(Process *p, BeamInstr *pc) { - Uint ms,s,us; + ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -1041,7 +1040,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) * from the process psd */ pbt = ERTS_PROC_GET_CALL_TIME(p); - get_sys_now(&ms,&s,&us); + time = get_mtime(p); /* get pbt * lookup bdt from code @@ -1057,7 +1056,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) */ ASSERT(pbt->pc); - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = time - pbt->time; sitem.pid = p->common.id; sitem.count = 0; @@ -1080,9 +1079,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } pbt->pc = pc; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = time; } } @@ -1183,10 +1180,14 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { for(ix = 0; ix < hash.n; ix++) { item = &(hash.item[ix]); if (item->pid != NIL) { + ErtsMonotonicTime sec, usec; + usec = ERTS_MONOTONIC_TO_USEC(item->time); + sec = usec / 1000000; + usec = usec - sec*1000000; t = TUPLE4(hp, item->pid, make_small(item->count), - make_small(item->s_time), - make_small(item->us_time)); + make_small((Uint) sec), + make_small((Uint) usec)); hp += 5; *retval = CONS(hp, t, *retval); hp += 2; } @@ -1266,8 +1267,7 @@ static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) { } item[hval].pid = hash->item[ix].pid; item[hval].count = hash->item[ix].count; - item[hval].s_time = hash->item[ix].s_time; - item[hval].us_time = hash->item[ix].us_time; + item[hval].time = hash->item[ix].time; } } @@ -1315,8 +1315,7 @@ static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_da item = &(hash->item[hval]); item->pid = sitem->pid; - item->s_time = sitem->s_time; - item->us_time = sitem->us_time; + item->time = sitem->time; item->count = sitem->count; hash->used++; @@ -1330,41 +1329,7 @@ static void bp_hash_delete(bp_time_hash_t *hash) { hash->item = NULL; } -static void bp_time_diff(bp_data_time_item_t *item, /* out */ - process_breakpoint_time_t *pbt, /* in */ - Uint ms, Uint s, Uint us) { - int ds,dus; -#ifdef DEBUG - int dms; - - - dms = ms - pbt->ms; -#endif - ds = s - pbt->s; - dus = us - pbt->us; - - /* get_sys_now may return zero difftime, - * this is ok. - */ - -#ifdef DEBUG - ASSERT(dms >= 0 || ds >= 0 || dus >= 0); -#endif - - if (dus < 0) { - dus += 1000000; - ds -= 1; - } - if (ds < 0) { - ds += 1000000; - } - - item->s_time = ds; - item->us_time = dus; -} - void erts_schedule_time_break(Process *p, Uint schedule) { - Uint ms, s, us; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -1387,8 +1352,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { pbdt = get_time_break(pbt->pc); if (pbdt) { - get_sys_now(&ms,&s,&us); - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = get_mtime(p) - pbt->time; sitem.pid = p->common.id; sitem.count = 0; @@ -1410,10 +1374,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * timestamp it and remove the previous * timestamp in the psd. */ - get_sys_now(&ms,&s,&us); - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = get_mtime(p); break; default : ASSERT(0); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 97d0539ac7..2b89d6fc71 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -29,8 +29,7 @@ typedef struct { Eterm pid; Sint count; - Uint s_time; - Uint us_time; + ErtsMonotonicTime time; } bp_data_time_item_t; typedef struct { @@ -46,9 +45,7 @@ typedef struct bp_data_time { /* Call time */ } BpDataTime; typedef struct { - Uint ms; - Uint s; - Uint us; + ErtsMonotonicTime time; BeamInstr *pc; } process_breakpoint_time_t; /* used within psd */ diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index c1fd17c65d..7a1f4901aa 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -143,7 +143,7 @@ BeamInstr *beam_catches_car(unsigned i) struct bc_pool* p = &bccix[erts_active_code_ix()]; if (i >= p->tabsize ) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } return p->beam_catches[i].cp; } @@ -157,10 +157,10 @@ void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes, ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging); for(i = head; i != (unsigned)-1;) { if (i >= p->tabsize) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: item %#x has cp %p which is not " "in module's range [%p,%p[\r\n", i, p->beam_catches[i].cp, code, ((char*)code + code_bytes)); diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 38def5d89f..3a15287555 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -2182,7 +2182,10 @@ void process_main(void) c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; - erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + + if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) + erts_smp_atomic32_read_band_relb(&c_p->state, + ~ERTS_PSFLG_ACTIVE); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); c_p->current = NULL; @@ -4069,7 +4072,7 @@ do { \ tmp_arg1 += Arg1; store_bs_add_result: - if (MY_IS_SSMALL((Sint) tmp_arg1)) { + if (tmp_arg1 <= MAX_SMALL) { tmp_arg1 = make_small(tmp_arg1); } else { /* @@ -5018,7 +5021,7 @@ do { \ #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): - erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); + erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); # define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT # define ERTS_NO_FPE_ERROR ERTS_FP_ERROR #else @@ -5176,7 +5179,7 @@ do { \ I = handle_error(c_p, I, reg, NULL); goto post_error_handling; default: - erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); } } OpCase(hipe_call_count): { @@ -5256,7 +5259,7 @@ do { \ OpCase(label_L): OpCase(on_load): OpCase(line_I): - erl_exit(1, "meta op\n"); + erts_exit(ERTS_ERROR_EXIT, "meta op\n"); /* * One-time initialization of Beam emulator. @@ -5310,7 +5313,7 @@ do { \ } #ifdef NO_JUMP_TABLE default: - erl_exit(1, "unexpected op code %d\n",Go); + erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go); } #endif return; /* Never executed */ @@ -5355,7 +5358,7 @@ translate_gc_bif(void* gcf) } else if (gcf == erts_gc_binary_part_3) { return binary_part_3; } else { - erl_exit(1, "bad gc bif"); + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); } } @@ -5420,7 +5423,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erl_exit(). */ + c_p->i = pc; /* In case we call erts_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ @@ -5484,7 +5487,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) c_p->cp = 0; /* To avoid keeping stale references. */ return new_pc; } - if (c_p->catches > 0) erl_exit(1, "Catch not found"); + if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); terminate_proc(c_p, Value); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index b70e5b9a2d..a6dce2d1d2 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6249,6 +6249,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_LITERALS_END] = 0; code[MI_LITERALS_OFF_HEAP] = 0; code[MI_ON_LOAD_FUNCTION_PTR] = 0; + code[MI_LINE_TABLE] = 0; code[MI_MD5_PTR] = 0; ci = MI_FUNCTIONS + n + 1; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index a8cc19ee1f..095df36a43 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1605,14 +1605,17 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) * true. For more info, see implementation of * erts_send_exit_signal(). */ + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); if (trap_exit) state = erts_smp_atomic32_read_bor_mb(&BIF_P->state, ERTS_PSFLG_TRAP_EXIT); else state = erts_smp_atomic32_read_band_mb(&BIF_P->state, ~ERTS_PSFLG_TRAP_EXIT); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); + #ifdef ERTS_SMP - if (ERTS_PROC_PENDING_EXIT(BIF_P)) { + if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); ERTS_BIF_EXITED(BIF_P); } @@ -2216,7 +2219,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3) erts_dsend_export_trap_context(p, ctx)); break; default: - erl_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result); + erts_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result); break; } @@ -2260,7 +2263,7 @@ static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1); } default: - erl_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result); + erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result); break; } ASSERT(! "Can not arrive here"); @@ -2332,7 +2335,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) erts_dsend_export_trap_context(p, ctx)); break; default: - erl_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result); + erts_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result); break; } @@ -3883,7 +3886,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1); if (pres < 0) - erl_exit(1, "Failed to convert term to string: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to string: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -3905,7 +3908,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) } str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); if (intlist_to_buf(string, str, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); str[len] = '\0'; erts_fprintf(stderr, "%s", str); erts_free(ERTS_ALC_T_TMP, (void *) str); @@ -3925,7 +3928,7 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) BIF_RETTYPE halt_0(BIF_ALIST_0) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n")); - erl_halt(0); + erts_halt(0); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } @@ -3938,17 +3941,18 @@ static char halt_msg[HALT_MSG_SIZE]; /* ARGSUSED */ BIF_RETTYPE halt_1(BIF_ALIST_1) { - Sint code; + Uint code; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { int i; @@ -3959,11 +3963,11 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) halt_msg[i] = '\0'; VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -3974,7 +3978,7 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) /* ARGSUSED */ BIF_RETTYPE halt_2(BIF_ALIST_2) { - Sint code; + Uint code; Eterm optlist = BIF_ARG_2; int flush = 1; @@ -4001,23 +4005,24 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) if (is_not_nil(optlist)) goto error; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); if (flush) { - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit((int)(- code), ""); + erts_exit(pos_int_code, ""); } } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { int i; @@ -4029,11 +4034,11 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -4089,7 +4094,7 @@ term2list_dsprintf(Process *p, Eterm term) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%T", term); if (pres < 0) - erl_exit(1, "Failed to convert term to list: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to list: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 4f0656d174..3177d5dae7 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -174,6 +174,8 @@ bif erts_internal:time_unit/0 bif erts_internal:is_system_process/1 +bif erts_internal:system_check/1 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 15bcd44fb9..87d3be2b0f 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -2028,6 +2028,32 @@ term_to_Uint(Eterm term, Uint *up) } } +/* same as term_to_Uint() + but also accept larger bignums by masking + */ +int +term_to_Uint_mask(Eterm term, Uint *up) +{ + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (Uint) i; + return 1; + } else if (is_big(term) && !big_sign(term)) { + ErtsDigit* xr = big_v(term); + + ERTS_CT_ASSERT(sizeof(ErtsDigit) == sizeof(Uint)); + *up = (Uint)*xr; /* just pick first word */ + return 1; + } else { + *up = BADARG; + return 0; + } +} + int term_to_UWord(Eterm term, UWord *up) { diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 4aa9724ae3..3c1f1e6b23 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -161,6 +161,7 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*); byte* big_to_bytes(Eterm, byte*); int term_to_Uint(Eterm, Uint*); +int term_to_Uint_mask(Eterm, Uint*); int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); #if HAVE_INT64 diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index e670fbf31c..247c8f1122 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -47,7 +47,7 @@ erts_init_binary(void) if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) { /* I assume that any compiler should be able to optimize this away. If not, this test is not very expensive... */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error: Address of orig_bytes[0] of a Binary" " is *not* 8-byte aligned\n"); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 4ce9d24479..298b30fff3 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -104,7 +104,7 @@ process_killer(void) erts_printf("(k)ill (n)ext (r)eturn:\n"); while(1) { if ((j = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch(j) { case 'k': { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; @@ -493,7 +493,7 @@ do_break(void) halt immediately if break is called */ mode = erts_read_env("ERL_CONSOLE_MODE"); if (mode && strcmp(mode, "window") != 0) - erl_exit(0, ""); + erts_exit(0, ""); erts_free_read_env(mode); #endif /* __WIN32__ */ @@ -503,7 +503,7 @@ do_break(void) while (1) { if ((i = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch (i) { case 'q': case 'a': @@ -513,9 +513,9 @@ do_break(void) * The usual reason for a read error is Ctrl-C. Treat this as * 'a' to avoid infinite loop. */ - erl_exit(0, ""); + erts_exit(0, ""); case 'A': /* Halt generating crash dump */ - erl_exit(1, "Crash dump requested by user"); + erts_exit(ERTS_ERROR_EXIT, "Crash dump requested by user"); case 'c': return; case 'p': @@ -684,7 +684,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) crash dump. */ erts_thr_progress_fatal_error_block(&tpd_buf); -#ifdef ERTS_THR_HAVE_SIG_FUNCS +#ifdef ERTS_SYS_SUSPEND_SIGNAL /* * We suspend all scheduler threads so that we can dump some * data about the currently running processes and scheduler data. @@ -785,7 +785,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); #ifdef USE_THREADS - /* We want to note which thread it was that called erl_exit */ + /* We want to note which thread it was that called erts_exit */ if (erts_get_scheduler_data()) { erts_fdprintf(fd, "Calling Thread: scheduler:%d\n", erts_get_scheduler_data()->no); @@ -818,7 +818,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) #ifdef ERTS_SMP -#if defined(ERTS_THR_HAVE_SIG_FUNCS) +#ifdef ERTS_SYS_SUSPEND_SIGNAL /* We resume all schedulers so that we are in a known safe state when we write the rest of the crash dump */ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 8849dadd00..64be43edb4 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -59,7 +59,7 @@ copy_object(Eterm obj, Process* to) res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG if (eq(obj, res) == 0) { - erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); + erts_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); } #endif return res; @@ -171,7 +171,7 @@ Uint size_object(Eterm obj) } break; default: - erl_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } break; case SUB_BINARY_SUBTAG: @@ -202,7 +202,7 @@ Uint size_object(Eterm obj) } break; case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "size_object: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; @@ -219,7 +219,7 @@ Uint size_object(Eterm obj) obj = ESTACK_POP(s); break; default: - erl_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); } } } @@ -272,7 +272,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } @@ -331,7 +331,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; case TAG_PRIMARY_BOXED: argp = tailp; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } @@ -512,11 +512,11 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *argp = make_hashmap_rel(tp, dst_base); break; default: - erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } break; case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "copy_struct: matchstate term not allowed"); default: i = thing_arityval(hdr)+1; @@ -540,13 +540,13 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) #ifdef DEBUG if (htop != hbot) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" " htop=%p != hbot=%p (sz=%beu)\n", org_obj, htop, hbot, org_sz); #else if (htop > hbot) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct(): htop, hbot overrun\n"); } #endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 7be2b77a3b..787241b960 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1959,7 +1959,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) goto done; } default: - erl_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase); + erts_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase); } } @@ -1980,7 +1980,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -2020,7 +2020,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -3382,7 +3382,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas continue; break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } } @@ -3691,7 +3691,7 @@ erts_processes_monitoring_nodes(Process *c_p) case ERTS_NODES_MON_OPT_TYPES: type = am_all; break; case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break; case ERTS_NODES_MON_OPT_TYPE_HIDDEN: type = am_hidden; break; - default: erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } olist = erts_bld_cons(hpp, szp, erts_bld_tuple(hpp, szp, 2, diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 47dafa53c0..4b0541c10e 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -241,7 +241,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index c3f4fe5a63..5f8cd2bbf7 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -653,11 +653,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) { int err = errno; char *errstr = err ? strerror(err) : "unknown"; - erl_exit(-1, "Failed to lock physical memory: %s (%d)\n", + erts_exit(1, "Failed to lock physical memory: %s (%d)\n", errstr, err); } #else - erl_exit(-1, "Failed to lock physical memory: Not supported\n"); + erts_exit(1, "Failed to lock physical memory: Not supported\n"); #endif } @@ -806,13 +806,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing alloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].realloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing realloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].free) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing free function for %s\n", ERTS_ALC_A2AD(i)); } @@ -890,7 +890,7 @@ erts_alloc_late_init(void) static void * erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Attempt to reallocate a block of the fixed size type %s\n", ERTS_ALC_T2TD(type)); } @@ -1012,7 +1012,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, * tspec->size) + ERTS_CACHE_LINE_SIZE - 1)); if (!states) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate allocator states for %salloc\n", init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; @@ -1040,7 +1040,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, (tot_fix_list_size + ERTS_CACHE_LINE_SIZE - 1)); if (!fix_lists) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate fix lists for %salloc\n", init->init.util.name_prefix); @@ -1114,7 +1114,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, } if (!as) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to start %salloc\n", init->init.util.name_prefix); ASSERT(as == (void *) as0); @@ -1909,7 +1909,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) case ERTS_ALC_O_FREE: op_str = "free"; break; default: op_str = "UNKNOWN"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s: %s operation not supported (memory type: \"%s\")\n", allctr_str, op_str, t_str); break; @@ -1923,18 +1923,18 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) va_start(argp, n); size = va_arg(argp, Uint); va_end(argp); - erl_exit(1, + erts_exit(ERTS_DUMP_EXIT, "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n", allctr_str, op, size, t_str); break; } case ERTS_ALC_E_NOALLCTR: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown allocator type: %d\n", ERTS_ALC_T2A(ERTS_ALC_N2T(n))); break; default: - erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); break; } } @@ -3189,7 +3189,7 @@ reply_alloc_info(void *vair) make_small(0), ainfo); } else { - erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", __FILE__, __LINE__); } } @@ -3426,7 +3426,7 @@ void *safe_realloc(void *ptr, Uint sz) * Keep alloc_SUITE_data/allocator_test.h updated if changes are made * * to erts_alc_test() * \* */ -#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") +#define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) { @@ -3888,7 +3888,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at beginning of memory block (p=0x%u) " "clobbered.\n", (UWord) ptr); @@ -3905,12 +3905,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) char *op_str; if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at end of memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); if (found_type != GET_TYPE_OF_PATTERN(post_pattern)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence around memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); @@ -3933,7 +3933,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) default: op_str = "???"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\"," " but %s as type \"%s\".\n", (UWord) ptr, (UWord) sz, ftype, op_str, otype); diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 1ecebdeb07..7738531142 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -369,6 +369,7 @@ type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD_LOW SYSTEM port_data_heap +type SYS_CHECK_REQ STANDARD_LOW SYSTEM system_check_request +else # "fullword" @@ -389,6 +390,7 @@ type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 8229a15824..d9eaa7b32c 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -4518,7 +4518,7 @@ make_name_atoms(Allctr_t *allctr) size_t prefix_len = strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) - erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix); + erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); @@ -5720,7 +5720,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) /* erts_alcu_start assumes that allctr has been zeroed */ if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", __FILE__, __LINE__); } @@ -5904,7 +5904,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->thread_safe) erts_mtx_destroy(&allctr->mutex); #endif - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to create main carrier for %salloc\n", init->name_prefix); } @@ -6049,7 +6049,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x019: return (UWord) PREV_BLK((Block_t *) a1); case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); case 0x01b: return (UWord) sizeof(Unit_t); - case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1); case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; #ifdef ERTS_SMP @@ -6120,7 +6120,7 @@ erts_alcu_verify_unused(Allctr_t *allctr) if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" "Total amount of bytes allocated: %bpu\n", diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 19420af8ab..5cd067ff54 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -1035,7 +1035,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index fb853b65ab..f39a18ac88 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -940,7 +940,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 28bec6325c..f4ef59993a 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1309,7 +1309,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro case ERL_DE_FORCE_RELOAD: break; default: - erl_exit(1,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); + erts_exit(ERTS_ERROR_EXIT,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); } p->flags |= F_USING_DDLL; r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index b44382cde8..ac7a70c642 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -65,6 +65,7 @@ static Export* gather_io_bytes_trap = NULL; static Export *gather_sched_wall_time_res_trap; static Export *gather_gc_info_res_trap; +static Export *gather_system_check_res_trap; #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -914,7 +915,7 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -954,7 +955,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -1533,7 +1534,7 @@ process_info_aux(Process *BIF_P, } case am_last_calls: { - struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(BIF_P); + struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp); if (!scb) { hp = HAlloc(BIF_P, 3); res = am_false; @@ -1759,7 +1760,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(*tp, buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_memory_map(buf) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -1778,7 +1779,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(tp[1], buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_stat(buf, 1) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -3234,6 +3235,39 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) if (is_non_value(res)) BIF_RET(am_undefined); BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); + } else if (BIF_ARG_1 == am_total_active_tasks + || BIF_ARG_1 == am_total_run_queue_lengths) { + Uint no = erts_run_queues_len(NULL, 0, BIF_ARG_1 == am_total_active_tasks); + if (IS_USMALL(0, no)) + res = make_small(no); + else { + Eterm *hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); + res = uint_to_big(no, hp); + } + BIF_RET(res); + } else if (BIF_ARG_1 == am_active_tasks + || BIF_ARG_1 == am_run_queue_lengths) { + Eterm res, *hp, **hpp; + Uint sz, *szp; + int no_qs = erts_no_run_queues; + Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); + (void) erts_run_queues_len(qszs, 0, BIF_ARG_1 == am_active_tasks); + sz = 0; + szp = &sz; + hpp = NULL; + while (1) { + int i; + for (i = 0; i < no_qs; i++) + qszs[no_qs+i] = erts_bld_uint(hpp, szp, qszs[i]); + res = erts_bld_list(hpp, szp, no_qs, &qszs[no_qs]); + if (hpp) { + erts_free(ERTS_ALC_T_TMP, qszs); + BIF_RET(res); + } + hp = HAlloc(BIF_P, sz); + szp = NULL; + hpp = &hp; + } } else if (BIF_ARG_1 == am_context_switches) { Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P); hp = HAlloc(BIF_P, 3); @@ -3282,7 +3316,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_run_queue) { - res = erts_run_queues_len(NULL); + res = erts_run_queues_len(NULL, 1, 0); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { UWord w1, w2; @@ -3302,7 +3336,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) Uint sz, *szp; int no_qs = erts_no_run_queues; Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); - (void) erts_run_queues_len(qszs); + (void) erts_run_queues_len(qszs, 0, 0); sz = 0; szp = &sz; hpp = NULL; @@ -3751,6 +3785,18 @@ BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1) +{ + Eterm res; + if (ERTS_IS_ATOM_STR("schedulers", BIF_ARG_1)) { + res = erts_system_check_request(BIF_P); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_system_check_res_trap, BIF_P, res); + } + + BIF_ERROR(BIF_P, BADARG); +} static erts_smp_atomic_t hipe_test_reschedule_flag; @@ -3765,7 +3811,7 @@ static void broken_halt_test(Eterm bif_arg_2) #if defined(ERTS_HAVE_TRY_CATCH) erts_get_scheduler_data()->run_queue = NULL; #endif - erl_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); + erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); } @@ -4012,7 +4058,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_true); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { - erl_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); + erts_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); } else if (ERTS_IS_ATOM_STR("kill_dist_connection", BIF_ARG_1)) { DistEntry *dep = erts_sysname_to_connected_dist_entry(BIF_ARG_2); @@ -4358,6 +4404,8 @@ erts_bif_info_init(void) = erts_export_put(am_erlang, am_gather_gc_info_result, 1); gather_io_bytes_trap = erts_export_put(am_erts_internal, am_gather_io_bytes, 2); + gather_system_check_res_trap + = erts_export_put(am_erts_internal, am_gather_system_check_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index e47d7bcbbb..27c24197ea 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -791,7 +791,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) } else { name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); name_buf[i] = '\0'; } driver = &vanilla_driver; @@ -1169,7 +1169,7 @@ static Eterm http_bld_uri(struct packet_callback_args* pca, return erts_bld_tuple(hpp, szp, 3, am_scheme, s1, s2); default: - erl_exit(1, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); } } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 03f51132b1..08807d72c9 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -431,6 +431,9 @@ Uint erts_trace_flag2bit(Eterm flag) { switch (flag) { + case am_timestamp: return F_NOW_TS; + case am_strict_monotonic_timestamp: return F_STRICT_MON_TS; + case am_monotonic_timestamp: return F_MON_TS; case am_all: return TRACEE_FLAGS; case am_send: return F_TRACE_SEND; case am_receive: return F_TRACE_RECEIVE; @@ -439,7 +442,6 @@ erts_trace_flag2bit(Eterm flag) case am_set_on_first_spawn: return F_TRACE_SOS1; case am_set_on_link: return F_TRACE_SOL; case am_set_on_first_link: return F_TRACE_SOL1; - case am_timestamp: return F_TIMESTAMP; case am_running: return F_TRACE_SCHED; case am_exiting: return F_TRACE_SCHED_EXIT; case am_garbage_collection: return F_TRACE_GC; @@ -592,7 +594,7 @@ Eterm trace_3(BIF_ALIST_3) ERTS_TRACE_FLAGS(tracee_port) |= mask; else ERTS_TRACE_FLAGS(tracee_port) &= ~mask; - + if (!ERTS_TRACE_FLAGS(tracee_port)) ERTS_TRACER_PROC(tracee_port) = NIL; else if (tracer != NIL) @@ -978,7 +980,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) } if (key == am_flags) { - int num_flags = 19; /* MAXIMUM number of flags. */ + int num_flags = 21; /* MAXIMUM number of flags. */ Uint needed = 3+2*num_flags; Eterm flag_list = NIL; Eterm* limit; @@ -996,6 +998,9 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) #endif hp = HAlloc(p, needed); limit = hp+needed; + FLAG(F_NOW_TS, am_timestamp); + FLAG(F_STRICT_MON_TS, am_strict_monotonic_timestamp); + FLAG(F_MON_TS, am_monotonic_timestamp); FLAG(F_TRACE_SEND, am_send); FLAG(F_TRACE_RECEIVE, am_receive); FLAG(F_TRACE_SOS, am_set_on_spawn); @@ -1007,7 +1012,6 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) FLAG(F_TRACE_SCHED, am_running); FLAG(F_TRACE_SCHED_EXIT, am_exiting); FLAG(F_TRACE_GC, am_garbage_collection); - FLAG(F_TIMESTAMP, am_timestamp); FLAG(F_TRACE_ARITY_ONLY, am_arity); FLAG(F_TRACE_RETURN_TO, am_return_to); FLAG(F_TRACE_SILENT, am_silent); @@ -1798,7 +1802,11 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2, } else if (arg1 == am_print) { current_flag = SEQ_TRACE_PRINT; } else if (arg1 == am_timestamp) { - current_flag = SEQ_TRACE_TIMESTAMP; + current_flag = SEQ_TRACE_NOW_TS; + } else if (arg1 == am_strict_monotonic_timestamp) { + current_flag = SEQ_TRACE_STRICT_MON_TS; + } else if (arg1 == am_monotonic_timestamp) { + current_flag = SEQ_TRACE_MON_TS; } else current_flag = 0; @@ -1909,7 +1917,9 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) #endif ) { if ((item == am_send) || (item == am_receive) || - (item == am_print) || (item == am_timestamp)) { + (item == am_print) || (item == am_timestamp) + || (item == am_monotonic_timestamp) + || (item == am_strict_monotonic_timestamp)) { hp = HAlloc(p,3); res = TUPLE2(hp, item, am_false); BIF_RET(res); @@ -1927,7 +1937,11 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) } else if (item == am_print) { current_flag = SEQ_TRACE_PRINT; } else if (item == am_timestamp) { - current_flag = SEQ_TRACE_TIMESTAMP; + current_flag = SEQ_TRACE_NOW_TS; + } else if (item == am_strict_monotonic_timestamp) { + current_flag = SEQ_TRACE_STRICT_MON_TS; + } else if (item == am_monotonic_timestamp) { + current_flag = SEQ_TRACE_MON_TS; } else { current_flag = 0; } @@ -2237,6 +2251,7 @@ static Eterm system_profile_get(Process *p) { if (erts_system_profile_flags.exclusive) { res = CONS(hp, am_exclusive, res); hp += 2; } + return TUPLE2(hp, system_profile, res); } } @@ -2255,6 +2270,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) int system_blocked = 0; Process *profiler_p = NULL; Port *profiler_port = NULL; + int ts; if (profiler == am_undefined || list == NIL) { prev = system_profile_get(p); @@ -2286,7 +2302,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) goto error; } - for (scheduler = 0, runnable_ports = 0, runnable_procs = 0, exclusive = 0; + for (ts = ERTS_TRACE_FLG_NOW_TIMESTAMP, scheduler = 0, + runnable_ports = 0, runnable_procs = 0, exclusive = 0; is_list(list); list = CDR(list_val(list))) { @@ -2299,6 +2316,12 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) exclusive = !0; } else if (t == am_scheduler) { scheduler = !0; + } else if (t == am_timestamp) { + ts = ERTS_TRACE_FLG_NOW_TIMESTAMP; + } else if (t == am_strict_monotonic_timestamp) { + ts = ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP; + } else if (t == am_monotonic_timestamp) { + ts = ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP; } else goto error; } if (is_not_nil(list)) goto error; @@ -2311,7 +2334,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) erts_system_profile_flags.runnable_ports = !!runnable_ports; erts_system_profile_flags.runnable_procs = !!runnable_procs; erts_system_profile_flags.exclusive = !!exclusive; - + erts_system_profile_ts_type = ts; erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 8395f6ecc6..c263eebfcc 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -402,7 +402,7 @@ cpu_bind_order_sort(erts_cpu_topology_t *cpudata, break; default: cmp_func = NULL; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Bad cpu bind type: %d\n", (int) cpu_bind_order); break; @@ -1590,7 +1590,7 @@ get_cpu_topology_term(Process *c_p, int type) } break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + erts_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); break; } @@ -1967,7 +1967,7 @@ cpu_group_insert(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); + erts_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); } @@ -2290,7 +2290,7 @@ remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) prev_cgm = cgm; } - erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); + erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); } static int @@ -2320,7 +2320,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); + erts_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); } static void diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 878ee32b47..870b556851 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1295,7 +1295,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) goto badarg; if (!remove_named_tab(tb, 1)) - erl_exit(1,"Could not find named tab %s", tb->common.id); + erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id); tb->common.id = tb->common.the_name = BIF_ARG_2; @@ -1376,7 +1376,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) status |= DB_ORDERED_SET; status &= ~(DB_SET | DB_BAG | DB_DUPLICATE_BAG); } - /*TT*/ else if (is_tuple(val)) { Eterm *tp = tuple_val(val); if (arityval(tp[0]) == 2) { @@ -1581,7 +1580,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_P->common.id, make_small(slot)), 0) != DB_ERROR_NONE) { - erl_exit(1,"Could not update ets metadata."); + erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata."); } db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); @@ -2940,7 +2939,7 @@ void init_db(ErtsDbSpinCount db_spin_count) bits = erts_fit_in_bits_int32(db_max_tabs-1); if (bits > SMALL_BITS) { - erl_exit(1,"Max limit for ets tabled too high %u (max %u).", + erts_exit(ERTS_ERROR_EXIT,"Max limit for ets tabled too high %u (max %u).", db_max_tabs, ((Uint)1)<<SMALL_BITS); } meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1; @@ -3001,7 +3000,7 @@ void init_db(ErtsDbSpinCount db_spin_count) db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0); @@ -3032,7 +3031,7 @@ void init_db(ErtsDbSpinCount db_spin_count) db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } /* Non visual BIF to trap to. */ @@ -3229,7 +3228,7 @@ retry: * yielding. */ #define ERTS_DB_INTERNAL_ERROR(LSTR) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ __FILE__, __LINE__) int @@ -3466,10 +3465,10 @@ static void fix_table_locked(Process* p, DbTable* tb) #endif erts_refc_inc(&tb->common.ref,1); fix = tb->common.fixations; - if (fix == NULL) { - get_now(&(tb->common.megasec), - &(tb->common.sec), - &(tb->common.microsec)); + if (fix == NULL) { + tb->common.time.monotonic + = erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(p)); + tb->common.time.offset = erts_get_time_offset(); } else { for (; fix != NULL; fix = fix->next) { @@ -3501,7 +3500,7 @@ static void fix_table_locked(Process* p, DbTable* tb) make_small(tb->common.slot)), 0) != DB_ERROR_NONE) { UnUseTmpHeap(3,p); - erl_exit(1,"Could not insert ets metadata in safe_fixtable."); + erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable."); } UnUseTmpHeap(3,p); db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); @@ -3731,6 +3730,7 @@ static int free_table_cont(Process *p, static Eterm table_info(Process* p, DbTable* tb, Eterm What) { Eterm ret = THE_NON_VALUE; + int use_monotonic; if (What == am_size) { ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems)); @@ -3788,7 +3788,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_true; else ret = am_false; - } else if (What == am_atom_put("safe_fixed",10)) { + } else if ((use_monotonic + = ERTS_IS_ATOM_STR("safe_fixed_monotonic_time", + What)) + || ERTS_IS_ATOM_STR("safe_fixed", What)) { #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif @@ -3797,7 +3800,19 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) Eterm *hp; Eterm tpl, lst; DbFixation *fix; - need = 7; + Sint64 mtime; + + need = 3; + if (use_monotonic) { + mtime = (Sint64) tb->common.time.monotonic; + mtime += ERTS_MONOTONIC_OFFSET_NATIVE; + if (!IS_SSMALL(mtime)) + need += ERTS_SINT64_HEAP_SIZE(mtime); + } + else { + mtime = 0; + need += 4; + } for (fix = tb->common.fixations; fix != NULL; fix = fix->next) { need += 5; } @@ -3809,11 +3824,18 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) lst = CONS(hp,tpl,lst); hp += 2; } - tpl = TUPLE3(hp, - make_small(tb->common.megasec), - make_small(tb->common.sec), - make_small(tb->common.microsec)); - hp += 4; + if (use_monotonic) + tpl = (IS_SSMALL(mtime) + ? make_small(mtime) + : erts_sint64_to_big(mtime, &hp)); + else { + Uint ms, s, us; + erts_make_timestamp_value(&ms, &s, &us, + tb->common.time.monotonic, + tb->common.time.offset); + tpl = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); + hp += 4; + } ret = TUPLE2(hp, tpl, lst); } else { ret = am_false; diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 98a2e2842a..0b31eb3bcd 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -2869,15 +2869,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, q->hvalue = hval; q->next = NULL; *bp = b = q; - - { - int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); - int nactive = NACTIVE(tb); - - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { - grow(tb, nactive); - } - } + flags |= DB_INC_TRY_GROW; } else { HashDbTerm *q, *next = b->next; @@ -2916,6 +2908,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) HashDbTerm **bp = (HashDbTerm **) handle->bp; HashDbTerm *b = *bp; erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; + HashDbTerm* free_me = NULL; ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ @@ -2927,21 +2920,34 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) b->hvalue = INVALID_HASH; } else { *bp = b->next; - free_term(tb, b); + free_me = b; } WUNLOCK_HASH(lck); erts_smp_atomic_dec_nob(&tb->common.nitems); try_shrink(tb); - } else if (handle->flags & DB_MUST_RESIZE) { - db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); - WUNLOCK_HASH(lck); - - free_term(tb, b); - } - else { - WUNLOCK_HASH(lck); + } else { + if (handle->flags & DB_MUST_RESIZE) { + db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); + free_me = b; + } + if (handle->flags & DB_INC_TRY_GROW) { + int nactive; + int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); + WUNLOCK_HASH(lck); + nactive = NACTIVE(tb); + + if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + grow(tb, nactive); + } + } else { + WUNLOCK_HASH(lck); + } } + + if (free_me) + free_term(tb, free_me); + #ifdef DEBUG handle->dbterm = 0; #endif @@ -3028,7 +3034,7 @@ void db_check_table_hash(DbTable *tbl) if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { - erl_exit(1, "Bad term in slot %d of ets table", j); + erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j); } list = list->next; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 465aa566ad..61293fbf9a 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1213,7 +1213,7 @@ static int db_select_count_continue_tree(Process *p, tptr = tuple_val(continuation); if (arityval(*tptr) != 5) - erl_exit(1,"Internal error in ets:select_count/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_count/1"); lastkey = tptr[2]; end_condition = tptr[3]; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index dab357a079..af5b611afd 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1946,11 +1946,11 @@ restart: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -2615,7 +2615,7 @@ restart: case matchHalt: goto success; default: - erl_exit(1, "Internal error: unexpected opcode in match program."); + erts_exit(ERTS_ERROR_EXIT, "Internal error: unexpected opcode in match program."); } } fail: @@ -2639,11 +2639,11 @@ success: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -3683,7 +3683,7 @@ static DMCRet dmc_one_term(DMCContext *context, break; } default: - erl_exit(1, "db_match_compile: " + erts_exit(ERTS_ERROR_EXIT, "db_match_compile: " "Bad object on heap: 0x%bex\n", c); } return retOk; @@ -4930,7 +4930,7 @@ static DMCRet dmc_fun(DMCContext *context, DMC_PUSH(*text, matchCall3); break; default: - erl_exit(1,"ets:match() internal error, " + erts_exit(ERTS_ERROR_EXIT,"ets:match() internal error, " "guard with more than 3 arguments."); } DMC_PUSH(*text, (UWord) b->biff); @@ -5210,7 +5210,7 @@ static Uint my_size_object(Eterm t) tmp == am_const) { sum += size_object(tuple_val(t)[2]); } else { - erl_exit(1,"Internal error, sizing unrecognized object in " + erts_exit(ERTS_ERROR_EXIT,"Internal error, sizing unrecognized object in " "(d)ets:match compilation."); } break; @@ -5255,7 +5255,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) sz = size_object(b); ret = copy_struct(b,sz,hp,off_heap); } else { - erl_exit(1, "Trying to constant-copy non constant expression " + erts_exit(ERTS_ERROR_EXIT, "Trying to constant-copy non constant expression " "0x%bex in (d)ets:match compilation.", t); } } else { diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 1ccdc0305b..bfd81f5d86 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -79,6 +79,7 @@ typedef union db_table DbTable; #define DB_MUST_RESIZE 1 #define DB_NEW_OBJECT 2 +#define DB_INC_TRY_GROW 4 /* Info about a database entry while it's being updated * (by update_counter or update_element) @@ -229,7 +230,10 @@ typedef struct db_table_common { DbTableMethod* meth; /* table methods */ erts_smp_atomic_t nitems; /* Total number of items in table */ erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */ - Uint megasec,sec,microsec; /* Last fixation time */ + struct { /* Last fixation time */ + ErtsMonotonicTime monotonic; + ErtsMonotonicTime offset; + } time; DbFixation* fixations; /* List of processes who have done safe_fixtable, "local" fixations not included. */ /* All 32-bit fields */ diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 2dcfb79f00..4928aae9c2 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -255,14 +255,14 @@ void erts_check_stack(Process *p) Eterm *stack_end = p->htop; if (p->stop > stack_start) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack underflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), internal_pid_serial(p->common.id)); if (p->stop < stack_end) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack overflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -287,7 +287,7 @@ void erts_check_stack(Process *p) if (in_mbuf) continue; - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Wild stack pointer\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -372,7 +372,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) #ifdef DEBUG if (hval == DEBUG_BAD_WORD) { print_untagged_memory(start, end); - erl_exit(1, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } #endif @@ -385,7 +385,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) if (verify_eterm(p,hval)) continue; - erl_exit(1, "Wild pointer found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Wild pointer found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } } @@ -395,11 +395,11 @@ void verify_process(Process *p) #define VERIFY_AREA(name,ptr,sz) { \ int n = (sz); \ while (n--) if(!verify_eterm(p,*(ptr+n))) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } #define VERIFY_ETERM(name,eterm) { \ if(!verify_eterm(p,eterm)) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } ErlMessage* mp = p->msg.first; diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index dbb4d719c1..7ab58e336b 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -37,47 +37,6 @@ # endif #endif -#ifdef SIZEOF_CHAR -# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR -# undef SIZEOF_CHAR -#endif -#ifdef SIZEOF_SHORT -# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT -# undef SIZEOF_SHORT -#endif -#ifdef SIZEOF_INT -# define SIZEOF_INT_SAVED__ SIZEOF_INT -# undef SIZEOF_INT -#endif -#ifdef SIZEOF_LONG -# define SIZEOF_LONG_SAVED__ SIZEOF_LONG -# undef SIZEOF_LONG -#endif -#ifdef SIZEOF_LONG_LONG -# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG -# undef SIZEOF_LONG_LONG -#endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif -#include "erl_int_sizes_config.h" -#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR -# error SIZEOF_CHAR mismatch -#endif -#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT -# error SIZEOF_SHORT mismatch -#endif -#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT -# error SIZEOF_INT mismatch -#endif -#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG -# error SIZEOF_LONG mismatch -#endif -#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG -# error SIZEOF_LONG_LONG mismatch -#endif - /* This is OK to override by the NIF/driver implementor */ #if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR) #define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__ @@ -134,7 +93,7 @@ typedef struct { #define ERL_DRV_EXTENDED_MARKER (0xfeeeeeed) #define ERL_DRV_EXTENDED_MAJOR_VERSION 3 -#define ERL_DRV_EXTENDED_MINOR_VERSION 2 +#define ERL_DRV_EXTENDED_MINOR_VERSION 3 /* * The emulator will refuse to load a driver with a major version @@ -176,28 +135,12 @@ typedef struct { /* * Integer types */ -#if defined(__WIN32__) && (SIZEOF_VOID_P == 8) -typedef unsigned __int64 ErlDrvTermData; -typedef unsigned __int64 ErlDrvUInt; -typedef signed __int64 ErlDrvSInt; -#else -typedef unsigned long ErlDrvTermData; -typedef unsigned long ErlDrvUInt; -typedef signed long ErlDrvSInt; -#endif -#if defined(__WIN32__) -typedef unsigned __int64 ErlDrvUInt64; -typedef __int64 ErlDrvSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlDrvUInt64; -typedef long ErlDrvSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlDrvUInt64; -typedef long long ErlDrvSInt64; -#else -#error No 64-bit integer type -#endif +typedef ErlNapiUInt64 ErlDrvUInt64; +typedef ErlNapiSInt64 ErlDrvSInt64; +typedef ErlNapiUInt ErlDrvUInt; +typedef ErlNapiSInt ErlDrvSInt; +typedef ErlNapiUInt ErlDrvTermData; #if defined(__WIN32__) || defined(_WIN32) typedef ErlDrvUInt ErlDrvSizeT; @@ -250,6 +193,17 @@ typedef struct { unsigned long microsecs; } ErlDrvNowData; +typedef ErlDrvSInt64 ErlDrvTime; + +#define ERL_DRV_TIME_ERROR ((ErlDrvSInt64) ERTS_NAPI_TIME_ERROR__) + +typedef enum { + ERL_DRV_SEC = ERTS_NAPI_SEC__, + ERL_DRV_MSEC = ERTS_NAPI_MSEC__, + ERL_DRV_USEC = ERTS_NAPI_USEC__, + ERL_DRV_NSEC = ERTS_NAPI_NSEC__ +} ErlDrvTimeUnit; + /* * Error codes that can be return from driver. */ @@ -685,8 +639,16 @@ EXTERN long driver_async(ErlDrvPort ix, EXTERN int driver_lock_driver(ErlDrvPort ix); /* Get the current 'now' timestamp (analogue to erlang:now()) */ -EXTERN int driver_get_now(ErlDrvNowData *now); - +EXTERN int driver_get_now(ErlDrvNowData *now) ERL_DRV_DEPRECATED_FUNC; + +/* Erlang Monotonic Time */ +EXTERN ErlDrvTime erl_drv_monotonic_time(ErlDrvTimeUnit time_unit); +/* Time offset between Erlang Monotonic Time and Erlang System Time */ +EXTERN ErlDrvTime erl_drv_time_offset(ErlDrvTimeUnit time_unit); +/* Time unit conversion */ +EXTERN ErlDrvTime erl_drv_convert_time_unit(ErlDrvTime val, + ErlDrvTimeUnit from, + ErlDrvTimeUnit to); /* These were removed from the ANSI version, now they're back. */ diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index e2385f63f4..f6b946ae82 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -50,6 +50,90 @@ typedef enum { } ErlDrvDirtyJobFlags; #endif +#ifdef SIZEOF_CHAR +# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR +# undef SIZEOF_CHAR +#endif +#ifdef SIZEOF_SHORT +# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT +# undef SIZEOF_SHORT +#endif +#ifdef SIZEOF_INT +# define SIZEOF_INT_SAVED__ SIZEOF_INT +# undef SIZEOF_INT +#endif +#ifdef SIZEOF_LONG +# define SIZEOF_LONG_SAVED__ SIZEOF_LONG +# undef SIZEOF_LONG +#endif +#ifdef SIZEOF_LONG_LONG +# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG +# undef SIZEOF_LONG_LONG +#endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif +#include "erl_int_sizes_config.h" +#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR +# error SIZEOF_CHAR mismatch +#endif +#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT +# error SIZEOF_SHORT mismatch +#endif +#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT +# error SIZEOF_INT mismatch +#endif +#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG +# error SIZEOF_LONG mismatch +#endif +#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG +# error SIZEOF_LONG_LONG mismatch +#endif + +#if !defined(__GNUC__) && (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef unsigned __int64 ErlNapiUInt64; +typedef signed __int64 ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807i64 +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1i64) +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlNapiUInt64; +typedef signed long ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807L +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1L) +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlNapiUInt64; +typedef signed long long ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807LL +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1LL) +#else +# error No 64-bit integer type +#endif + +#if SIZEOF_VOID_P == 8 +typedef ErlNapiUInt64 ErlNapiUInt; +typedef ErlNapiSInt64 ErlNapiSInt; +#elif SIZEOF_VOID_P == 4 +# if SIZEOF_LONG == SIZEOF_VOID_P +typedef unsigned long ErlNapiUInt; +typedef signed long ErlNapiSInt; +# elif SIZEOF_INT == SIZEOF_VOID_P +typedef unsigned int ErlNapiUInt; +typedef signed int ErlNapiSInt; +# else +# error No 32-bit integer type +# endif +#else +# error Not support arch +#endif + +#define ERTS_NAPI_TIME_ERROR__ ERL_NAPI_SINT64_MIN__ + +#define ERTS_NAPI_SEC__ 0 +#define ERTS_NAPI_MSEC__ 1 +#define ERTS_NAPI_USEC__ 2 +#define ERTS_NAPI_NSEC__ 3 + #endif /* __ERL_DRV_NIF_H__ */ diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index e0404eb5c9..184f8e8931 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -43,7 +43,7 @@ fatal_error(int err, char *func) else estr = "Unknown error"; } - erl_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); + erts_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); } #define ERL_DRV_TSD_KEYS_INC 10 diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 4268e2d40a..2f13aa364f 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -185,7 +185,7 @@ erts_erase_fun_entry(ErlFunEntry* fe) #endif { if (fe->address != unloaded_fun) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Internal error: " "Invalid reference count found on #Fun<%T.%d.%d>: " " About to erase fun still referred by code.\n", diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 2f21111a2e..f48c46ca33 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -59,7 +59,7 @@ erts_fprintf(stderr, "stop=%p\n", (p)->stop); \ erts_fprintf(stderr, "htop=%p\n", (p)->htop); \ erts_fprintf(stderr, "heap=%p\n", (p)->heap); \ - erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ __FILE__,__LINE__,(P)->common.id); \ } @@ -268,7 +268,7 @@ erts_next_heap_size(Uint size, Uint offset) low = mid + 1; } } - erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset); + erts_exit(ERTS_ERROR_EXIT, "no next heap size found: %beu, offset %beu\n", size, offset); } return 0; } @@ -2788,7 +2788,7 @@ within(Eterm *ptr, Process *p) #define ERTS_CHK_OFFHEAP_ASSERT(EXP) \ do { \ if (!(EXP)) \ - erl_exit(ERTS_ABORT_EXIT, \ + erts_exit(ERTS_ABORT_EXIT, \ "%s:%d: Assertion failed: %s\n", \ __FILE__, __LINE__, #EXP); \ } while (0) diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index f89f8723d9..9b4aad9d91 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -571,8 +571,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", - __FILE__, __LINE__);; + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", + __FILE__, __LINE__); res = NIL; add_2tup(hpp, szp, &res, am.as, am.gf); diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 51a0d68247..fbfde83dba 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -735,7 +735,10 @@ proc_timeout_common(Process *proc, void *tmr) if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&proc->common.timer, ERTS_PTMR_TIMEDOUT, (erts_aint_t) tmr)) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + erts_aint32_t state; + erts_smp_proc_lock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); + state = erts_smp_atomic32_read_acqb(&proc->state); + erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); if (!(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_EXITING))) erts_schedule_process(proc, state, 0); return 1; @@ -1055,6 +1058,8 @@ create_hl_timer(ErtsSchedulerData *esdp, erts_aint32_t refc; Uint32 roflgs; + ERTS_HLT_HDBG_CHK_SRV(srv); + check_canceled_queue(esdp, srv); ERTS_HLT_ASSERT((esdp->no & ~ERTS_TMR_ROFLG_SID_MASK) == 0); @@ -1179,8 +1184,6 @@ create_hl_timer(ErtsSchedulerData *esdp, erts_smp_atomic32_init_nob(&tmr->head.refc, refc); erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE); - ERTS_HLT_HDBG_CHK_SRV(srv); - if (!srv->next_timeout || tmr->timeout < srv->next_timeout->timeout) { if (srv->next_timeout) @@ -3099,7 +3102,8 @@ tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) & ~ERTS_HLT_PFLGS_MASK); ERTS_HLT_ASSERT(tmr == prnt); } - ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); + if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); if (tmr->time.tree.same_time) { ErtsHdbgHLT st_hdbg; st_hdbg.srv = hdbg->srv; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index d9c3b0dcf4..8b85613e18 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -269,7 +269,7 @@ this_rel_num(void) i++; this_rel = atoi(&this_rel_str[i]); if (this_rel < 1) - erl_exit(-1, "Unexpected ERLANG_OTP_RELEASE format\n"); + erts_exit(1, "Unexpected ERLANG_OTP_RELEASE format\n"); } return this_rel; } @@ -415,7 +415,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); if (erts_find_function(start_mod, am_start, 2, erts_active_code_ix()) == NULL) { - erl_exit(5, "No function %s:start/2\n", modname); + erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname); } /* @@ -512,12 +512,12 @@ load_preloaded(void) length = preload_p[i].size; module_name = erts_atom_put((byte *) name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1); if ((code = sys_preload_begin(&preload_p[i])) == 0) - erl_exit(1, "Failed to find preloaded code for module %s\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to find preloaded code for module %s\n", name); res = erts_preload_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); if (res != NIL) - erl_exit(1,"Failed loading preloaded module %s (%T)\n", + erts_exit(ERTS_ERROR_EXIT,"Failed loading preloaded module %s (%T)\n", name, res); i++; } @@ -648,7 +648,7 @@ void erts_usage(void) erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n"); erts_fprintf(stderr, "\n\n"); - erl_exit(-1, ""); + erts_exit(1, ""); } #ifdef USE_THREADS @@ -1445,7 +1445,7 @@ erl_start(int argc, char **argv) } erts_fprintf(stderr, "(" EMULATOR ") emulator version " ERLANG_VERSION "\n"); - erl_exit(0, ""); + erts_exit(0, ""); } break; @@ -2134,6 +2134,7 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); + sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ @@ -2247,58 +2248,53 @@ system_cleanup(int flush_async) } static __decl_noreturn void __noreturn -erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) +erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { - unsigned int an; - system_cleanup(flush_async); save_statistics(); - if (n < 0) - an = -(unsigned int)n; - else - an = n; if (erts_mtrace_enabled) - erts_mtrace_exit((Uint32) an); + erts_mtrace_exit((Uint32) n); + + if (fmt != NULL && *fmt != '\0') + erl_error(fmt, args2); /* Print error message. */ /* Produce an Erlang core dump if error */ - if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) + if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args2); /* Print error message. */ sys_tty_reset(n); if (n == ERTS_INTR_EXIT) exit(0); else if (n == ERTS_DUMP_EXIT) ERTS_EXIT_AFTER_DUMP(1); - else if (n > 0 || n == ERTS_ABORT_EXIT) + else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT) abort(); - exit(an); + exit(n); } /* Exit without flushing async threads */ -__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 0, fmt, args1, args2); + erts_exit_vv(n, 0, fmt, args1, args2); va_end(args2); va_end(args1); } /* Exit after flushing async threads */ -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 1, fmt, args1, args2); + erts_exit_vv(n, 1, fmt, args1, args2); va_end(args2); va_end(args1); } diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index ff2a355309..3c066cea7b 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1274,7 +1274,7 @@ recurse: break; } default: - erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); } } @@ -1301,7 +1301,7 @@ recurse: break; } default: - erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); } } } @@ -1391,7 +1391,7 @@ resume_from_trap: res = make_boxed(nhp); break; default: - erl_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); + erts_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); } } @@ -1886,7 +1886,7 @@ void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(ERTS_ABORT_EXIT, "bad header"); + erts_exit(ERTS_ABORT_EXIT, "bad header"); } WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ @@ -1923,7 +1923,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) { ASSERT(sz < 17); break; default: - erl_exit(ERTS_ABORT_EXIT, "bad header"); + erts_exit(ERTS_ABORT_EXIT, "bad header"); } idx++; @@ -1973,7 +1973,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) { ASSERT(sz < 17); break; default: - erl_exit(1, "bad header"); + erts_exit(ERTS_ERROR_EXIT, "bad header"); } if (idx > sz) @@ -2161,12 +2161,12 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, size += HAMT_HEAD_BITMAP_SZ(n+1); goto unroll; default: - erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } break; default: - erl_exit(1, "bad primary tag %p\r\n", node); + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); break; } } @@ -2281,12 +2281,12 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, res = make_hashmap(nhp); break; default: - erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } break; default: - erl_exit(1, "bad primary tag %x\r\n", primary_tag(node)); + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %x\r\n", primary_tag(node)); break; } @@ -2404,12 +2404,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { /* not occupied */ goto not_found; default: - erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } break; default: - erl_exit(1, "bad primary tag %p\r\n", node); + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); break; } } @@ -2586,7 +2586,7 @@ unroll: res = make_hashmap(nhp); break; default: - erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } } while(!ESTACK_ISEMPTY(stack)); @@ -2727,7 +2727,7 @@ BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) { case HAMT_SUBTAG_NODE_BITMAP: BIF_RET(AM_hashmap_node); default: - erl_exit(1, "bad header"); + erts_exit(ERTS_ERROR_EXIT, "bad header"); } } BIF_P->fvalue = BIF_ARG_1; @@ -2763,7 +2763,7 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) { ptr += 2; break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); break; } ASSERT(sz < 17); @@ -2841,7 +2841,7 @@ static Eterm hashmap_info(Process *p, Eterm node) { } break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); break; } } diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 7dfa01c8ac..bd899fa2f9 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -356,7 +356,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, tstack[tpos++] = this; this = &((*this)->right); } else { /* Equal key is an error for monitors */ - erl_exit(1,"Insertion of already present monitor!"); + erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!"); break; } } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index d7a2076d85..1097916c97 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -330,7 +330,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #ifdef ERTS_SMP c_p = NULL; #else - erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); + erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); #endif } @@ -1175,6 +1175,26 @@ void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); } int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); } int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); } +ErlNifTime enif_monotonic_time(ErlNifTimeUnit time_unit) +{ + return (ErlNifTime) erts_napi_monotonic_time((int) time_unit); +} + +ErlNifTime enif_time_offset(ErlNifTimeUnit time_unit) +{ + return (ErlNifTime) erts_napi_time_offset((int) time_unit); +} + +ErlNifTime +enif_convert_time_unit(ErlNifTime val, + ErlNifTimeUnit from, + ErlNifTimeUnit to) +{ + return (ErlNifTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val, + (int) from, + (int) to); +} + int enif_fprintf(void* filep, const char* format, ...) { int ret; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 5e39343e9b..75070ad901 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -49,9 +49,10 @@ ** add ErlNifFunc flags ** 2.8: 18.0 add enif_has_pending_exception ** 2.9: 18.2 enif_getenv +** 2.10: Time API */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 9 +#define ERL_NIF_MINOR_VERSION 10 /* * The emulator will refuse to load a nif-lib with a major version @@ -67,63 +68,36 @@ #include <stdlib.h> -#ifdef SIZEOF_CHAR -# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR -# undef SIZEOF_CHAR -#endif -#ifdef SIZEOF_SHORT -# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT -# undef SIZEOF_SHORT -#endif -#ifdef SIZEOF_INT -# define SIZEOF_INT_SAVED__ SIZEOF_INT -# undef SIZEOF_INT -#endif -#ifdef SIZEOF_LONG -# define SIZEOF_LONG_SAVED__ SIZEOF_LONG -# undef SIZEOF_LONG -#endif -#ifdef SIZEOF_LONG_LONG -# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG -# undef SIZEOF_LONG_LONG -#endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif -#include "erl_int_sizes_config.h" - #ifdef __cplusplus extern "C" { #endif -#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) -typedef unsigned __int64 ErlNifUInt64; -typedef __int64 ErlNifSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlNifUInt64; -typedef long ErlNifSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlNifUInt64; -typedef long long ErlNifSInt64; -#else -#error No 64-bit integer type -#endif +typedef ErlNapiUInt64 ErlNifUInt64; +typedef ErlNapiSInt64 ErlNifSInt64; +typedef ErlNapiUInt ErlNifUInt; +typedef ErlNapiSInt ErlNifSInt; #ifdef HALFWORD_HEAP_EMULATOR # define ERL_NIF_VM_VARIANT "beam.halfword" typedef unsigned int ERL_NIF_TERM; #else # define ERL_NIF_VM_VARIANT "beam.vanilla" -# if SIZEOF_LONG == SIZEOF_VOID_P -typedef unsigned long ERL_NIF_TERM; -# elif SIZEOF_LONG_LONG == SIZEOF_VOID_P -typedef unsigned long long ERL_NIF_TERM; -# endif +typedef ErlNifUInt ERL_NIF_TERM; #endif typedef ERL_NIF_TERM ERL_NIF_UINT; +typedef ErlNifSInt64 ErlNifTime; + +#define ERL_NIF_TIME_ERROR ((ErlNifSInt64) ERTS_NAPI_TIME_ERROR__) + +typedef enum { + ERL_NIF_SEC = ERTS_NAPI_SEC__, + ERL_NIF_MSEC = ERTS_NAPI_MSEC__, + ERL_NIF_USEC = ERTS_NAPI_USEC__, + ERL_NIF_NSEC = ERTS_NAPI_NSEC__ +} ErlNifTimeUnit; + struct enif_environment_t; typedef struct enif_environment_t ErlNifEnv; diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 08b9afc6af..1448a508a2 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -160,6 +160,9 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason)); ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* value_size)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_monotonic_time, (ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_time_offset, (ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTimeUnit, ErlNifTimeUnit)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -312,6 +315,9 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); # define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception) # define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception) # define enif_getenv ERL_NIF_API_FUNC_MACRO(enif_getenv) +# define enif_monotonic_time ERL_NIF_API_FUNC_MACRO(enif_monotonic_time) +# define enif_time_offset ERL_NIF_API_FUNC_MACRO(enif_time_offset) +# define enif_convert_time_unit ERL_NIF_API_FUNC_MACRO(enif_convert_time_unit) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 707de39556..e69bd49c42 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1050,7 +1050,7 @@ insert_dist_entry(DistEntry *dist, int type, Eterm id, Uint creation) } if(!rdp) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing distribution table entry found!\n"); insert_dist_referrer(rdp, type, id, creation); @@ -1111,7 +1111,7 @@ insert_node(ErlNode *node, int type, Eterm id) } if (!rnp) - erl_exit(1, "Reference to non-existing node table entry found!\n"); + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing node table entry found!\n"); insert_node_referrer(rnp, type, id); } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 5c38db1cbc..ddcca06b0a 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1784,7 +1784,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds); break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); break; @@ -2048,7 +2048,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) break; } default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index d583118e7b..4ec98d5cc7 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -353,6 +353,7 @@ struct erts_system_monitor_flags_t erts_system_monitor_flags; /* system performance monitor */ Eterm erts_system_profile; struct erts_system_profile_flags_t erts_system_profile_flags; +int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; #if ERTS_MAX_PROCESSES > 0x7fffffff #error "Need to store process_count in another type" @@ -515,7 +516,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; if (~valid & value) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid aux_work value found: 0x%x\n", ~valid & value); } @@ -966,11 +967,25 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsSchedWallTimeReq; +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSystemCheckReq; + + #if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, ErtsSchedWallTimeReq, 5, ERTS_ALC_T_SCHED_WTIME_REQ) + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, + ErtsSystemCheckReq, + 5, + ERTS_ALC_T_SYS_CHECK_REQ) #else static ERTS_INLINE ErtsSchedWallTimeReq * swtreq_alloc(void) @@ -984,6 +999,19 @@ swtreq_free(ErtsSchedWallTimeReq *ptr) { erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); } + +static ERTS_INLINE ErtsSystemCheckReq * +screq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_SYS_CHECK_REQ, + sizeof(ErtsSystemCheckReq)); +} + +static ERTS_INLINE void +screq_free(ErtsSystemCheckReq *ptr) +{ + erts_free(ERTS_ALC_T_SYS_CHECK_REQ, ptr); +} #endif static void @@ -1117,6 +1145,75 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) return ref; } +static void +reply_system_check(void *vscrp) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSystemCheckReq *scrp = (ErtsSystemCheckReq *) vscrp; + ErtsProcLocks rp_locks = (scrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0); + Process *rp = scrp->proc; + Eterm msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif + + sz = REF_THING_SIZE; + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + hpp = &hp; + msg = STORE_NC(hpp, ohp, scrp->ref); + + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (scrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0) + screq_free(vscrp); +} + + +Eterm erts_system_check_request(Process *c_p) { + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSystemCheckReq *scrp; + Eterm *hp; + + scrp = screq_alloc(); + ref = erts_make_ref(c_p); + hp = &scrp->ref_heap[0]; + + scrp->proc = c_p; + scrp->ref = STORE_NC(&hp, NULL, ref); + scrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers); + + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_system_check, + (void *) scrp); +#endif + + reply_system_check((void *) scrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -1211,7 +1308,7 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case 0: break; default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); break; } @@ -2195,7 +2292,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) erts_port_release(prt); } if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } } return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; @@ -2239,6 +2336,7 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) erts_aint32_t aux_work = orig_aux_work; erts_aint32_t ignore = 0; + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #ifdef ERTS_SMP haw_thr_prgr_current_reset(awdp); #endif @@ -2972,14 +3070,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ErtsMonotonicTime current_time; aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp) - && erts_thr_progress_update(esdp)) + if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } @@ -3131,25 +3228,22 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #endif aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (!working) - sched_wall_time_change(esdp, working = 1); + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!working) + sched_wall_time_change(esdp, working = 1); #ifdef ERTS_SMP - if (!thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 1); + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif - } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); #ifdef ERTS_SMP - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work && - erts_thr_progress_update(esdp)) + if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); #endif } #ifndef ERTS_SMP - if (rq->len != 0 || rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) != 0 || rq->misc.start) goto sys_woken; #else flgs = erts_smp_atomic32_read_acqb(&ssi->flags); @@ -3248,7 +3342,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } #ifndef ERTS_SMP - if (rq->len == 0 && !rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) == 0 && !rq->misc.start) goto sys_aux_work; sys_woken: #else @@ -3717,7 +3811,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) prio = ERTS_PORT_PRIO_LEVEL; break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Invalid immigrate queue mask", __FILE__, __LINE__, __func__); prio = 0; @@ -3741,7 +3835,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) rq = erts_port_runq(prt); if (rq) { if (rq != c_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error", __FILE__, __LINE__, __func__); erts_enqueue_port(c_rq, prt); @@ -3932,7 +4026,7 @@ evacuate_run_queue(ErtsRunQueue *rq, prt_rq = erts_port_runq(prt); if (prt_rq) { if (prt_rq != to_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); erts_enqueue_port(to_rq, prt); @@ -4133,7 +4227,7 @@ no_procs: return 0; else { if (prt_rq != rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); *rq_lockedp = 1; @@ -4965,7 +5059,7 @@ erts_fprintf(stderr, "--------------------------------\n"); rq->out_of_work_count = 0; (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); - rq->max_len = rq->len; + rq->max_len = erts_smp_atomic32_read_dirty(&rq->len); for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { ErtsRunQueueInfo *rqi; rqi = (pix == ERTS_PORT_PRIO_LEVEL @@ -5123,7 +5217,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - int left_len = rq->len - 1; + int left_len = erts_smp_atomic32_read_dirty(&rq->len) - 1; if (left_len < 1) { int wo_reduce = wo_reds << wakeup_other.dec_shift; wo_reduce &= wakeup_other.dec_mask; @@ -5196,7 +5290,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - erts_aint32_t len = rq->len; + erts_aint32_t len = erts_smp_atomic32_read_dirty(&rq->len); if (len < 2) { rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; if (rq->wakeup_other < 0) @@ -5292,7 +5386,7 @@ runq_supervisor(void *unused) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) { erts_smp_runq_lock(rq); - if (rq->len != 0) + if (erts_smp_atomic32_read_dirty(&rq->len) != 0) wake_scheduler_on_empty_runq(rq); /* forced wakeup... */ erts_smp_runq_unlock(rq); } @@ -5642,7 +5736,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online } rq->out_of_work_count = 0; rq->max_len = 0; - rq->len = 0; + erts_smp_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; rq->halt_in_progress = 0; @@ -5794,6 +5888,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online init_misc_aux_work(); #if !HALFWORD_HEAP init_swtreq_alloc(); + init_screq_alloc(); #endif erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ @@ -6464,9 +6559,6 @@ suspend_process(Process *c_p, Process *p) if (suspended) { - ASSERT(!(ERTS_PSFLG_RUNNING & state) - || p == erts_get_current_process()); - if (suspended > 0 && erts_system_profile_flags.runnable_procs) { /* 'state' is before our change... */ @@ -6798,18 +6890,19 @@ suspend_scheduler(ErtsSchedulerData *esdp) & ERTS_RUNQ_FLGS_QMASK); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work|qmask) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); + + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && - (aux_work && erts_thr_progress_update(esdp))) - erts_thr_progress_leader_update(esdp); if (qmask) { #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { @@ -7026,17 +7119,18 @@ suspend_scheduler(ErtsSchedulerData *esdp) & ERTS_RUNQ_FLGS_QMASK); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work|qmask) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work && - erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); if (qmask) { erts_smp_runq_lock(esdp->run_queue); evacuate_run_queue(esdp->run_queue, &sbp); @@ -7939,6 +8033,9 @@ sched_thread_func(void *vesdp) erts_sched_init_time_sup(esdp); + (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, + ERTS_RUNQ_FLG_EXEC); + #ifdef ERTS_SMP tse = erts_tse_fetch(); erts_tse_prepare_timed(tse); @@ -8020,7 +8117,7 @@ sched_thread_func(void *vesdp) process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", no); return NULL; @@ -8085,7 +8182,7 @@ sched_dirty_cpu_thread_func(void *vesdp) process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty CPU scheduler thread number %beu terminated\n", no); return NULL; @@ -8148,7 +8245,7 @@ sched_dirty_io_thread_func(void *vesdp) process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty I/O scheduler thread number %beu terminated\n", no); return NULL; @@ -8178,12 +8275,12 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "runq_supervisor"); erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) - erl_exit(1, "Failed to create run-queue supervision event\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); if (0 != ethr_thr_create(&runq_supervisor_tid, runq_supervisor, NULL, &opts)) - erl_exit(1, "Failed to create run-queue supervision thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n"); } #endif @@ -8227,14 +8324,14 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); } } #endif @@ -8250,10 +8347,10 @@ erts_start_schedulers(void) res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); if (res != 0) - erl_exit(1, "Failed to create aux thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n"); if (actual < 1) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Failed to create any scheduler-threads: %s (%d)\n", erl_errno_id(res), res); @@ -8408,9 +8505,8 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, resume_process(rp, rp_locks); } else { - rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, pid_locks|ERTS_PROC_LOCK_STATUS); + pid, ERTS_PROC_LOCK_STATUS); if (!rp) { c_p->flags &= ~F_P2PNR_RESCHED; @@ -8419,40 +8515,84 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); - if (suspend) { - if (suspend_process(c_p, rp)) - goto done; - } - else { - if (!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) - & erts_smp_atomic32_read_acqb(&rp->state))) - goto done; + /* + * Suspend the other process in order to prevent + * it from being selected for normal execution. + * This will however not prevent it from being + * selected for execution of a system task. If + * it is selected for execution of a system task + * we might be blocked for quite a while if the + * try-lock below fails. That is, there is room + * for improvement here... + */ - } + if (!suspend_process(c_p, rp)) { + /* Other process running */ - /* Other process running */ + ASSERT(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state)); - /* - * If we got pending suspenders and suspend ourselves waiting - * to suspend another process we might deadlock. - * In this case we have to yield, be suspended by - * someone else and then do it all over again. - */ - if (!c_p->pending_suspenders) { - /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); - ASSERT(is_nil(c_p->suspendee)); + running: - /* Suspend c_p; when rp is suspended c_p will be resumed. */ - suspend_process(c_p, c_p); - c_p->flags |= F_P2PNR_RESCHED; + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(c_p, c_p); + c_p->flags |= F_P2PNR_RESCHED; + } + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; + } + else { + ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; + if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + if (ERTS_PSFLG_RUNNING_SYS + & erts_smp_atomic32_read_nob(&rp->state)) { + /* Executing system task... */ + resume_process(rp, ERTS_PROC_LOCK_STATUS); + goto running; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + /* + * If we are unlucky, the process just got selected for + * execution of a system task. In this case we may be + * blocked here for quite a while... Execution of system + * tasks are fortunately quite rare events. We try to + * avoid this by checking if it is in a state executing + * system tasks (above), but it will not prevent all + * scenarios for a long block here... + */ + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, pid_locks|ERTS_PROC_LOCK_STATUS); + if (!rp) + goto done; + } + + /* + * The previous suspend has prevented the process + * from being selected for normal execution regardless + * of locks held or not held on it... + */ + ASSERT(!(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state))); + + if (!suspend) + resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); } - /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS); - rp = ERTS_PROC_LOCK_BUSY; } done: + if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); if (unlock_c_p_status) @@ -8947,24 +9087,39 @@ resume_process_1(BIF_ALIST_1) } Uint -erts_run_queues_len(Uint *qlen) +erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) { int i = 0; Uint len = 0; - ERTS_ATOMIC_FOREACH_RUNQ(rq, - { - Sint pqlen = 0; - int pix; - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) - pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len); + if (atomic_queues_read) + ERTS_ATOMIC_FOREACH_RUNQ(rq, + { + Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i++] = rq_len; + len += (Uint) rq_len; + } + ); + else { + for (i = 0; i < erts_no_run_queues; i++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(i); + Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i] = rq_len; + len += (Uint) rq_len; + } - if (pqlen < 0) - pqlen = 0; - if (qlen) - qlen[i++] = pqlen; - len += pqlen; } - ); return len; } @@ -9391,8 +9546,10 @@ Process *schedule(Process *p, int calls) if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); suspend_scheduler(esdp); - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; } if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); @@ -9407,10 +9564,9 @@ Process *schedule(Process *p, int calls) suspend_scheduler(esdp); #endif - { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { erts_aint32_t aux_work; - int leader_update = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 - : erts_thr_progress_update(esdp); + int leader_update = erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); if (aux_work | leader_update | ERTS_SCHED_FAIR) { erts_smp_runq_unlock(rq); @@ -9423,8 +9579,7 @@ Process *schedule(Process *p, int calls) erts_smp_runq_lock(rq); } - ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) - || !erts_thr_progress_is_blocking()); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); } ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -9483,7 +9638,10 @@ Process *schedule(Process *p, int calls) } #endif + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_EXEC); scheduler_wait(&fcalls, esdp, rq); + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; #ifdef ERTS_SMP non_empty_runq(rq); @@ -9649,12 +9807,14 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_ACTIVE_SYS)) == ERTS_PSFLG_SUSPENDED)) { - if (state & ERTS_PSFLG_FREE) - erts_proc_dec_refc(p); if (proxy_p) { free_proxy_proc(proxy_p); proxy_p = NULL; } + else if (state & ERTS_PSFLG_FREE) { + /* free and not queued by proxy */ + erts_proc_dec_refc(p); + } goto pick_next_process; } state = new; @@ -9678,6 +9838,8 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_nob(&p->state); + if (erts_sched_stat.enabled) { int prio; UWord old = ERTS_PROC_SCHED_ID(p, @@ -12092,7 +12254,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) break; default: - erl_exit(1, "bad type in link list\n"); + erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n"); break; } erts_destroy_link(lnk); @@ -12133,7 +12295,7 @@ erts_do_exit_process(Process* p, Eterm reason) #endif if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC) - erl_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", + erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", p->common.id, reason); #ifdef ERTS_SMP @@ -12251,7 +12413,7 @@ erts_continue_exit_process(Process *p) break; case ERTS_SCHDLR_SSPND_EINVAL: default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", __FILE__, __LINE__, (int) ssr); } } @@ -12748,10 +12910,10 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { * The same global atomic is used as refcount. * * A BIF that calls this should make sure to schedule out to never come back: - * erl_halt((int)(- code)); + * erts_halt(code); * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); */ -void erl_halt(int code) +void erts_halt(int code) { if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 10c6fa4a67..b536426b0f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -60,6 +60,9 @@ typedef struct process Process; #include "erl_mseg.h" #include "erl_async.h" #include "erl_gc.h" +#define ERTS_ONLY_INCLUDE_TRACE_FLAGS +#include "erl_trace.h" +#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS #ifdef HIPE #include "hipe_process.h" @@ -170,8 +173,10 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5)) #define ERTS_RUNQ_FLG_PROTECTED \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6)) +#define ERTS_RUNQ_FLG_EXEC \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7)) -#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 7) +#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ @@ -215,6 +220,9 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \ ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags, \ (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags, \ + (erts_aint32_t) (FLGS))) #define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS) \ ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \ (erts_aint32_t) (MSK), \ @@ -222,6 +230,9 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \ ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags, \ (erts_aint32_t) ~(FLGS))) +#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags, \ + (erts_aint32_t) ~(FLGS))) #define ERTS_RUNQ_FLGS_GET(RQ) \ ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags)) #define ERTS_RUNQ_FLGS_GET_NOB(RQ) \ @@ -467,7 +478,7 @@ struct ErtsRunQueue_ { int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE]; int out_of_work_count; erts_aint32_t max_len; - erts_aint32_t len; + erts_smp_atomic32_t len; int wakeup_other; int wakeup_other_reds; int halt_in_progress; @@ -607,7 +618,7 @@ typedef enum { typedef union { struct { ErtsDirtySchedulerType type: 1; - unsigned num: 31; + Uint num: sizeof(Uint)*8 - 1; } s; Uint no; } ErtsDirtySchedId; @@ -728,7 +739,19 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rq->len); + +#ifdef ERTS_SMP + if (len == 0) + erts_non_empty_runq(rq); +#endif + len++; + if (rq->max_len < len) + rq->max_len = len; + ASSERT(len > 0); + erts_smp_atomic32_set_nob(&rq->len, len); + + len = erts_smp_atomic32_read_dirty(&rqi->len); ASSERT(len >= 0); if (len == 0) { ASSERT((erts_smp_atomic32_read_nob(&rq->flags) @@ -741,15 +764,6 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) rqi->max_len = len; erts_smp_atomic32_set_relb(&rqi->len, len); - -#ifdef ERTS_SMP - if (rq->len == 0) - erts_non_empty_runq(rq); -#endif - rq->len++; - if (rq->max_len < rq->len) - rq->max_len = len; - ASSERT(rq->len > 0); } ERTS_GLB_INLINE void @@ -759,7 +773,12 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rq->len); + len--; + ASSERT(len >= 0); + erts_smp_atomic32_set_nob(&rq->len, len); + + len = erts_smp_atomic32_read_dirty(&rqi->len); len--; ASSERT(len >= 0); if (len == 0) { @@ -770,8 +789,6 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) } erts_smp_atomic32_set_relb(&rqi->len, len); - rq->len--; - ASSERT(rq->len >= 0); } ERTS_GLB_INLINE void @@ -781,7 +798,7 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rqi->len); ASSERT(rqi->max_len >= len); rqi->max_len = len; } @@ -1170,7 +1187,7 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \ (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \ - (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) + (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) /* * Static flags that do not change after process creation. @@ -1278,6 +1295,7 @@ struct erts_system_profile_flags_t { unsigned int exclusive : 1; }; extern struct erts_system_profile_flags_t erts_system_profile_flags; +extern int erts_system_profile_ts_type; /* process flags */ #define F_HIBERNATE_SCHED (1 << 0) /* Schedule out after hibernate op */ @@ -1293,61 +1311,90 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ #define F_DISABLE_GC (1 << 11) /* Disable GC */ +#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT 0 + +#define F_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) + /* process trace_flags */ -#define F_SENSITIVE (1 << 0) -#define F_TRACE_SEND (1 << 1) -#define F_TRACE_RECEIVE (1 << 2) -#define F_TRACE_SOS (1 << 3) /* Set on spawn */ -#define F_TRACE_SOS1 (1 << 4) /* Set on first spawn */ -#define F_TRACE_SOL (1 << 5) /* Set on link */ -#define F_TRACE_SOL1 (1 << 6) /* Set on first link */ -#define F_TRACE_CALLS (1 << 7) -#define F_TIMESTAMP (1 << 8) -#define F_TRACE_PROCS (1 << 9) -#define F_TRACE_FIRST_CHILD (1 << 10) -#define F_TRACE_SCHED (1 << 11) -#define F_TRACE_GC (1 << 12) -#define F_TRACE_ARITY_ONLY (1 << 13) -#define F_TRACE_RETURN_TO (1 << 14) /* Return_to trace when breakpoint tracing */ -#define F_TRACE_SILENT (1 << 15) /* No call trace msg suppress */ -#define F_TRACER (1 << 16) /* May be (has been) tracer */ -#define F_EXCEPTION_TRACE (1 << 17) /* May have exception trace on stack */ + +#define F_NOW_TS (ERTS_TRACE_FLG_NOW_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_SENSITIVE F_TRACE_FLAG(0) +#define F_TRACE_SEND F_TRACE_FLAG(1) +#define F_TRACE_RECEIVE F_TRACE_FLAG(2) +#define F_TRACE_SOS F_TRACE_FLAG(3) /* Set on spawn */ +#define F_TRACE_SOS1 F_TRACE_FLAG(4) /* Set on first spawn */ +#define F_TRACE_SOL F_TRACE_FLAG(5) /* Set on link */ +#define F_TRACE_SOL1 F_TRACE_FLAG(6) /* Set on first link */ +#define F_TRACE_CALLS F_TRACE_FLAG(7) +#define F_TRACE_PROCS F_TRACE_FLAG(8) +#define F_TRACE_FIRST_CHILD F_TRACE_FLAG(9) +#define F_TRACE_SCHED F_TRACE_FLAG(10) +#define F_TRACE_GC F_TRACE_FLAG(11) +#define F_TRACE_ARITY_ONLY F_TRACE_FLAG(12) +#define F_TRACE_RETURN_TO F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */ +#define F_TRACE_SILENT F_TRACE_FLAG(14) /* No call trace msg suppress */ +#define F_TRACER F_TRACE_FLAG(15) /* May be (has been) tracer */ +#define F_EXCEPTION_TRACE F_TRACE_FLAG(16) /* May have exception trace on stack */ /* port trace flags, currently the same as process trace flags */ -#define F_TRACE_SCHED_PORTS (1 << 18) /* Trace of port scheduling */ -#define F_TRACE_SCHED_PROCS (1 << 19) /* With virtual scheduling */ -#define F_TRACE_PORTS (1 << 20) /* Ports equivalent to F_TRACE_PROCS */ -#define F_TRACE_SCHED_NO (1 << 21) /* Trace with scheduler id */ -#define F_TRACE_SCHED_EXIT (1 << 22) +#define F_TRACE_SCHED_PORTS F_TRACE_FLAG(17) /* Trace of port scheduling */ +#define F_TRACE_SCHED_PROCS F_TRACE_FLAG(18) /* With virtual scheduling */ +#define F_TRACE_PORTS F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */ +#define F_TRACE_SCHED_NO F_TRACE_FLAG(20) /* Trace with scheduler id */ +#define F_TRACE_SCHED_EXIT F_TRACE_FLAG(21) -#define F_NUM_FLAGS 23 +#define F_NUM_FLAGS (ERTS_TRACE_TS_TYPE_BITS + 22) #ifdef DEBUG # define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS) #else # define F_INITIAL_TRACE_FLAGS 0 #endif +/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */ +#define F_TIMESTAMP_MASK \ + (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) + #define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \ | F_TRACE_SOS | F_TRACE_SOS1| F_TRACE_RECEIVE \ | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \ - | F_TRACE_SCHED | F_TIMESTAMP | F_TRACE_GC \ + | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \ | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \ | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \ | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \ | F_TRACE_SCHED_EXIT) #define ERTS_TRACEE_MODIFIER_FLAGS \ - (F_TRACE_SILENT | F_TIMESTAMP | F_TRACE_SCHED_NO) + (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO) #define ERTS_PORT_TRACEE_FLAGS \ (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS) #define ERTS_PROC_TRACEE_FLAGS \ ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS) +#define SEQ_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) + /* Sequential trace flags */ + +/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */ +#define SEQ_TRACE_TIMESTAMP_MASK \ + (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) + #define SEQ_TRACE_SEND (1 << 0) #define SEQ_TRACE_RECEIVE (1 << 1) #define SEQ_TRACE_PRINT (1 << 2) -#define SEQ_TRACE_TIMESTAMP (1 << 3) + +#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3 + +#define SEQ_TRACE_NOW_TS (ERTS_TRACE_FLG_NOW_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) +#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) +#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) #ifdef USE_VM_PROBES #define DT_UTAG_PERMANENT (1 << 0) @@ -1436,6 +1483,7 @@ void erts_init_scheduling(int, int int erts_set_gc_state(Process *c_p, int enable); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_system_check_request(Process *c_p); Eterm erts_gc_info_request(Process *c_p); Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); @@ -1678,7 +1726,7 @@ void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int, int, int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); -Uint erts_run_queues_len(Uint *); +Uint erts_run_queues_len(Uint *, int, int); void erts_add_to_runq(Process *); Eterm erts_bound_schedulers_term(Process *c_p); Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); @@ -2303,6 +2351,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) #endif -void erl_halt(int code); +void erts_halt(int code); extern erts_smp_atomic32_t erts_halt_progress; extern int erts_halt_code; diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 8606371bdf..3253ee6b70 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -374,7 +374,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) "display term found in line %d:\n" "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during erase/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during erase/1."); } if ((range = HASH_RANGE(p->dictionary)) > INITIAL_SIZE && range / 2 > (p->dictionary->numElements)) { @@ -419,7 +419,7 @@ Eterm erts_pd_hash_get(Process *p, Eterm id) "display term found in line %d:\n" "%T\n", p->common.id, __LINE__, tmp); #endif - erl_exit(1, "Damaged process dictionary found during get/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during get/1."); } return am_undefined; } @@ -670,7 +670,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during put/2."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2."); } if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) { grow(p); @@ -1024,7 +1024,7 @@ static void pd_check(ProcDict *pd) } continue; } else { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Found tag 0x%08x in process dictionary at position %d", (unsigned long) t, (int) i); } diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index a64c993e8f..9c59301086 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -523,6 +523,10 @@ erts_smp_proc_lock__(Process *p, ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_proc_lc_lock(p, locks, file, line); +#endif + old_lflgs = erts_smp_proc_raw_trylock__(p, locks); if (old_lflgs != 0) { @@ -544,9 +548,6 @@ erts_smp_proc_lock__(Process *p, #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line); #endif -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_lock(p, locks, file, line); -#endif #ifdef ERTS_PROC_LOCK_DEBUG erts_proc_lock_op_debug(p, locks, 1); diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index f7997df051..9ed175fe01 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -1255,7 +1255,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp) return 1; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:ptab_list_bif_engine(): Invalid state: %d\n", __FILE__, __LINE__, (int) ptlbdp->state); } diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index e5050bfaa5..bae3385d3f 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -41,7 +41,7 @@ et_abort(const char *expr, const char *file, unsigned line) * Prevent infinite loop. */ have_been_called = 1; - erl_exit(1, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); + erts_exit(ERTS_ERROR_EXIT, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); } #else erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 7148b756e7..232a2d0fdc 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -502,7 +502,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -524,7 +524,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) #endif ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->unmanaged.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many unmanaged registered threads\n", __FILE__, __LINE__, __func__); @@ -547,7 +547,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -568,7 +568,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id); ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many managed registered threads\n", __FILE__, __LINE__, __func__); @@ -700,6 +700,7 @@ leader_update(ErtsThrPrgrData *tpd) tpd->leader_state.chk_next_ix = no_managed; erts_atomic32_set_nob(&intrnl->misc.data.umrefc_ix.current, (erts_aint32_t) new_umrefc_ix); + tpd->leader_state.umrefc_ix.current = new_umrefc_ix; ETHR_MEMBAR(ETHR_StoreLoad); refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); ASSERT(refc >= 0); @@ -1033,7 +1034,7 @@ has_reached_wakeup(ErtsThrPrgrVal wakeup) limit += 1; if (!erts_thr_progress_has_passed__(limit, wakeup)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid wakeup request value found:" " current=%b64u, wakeup=%b64u, limit=%b64u", current, wakeup, limit); @@ -1102,7 +1103,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) ix = erts_atomic32_inc_read_nob(&mwd->len) - 1; #if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE if (ix >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); + erts_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); #endif mwd->id[ix] = tpd->id; diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index 3a91ca9dbe..7ff456b915 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -224,7 +224,7 @@ ErtsThrQCleanState_t erts_thr_q_destroy(ErtsThrQ_t *q) { if (!q->q.blk) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Trying to destroy not created thread queue\n"); return erts_thr_q_finalize(q); } @@ -589,7 +589,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) #if ERTS_THR_Q_DBG_CHK_DATA if (!data) - erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); #endif ASSERT(!q->q.finalizing); @@ -771,7 +771,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) #if ERTS_THR_Q_DBG_CHK_DATA head->data.ptr = NULL; if (!res) - erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); #endif clean(q, (q->head.deq_fini.automatic diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 43e543e035..93a0d556bf 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -133,6 +133,10 @@ typedef struct { extern ErtsTimeSupData erts_time_sup__; +ErtsMonotonicTime erts_napi_monotonic_time(int time_unit); +ErtsMonotonicTime erts_napi_time_offset(int time_unit); +ErtsMonotonicTime erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to); + ERTS_GLB_INLINE Uint64 erts_time_unit_conversion(Uint64 value, Uint32 from_time_unit, diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 7327e0b48c..99005356ed 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -33,6 +33,8 @@ #include "global.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" +#include "erl_driver.h" +#include "erl_nif.h" static erts_smp_mtx_t erts_timeofday_mtx; static erts_smp_mtx_t erts_get_time_mtx; @@ -57,6 +59,7 @@ static int time_sup_initialized = 0; #define ERTS_MONOTONIC_TIME_TERA \ (ERTS_MONOTONIC_TIME_GIGA*ERTS_MONOTONIC_TIME_KILO) +static void init_time_napi(void); static void schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset); @@ -286,7 +289,7 @@ read_corrected_time(int os_drift_corrected) ci = time_sup.inf.c.parmon.cdata.insts.curr; else { if (os_mtime < time_sup.inf.c.parmon.cdata.insts.prev.os_mtime) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "OS monotonic time stepped backwards\n"); ci = time_sup.inf.c.parmon.cdata.insts.prev; } @@ -378,7 +381,7 @@ check_time_correction(void *vesdp) erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); if (os_mtime < ci.os_mtime) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "OS monotonic time stepped backwards\n"); erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, &mdiff, @@ -795,7 +798,7 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep) erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); if (os_mtime < ci.os_mtime) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "OS monotonic time stepped backwards\n"); return calc_corrected_erl_mtime(os_mtime, &ci, NULL, @@ -948,6 +951,8 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) ErtsMonotonicTime abs_native_offset, native_offset; #endif + init_time_napi(); + erts_hl_timer_init(); ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX); @@ -1747,6 +1752,39 @@ erts_get_monotonic_time(ErtsSchedulerData *esdp) return mtime; } +ErtsMonotonicTime +erts_get_time_offset(void) +{ + return get_time_offset(); +} + +static ERTS_INLINE void +make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset) +{ + ErtsMonotonicTime stime, as; + Uint ms; + + stime = ERTS_MONOTONIC_TO_USEC(mtime + offset); + + as = stime / ERTS_MONOTONIC_TIME_MEGA; + *megasec = ms = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA); + *sec = (Uint) (as - (((ErtsMonotonicTime) ms) + * ERTS_MONOTONIC_TIME_MEGA)); + *microsec = (Uint) (stime - as*ERTS_MONOTONIC_TIME_MEGA); + + ASSERT(((ErtsMonotonicTime) ms)*ERTS_MONOTONIC_TIME_TERA + + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA + + *microsec == stime); +} + +void +erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset) +{ + make_timestamp_value(megasec, sec, microsec, mtime, offset); +} + void get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) { @@ -2164,6 +2202,146 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto return ret; } + +/* + * Time Native API (drivers and NIFs) + */ + +#define ERTS_NAPI_TIME_ERROR ((ErtsMonotonicTime) ERTS_NAPI_TIME_ERROR__) + +static void +init_time_napi(void) +{ + /* Verify that time native api constants are as expected... */ + + ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlDrvTime)); + ASSERT(ERL_DRV_TIME_ERROR == (ErlDrvTime) ERTS_NAPI_TIME_ERROR); + ASSERT(ERL_DRV_TIME_ERROR < (ErlDrvTime) 0); + ASSERT(ERTS_NAPI_SEC__ == (int) ERL_DRV_SEC); + ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_DRV_MSEC); + ASSERT(ERTS_NAPI_USEC__ == (int) ERL_DRV_USEC); + ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_DRV_NSEC); + + ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlNifTime)); + ASSERT(ERL_NIF_TIME_ERROR == (ErlNifTime) ERTS_NAPI_TIME_ERROR); + ASSERT(ERL_NIF_TIME_ERROR < (ErlNifTime) 0); + ASSERT(ERTS_NAPI_SEC__ == (int) ERL_NIF_SEC); + ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_NIF_MSEC); + ASSERT(ERTS_NAPI_USEC__ == (int) ERL_NIF_USEC); + ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_NIF_NSEC); +} + +ErtsMonotonicTime +erts_napi_monotonic_time(int time_unit) +{ + ErtsSchedulerData *esdp; + ErtsMonotonicTime mtime; + + /* At least for now only allow schedulers to do this... */ + esdp = erts_get_scheduler_data(); + if (!esdp) + return ERTS_NAPI_TIME_ERROR; + + mtime = time_sup.r.o.get_time(); + update_last_mtime(esdp, mtime); + + switch (time_unit) { + case ERTS_NAPI_SEC__: + mtime = ERTS_MONOTONIC_TO_SEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_SEC; + break; + case ERTS_NAPI_MSEC__: + mtime = ERTS_MONOTONIC_TO_MSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_MSEC; + break; + case ERTS_NAPI_USEC__: + mtime = ERTS_MONOTONIC_TO_USEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_USEC; + break; + case ERTS_NAPI_NSEC__: + mtime = ERTS_MONOTONIC_TO_NSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_NSEC; + break; + default: + return ERTS_NAPI_TIME_ERROR; + } + + return mtime; +} + +ErtsMonotonicTime +erts_napi_time_offset(int time_unit) +{ + ErtsSchedulerData *esdp; + ErtsSystemTime offs; + + /* At least for now only allow schedulers to do this... */ + esdp = erts_get_scheduler_data(); + if (!esdp) + return ERTS_NAPI_TIME_ERROR; + + offs = get_time_offset(); + switch (time_unit) { + case ERTS_NAPI_SEC__: + offs = ERTS_MONOTONIC_TO_SEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_SEC; + break; + case ERTS_NAPI_MSEC__: + offs = ERTS_MONOTONIC_TO_MSEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_MSEC; + break; + case ERTS_NAPI_USEC__: + offs = ERTS_MONOTONIC_TO_USEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_USEC; + break; + case ERTS_NAPI_NSEC__: + offs = ERTS_MONOTONIC_TO_NSEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_NSEC; + break; + default: + return ERTS_NAPI_TIME_ERROR; + } + return offs; +} + +ErtsMonotonicTime +erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to) +{ + ErtsMonotonicTime ffreq, tfreq, denom; + /* + * Convertion between time units using floor function. + * + * Note that this needs to work also for negative + * values. Ordinary integer division on a negative + * value will give ceiling... + */ + + switch ((int) from) { + case ERTS_NAPI_SEC__: ffreq = 1; break; + case ERTS_NAPI_MSEC__: ffreq = 1000; break; + case ERTS_NAPI_USEC__: ffreq = 1000*1000; break; + case ERTS_NAPI_NSEC__: ffreq = 1000*1000*1000; break; + default: return ERTS_NAPI_TIME_ERROR; + } + + switch ((int) to) { + case ERTS_NAPI_SEC__: tfreq = 1; break; + case ERTS_NAPI_MSEC__: tfreq = 1000; break; + case ERTS_NAPI_USEC__: tfreq = 1000*1000; break; + case ERTS_NAPI_NSEC__: tfreq = 1000*1000*1000; break; + default: return ERTS_NAPI_TIME_ERROR; + } + + if (tfreq >= ffreq) + return val * (tfreq / ffreq); + + denom = ffreq / tfreq; + if (val >= 0) + return val / denom; + + return (val - (denom - 1)) / denom; +} + /* Built in functions */ BIF_RETTYPE monotonic_time_0(BIF_ALIST_0) @@ -2220,22 +2398,14 @@ BIF_RETTYPE time_offset_1(BIF_ALIST_1) BIF_RETTYPE timestamp_0(BIF_ALIST_0) { Eterm *hp, res; - ErtsMonotonicTime stime, mtime, all_sec, offset; + ErtsMonotonicTime mtime, offset; Uint mega_sec, sec, micro_sec; mtime = time_sup.r.o.get_time(); offset = get_time_offset(); update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); - stime = ERTS_MONOTONIC_TO_USEC(mtime + offset); - all_sec = stime / ERTS_MONOTONIC_TIME_MEGA; - mega_sec = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA); - sec = (Uint) (all_sec - (((ErtsMonotonicTime) mega_sec) - * ERTS_MONOTONIC_TIME_MEGA)); - micro_sec = (Uint) (stime - all_sec*ERTS_MONOTONIC_TIME_MEGA); - - ASSERT(((ErtsMonotonicTime) mega_sec)*ERTS_MONOTONIC_TIME_TERA - + ((ErtsMonotonicTime) sec)*ERTS_MONOTONIC_TIME_MEGA - + micro_sec == stime); + + make_timestamp_value(&mega_sec, &sec, µ_sec, mtime, offset); /* * Mega seconds is the only value that potentially diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index e1b03a057f..114853cac4 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -38,6 +38,7 @@ #include "erl_binary.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #if 0 #define DEBUG_PRINTOUTS @@ -77,6 +78,264 @@ enum ErtsSysMsgType { SYS_MSG_TYPE_SYSPROF }; +#define ERTS_TRACE_TS_NOW_MAX_SIZE \ + 4 +#define ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + ERTS_MAX_SINT64_HEAP_SIZE +#define ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE \ + (3 + ERTS_MAX_SINT64_HEAP_SIZE \ + + ERTS_MAX_UINT64_HEAP_SIZE) + +#define ERTS_TRACE_PATCH_TS_MAX_SIZE \ + (1 + ((ERTS_TRACE_TS_NOW_MAX_SIZE \ + > ERTS_TRACE_TS_MONOTONIC_MAX_SIZE) \ + ? ((ERTS_TRACE_TS_NOW_MAX_SIZE \ + > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + ? ERTS_TRACE_TS_NOW_MAX_SIZE \ + : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + : ((ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + ? ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE))) + +#define TFLGS_TS_TYPE(p) ERTS_TFLGS2TSTYPE(ERTS_TRACE_FLAGS((p))) + +/* + * FUTURE CHANGES: + * + * The timestamp functionality has intentionally been + * split in two parts for future use even though it + * is not used like this today. take_timestamp() takes + * the timestamp and calculate heap need for it (which + * is not constant). write_timestamp() writes the + * timestamp to the allocated heap. That is, one typically + * want to take the timestamp before allocating the heap + * and then write it to the heap. + * + * The trace output functionality now use patch_ts_size(), + * write_ts(), and patch_ts(). write_ts() both takes the + * timestamp and writes it. Since we don't know the + * heap need when allocating the heap area we need to + * over allocate (maximum size from patch_ts_size()) and + * then potentially (often) shrink the heap area after the + * timestamp has been written. The only reason it is + * currently done this way is because we do not want to + * make major changes of the trace behavior in a patch. + * This is planned to be changed in next major release. + */ + +typedef struct { + int ts_type_flag; + union { + struct { + Uint ms; + Uint s; + Uint us; + } now; + struct { + ErtsMonotonicTime time; + Sint64 raw_unique; + } monotonic; + } u; +} ErtsTraceTimeStamp; + +static ERTS_INLINE Uint +take_timestamp(ErtsTraceTimeStamp *tsp, int ts_type) +{ + int ts_type_flag = ts_type & -ts_type; /* least significant flag */ + + ASSERT(ts_type_flag == ERTS_TRACE_FLG_NOW_TIMESTAMP + || ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP + || ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP + || ts_type_flag == 0); + + tsp->ts_type_flag = ts_type_flag; + switch (ts_type_flag) { + case 0: + return (Uint) 0; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: +#ifdef HAVE_ERTS_NOW_CPU + if (erts_cpu_timestamp) + erts_get_now_cpu(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us); + else +#endif + get_now(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us); + return (Uint) 4; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: { + Uint hsz = 0; + ErtsMonotonicTime mtime = erts_get_monotonic_time(NULL); + mtime = ERTS_MONOTONIC_TO_NSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_NSEC; + hsz = (IS_SSMALL(mtime) ? + (Uint) 0 + : ERTS_SINT64_HEAP_SIZE((Sint64) mtime)); + tsp->u.monotonic.time = mtime; + if (ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP) { + Sint64 raw_unique; + hsz += 3; /* 2-tuple */ + raw_unique = erts_raw_get_unique_monotonic_integer(); + tsp->u.monotonic.raw_unique = raw_unique; + hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique); + } + return hsz; + } + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return 0; + } +} + +static ERTS_INLINE Eterm +write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp) +{ + int ts_type_flag = tsp->ts_type_flag; + Eterm res; + + switch (ts_type_flag) { + case 0: + return NIL; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: + res = TUPLE3(*hpp, + make_small(tsp->u.now.ms), + make_small(tsp->u.now.s), + make_small(tsp->u.now.us)); + *hpp += 4; + return res; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: { + Sint64 mtime, raw; + Eterm unique, emtime; + + mtime = (Sint64) tsp->u.monotonic.time; + emtime = (IS_SSMALL(mtime) + ? make_small((Sint64) mtime) + : erts_sint64_to_big((Sint64) mtime, hpp)); + + if (ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP) + return emtime; + + raw = tsp->u.monotonic.raw_unique; + unique = erts_raw_make_unique_monotonic_integer_value(hpp, + raw); + res = TUPLE2(*hpp, emtime, unique); + *hpp += 3; + return res; + } + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return THE_NON_VALUE; + } +} + +#define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p)) + +static ERTS_INLINE Uint +patch_ts_size(int ts_type) +{ + int ts_type_flag = ts_type & -ts_type; /* least significant flag */ + switch (ts_type_flag) { + case 0: + return 0; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: + return 1 + ERTS_TRACE_TS_NOW_MAX_SIZE; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + return 1 + ERTS_TRACE_TS_MONOTONIC_MAX_SIZE; + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: + return 1 + ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE; + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return 0; + } +} + +/* + * Write a timestamp. The timestamp MUST be the last + * thing built on the heap. This since write_ts() might + * adjust the size of the used area. + */ +static Eterm +write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer) +{ + ErtsTraceTimeStamp ts; + Sint shrink; + Eterm res, *ts_hp = hp; + Uint hsz; + + ASSERT(ts_type); + + hsz = take_timestamp(&ts, ts_type); + + res = write_timestamp(&ts, &ts_hp); + + ASSERT(ts_hp == hp + hsz); + + switch (ts.ts_type_flag) { + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + shrink = ERTS_TRACE_TS_MONOTONIC_MAX_SIZE; + break; + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: + shrink = ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE; + break; + default: + return res; + } + + shrink -= hsz; + + ASSERT(shrink >= 0); + + if (shrink) { + if (bp) + bp->used_size -= shrink; +#ifndef ERTS_SMP + else if (tracer) { + Eterm *endp = ts_hp + shrink; + HRelease(tracer, endp, ts_hp); + } +#endif + } + + return res; +} + +/* + * Patch a timestamp into a tuple. The tuple MUST be the last thing + * built on the heap before the call, and the timestamp MUST be + * the last thing after the call. This since patch_ts() might adjust + * the size of the used area. + */ + +#define PATCH_TS__(Type, Tuple, Hp, Bp, Tracer) \ + do { \ + int ts_type__ = (Type); \ + if (ts_type__) \ + patch_ts(ts_type__, (Tuple), (Hp), (Bp), (Tracer)); \ + } while (0) + +#ifdef ERTS_SMP +#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ + PATCH_TS__((Type), (Tuple), (Hp), (Bp), NULL) +#else +#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ + PATCH_TS__((Type), (Tuple), (Hp), (Bp), (Tracer)) +#endif + +static ERTS_INLINE void +patch_ts(int ts_type, Eterm tuple, Eterm* hp, ErlHeapFragment *bp, Process *tracer) +{ + Eterm *tptr = tuple_val(tuple); + int arity = arityval(*tptr); + + ASSERT(ts_type); + ASSERT((tptr+arity+1) == hp); + + tptr[0] = make_arityval(arity+1); + tptr[1] = am_trace_ts; + + *hp = write_ts(ts_type, hp+1, bp, tracer); +} + #ifdef ERTS_SMP static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type, Eterm from, @@ -365,23 +624,6 @@ erts_get_system_profile(void) { return profile; } - -#ifdef HAVE_ERTS_NOW_CPU -# define GET_NOW(m, s, u) \ -do { \ - if (erts_cpu_timestamp) \ - erts_get_now_cpu(m, s, u); \ - else \ - get_now(m, s, u); \ -} while (0) -#else -# define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0) -#endif - - - -static Eterm* patch_ts(Eterm tuple4, Eterm* hp); - #ifdef ERTS_SMP static void do_send_to_port(Eterm to, @@ -420,7 +662,7 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, erts_encode_ext(message, &ptr); if (!(ptr <= buffer+size)) { - erl_exit(1, "Internal error in do_send_to_port: %d\n", ptr-buffer); + erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer); } #ifndef ERTS_SMP @@ -436,11 +678,11 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, /* Send {trace_ts, Pid, out, 0, Timestamp} * followed by {trace_ts, Pid, in, 0, NewTimestamp} * - * 'NewTimestamp' is fetched from GET_NOW() through patch_ts(). + * 'NewTimestamp' through patch_ts(). */ static void -do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { -#define LOCAL_HEAP_SIZE (4+5+5) +do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp, int ts_type) { +#define LOCAL_HEAP_SIZE (5+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); Eterm message; Eterm *hp; @@ -462,9 +704,11 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { SYS_MSG_TYPE_UNDEFINED, message); - message = TUPLE4(hp, am_trace_ts, pid, am_in, mfarity); - hp += 5; - hp = patch_ts(message, hp); + + message = TUPLE5(hp, am_trace_ts, pid, am_in, mfarity, + NIL /* Will be overwritten by timestamp */); + hp += 6; + hp[-1] = write_ts(ts_type, hp, NULL, NULL); do_send_to_port(trace_port->common.id, trace_port, @@ -481,7 +725,7 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { * It is assumed that 'message' is not an 'out' message. * * 'c_p' is the currently executing process, "tracee" is the traced process - * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP), + * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP_MASK), * 'message' must contain a timestamp. */ static void @@ -489,8 +733,9 @@ send_to_port(Process *c_p, Eterm message, Eterm *tracer_pid, Uint *tracee_flags) { Port* trace_port; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4) - Eterm ts, *hp; + int ts_type; +#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE + Eterm ts; DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); #endif @@ -519,7 +764,7 @@ send_to_port(Process *c_p, Eterm message, */ if ( c_p == NULL || - (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) { + (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { #endif do_send_to_port(*tracer_pid, trace_port, @@ -538,22 +783,12 @@ send_to_port(Process *c_p, Eterm message, */ UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - if (*tracee_flags & F_TIMESTAMP) { - ASSERT(is_tuple(message)); - hp = tuple_val(message); - ts = hp[arityval(hp[0])]; - } else { - /* A fake schedule might be needed, - * but this message does not contain a timestamp. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - Uint ms,s,us; - GET_NOW(&ms, &s, &us); - hp = local_heap; - ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); - hp += 4; - } + /* A fake schedule might be needed. + * Create a dummy trace message with timestamp to be + * passed to do_send_schedfix_to_port(). + */ + ts_type = TFLGS_TS_TYPE(c_p); + ts = write_ts(ts_type, local_heap, NULL, NULL); trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(*tracer_pid, @@ -572,7 +807,7 @@ send_to_port(Process *c_p, Eterm message, * just after writning the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); } erts_port_release(trace_port); @@ -641,20 +876,19 @@ profile_send(Eterm from, Eterm message) { /* A fake schedule out/in message pair will be sent, * if the driver so requests. - * If (timestamp == NIL), one is fetched from GET_NOW(). * * 'c_p' is the currently executing process, may be NULL. */ static void seq_trace_send_to_port(Process *c_p, Eterm seq_tracer, - Eterm message, - Eterm timestamp) + Eterm message) { Port* trace_port; #ifndef ERTS_SMP - Eterm ts, *hp; -#define LOCAL_HEAP_SIZE (4) + int ts_type; + Eterm ts; +#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); #endif @@ -679,7 +913,7 @@ seq_trace_send_to_port(Process *c_p, } if (c_p == NULL - || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) { + || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { #endif do_send_to_port(seq_tracer, trace_port, @@ -696,20 +930,12 @@ seq_trace_send_to_port(Process *c_p, * with 'running' and 'timestamp'. */ - if (timestamp != NIL) { - ts = timestamp; - } else { - /* A fake schedule might be needed, - * but this message does not contain a timestamp. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - Uint ms,s,us; - GET_NOW(&ms, &s, &us); - hp = local_heap; - ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); - hp += 4; - } + /* A fake schedule might be needed. + * Create a dummy trace message with timestamp to be + * passed to do_send_schedfix_to_port(). + */ + ts_type = TFLGS_TS_TYPE(c_p); + ts = write_ts(ts_type, local_heap, NULL, NULL); trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(seq_tracer, @@ -728,7 +954,7 @@ seq_trace_send_to_port(Process *c_p, * just after writing the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); } erts_port_release(trace_port); @@ -738,32 +964,6 @@ seq_trace_send_to_port(Process *c_p, #endif } -#define TS_HEAP_WORDS 5 -#define TS_SIZE(p) ((ERTS_TRACE_FLAGS((p)) & F_TIMESTAMP) \ - ? TS_HEAP_WORDS \ - : 0) - -/* - * Patch a timestamp into a tuple. The tuple must be the last thing - * built on the heap. - * - * Returns the new hp pointer. -*/ -static Eterm* -patch_ts(Eterm tuple, Eterm* hp) -{ - Uint ms, s, us; - Eterm* ptr = tuple_val(tuple); - int arity = arityval(*ptr); - - ASSERT((ptr+arity+1) == hp); - ptr[0] = make_arityval(arity+1); - ptr[1] = am_trace_ts; - GET_NOW(&ms, &s, &us); - *hp = TUPLE3(hp+1, make_small(ms), make_small(s), make_small(us)); - return hp+5; -} - static ERTS_INLINE void send_to_tracer(Process *tracee, ERTS_TRACER_REF_TYPE tracer_ref, @@ -776,13 +976,13 @@ send_to_tracer(Process *tracee, erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(tracee) & F_TIMESTAMP) - *hpp = patch_ts(msg, *hpp); - - if (is_internal_pid(ERTS_TRACER_PROC(tracee))) + if (is_internal_pid(ERTS_TRACER_PROC(tracee))) { + PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee->common.id, tracer_ref, msg, bp); + } else { ASSERT(is_internal_port(ERTS_TRACER_PROC(tracee))); + PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, NULL, NULL); send_to_port(no_fake_sched ? NULL : tracee, msg, &ERTS_TRACER_PROC(tracee), @@ -796,7 +996,7 @@ send_to_tracer(Process *tracee, static void trace_sched_aux(Process *p, Eterm what, int never_fake_sched) { -#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS) +#define LOCAL_HEAP_SIZE (5+4+1+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); Eterm tmp, mess, *hp; ErlHeapFragment *bp = NULL; @@ -852,7 +1052,7 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) size += 4; if (sched_no) size += 1; - size += TS_SIZE(p); + size += PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); } @@ -926,7 +1126,7 @@ trace_send(Process *p, Eterm to, Eterm msg) } if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (11) +#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -934,9 +1134,7 @@ trace_send(Process *p, Eterm to, Eterm msg) mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -955,7 +1153,7 @@ trace_send(Process *p, Eterm to, Eterm msg) sz_msg = size_object(msg); sz_to = size_object(to); - need = sz_msg + sz_to + 6 + TS_SIZE(p); + need = sz_msg + sz_to + 6 + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -972,10 +1170,7 @@ trace_send(Process *p, Eterm to, Eterm msg) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -992,7 +1187,7 @@ trace_receive(Process *rp, Eterm msg) Eterm* hp; if (is_internal_port(ERTS_TRACER_PROC(rp))) { -#define LOCAL_HEAP_SIZE (10) +#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1000,9 +1195,7 @@ trace_receive(Process *rp, Eterm msg) mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, NULL, NULL); send_to_port(rp, mess, &ERTS_TRACER_PROC(rp), &ERTS_TRACE_FLAGS(rp)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1021,7 +1214,7 @@ trace_receive(Process *rp, Eterm msg) sz_msg = size_object(msg); - hsz = sz_msg + 5 + TS_SIZE(rp); + hsz = sz_msg + 5 + PATCH_TS_SIZE(rp); hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref); @@ -1031,10 +1224,7 @@ trace_receive(Process *rp, Eterm msg) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { - patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(rp->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -1084,6 +1274,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, Eterm type_atom; int sz_exit; Eterm seq_tracer; + int ts_type; seq_tracer = erts_get_system_seq_tracer(); @@ -1098,7 +1289,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, case SEQ_TRACE_PRINT: type_atom = am_print; break; case SEQ_TRACE_RECEIVE: type_atom = am_receive; break; default: - erl_exit(1, "invalid type in seq_trace_output_generic: %d:\n", type); + erts_exit(ERTS_ERROR_EXIT, "invalid type in seq_trace_output_generic: %d:\n", type); return; /* To avoid warning */ } @@ -1111,8 +1302,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, return; /* no need to send anything */ } + ts_type = ERTS_SEQTFLGS2TSTYPE(unsigned_val(SEQ_TRACE_T_FLAGS(token))); + if (is_internal_port(seq_tracer)) { -#define LOCAL_HEAP_SIZE (64) +#define LOCAL_HEAP_SIZE (60 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1128,17 +1321,17 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), receiver, msg); hp += 6; + erts_smp_mtx_lock(&smq_mtx); - if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) == 0) { + if (!ts_type) { mess = TUPLE3(hp, am_seq_trace, label, mess); - seq_trace_send_to_port(NULL, seq_tracer, mess, NIL); + seq_trace_send_to_port(NULL, seq_tracer, mess); } else { - Uint ms,s,us,ts; - GET_NOW(&ms, &s, &us); - ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us)); - hp += 4; - mess = TUPLE4(hp, am_seq_trace, label, mess, ts); - seq_trace_send_to_port(process, seq_tracer, mess, ts); + mess = TUPLE4(hp, am_seq_trace, label, mess, + NIL /* Will be overwritten by timestamp */); + hp += 5; + hp[-1] = write_ts(ts_type, hp, NULL, NULL); + seq_trace_send_to_port(process, seq_tracer, mess); } UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1173,8 +1366,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, sz_lastcnt_serial = 3; /* TUPLE2 */ sz_msg = size_object(msg); - sz_ts = ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) ? - 5 : 0); + sz_ts = patch_ts_size(ts_type); if (exitfrom != NIL) { sz_exit = 4; /* create {'EXIT',exitfrom,msg} */ sz_exitfrom = size_object(exitfrom); @@ -1218,14 +1410,20 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, erts_smp_mtx_lock(&smq_mtx); - if (sz_ts) {/* timestamp should be included */ - Uint ms,s,us,ts; - GET_NOW(&ms, &s, &us); - ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us)); - hp += 4; - mess = TUPLE4(hp, am_seq_trace, label, mess, ts); - } else { + if (!ts_type) mess = TUPLE3(hp, am_seq_trace, label, mess); + else { + mess = TUPLE4(hp, am_seq_trace, label, mess, + NIL /* Will be overwritten by timestamp */); + hp += 5; + /* Write timestamp in element 6 of the 'msg' tuple */ + hp[-1] = write_ts(ts_type, hp, bp, +#ifndef ERTS_SMP + tracer +#else + NULL +#endif + ); } #ifdef ERTS_SMP @@ -1244,7 +1442,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, void erts_trace_return_to(Process *p, BeamInstr *pc) { -#define LOCAL_HEAP_SIZE (4+5+5) +#define LOCAL_HEAP_SIZE (4+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) Eterm* hp; Eterm mfa; Eterm mess; @@ -1269,9 +1467,7 @@ erts_trace_return_to(Process *p, BeamInstr *pc) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); if (is_internal_port(ERTS_TRACER_PROC(p))) { send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); @@ -1318,6 +1514,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) Eterm mod, name; int arity; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1353,7 +1550,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) * meta trace => * use fixed flag set instead of process flags */ - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; @@ -1367,8 +1564,10 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) name = fi[1]; arity = fi[2]; + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+6+5) +#define LOCAL_HEAP_SIZE (4+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; @@ -1377,9 +1576,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1390,24 +1587,15 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) ERTS_TRACER_REF_TYPE tracer_ref; unsigned size; unsigned retval_size; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); - + retval_size = size_object(retval); - size = 6 + 4 + retval_size; - if (*tracee_flags & F_TIMESTAMP) { - size += 1+4; - } + size = 6 + 4 + retval_size + patch_ts_size(ts_type); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the trace tuple and put it into receive queue of the tracer process. @@ -1421,11 +1609,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -1448,6 +1632,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, Eterm cv; Eterm mess; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1486,15 +1671,17 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, * meta trace => * use fixed flag set instead of process flags */ - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; #endif } + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+3+6+5) +#define LOCAL_HEAP_SIZE (4+3+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1507,10 +1694,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, hp += 6; ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE); erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); /* hp += 5 */ - ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1521,24 +1705,15 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, ERTS_TRACER_REF_TYPE tracer_ref; unsigned size; unsigned value_size; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); value_size = size_object(value); - size = 6 + 4 + 3 + value_size; - if (*tracee_flags & F_TIMESTAMP) { - size += 1+4; - } + size = 6 + 4 + 3 + value_size + patch_ts_size(ts_type); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the trace tuple and put it into receive queue of the tracer process. @@ -1555,11 +1730,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -1592,6 +1763,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm pam_result = am_true; Eterm mess; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1633,7 +1805,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* No trace messages for sensitive processes. */ return 0; } - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; @@ -1676,12 +1848,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } args = transformed_args; + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { #if HEAP_ON_C_STACK - Eterm local_heap[64+MAX_ARG]; + Eterm local_heap[60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG]; #else Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*(64+MAX_ARG)); + sizeof(Eterm)*(60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG)); #endif hp = local_heap; @@ -1796,9 +1970,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, *hp++ = pam_result; } erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); erts_smp_mtx_unlock(&smq_mtx); erts_match_set_release_result(p); @@ -1820,9 +1992,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, unsigned sizes[MAX_ARG]; unsigned pam_result_size = 0; int invalid_tracer; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); @@ -1915,10 +2084,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, size += sizes[i]; } } - if (*tracee_flags & F_TIMESTAMP) { - size += 1 + 4; - /* One element in trace tuple + timestamp tuple. */ - } + size += patch_ts_size(ts_type); if (pam_result != am_true) { pam_result_size = size_object(pam_result); size += 1 + pam_result_size; @@ -1926,9 +2092,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the the {M,F,A} tuple in the message buffer. @@ -1971,11 +2134,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); @@ -2010,9 +2169,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); send_to_port( #ifndef ERTS_SMP /* No fake schedule out and in again after an exit */ @@ -2042,7 +2199,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) sz_data = size_object(data); - need = sz_data + 5 + TS_SIZE(t_p); + need = sz_data + 5 + PATCH_TS_SIZE(t_p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -2052,9 +2209,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2088,9 +2243,7 @@ trace_proc_spawn(Process *p, Eterm pid, mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, pid, mfa); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -2111,7 +2264,7 @@ trace_proc_spawn(Process *p, Eterm pid, sz_args = size_object(args); sz_pid = size_object(pid); - need = sz_args + 4 + 6 + TS_SIZE(p); + need = sz_args + 4 + 6 + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -2124,9 +2277,7 @@ trace_proc_spawn(Process *p, Eterm pid, erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2208,11 +2359,8 @@ trace_gc(Process *p, Eterm what) #define LOCAL_HEAP_SIZE \ (sizeof(values)/sizeof(*values)) * \ (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \ - 5/*4-tuple */ + TS_HEAP_WORDS + 5/*4-tuple */ + ERTS_TRACE_PATCH_TS_MAX_SIZE DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); -#ifdef DEBUG - Eterm* limit; -#endif ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm)); @@ -2227,7 +2375,7 @@ trace_gc(Process *p, Eterm what) sizeof(values)/sizeof(*values), tags, values); - size += 5/*4-tuple*/ + TS_SIZE(p); + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); #endif } else { ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); @@ -2242,15 +2390,12 @@ trace_gc(Process *p, Eterm what) sizeof(values)/sizeof(*values), tags, values); - size += 5/*4-tuple*/ + TS_SIZE(p); + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); } -#ifdef DEBUG - limit = hp + size; ASSERT(size <= LOCAL_HEAP_SIZE); -#endif msg = erts_bld_atom_uword_2tup_list(&hp, NULL, @@ -2263,14 +2408,14 @@ trace_gc(Process *p, Eterm what) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(msg, hp); - } - ASSERT(hp == limit); - if (is_internal_port(ERTS_TRACER_PROC(p))) + if (is_internal_port(ERTS_TRACER_PROC(p))) { + PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, NULL, NULL); send_to_port(p, msg, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - else + } + else { + PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, msg, bp); + } erts_smp_mtx_unlock(&smq_mtx); UnUseTmpHeap(LOCAL_HEAP_SIZE,p); #undef LOCAL_HEAP_SIZE @@ -2574,19 +2719,18 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { void profile_scheduler(Eterm scheduler_id, Eterm state) { - Eterm *hp, msg, timestamp; - Uint Ms, s, us; + Eterm *hp, msg; + ErlHeapFragment *bp = NULL; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 7) +#define LOCAL_HEAP_SIZE (7 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else - ErlHeapFragment *bp; Uint hsz; - hsz = 4 + 7; + hsz = 7 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; @@ -2606,10 +2750,13 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { break; } - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, - make_small(active_sched), timestamp); hp += 7; + msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, + state, make_small(active_sched), + NIL /* Will be overwritten by timestamp */); + hp += 7; + + /* Write timestamp in element 6 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); #ifndef ERTS_SMP profile_send(NIL, msg); @@ -2680,7 +2827,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { Eterm* hp; if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (5+6) +#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2689,9 +2836,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); /* No fake schedule */ send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2705,7 +2850,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - sz_data = 6 + TS_SIZE(p); + sz_data = 6 + PATCH_TS_SIZE(p); ERTS_GET_TRACER_REF(tracer_ref, ERTS_TRACER_PROC(p), @@ -2718,9 +2863,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2744,7 +2887,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { || erts_thr_progress_is_blocking()); if (is_internal_port(ERTS_TRACER_PROC(t_p))) { -#define LOCAL_HEAP_SIZE (5+5) +#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2752,9 +2895,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); /* No fake schedule */ send_to_port(NULL,mess,&ERTS_TRACER_PROC(t_p),&ERTS_TRACE_FLAGS(t_p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2768,7 +2909,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p))); - sz_data = 5 + TS_SIZE(t_p); + sz_data = 5 + PATCH_TS_SIZE(t_p); ERTS_GET_TRACER_REF(tracer_ref, ERTS_TRACER_PROC(t_p), @@ -2781,9 +2922,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2811,7 +2950,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { Eterm sched_id = am_undefined; if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (5+6) +#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2834,9 +2973,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { hp += ws; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); /* No fake scheduling */ send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); @@ -2856,7 +2994,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { ERTS_TRACER_PROC(p), ERTS_TRACE_FLAGS(p)); - hp = ERTS_ALLOC_SYSMSG_HEAP(ws+TS_SIZE(p), &bp, &off_heap, tracer_ref); + hp = ERTS_ALLOC_SYSMSG_HEAP(ws+PATCH_TS_SIZE(p), &bp, &off_heap, tracer_ref); if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) { #ifdef ERTS_SMP @@ -2874,10 +3012,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -2887,13 +3022,12 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { void profile_runnable_port(Port *p, Eterm status) { - Uint Ms, s, us; - Eterm *hp, msg, timestamp; - + Eterm *hp, msg; + ErlHeapFragment *bp = NULL; Eterm count = make_small(0); #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 6) +#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2901,10 +3035,9 @@ profile_runnable_port(Port *p, Eterm status) { hp = local_heap; #else - ErlHeapFragment *bp; Uint hsz; - hsz = 4 + 6; + hsz = 6 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; @@ -2912,9 +3045,12 @@ profile_runnable_port(Port *p, Eterm status) { erts_smp_mtx_lock(&smq_mtx); - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->common.id, status, count, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, count, + NIL /* Will be overwritten by timestamp */); + hp += 6; + + /* Write timestamp in element 5 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); #ifndef ERTS_SMP profile_send(p->common.id, msg); @@ -2929,20 +3065,19 @@ profile_runnable_port(Port *p, Eterm status) { /* Process profiling */ void profile_runnable_proc(Process *p, Eterm status){ - Uint Ms, s, us; - Eterm *hp, msg, timestamp; + Eterm *hp, msg; Eterm where = am_undefined; + ErlHeapFragment *bp = NULL; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 6 + 4) +#define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else - ErlHeapFragment *bp; - Uint hsz = 4 + 6 + 4; + Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1; #endif if (!p->current) { @@ -2951,7 +3086,7 @@ profile_runnable_proc(Process *p, Eterm status){ #ifdef ERTS_SMP if (!p->current) { - hsz = 4 + 6; + hsz -= 4; } bp = new_message_buffer(hsz); @@ -2966,9 +3101,13 @@ profile_runnable_proc(Process *p, Eterm status){ erts_smp_mtx_lock(&smq_mtx); - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->common.id, status, where, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, where, + NIL /* Will be overwritten by timestamp */); + hp += 6; + + /* Write timestamp in element 5 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); + #ifndef ERTS_SMP profile_send(p->common.id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 7405490f76..a0058264d7 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -18,8 +18,38 @@ * %CopyrightEnd% */ +#ifndef ERL_TRACE_H__FLAGS__ +#define ERL_TRACE_H__FLAGS__ +/* + * NOTE! The bits used for these flags matter. The flag with + * the least significant bit will take precedence! + * + * The "now timestamp" has highest precedence due to + * compatibility reasons. + */ +#define ERTS_TRACE_FLG_NOW_TIMESTAMP (1 << 0) +#define ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP (1 << 1) +#define ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP (1 << 2) + +/* + * The bits used effects trace flags (of processes and ports) + * as well as sequential trace flags. If changed make sure + * these arn't messed up... + */ +#define ERTS_TRACE_TS_TYPE_BITS 3 +#define ERTS_TRACE_TS_TYPE_MASK \ + ((1 << ERTS_TRACE_TS_TYPE_BITS) - 1) + +#define ERTS_TFLGS2TSTYPE(TFLGS) \ + ((int) (((TFLGS) >> ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) \ + & ERTS_TRACE_TS_TYPE_MASK)) +#define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS) \ + ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) \ + & ERTS_TRACE_TS_TYPE_MASK)) + +#endif /* ERL_TRACE_H__FLAGS__ */ -#ifndef ERL_TRACE_H__ +#if !defined(ERL_TRACE_H__) && !defined(ERTS_ONLY_INCLUDE_TRACE_FLAGS) #define ERL_TRACE_H__ struct binary; diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 551717139d..36d85b0a22 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -2507,7 +2507,7 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) ((Uint) (bytes[3] & ((byte) 0x3F))); bytes += 4; } else { - erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal unicode error in prim_file:internal_name2native/1"); } *target++ = (byte) (unipoint & 0xFF); *target++ = (byte) ((unipoint >> 8) & 0xFF); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a85aa15403..31ec12cccf 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -570,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext) *ep++ = VERSION_MAGIC; ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", __FILE__, __LINE__); *ext = ep; @@ -1206,6 +1206,7 @@ typedef struct B2TContext_t { } u; } B2TContext; +static B2TContext* b2t_export_context(Process*, B2TContext* src); static uLongf binary2term_uncomp_size(byte* data, Sint size) { @@ -1238,7 +1239,7 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size) static ERTS_INLINE int binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, - B2TContext* ctx) + B2TContext** ctxp, Process* p) { byte *bytes = data; Sint size = data_size; @@ -1252,8 +1253,8 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, size--; if (size < 5 || *bytes != COMPRESSED) { state->extp = bytes; - if (ctx) - ctx->state = B2TSizeInit; + if (ctxp) + (*ctxp)->state = B2TSizeInit; } else { uLongf dest_len = (Uint32) get_int32(bytes+1); @@ -1270,16 +1271,26 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, return -1; } state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len); - ctx->reds -= dest_len; + if (ctxp) + (*ctxp)->reds -= dest_len; } state->exttmp = 1; - if (ctx) { + if (ctxp) { + /* + * Start decompression by exporting trap context + * so we don't have to deal with deep-copying z_stream. + */ + B2TContext* ctx = b2t_export_context(p, *ctxp); + ASSERT(state = &(*ctxp)->b2ts); + state = &ctx->b2ts; + if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK) return -1; ctx->u.uc.dbytes = state->extp; ctx->u.uc.dleft = dest_len; ctx->state = B2TUncompressChunk; + *ctxp = ctx; } else { uLongf dlen = dest_len; @@ -1323,7 +1334,7 @@ erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size { Sint res; - if (binary2term_prepare(state, data, data_size, NULL) < 0 || + if (binary2term_prepare(state, data, data_size, NULL, NULL) < 0 || (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) { if (state->exttmp) @@ -1469,7 +1480,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar if (ctx->aligned_alloc) { ctx->reds -= bin_size / 8; } - if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) { + if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, &ctx, p) < 0) { ctx->state = B2TBadArg; } break; @@ -1559,7 +1570,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar b2t_destroy_context(ctx); if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", + erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n", __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); } erts_factory_close(&ctx->u.dc.factory); @@ -1708,12 +1719,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, real_size - size); } @@ -1753,12 +1764,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl bytes[0] = VERSION_MAGIC; if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } return erts_realloc_binary(bin, real_size); @@ -2650,7 +2661,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ASSERT(node_sz < 17); break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); } ptr++; @@ -4110,7 +4121,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, ASSERT(node_sz < 17); break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); } ptr++; @@ -4202,7 +4213,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, break; default: - erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n", + erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n", obj); } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index ec9296d034..1b8595fe57 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -428,7 +428,7 @@ void erl_grow_estack(ErtsEStack*, Uint need); #define ESTACK_CHANGE_ALLOCATOR(s,t) \ do { \ if ((s).start != ESTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active estack\n"); \ } \ (s).alloc_type = (t); \ @@ -589,7 +589,7 @@ do { \ #define WSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ if (s.wstart != WSTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active wstack\n"); \ } \ s.alloc_type = (t); \ @@ -781,7 +781,7 @@ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ #define PSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active pstack\n"); \ } \ s.alloc_type = (t); \ @@ -952,8 +952,8 @@ double erts_get_positive_zero_float(void); /* config.c */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...); void erl_error(char*, va_list); /* copy.c */ @@ -1398,7 +1398,7 @@ erts_alloc_message_heap_state(Uint size, #endif if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size); + erts_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size); if ( #if defined(ERTS_SMP) diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index e0fde337f2..895fe657d1 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -133,7 +133,7 @@ Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions while (h_size_table[ix] != -1 && h_size_table[ix] < size) ix++; if (h_size_table[ix] == -1) - erl_exit(1, "panic: too large hash table size (%d)\n", size); + erts_exit(ERTS_ERROR_EXIT, "panic: too large hash table size (%d)\n", size); size = h_size_table[ix]; sz = size*sizeof(HashBucket*); diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 06d0b5123d..5f6ea14732 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -84,7 +84,7 @@ index_put_entry(IndexTable* t, void* tmpl) Uint sz; if (ix >= t->limit) { /* A core dump is unnecessary */ - erl_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", t->htable.name, t->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); @@ -123,7 +123,7 @@ void erts_index_merge(Hash* src, IndexTable* dst) ix = dst->entries++; if (ix >= dst->size) { if (ix >= dst->limit) { - erl_exit(1, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", dst->htable.name, dst->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index c64c8802b9..538015b242 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -50,6 +50,7 @@ #include "erl_map.h" #include "erl_bif_unique.h" #include "erl_hl_timer.h" +#include "erl_time.h" extern ErlDrvEntry fd_driver_entry; #ifndef __OSE__ @@ -1528,8 +1529,19 @@ erts_schedule_proc2port_signal(Process *c_p, erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); if (sched_res != 0) { - if (refp) + if (refp) { + /* + * We need to restore the message queue save + * pointer to the beginning of the message queue + * since the caller now wont wait for a message + * containing the reference created above... + */ + ASSERT(c_p); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + JOIN_MESSAGE(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); *refp = NIL; + } return ERTS_PORT_OP_DROPPED; } return ERTS_PORT_OP_SCHEDULED; @@ -3466,7 +3478,7 @@ terminate_port(Port *prt) if ((state & ERTS_PORT_SFLG_HALT) && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { erts_port_release(prt); /* We will exit and never return */ - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } if (is_internal_port(send_closed_port_id)) deliver_result(send_closed_port_id, connected_id, am_closed); @@ -4881,7 +4893,7 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (len > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large data buffer (%beu bytes) passed to" "output callback of %s driver.\n", len, @@ -5755,16 +5767,20 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p) ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); #endif erts_aint32_t state; + int res = 1; Port *prt = erts_port_lookup_raw((Eterm) port_id); - if (!prt) - return -1; + if (!prt) { + res = -1; + goto done; + } state = erts_atomic32_read_nob(&prt->state); if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP | ERTS_PORT_SFLG_CLOSING)) { if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - return -1; + res = -1; else - return 0; + res = 0; + goto done; } if (connected_p) { #ifdef ERTS_SMP @@ -5773,22 +5789,25 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p) #endif *connected_p = ERTS_PORT_GET_CONNECTED(prt); } + +done: + #ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { + ERTS_SMP_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt)); erts_thr_progress_unmanaged_continue(dhndl); ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); - } + } else #endif - ERTS_SMP_LC_ASSERT(dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED - ? erts_lc_is_port_locked(prt) - : !erts_lc_is_port_locked(prt)); - return 1; + ERTS_SMP_LC_ASSERT(res != 1 || erts_lc_is_port_locked(prt)); + + return res; } int erl_drv_output_term(ErlDrvTermData port_id, ErlDrvTermData* data, int len) { /* May be called from arbitrary thread */ - Eterm connected; + Eterm connected = NIL; /* Shut up faulty warning... */ int res = deliver_term_check_port(port_id, &connected); if (res <= 0) return res; @@ -6762,6 +6781,28 @@ driver_get_now(ErlDrvNowData *now_data) return 0; } +ErlDrvTime +erl_drv_monotonic_time(ErlDrvTimeUnit time_unit) +{ + return (ErlDrvTime) erts_napi_monotonic_time((int) time_unit); +} + +ErlDrvTime +erl_drv_time_offset(ErlDrvTimeUnit time_unit) +{ + return (ErlDrvTime) erts_napi_time_offset((int) time_unit); +} + +ErlDrvTime +erl_drv_convert_time_unit(ErlDrvTime val, + ErlDrvTimeUnit from, + ErlDrvTimeUnit to) +{ + return (ErlDrvTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val, + (int) from, + (int) to); +} + static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) { RefThing *refp; @@ -7289,7 +7330,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) * of ErlDrvSysInfo (introduced in driver version 1.0). */ if (!sip || si_size < ERL_DRV_SYS_INFO_SIZE(smp_support)) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "driver_system_info(%p, %ld) called with invalid arguments\n", sip, si_size); diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 7ade8bca0f..020e61d136 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -125,7 +125,7 @@ static RegProc* reg_alloc(RegProc *tmpl) { RegProc* obj = (RegProc*) erts_alloc(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); if (!obj) { - erl_exit(1, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); + erts_exit(ERTS_ERROR_EXIT, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); } obj->name = tmpl->name; obj->p = tmpl->p; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index ec94e3a596..6497a1e648 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -611,15 +611,16 @@ static unsigned long zero_value = 0, one_value = 1; # endif /* !__WIN32__ */ #endif /* WANT_NONBLOCKING */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); -/* Some special erl_exit() codes: */ -#define ERTS_INTR_EXIT INT_MIN /* called from signal handler */ -#define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ -#define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */ +/* Some special erts_exit() codes: */ +#define ERTS_INTR_EXIT -1 /* called from signal handler */ +#define ERTS_ABORT_EXIT -2 /* no crash dump; only abort() */ +#define ERTS_DUMP_EXIT -3 /* crash dump; then exit() */ +#define ERTS_ERROR_EXIT -4 /* crash dump; then abort() */ #define ERTS_INTERNAL_ERROR(What) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ __FILE__, __LINE__, __func__, What) Eterm erts_check_io_info(void *p); @@ -819,6 +820,10 @@ int local_to_univ(Sint *year, Sint *month, Sint *day, void get_now(Uint*, Uint*, Uint*); struct ErtsSchedulerData_; ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *); +ErtsMonotonicTime erts_get_time_offset(void); +void +erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset); void get_sys_now(Uint*, Uint*, Uint*); void set_break_quit(void (*)(void), void (*)(void)); @@ -929,7 +934,7 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -943,7 +948,7 @@ erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -956,7 +961,7 @@ erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -970,7 +975,7 @@ erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -983,7 +988,7 @@ erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", diff, val, min_val); #else @@ -997,7 +1002,7 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 0ab6661c9f..988338ed81 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -555,7 +555,7 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) itime = erts_init_time_sup(time_correction, time_warp_mode); #ifdef TIW_ITIME_IS_CONSTANT if (itime != TIW_ITIME) { - erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); + erts_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); } #else tiw_itime = itime; diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 5286391746..b9ce70e364 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1055,7 +1055,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -1328,7 +1328,7 @@ make_hash2(Eterm term) i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(1, "bad header"); + erts_exit(ERTS_ERROR_EXIT, "bad header"); } while (i) { if (is_list(*ptr)) { @@ -1491,7 +1491,7 @@ make_hash2(Eterm term) break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } } break; @@ -1522,7 +1522,7 @@ make_hash2(Eterm term) UINT32_HASH(NIL_DEF, HCONST_2); goto hash2_common; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } case _TAG_IMMED1_SMALL: { @@ -1538,7 +1538,7 @@ make_hash2(Eterm term) } break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); hash2_common: /* Uint32 hash always has the hash value of the previous term, @@ -1733,7 +1733,7 @@ make_internal_hash(Eterm term) i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(1, "bad header"); + erts_exit(ERTS_ERROR_EXIT, "bad header"); } while (i) { if (is_list(*ptr)) { @@ -1909,7 +1909,7 @@ make_internal_hash(Eterm term) goto pop_next; } default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } } break; @@ -1922,7 +1922,7 @@ make_internal_hash(Eterm term) goto pop_next; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); pop_next: if (ESTACK_ISEMPTY(s)) { @@ -2205,7 +2205,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_broken_hash\n"); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n"); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -2879,7 +2879,7 @@ tailrecur_ne: ASSERT(sz > 0 && sz < 17); break; default: - erl_exit(1, "Unknown hashmap subsubtag\n"); + erts_exit(ERTS_ERROR_EXIT, "Unknown hashmap subsubtag\n"); } goto term_array; } |