diff options
Diffstat (limited to 'erts/emulator/beam')
128 files changed, 26827 insertions, 14892 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index d7c7f117cf..84d2d5e3ed 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -111,7 +111,7 @@ atom_text_alloc(int bytes) { byte *res; - ASSERT(bytes <= MAX_ATOM_LENGTH); + ASSERT(bytes <= MAX_ATOM_SZ_LIMIT); if (atom_text_pos + bytes >= atom_text_end) { more_atom_space(); } @@ -132,9 +132,17 @@ atom_hash(Atom* obj) byte* p = obj->name; int len = obj->len; HashValue h = 0, g; + byte v; while(len--) { - h = (h << 4) + *p++; + v = *p++; + /* latin1 clutch for r16 */ + if ((v & 0xFE) == 0xC2 && (*p & 0xC0) == 0x80) { + v = (v << 6) | (*p & 0x3F); + p++; len--; + } + /* normal hashpjw follows for v */ + h = (h << 4) + v; if ((g = h & 0xf0000000)) { h ^= (g >> 24); h ^= g; @@ -162,6 +170,7 @@ atom_alloc(Atom* tmpl) obj->name = atom_text_alloc(tmpl->len); sys_memcpy(obj->name, tmpl->name, tmpl->len); obj->len = tmpl->len; + obj->latin1_chars = tmpl->latin1_chars; obj->slot.index = -1; /* @@ -192,44 +201,146 @@ atom_free(Atom* obj) erts_free(ERTS_ALC_T_ATOM, (void*) obj); } +static void latin1_to_utf8(byte* conv_buf, const byte** srcp, int* lenp) +{ + byte* dst; + const byte* src = *srcp; + int i, len = *lenp; + + for (i=0 ; i < len; ++i) { + if (src[i] & 0x80) { + goto need_convertion; + } + } + return; + +need_convertion: + sys_memcpy(conv_buf, src, i); + dst = conv_buf + i; + for ( ; i < len; ++i) { + unsigned char chr = src[i]; + if (!(chr & 0x80)) { + *dst++ = chr; + } + else { + *dst++ = 0xC0 | (chr >> 6); + *dst++ = 0x80 | (chr & 0x3F); + } + } + *srcp = conv_buf; + *lenp = dst - conv_buf; +} + +/* + * erts_atom_put() may fail. If it fails THE_NON_VALUE is returned! + */ Eterm -am_atom_put(const char* name, int len) +erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc) { + byte utf8_copy[MAX_ATOM_SZ_FROM_LATIN1]; + const byte *text = name; + int tlen = len; + Sint no_latin1_chars; Atom a; - Eterm ret; int aix; - /* - * Silently truncate the atom if it is too long. Overlong atoms - * could occur in situations where we have no good way to return - * an error, such as in the I/O system. (Unfortunately, many - * drivers don't check for errors.) - * - * If an error should be produced for overlong atoms (such in - * list_to_atom/1), the caller should check the length before - * calling this function. - */ - if (len > MAX_ATOM_LENGTH) { - len = MAX_ATOM_LENGTH; - } #ifdef ERTS_ATOM_PUT_OPS_STAT erts_smp_atomic_inc_nob(&atom_put_ops); #endif - a.len = len; - a.name = (byte*)name; + + if (tlen < 0) { + if (trunc) + tlen = 0; + else + return THE_NON_VALUE; + } + + switch (enc) { + case ERTS_ATOM_ENC_7BIT_ASCII: + if (tlen > MAX_ATOM_CHARACTERS) { + if (trunc) + tlen = MAX_ATOM_CHARACTERS; + else + return THE_NON_VALUE; + } +#ifdef DEBUG + for (aix = 0; aix < len; aix++) { + ASSERT((name[aix] & 0x80) == 0); + } +#endif + no_latin1_chars = tlen; + break; + case ERTS_ATOM_ENC_LATIN1: + if (tlen > MAX_ATOM_CHARACTERS) { + if (trunc) + tlen = MAX_ATOM_CHARACTERS; + else + return THE_NON_VALUE; + } + no_latin1_chars = tlen; + latin1_to_utf8(utf8_copy, &text, &tlen); + break; + case ERTS_ATOM_ENC_UTF8: + /* First sanity check; need to verify later */ + if (tlen > MAX_ATOM_SZ_LIMIT && !trunc) + return THE_NON_VALUE; + break; + } + + a.len = tlen; + a.name = (byte *) text; atom_read_lock(); aix = index_get(&erts_atom_table, (void*) &a); atom_read_unlock(); - if (aix >= 0) - ret = make_atom(aix); - else { - atom_write_lock(); - ret = make_atom(index_put(&erts_atom_table, (void*) &a)); - atom_write_unlock(); + if (aix >= 0) { + /* Already in table no need to verify it */ + return make_atom(aix); } - return ret; + + if (enc == ERTS_ATOM_ENC_UTF8) { + /* Need to verify encoding and length */ + byte *err_pos; + Uint no_chars; + switch (erts_analyze_utf8_x((byte *) text, + (Uint) tlen, + &err_pos, + &no_chars, NULL, + &no_latin1_chars, + MAX_ATOM_CHARACTERS)) { + case ERTS_UTF8_OK: + ASSERT(no_chars <= MAX_ATOM_CHARACTERS); + break; + case ERTS_UTF8_OK_MAX_CHARS: + /* Truncated... */ + if (!trunc) + return THE_NON_VALUE; + ASSERT(no_chars == MAX_ATOM_CHARACTERS); + tlen = err_pos - text; + break; + default: + /* Bad utf8... */ + return THE_NON_VALUE; + } + } + + ASSERT(tlen <= MAX_ATOM_SZ_LIMIT); + ASSERT(-1 <= no_latin1_chars && no_latin1_chars <= MAX_ATOM_CHARACTERS); + + a.len = tlen; + a.latin1_chars = (Sint16) no_latin1_chars; + a.name = (byte *) text; + atom_write_lock(); + aix = index_put(&erts_atom_table, (void*) &a); + atom_write_unlock(); + return make_atom(aix); } +Eterm +am_atom_put(const char* name, int len) +{ + /* Assumes 7-bit ascii; use erts_atom_put() for other encodings... */ + return erts_atom_put((byte *) name, len, ERTS_ATOM_ENC_7BIT_ASCII, 1); +} int atom_table_size(void) { @@ -264,14 +375,19 @@ int atom_table_sz(void) } int -erts_atom_get(const char *name, int len, Eterm* ap) +erts_atom_get(const char *name, int len, Eterm* ap, ErtsAtomEncoding enc) { + byte utf8_copy[MAX_ATOM_SZ_FROM_LATIN1]; Atom a; int i; int res; - a.len = len; + a.len = (Sint16) len; a.name = (byte *)name; + if (enc == ERTS_ATOM_ENC_LATIN1) { + latin1_to_utf8(utf8_copy, (const byte**)&a.name, &len); + a.len = (Sint16) len; + } atom_read_lock(); i = index_get(&erts_atom_table, (void*) &a); res = i < 0 ? 0 : (*ap = make_atom(i), 1); @@ -333,8 +449,15 @@ init_atom_table(void) for (i = 0; erl_atom_names[i] != 0; i++) { int ix; a.len = strlen(erl_atom_names[i]); + a.latin1_chars = a.len; a.name = (byte*)erl_atom_names[i]; a.slot.index = i; +#ifdef DEBUG + /* Verify 7-bit ascii */ + for (ix = 0; ix < a.len; ix++) { + ASSERT((a.name[ix] & 0x80) == 0); + } +#endif ix = index_put(&erts_atom_table, (void*) &a); atom_text_pos -= a.len; atom_space -= a.len; diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index fd9c04d3d0..5904ae0f7e 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,7 +26,9 @@ #include "erl_atom_table.h" -#define MAX_ATOM_LENGTH 255 +#define MAX_ATOM_CHARACTERS 255 +#define MAX_ATOM_SZ_FROM_LATIN1 (2*MAX_ATOM_CHARACTERS) +#define MAX_ATOM_SZ_LIMIT (4*MAX_ATOM_CHARACTERS) /* theoretical byte limit */ #define ATOM_LIMIT (1024*1024) #define MIN_ATOM_TABLE_SIZE 8192 @@ -45,7 +47,8 @@ */ typedef struct atom { IndexSlot slot; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ - int len; /* length of atom name */ + Sint16 len; /* length of atom name (UTF-8 encoded) */ + Sint16 latin1_chars; /* 0-255 if atom can be encoded in latin1; otherwise, -1 */ int ord0; /* ordinal value of first 3 bytes + 7 bits */ byte* name; /* name of atom */ } Atom; @@ -53,8 +56,8 @@ typedef struct atom { extern IndexTable erts_atom_table; ERTS_GLB_INLINE Atom* atom_tab(Uint i); -ERTS_GLB_INLINE int erts_is_atom_bytes(byte *text, size_t len, Eterm term); -ERTS_GLB_INLINE int erts_is_atom_str(char *str, Eterm term); +ERTS_GLB_INLINE int erts_is_atom_utf8_bytes(byte *text, size_t len, Eterm term); +ERTS_GLB_INLINE int erts_is_atom_str(const char *str, Eterm term, int is_latin1); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE Atom* @@ -63,7 +66,7 @@ atom_tab(Uint i) return (Atom *) erts_index_lookup(&erts_atom_table, i); } -ERTS_GLB_INLINE int erts_is_atom_bytes(byte *text, size_t len, Eterm term) +ERTS_GLB_INLINE int erts_is_atom_utf8_bytes(byte *text, size_t len, Eterm term) { Atom *a; if (!is_atom(term)) @@ -73,43 +76,70 @@ ERTS_GLB_INLINE int erts_is_atom_bytes(byte *text, size_t len, Eterm term) && sys_memcmp((void *) a->name, (void *) text, len) == 0); } -ERTS_GLB_INLINE int erts_is_atom_str(char *str, Eterm term) +ERTS_GLB_INLINE int erts_is_atom_str(const char *str, Eterm term, int is_latin1) { Atom *a; int i, len; - char *aname; + const byte* aname; + const byte* s = (const byte*) str; + if (!is_atom(term)) return 0; a = atom_tab(atom_val(term)); len = a->len; - aname = (char *) a->name; - for (i = 0; i < len; i++) - if (aname[i] != str[i] || str[i] == '\0') - return 0; - return str[len] == '\0'; + aname = a->name; + if (is_latin1) { + for (i = 0; i < len; s++) { + if (aname[i] < 0x80) { + if (aname[i] != *s || *s == '\0') + return 0; + i++; + } + else { + if (aname[i] != (0xC0 | (*s >> 6)) || + aname[i+1] != (0x80 | (*s & 0x3F))) { + return 0; + } + i += 2; + } + } + } + else { + for (i = 0; i < len; i++, s++) + if (aname[i] != *s || *s == '\0') + return 0; + } + return *s == '\0'; } #endif +typedef enum { + ERTS_ATOM_ENC_7BIT_ASCII, + ERTS_ATOM_ENC_LATIN1, + ERTS_ATOM_ENC_UTF8 +} ErtsAtomEncoding; + /* * Note, ERTS_IS_ATOM_STR() expects the first argument to be a - * string literal. + * 7-bit ASCII string literal. */ #define ERTS_IS_ATOM_STR(LSTR, TERM) \ - (erts_is_atom_bytes((byte *) LSTR, sizeof(LSTR) - 1, (TERM))) + (erts_is_atom_utf8_bytes((byte *) LSTR, sizeof(LSTR) - 1, (TERM))) #define ERTS_DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) #define ERTS_INIT_AM(S) AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) int atom_table_size(void); /* number of elements */ int atom_table_sz(void); /* table size in bytes, excluding stored objects */ -Eterm am_atom_put(const char*, int); /* most callers pass plain char*'s */ +Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */ +Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc); int atom_erase(byte*, int); int atom_static_put(byte*, int); void init_atom_table(void); void atom_info(int, void *); void dump_atoms(int, void *); -int erts_atom_get(const char* name, int len, Eterm* ap); +int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc); void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used); #endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 89249cee2f..c2f32ba089 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2012. All Rights Reserved. +# Copyright Ericsson AB 1996-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -18,6 +18,8 @@ # # +# IMPORTANT! All atoms defined here *need* to be in 7-bit ascii! +# # File format: # # Lines starting with '#' are ignored. @@ -94,6 +96,7 @@ atom asynchronous atom atom atom atom_used atom attributes +atom await_port_send_result atom await_proc_exit atom await_sched_wall_time_modifications atom awaiting_load @@ -136,6 +139,7 @@ atom caseless atom catchlevel atom cd atom cdr +atom cflags atom characters_to_binary_int atom characters_to_list_int atom clear @@ -143,15 +147,18 @@ atom close atom closed atom code atom command +atom compact atom compat_rel atom compile atom compressed +atom config_h atom connect atom connected atom connection_closed atom cons atom const atom context_switches +atom control atom copy atom cpu atom cpu_timestamp @@ -163,6 +170,7 @@ atom current_location atom current_stacktrace atom data atom debug_flags +atom decimals atom delay_trap atom dexit atom depth @@ -205,6 +213,7 @@ atom erlang atom ERROR='ERROR' atom error_handler atom error_logger +atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' atom event @@ -238,9 +247,11 @@ atom gc_end atom gc_start atom Ge='>=' atom generational +atom get_data atom get_seq_token atom get_tcw atom getenv +atom gather_gc_info_result atom gather_sched_wall_time_result atom getting_linked atom getting_unlinked @@ -262,6 +273,7 @@ atom hipe_architecture atom http httph https http_response http_request http_header http_eoh http_error http_bin httph_bin atom id atom if_clause +atom ignore atom imports atom in atom in_exiting @@ -291,6 +303,7 @@ atom label atom large_heap atom last_calls atom latin1 +atom ldflags atom Le='=<' atom lf atom line @@ -315,6 +328,7 @@ atom maximum atom max_tables max_processes atom mbuf_size atom memory +atom memory_internal atom memory_types atom message atom message_binary @@ -410,6 +424,7 @@ atom overlapped_io atom owner atom packet atom packet_size +atom parallelism atom Plus='+' atom pause atom pending @@ -421,22 +436,24 @@ atom pid atom port atom ports atom port_count +atom port_limit atom port_op atom print atom priority atom private atom process atom processes -atom processes_trap atom processes_used atom process_count atom process_display atom process_limit atom process_dump atom procs +atom proc_sig atom profile atom protected atom protection +atom ptab_list_continue atom public atom purify atom quantify @@ -477,6 +494,7 @@ atom scheduler atom scheduler_id atom schedulers_online atom scheme +atom scientific atom scope atom sensitive atom sequential_tracer @@ -484,6 +502,7 @@ atom sequential_trace_token atom serial atom set atom set_cpu_topology +atom set_data atom set_on_first_link atom set_on_first_spawn atom set_on_link @@ -518,6 +537,7 @@ atom system_version atom system_architecture atom SYSTEM='SYSTEM' atom table +atom term_to_binary_trap atom this atom thread_pool_size atom threads diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 02f37bf9bc..649594a334 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -37,25 +37,83 @@ static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp); -static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp); -static void delete_export_references(Eterm module); -static int purge_module(int module); +static void delete_code(Module* modp); static void decrement_refc(BeamInstr* code); static int is_native(BeamInstr* code); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); -static void remove_from_address_table(BeamInstr* code); -Eterm -load_module_2(BIF_ALIST_2) + + +BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { - Eterm reason; - Eterm* hp; - int sz; - byte* code; + Module* modp; Eterm res; + ErtsCodeIndex code_ix; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + code_ix = erts_active_code_ix(); + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { + return am_undefined; + } + erts_rlock_old_code(code_ix); + res = ((modp->curr.code && is_native(modp->curr.code)) || + (modp->old.code != 0 && is_native(modp->old.code))) ? + am_true : am_false; + erts_runlock_old_code(code_ix); + return res; +} + +BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) +{ + Module* modp; + Eterm res; + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], + BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + } + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + + if (modp && modp->curr.num_breakpoints > 0) { + ASSERT(modp->curr.code != NULL); + erts_clear_module_break(modp); + ASSERT(modp->curr.num_breakpoints == 0); + } + + erts_start_staging_code_ix(); + + res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + + if (res == BIF_ARG_1) { + erts_end_staging_code_ix(); + erts_commit_staging_code_ix(); + } + else { + erts_abort_staging_code_ix(); + } + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); + return res; +} + +BIF_RETTYPE +prepare_loading_2(BIF_ALIST_2) +{ byte* temp_alloc = NULL; - struct LoaderState* stp; + byte* code; + Uint sz; + Binary* magic; + Eterm reason; + Eterm* hp; + Eterm res; if (is_not_atom(BIF_ARG_1)) { error: @@ -65,108 +123,307 @@ load_module_2(BIF_ALIST_2) if ((code = erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc)) == NULL) { goto error; } - hp = HAlloc(BIF_P, 3); - /* - * Read the BEAM file and prepare the module for loading. - */ - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); sz = binary_size(BIF_ARG_2); - reason = erts_prepare_loading(stp, BIF_P, BIF_P->group_leader, + reason = erts_prepare_loading(magic, BIF_P, BIF_P->group_leader, &BIF_ARG_1, code, sz); erts_free_aligned_binary_bytes(temp_alloc); if (reason != NIL) { + hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_error, reason); BIF_RET(res); } + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->refc, 1); + BIF_RET(res); +} + +struct m { + Binary* code; + Eterm module; + Module* modp; + Uint exception; +}; + +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +#ifdef ERTS_SMP +static void smp_code_ix_commiter(void*); + +static struct /* Protected by code_write_permission */ +{ + Process* stager; + ErtsThrPrgrLaterOp lop; +}commiter_state; +#endif + +static Eterm +exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) +{ + Eterm* hp = HAlloc(p, 3 + 2*exceptions); + Eterm res = NIL; + + mp += exceptions - 1; + while (exceptions > 0) { + if (mp->exception) { + res = CONS(hp, mp->module, res); + hp += 2; + exceptions--; + } + mp--; + } + return TUPLE2(hp, tag, res); +} + + +BIF_RETTYPE +finish_loading_1(BIF_ALIST_1) +{ + int i; + int n; + struct m* p = NULL; + Uint exceptions; + Eterm res; + int is_blocking = 0; + int do_commit = 0; + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_finish_loading_1], BIF_P, BIF_ARG_1); + } /* - * Stop all other processes and finish the loading of the module. + * Validate the argument before we start loading; it must be a + * proper list where each element is a magic binary containing + * prepared (not previously loaded) code. + * + * First count the number of elements and allocate an array + * to keep the elements in. */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - reason = erts_finish_loading(stp, BIF_P, 0, &BIF_ARG_1); - if (reason != NIL) { - res = TUPLE2(hp, am_error, reason); - } else { - set_default_trace_pattern(BIF_ARG_1); - res = TUPLE2(hp, am_module, BIF_ARG_1); + n = list_length(BIF_ARG_1); + if (n == -1) { + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); + goto done; } + p = erts_alloc(ERTS_ALC_T_LOADER_TMP, n*sizeof(struct m)); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_RET(res); -} + /* + * We now know that the argument is a proper list. Validate + * and collect the binaries into the array. + */ -BIF_RETTYPE purge_module_1(BIF_ALIST_1) -{ - int purge_res; + for (i = 0; i < n; i++) { + Eterm* cons = list_val(BIF_ARG_1); + Eterm term = CAR(cons); + ProcBin* pb; - if (is_not_atom(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); + goto done; + } + pb = (ProcBin*) binary_val(term); + p[i].code = pb->val; + p[i].module = erts_module_for_prepared_code(p[i].code); + if (p[i].module == NIL) { + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); + goto done; + } + BIF_ARG_1 = CDR(cons); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + /* + * Since we cannot handle atomic loading of a group of modules + * if one or more of them uses on_load, we will only allow one + * element in the list. This limitation is intended to be + * lifted in the future. + */ - erts_export_consolidate(); - purge_res = purge_module(atom_val(BIF_ARG_1)); + if (n > 1) { + ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT); + goto done; + } - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + /* + * All types are correct. There cannot be a BADARG from now on. + * Before we can start loading, we must check whether any of + * the modules already has old code. To avoid a race, we must + * not allow other process to initiate a code loading operation + * from now on. + */ - if (purge_res < 0) { - BIF_ERROR(BIF_P, BADARG); + res = am_ok; + erts_start_staging_code_ix(); + + for (i = 0; i < n; i++) { + p[i].modp = erts_put_module(p[i].module); + } + for (i = 0; i < n; i++) { + if (p[i].modp->curr.num_breakpoints > 0 || + p[i].modp->curr.num_traced_exports > 0 || + erts_is_default_trace_enabled()) { + /* tracing involved, fallback with thread blocking */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + break; + } } - BIF_RET(am_true); -} -BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) -{ - Module* modp; + if (is_blocking) { + for (i = 0; i < n; i++) { + if (p[i].modp->curr.num_breakpoints) { + erts_clear_module_break(p[i].modp); + ASSERT(p[i].modp->curr.num_breakpoints == 0); + } + } + } - if (is_not_atom(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); + exceptions = 0; + for (i = 0; i < n; i++) { + p[i].exception = 0; + if (p[i].modp->curr.code && p[i].modp->old.code) { + p[i].exception = 1; + exceptions++; + } } - if ((modp = erts_get_module(BIF_ARG_1)) == NULL) { - return am_undefined; + + if (exceptions) { + res = exception_list(BIF_P, am_not_purged, p, exceptions); + } else { + /* + * Now we can load all code. This can't fail. + */ + + exceptions = 0; + for (i = 0; i < n; i++) { + Eterm mod; + Eterm retval; + + erts_refc_inc(&p[i].code->refc, 1); + retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod); + ASSERT(retval == NIL || retval == am_on_load); + if (retval == am_on_load) { + p[i].exception = 1; + exceptions++; + } + } + + /* + * Check whether any module has an on_load_handler. + */ + + if (exceptions) { + res = exception_list(BIF_P, am_on_load, p, exceptions); + } + do_commit = 1; } - return ((modp->code && is_native(modp->code)) || - (modp->old_code != 0 && is_native(modp->old_code))) ? - am_true : am_false; + +done: + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); } -BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) -{ - Eterm res; +static Eterm +staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, + struct m* loaded, int nloaded) +{ +#ifdef ERTS_SMP + if (is_blocking || !commit) +#endif + { + if (commit) { + erts_end_staging_code_ix(); + erts_commit_staging_code_ix(); + if (loaded) { + int i; + for (i=0; i < nloaded; i++) { + set_default_trace_pattern(loaded[i].module); + } + } + } + else { + erts_abort_staging_code_ix(); + } + if (loaded) { + erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + erts_release_code_write_permission(); + return res; + } +#ifdef ERTS_SMP + else { + ASSERT(is_value(res)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + if (loaded) { + erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + } + erts_end_staging_code_ix(); + /* + * Now we must wait for all schedulers to do a memory barrier before + * we can commit and let them access the new staged code. This allows + * schedulers to read active code_ix in a safe way while executing + * without any memory barriers at all. + */ + ASSERT(commiter_state.stager == NULL); + commiter_state.stager = c_p; + erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &commiter_state.lop); + erts_smp_proc_inc_refc(c_p); + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + /* + * smp_code_ix_commiter() will do the rest "later" + * and resume this process to return 'res'. + */ + ERTS_BIF_YIELD_RETURN(c_p, res); + } +#endif +} - erts_export_consolidate(); - res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - return res; +#ifdef ERTS_SMP +static void smp_code_ix_commiter(void* null) +{ + Process* p = commiter_state.stager; + + erts_commit_staging_code_ix(); +#ifdef DEBUG + commiter_state.stager = NULL; +#endif + erts_release_code_write_permission(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(p)) { + erts_resume(p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_dec_refc(p); } +#endif /* ERTS_SMP */ + + BIF_RETTYPE check_old_code_1(BIF_ALIST_1) { + ErtsCodeIndex code_ix; Module* modp; + Eterm res = am_false; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } - modp = erts_get_module(BIF_ARG_1); - if (modp == NULL) { /* Doesn't exist. */ - BIF_RET(am_false); - } else if (modp->old_code == NULL) { /* No old code. */ - BIF_RET(am_false); + code_ix = erts_active_code_ix(); + modp = erts_get_module(BIF_ARG_1, code_ix); + if (modp != NULL) { + erts_rlock_old_code(code_ix); + if (modp->old.code != NULL) { + res = am_true; + } + erts_runlock_old_code(code_ix); } - BIF_RET(am_true); + BIF_RET(res); } Eterm @@ -180,14 +437,19 @@ check_process_code_2(BIF_ALIST_2) } if (is_internal_pid(BIF_ARG_1)) { Eterm res; - if (internal_pid_index(BIF_ARG_1) >= erts_max_processes) - goto error; - modp = erts_get_module(BIF_ARG_2); + ErtsCodeIndex code_ix; + + code_ix = erts_active_code_ix(); + modp = erts_get_module(BIF_ARG_2, code_ix); if (modp == NULL) { /* Doesn't exist. */ return am_false; - } else if (modp->old_code == NULL) { /* No old code. */ + } + erts_rlock_old_code(code_ix); + if (modp->old.code == NULL) { /* No old code. */ + erts_runlock_old_code(code_ix); return am_false; } + erts_runlock_old_code(code_ix); #ifdef ERTS_SMP rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, @@ -202,7 +464,14 @@ check_process_code_2(BIF_ALIST_2) ERTS_BIF_YIELD2(bif_export[BIF_check_process_code_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } - res = check_process_code(rp, modp); + erts_rlock_old_code(code_ix); + if (modp->old.code != NULL) { /* must check again */ + res = check_process_code(rp, modp); + } + else { + res = am_false; + } + erts_runlock_old_code(code_ix); #ifdef ERTS_SMP if (BIF_P != rp) { erts_resume(rp, ERTS_PROC_LOCK_MAIN); @@ -223,56 +492,71 @@ check_process_code_2(BIF_ALIST_2) BIF_RETTYPE delete_module_1(BIF_ALIST_1) { - int res; + ErtsCodeIndex code_ix; + Module* modp; + int is_blocking = 0; + int success = 0; + Eterm res = NIL; - if (is_not_atom(BIF_ARG_1)) - goto badarg; + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_delete_module_1], BIF_P, BIF_ARG_1); + } { - Module *modp = erts_get_module(BIF_ARG_1); + erts_start_staging_code_ix(); + code_ix = erts_staging_code_ix(); + modp = erts_get_module(BIF_ARG_1, code_ix); if (!modp) { res = am_undefined; } - else if (modp->old_code != 0) { + else if (modp->old.code != 0) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", BIF_ARG_1); erts_send_error_to_logger(BIF_P->group_leader, dsbufp); - res = am_badarg; + ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); } else { - delete_export_references(BIF_ARG_1); - delete_code(BIF_P, 0, modp); + if (modp->curr.num_breakpoints > 0 || + modp->curr.num_traced_exports > 0) { + /* we have tracing, retry single threaded */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + if (modp->curr.num_breakpoints) { + erts_clear_module_break(modp); + ASSERT(modp->curr.num_breakpoints == 0); + } + } + delete_code(modp); res = am_true; + success = 1; } } - - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - - if (res == am_badarg) { - badarg: - BIF_ERROR(BIF_P, BADARG); - } - BIF_RET(res); + return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) { Module* modp; + ErtsCodeIndex code_ix; + Eterm res = am_false; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } - if ((modp = erts_get_module(BIF_ARG_1)) == NULL || - modp->code == NULL || - modp->code[MI_ON_LOAD_FUNCTION_PTR] != 0) { - BIF_RET(am_false); + code_ix = erts_active_code_ix(); + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) != NULL) { + if (modp->curr.code != NULL + && modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] == 0) { + res = am_true; + } } - BIF_RET(am_true); + BIF_RET(res); } BIF_RETTYPE pre_loaded_0(BIF_ALIST_0) @@ -282,27 +566,28 @@ BIF_RETTYPE pre_loaded_0(BIF_ALIST_0) BIF_RETTYPE loaded_0(BIF_ALIST_0) { + ErtsCodeIndex code_ix = erts_active_code_ix(); + Module* modp; Eterm previous = NIL; Eterm* hp; int i; int j = 0; - - for (i = 0; i < module_code_size(); i++) { - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { + + for (i = 0; i < module_code_size(code_ix); i++) { + if ((modp = module_code(i, code_ix)) != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { j++; } } if (j > 0) { hp = HAlloc(BIF_P, j*2); - for (i = 0; i < module_code_size(); i++) { - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { - previous = CONS(hp, make_atom(module_code(i)->module), - previous); + for (i = 0; i < module_code_size(code_ix); i++) { + if ((modp=module_code(i,code_ix)) != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { + previous = CONS(hp, make_atom(modp->module), previous); hp += 2; } } @@ -312,54 +597,65 @@ BIF_RETTYPE loaded_0(BIF_ALIST_0) BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1) { - Module* modp = erts_get_module(BIF_ARG_1); - Eterm on_load; + Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); - if (!modp || modp->code == 0) { - error: - BIF_ERROR(BIF_P, BADARG); + if (modp && modp->curr.code) { + BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]); } - if ((on_load = modp->code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { - goto error; + else { + BIF_ERROR(BIF_P, BADARG); } - BIF_TRAP_CODE_PTR_0(BIF_P, on_load); } BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) { - Module* modp = erts_get_module(BIF_ARG_1); + ErtsCodeIndex code_ix; + Module* modp; Eterm on_load; - if (!modp || modp->code == 0) { + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + /* ToDo: Use code_ix staging instead of thread blocking */ + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + code_ix = erts_active_code_ix(); + modp = erts_get_module(BIF_ARG_1, code_ix); + + if (!modp || modp->curr.code == 0) { error: + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } - if ((on_load = modp->code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { + if ((on_load = modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { goto error; } if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) { goto error; } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - if (BIF_ARG_2 == am_true) { int i; /* * The on_load function succeded. Fix up export entries. */ - for (i = 0; i < export_list_size(); i++) { - Export *ep = export_list(i); + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i,code_ix); if (ep != NULL && ep->code[0] == BIF_ARG_1 && ep->code[4] != 0) { - ep->address = (void *) ep->code[4]; + ep->addressv[code_ix] = (void *) ep->code[4]; ep->code[4] = 0; } } - modp->code[MI_ON_LOAD_FUNCTION_PTR] = 0; + modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] = 0; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { BeamInstr* code; @@ -370,19 +666,21 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) * This is an combination of delete and purge. We purge * the current code; the old code is not touched. */ - erts_total_code_size -= modp->code_length; - code = modp->code; - end = (BeamInstr *)((char *)code + modp->code_length); + erts_total_code_size -= modp->curr.code_length; + code = modp->curr.code; + end = (BeamInstr *)((char *)code + modp->curr.code_length); erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->catches, code, modp->code_length); + beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length, + erts_active_code_ix()); erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->code = NULL; - modp->code_length = 0; - modp->catches = BEAM_CATCHES_NIL; - remove_from_address_table(code); + modp->curr.code = NULL; + modp->curr.code_length = 0; + modp->curr.catches = BEAM_CATCHES_NIL; + erts_remove_from_ranges(code); } erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); BIF_RET(am_true); } @@ -403,11 +701,11 @@ set_default_trace_pattern(Eterm module) if (trace_pattern_is_on) { Eterm mfa[1]; mfa[0] = module; - (void) erts_set_trace_pattern(mfa, 1, + (void) erts_set_trace_pattern(0, mfa, 1, match_spec, meta_match_spec, 1, trace_pattern_flags, - meta_tracer_pid); + meta_tracer_pid, 1); } } @@ -427,10 +725,10 @@ check_process_code(Process* rp, Module* modp) /* * Pick up limits for the module. */ - start = modp->old_code; - end = (BeamInstr *)((char *)start + modp->old_code_length); + start = modp->old.code; + end = (BeamInstr *)((char *)start + modp->old.code_length); mod_start = (char *) start; - mod_size = modp->old_code_length; + mod_size = modp->old.code_length; /* * Check if current instruction or continuation pointer points into module. @@ -562,10 +860,10 @@ check_process_code(Process* rp, Module* modp) done_gc = 1; FLAGS(rp) |= F_NEED_FULLSWEEP; (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - literals = (Eterm *) modp->old_code[MI_LITERALS_START]; - lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals; + literals = (Eterm *) modp->old.code[MI_LITERALS_START]; + lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals; oh = (struct erl_off_heap_header *) - modp->old_code[MI_LITERALS_OFF_HEAP]; + modp->old.code[MI_LITERALS_OFF_HEAP]; erts_garbage_collect_literals(rp, literals, lit_size, oh); } } @@ -624,53 +922,82 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) #undef in_area - -static int -purge_module(int module) +BIF_RETTYPE purge_module_1(BIF_ALIST_1) { + ErtsCodeIndex code_ix; BeamInstr* code; BeamInstr* end; Module* modp; + int is_blocking = 0; + Eterm ret; - /* - * Correct module? - */ - - if ((modp = erts_get_module(make_atom(module))) == NULL) { - return -2; + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); } - /* - * Any code to purge? - */ - if (modp->old_code == 0) { - return -1; + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD1(bif_export[BIF_purge_module_1], BIF_P, BIF_ARG_1); } + code_ix = erts_active_code_ix(); + /* - * Unload any NIF library + * Correct module? */ - if (modp->old_nif != NULL) { - erts_unload_nif(modp->old_nif); - modp->old_nif = NULL; + + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); } + else { + erts_rwlock_old_code(code_ix); - /* - * Remove the old code. - */ - ASSERT(erts_total_code_size >= modp->old_code_length); - erts_total_code_size -= modp->old_code_length; - code = modp->old_code; - end = (BeamInstr *)((char *)code + modp->old_code_length); - erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->old_catches, code, modp->old_code_length); - decrement_refc(code); - erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->old_code = NULL; - modp->old_code_length = 0; - modp->old_catches = BEAM_CATCHES_NIL; - remove_from_address_table(code); - return 0; + /* + * Any code to purge? + */ + if (modp->old.code == 0) { + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + } + else { + /* + * Unload any NIF library + */ + if (modp->old.nif != NULL) { + /* ToDo: Do unload nif without blocking */ + erts_rwunlock_old_code(code_ix); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + erts_rwlock_old_code(code_ix); + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } + + /* + * Remove the old code. + */ + ASSERT(erts_total_code_size >= modp->old.code_length); + erts_total_code_size -= modp->old.code_length; + code = modp->old.code; + end = (BeamInstr *)((char *)code + modp->old.code_length); + erts_cleanup_funs_on_purge(code, end); + beam_catches_delmod(modp->old.catches, code, modp->old.code_length, + code_ix); + decrement_refc(code); + erts_free(ERTS_ALC_T_CODE, (void *) code); + modp->old.code = NULL; + modp->old.code_length = 0; + modp->old.catches = BEAM_CATCHES_NIL; + erts_remove_from_ranges(code); + ERTS_BIF_PREP_RET(ret, am_true); + } + erts_rwunlock_old_code(code_ix); + } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + } + erts_release_code_write_permission(); + return ret; } static void @@ -690,92 +1017,48 @@ decrement_refc(BeamInstr* code) } } -static void -remove_from_address_table(BeamInstr* code) -{ - int i; - - for (i = 0; i < num_loaded_modules; i++) { - if (modules[i].start == code) { - num_loaded_modules--; - while (i < num_loaded_modules) { - modules[i] = modules[i+1]; - i++; - } - mid_module = &modules[num_loaded_modules/2]; - return; - } - } - ASSERT(0); /* Not found? */ -} - - /* - * Move code from current to old. + * Move code from current to old and null all export entries for the module */ -static void -delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp) -{ -#ifdef ERTS_ENABLE_LOCK_CHECK -#ifdef ERTS_SMP - if (c_p && c_p_locks) - erts_proc_lc_chk_only_proc_main(c_p); - else -#endif - erts_lc_check_exact(NULL, 0); -#endif - - /* - * Clear breakpoints if any - */ - if (modp->code != NULL && modp->code[MI_NUM_BREAKPOINTS] > 0) { - if (c_p && c_p_locks) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - erts_clear_module_break(modp); - modp->code[MI_NUM_BREAKPOINTS] = 0; - erts_smp_thr_progress_unblock(); - if (c_p && c_p_locks) - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); - } - modp->old_code = modp->code; - modp->old_code_length = modp->code_length; - modp->old_catches = modp->catches; - modp->old_nif = modp->nif; - modp->code = NULL; - modp->code_length = 0; - modp->catches = BEAM_CATCHES_NIL; - modp->nif = NULL; -} - - -/* null all references on the export table for the module called with the - atom index below */ - static void -delete_export_references(Eterm module) +delete_code(Module* modp) { + ErtsCodeIndex code_ix = erts_staging_code_ix(); + Eterm module = make_atom(modp->module); int i; - ASSERT(is_atom(module)); - - for (i = 0; i < export_list_size(); i++) { - Export *ep = export_list(i); + for (i = 0; i < export_list_size(code_ix); i++) { + Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->code[0] == module)) { - if (ep->address == ep->code+3 && - (ep->code[3] == (BeamInstr) em_apply_bif)) { - continue; + if (ep->addressv[code_ix] == ep->code+3) { + if (ep->code[3] == (BeamInstr) em_apply_bif) { + continue; + } + else if (ep->code[3] == + (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(modp->curr.num_traced_exports > 0); + erts_clear_export_break(modp, ep->code+3); + } + else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler + || !erts_initialized); } - ep->address = ep->code+3; + ep->addressv[code_ix] = ep->code+3; ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; - MatchSetUnref(ep->match_prog_set); - ep->match_prog_set = NULL; } } + + ASSERT(modp->curr.num_breakpoints == 0); + ASSERT(modp->curr.num_traced_exports == 0); + modp->old = modp->curr; + modp->curr.code = NULL; + modp->curr.code_length = 0; + modp->curr.catches = BEAM_CATCHES_NIL; + modp->curr.nif = NULL; } - + Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) @@ -787,11 +1070,10 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->code != NULL && modp->old_code != NULL) { + if (modp->curr.code != NULL && modp->old.code != NULL) { return am_not_purged; - } else if (modp->old_code == NULL) { /* Make the current version old. */ - delete_code(c_p, c_p_locks, modp); - delete_export_references(module); + } else if (modp->old.code == NULL) { /* Make the current version old. */ + delete_code(modp); } return NIL; } @@ -799,7 +1081,21 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) static int is_native(BeamInstr* code) { - return ((Eterm *)code[MI_FUNCTIONS])[1] != 0; + Uint i, num_functions = code[MI_NUM_FUNCTIONS]; + + /* Check NativeAdress of first real function in module + */ + for (i=0; i<num_functions; i++) { + BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + Eterm name = (Eterm) func_info[3]; + + if (is_atom(name)) { + return func_info[1] != 0; + } + else ASSERT(is_nil(name)); /* ignore BIF stubs */ + } + /* Not a single non-BIF function? */ + return 0; } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index d772bea02f..68907a771a 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2012. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,67 +44,34 @@ #define ReAlloc(P, SIZ) erts_realloc(ERTS_ALC_T_BPD, (P), (SZ)) #define Free(P) erts_free(ERTS_ALC_T_BPD, (P)) -/* -** Doubly linked ring macros -*/ - -#define BpInit(a,i) \ -do { \ - (a)->orig_instr = (i); \ - (a)->next = (a); \ - (a)->prev = (a); \ -} while (0) - -#define BpSpliceNext(a,b) \ -do { \ - register BpData *c = (a), *d = (b), *e; \ - e = c->next->prev; \ - c->next->prev = d->next->prev; \ - d->next->prev = e; \ - e = c->next; \ - c->next = d->next; \ - d->next = e; \ -} while (0) - -#define BpSplicePrev(a,b) \ -do { \ - register BpData *c = (a), *d = (b), *e; \ - e = c->prev->next; \ - c->prev->next = d->prev->next; \ - d->prev->next = e; \ - e = c->prev; \ - c->prev = d->prev; \ - d->prev = e; \ -} while (0) - -#ifdef DEBUG -# define BpSingleton(a) ((a)->next == (a) && (a)->prev == (a)) +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ + if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN) +# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ + if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) #else -# define BpSingleton(a) ((a)->next == (a)) +# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) +# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) #endif -#define BpInitAndSpliceNext(a,i,b) \ -do { \ - (a)->orig_instr = (i); \ - (a)->prev = (b); \ - (b)->next->prev = (a); \ - (a)->next = (b)->next; \ - (b)->next = (a); \ -} while (0) +#define ERTS_BPF_LOCAL_TRACE 0x01 +#define ERTS_BPF_META_TRACE 0x02 +#define ERTS_BPF_COUNT 0x04 +#define ERTS_BPF_COUNT_ACTIVE 0x08 +#define ERTS_BPF_DEBUG 0x10 +#define ERTS_BPF_TIME_TRACE 0x20 +#define ERTS_BPF_TIME_TRACE_ACTIVE 0x40 +#define ERTS_BPF_GLOBAL_TRACE 0x80 -#define BpInitAndSplicePrev(a,i,b) \ -do { \ - (a)->orig_instr = (i); \ - (a)->next = (b); \ - (b)->prev->next = (a); \ - (a)->prev = (b)->prev; \ - (b)->prev = (a); \ -} while (0) +#define ERTS_BPF_ALL 0xFF +extern BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +extern BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern BeamInstr beam_exception_trace[1]; /* OpCode(i_exception_trace) */ +extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ -#define BREAK_IS_BIF (1) -#define BREAK_IS_ERL (0) - +erts_smp_atomic32_t erts_active_bp_index; +erts_smp_atomic32_t erts_staging_bp_index; /* ************************************************************************* ** Local prototypes @@ -113,26 +80,30 @@ do { \ /* ** Helpers */ - -static int set_break(Eterm mfa[3], int specified, - Binary *match_spec, BeamInstr break_op, - enum erts_break_op count_op, Eterm tracer_pid); -static int set_module_break(Module *modp, Eterm mfa[3], int specified, - Binary *match_spec, BeamInstr break_op, - enum erts_break_op count_op, Eterm tracer_pid); -static int set_function_break(Module *modp, BeamInstr *pc, int bif, - Binary *match_spec, BeamInstr break_op, - enum erts_break_op count_op, Eterm tracer_pid); - -static int clear_break(Eterm mfa[3], int specified, - BeamInstr break_op); -static int clear_module_break(Module *modp, Eterm mfa[3], int specified, - BeamInstr break_op); -static int clear_function_break(Module *modp, BeamInstr *pc, int bif, - BeamInstr break_op); - -static BpData *is_break(BeamInstr *pc, BeamInstr break_op); -static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op); +static Eterm do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, + int local, Binary* ms, Eterm tracer_pid); +static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, + enum erts_break_op count_op, Eterm tracer_pid); +static void set_function_break(BeamInstr *pc, + Binary *match_spec, + Uint break_flags, + enum erts_break_op count_op, + Eterm tracer_pid); + +static void clear_break(BpFunctions* f, Uint break_flags); +static int clear_function_break(BeamInstr *pc, Uint break_flags); + +static BpDataTime* get_time_break(BeamInstr *pc); +static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); +static void bp_time_diff(bp_data_time_item_t *item, + process_breakpoint_time_t *pbt, + Uint ms, Uint s, Uint us); + +static void bp_meta_unref(BpMetaPid* bmp); +static void bp_count_unref(BpCount* bcp); +static void bp_time_unref(BpDataTime* bdt); +static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local); +static void uninstall_breakpoint(BeamInstr* pc); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ @@ -152,240 +123,996 @@ static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_da static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t *sitem); static void bp_hash_delete(bp_time_hash_t *hash); - /* ************************************************************************* ** External interfaces */ -erts_smp_spinlock_t erts_bp_lock; - void erts_bp_init(void) { - erts_smp_spinlock_init(&erts_bp_lock, "breakpoints"); + erts_smp_atomic32_init_nob(&erts_active_bp_index, 0); + erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1); } -int -erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, - Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return set_break(mfa, specified, match_spec, - (BeamInstr) BeamOp(op_i_trace_breakpoint), 0, tracer_pid); + +void +erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) +{ + ErtsCodeIndex code_ix = erts_active_code_ix(); + Uint max_funcs = 0; + int current; + int max_modules = module_code_size(code_ix); + int num_modules = 0; + Module* modp; + Module** module; + Uint i; + + module = (Module **) Alloc(max_modules*sizeof(Module *)); + num_modules = 0; + for (current = 0; current < max_modules; current++) { + modp = module_code(current, code_ix); + if (modp->curr.code) { + max_funcs += modp->curr.code[MI_NUM_FUNCTIONS]; + module[num_modules++] = modp; + } + } + + f->matching = (BpFunction *) Alloc(max_funcs*sizeof(BpFunction)); + i = 0; + for (current = 0; current < num_modules; current++) { + BeamInstr** code_base = (BeamInstr **) module[current]->curr.code; + BeamInstr* code; + Uint num_functions = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; + Uint fi; + + if (specified > 0) { + if (mfa[0] != make_atom(module[current]->module)) { + /* Wrong module name */ + continue; + } + } + + for (fi = 0; fi < num_functions; fi++) { + BeamInstr* pc; + int wi; + + code = code_base[MI_FUNCTIONS+fi]; + ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + pc = code+5; + if (erts_is_native_break(pc)) { + continue; + } + if (is_nil(code[3])) { /* Ignore BIF stub */ + continue; + } + for (wi = 0; + wi < specified && (Eterm) code[2+wi] == mfa[wi]; + wi++) { + /* Empty loop body */ + } + if (wi == specified) { + /* Store match */ + f->matching[i].pc = pc; + f->matching[i].mod = module[current]; + i++; + } + } + } + f->matched = i; + Free(module); } -int -erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, - Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return set_break(mfa, specified, match_spec, - (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); +void +erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) +{ + ErtsCodeIndex code_ix = erts_active_code_ix(); + int i; + int num_exps = export_list_size(code_ix); + int ne; + + f->matching = (BpFunction *) Alloc(num_exps*sizeof(BpFunction)); + ne = 0; + for (i = 0; i < num_exps; i++) { + Export* ep = export_list(i, code_ix); + BeamInstr* pc; + int j; + + for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { + /* Empty loop body */ + } + if (j < specified) { + continue; + } + pc = ep->code+3; + if (ep->addressv[code_ix] == pc) { + if ((*pc == (BeamInstr) em_apply_bif || + *pc == (BeamInstr) em_call_error_handler)) { + continue; + } + ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); + } else if (erts_is_native_break(ep->addressv[code_ix])) { + continue; + } + + f->matching[ne].pc = pc; + f->matching[ne].mod = erts_get_module(ep->code[0], code_ix); + ne++; + + } + f->matched = ne; } -/* set breakpoint data for on exported bif entry */ +void +erts_bp_free_matched_functions(BpFunctions* f) +{ + Free(f->matching); +} void -erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); +erts_consolidate_bp_data(BpFunctions* f, int local) +{ + BpFunction* fs = f->matching; + Uint i; + Uint n = f->matched; + + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + + for (i = 0; i < n; i++) { + consolidate_bp_data(fs[i].mod, fs[i].pc, local); + } } -void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) { - set_function_break(NULL, pc, BREAK_IS_BIF, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +void +erts_consolidate_bif_bp_data(void) +{ + int i; + + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + for (i = 0; i < BIF_SIZE; i++) { + Export *ep = bif_export[i]; + consolidate_bp_data(0, ep->code+3, 0); + } } -void erts_clear_time_trace_bif(BeamInstr *pc) { - clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_time_breakpoint)); +static void +consolidate_bp_data(Module* modp, BeamInstr* pc, int local) +{ + GenericBp* g = (GenericBp *) pc[-4]; + GenericBpData* src; + GenericBpData* dst; + Uint flags; + + if (g == 0) { + return; + } + + src = &g->data[erts_active_bp_ix()]; + dst = &g->data[erts_staging_bp_ix()]; + + /* + * The contents of the staging area may be out of date. + * Decrement all reference pointers. + */ + + flags = dst->flags; + if (flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE)) { + MatchSetUnref(dst->local_ms); + } + if (flags & ERTS_BPF_META_TRACE) { + bp_meta_unref(dst->meta_pid); + MatchSetUnref(dst->meta_ms); + } + if (flags & ERTS_BPF_COUNT) { + bp_count_unref(dst->count); + } + if (flags & ERTS_BPF_TIME_TRACE) { + bp_time_unref(dst->time); + } + + /* + * If all flags are zero, deallocate all breakpoint data. + */ + + flags = dst->flags = src->flags; + if (flags == 0) { + if (modp) { + if (local) { + modp->curr.num_breakpoints--; + } else { + modp->curr.num_traced_exports--; + } + ASSERT(modp->curr.num_breakpoints >= 0); + ASSERT(modp->curr.num_traced_exports >= 0); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_generic_breakpoint)); + } + pc[-4] = 0; + Free(g); + return; + } + + /* + * Copy the active data to the staging area (making it ready + * for the next time it will be used). + */ + + if (flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE)) { + dst->local_ms = src->local_ms; + MatchSetRef(dst->local_ms); + } + if (flags & ERTS_BPF_META_TRACE) { + dst->meta_pid = src->meta_pid; + erts_refc_inc(&dst->meta_pid->refc, 1); + dst->meta_ms = src->meta_ms; + MatchSetRef(dst->meta_ms); + } + if (flags & ERTS_BPF_COUNT) { + dst->count = src->count; + erts_refc_inc(&dst->count->refc, 1); + } + if (flags & ERTS_BPF_TIME_TRACE) { + dst->time = src->time; + erts_refc_inc(&dst->time->refc, 1); + ASSERT(dst->time->hash); + } } -int -erts_set_debug_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return set_break(mfa, specified, NULL, - (BeamInstr) BeamOp(op_i_debug_breakpoint), 0, NIL); +void +erts_commit_staged_bp(void) +{ + ErtsBpIndex staging = erts_staging_bp_ix(); + ErtsBpIndex active = erts_active_bp_ix(); + + erts_smp_atomic32_set_nob(&erts_active_bp_index, staging); + erts_smp_atomic32_set_nob(&erts_staging_bp_index, active); } -int -erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return set_break(mfa, specified, NULL, - (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL); +void +erts_install_breakpoints(BpFunctions* f) +{ + Uint i; + Uint n = f->matched; + BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint); + + for (i = 0; i < n; i++) { + BeamInstr* pc = f->matching[i].pc; + GenericBp* g = (GenericBp *) pc[-4]; + if (*pc != br && g) { + Module* modp = f->matching[i].mod; + + /* + * The breakpoint must be disabled in the active data + * (it will enabled later by switching bp indices), + * and enabled in the staging data. + */ + ASSERT(g->data[erts_active_bp_ix()].flags == 0); + ASSERT(g->data[erts_staging_bp_ix()].flags != 0); + + /* + * The following write is not protected by any lock. We + * assume that the hardware guarantees that a write of an + * aligned word-size (or half-word) writes is atomic + * (i.e. that other processes executing this code will not + * see a half pointer). + */ + *pc = br; + modp->curr.num_breakpoints++; + } + } } -int -erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return set_break(mfa, specified, NULL, - (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +void +erts_uninstall_breakpoints(BpFunctions* f) +{ + Uint i; + Uint n = f->matched; + + for (i = 0; i < n; i++) { + BeamInstr* pc = f->matching[i].pc; + uninstall_breakpoint(pc); + } } -int -erts_clear_trace_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return clear_break(mfa, specified, - (BeamInstr) BeamOp(op_i_trace_breakpoint)); +static void +uninstall_breakpoint(BeamInstr* pc) +{ + if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + GenericBp* g = (GenericBp *) pc[-4]; + if (g->data[erts_active_bp_ix()].flags == 0) { + /* + * The following write is not protected by any lock. We + * assume that the hardware guarantees that a write of an + * aligned word-size (or half-word) writes is atomic + * (i.e. that other processes executing this code will not + * see a half pointer). + */ + *pc = g->orig_instr; + } + } } -int -erts_clear_mtrace_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return clear_break(mfa, specified, - (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); +void +erts_set_trace_break(BpFunctions* f, Binary *match_spec) +{ + set_break(f, match_spec, ERTS_BPF_LOCAL_TRACE, 0, am_true); } void -erts_clear_mtrace_bif(BeamInstr *pc) { - clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); +erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, Eterm tracer_pid) +{ + set_break(f, match_spec, ERTS_BPF_META_TRACE, 0, tracer_pid); } -int -erts_clear_debug_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return clear_break(mfa, specified, - (BeamInstr) BeamOp(op_i_debug_breakpoint)); +void +erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local) +{ + Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; + + set_function_break(pc, match_spec, flags, 0, NIL); } -int -erts_clear_count_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return clear_break(mfa, specified, - (BeamInstr) BeamOp(op_i_count_breakpoint)); +void +erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) +{ + set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer_pid); } -int -erts_clear_time_break(Eterm mfa[3], int specified) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return clear_break(mfa, specified, - (BeamInstr) BeamOp(op_i_time_breakpoint)); +void +erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) +{ + set_function_break(pc, NULL, + ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, + count_op, NIL); } -int -erts_clear_break(Eterm mfa[3], int specified) { +void +erts_clear_time_trace_bif(BeamInstr *pc) { + clear_function_break(pc, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); +} + +void +erts_set_debug_break(BpFunctions* f) { + set_break(f, NULL, ERTS_BPF_DEBUG, 0, NIL); +} + +void +erts_set_count_break(BpFunctions* f, enum erts_break_op count_op) +{ + set_break(f, 0, ERTS_BPF_COUNT|ERTS_BPF_COUNT_ACTIVE, + count_op, NIL); +} + +void +erts_set_time_break(BpFunctions* f, enum erts_break_op count_op) +{ + set_break(f, 0, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, + count_op, NIL); +} + +void +erts_clear_trace_break(BpFunctions* f) +{ + clear_break(f, ERTS_BPF_LOCAL_TRACE); +} + +void +erts_clear_call_trace_bif(BeamInstr *pc, int local) +{ + GenericBp* g = (GenericBp *) pc[-4]; + + if (g) { + Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; + if (g->data[erts_staging_bp_ix()].flags & flags) { + clear_function_break(pc, flags); + } + } +} + +void +erts_clear_mtrace_break(BpFunctions* f) +{ + clear_break(f, ERTS_BPF_META_TRACE); +} + +void +erts_clear_mtrace_bif(BeamInstr *pc) +{ + clear_function_break(pc, ERTS_BPF_META_TRACE); +} + +void +erts_clear_debug_break(BpFunctions* f) +{ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - return clear_break(mfa, specified, 0); + clear_break(f, ERTS_BPF_DEBUG); +} + +void +erts_clear_count_break(BpFunctions* f) +{ + clear_break(f, ERTS_BPF_COUNT|ERTS_BPF_COUNT_ACTIVE); +} + +void +erts_clear_time_break(BpFunctions* f) +{ + clear_break(f, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); +} + +void +erts_clear_all_breaks(BpFunctions* f) +{ + clear_break(f, ERTS_BPF_ALL); } int erts_clear_module_break(Module *modp) { + BeamInstr** code_base; + Uint n; + Uint i; + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); - return clear_module_break(modp, NULL, 0, 0); + code_base = (BeamInstr **) modp->curr.code; + if (code_base == NULL) { + return 0; + } + n = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; + for (i = 0; i < n; ++i) { + BeamInstr* pc; + + pc = code_base[MI_FUNCTIONS+i] + 5; + if (erts_is_native_break(pc)) { + continue; + } + clear_function_break(pc, ERTS_BPF_ALL); + } + + erts_commit_staged_bp(); + + for (i = 0; i < n; ++i) { + BeamInstr* pc; + + pc = code_base[MI_FUNCTIONS+i] + 5; + if (erts_is_native_break(pc)) { + continue; + } + uninstall_breakpoint(pc); + consolidate_bp_data(modp, pc, 1); + ASSERT(pc[-4] == 0); + } + return n; } -int -erts_clear_function_break(Module *modp, BeamInstr *pc) { +void +erts_clear_export_break(Module* modp, BeamInstr* pc) +{ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - ASSERT(modp); - return clear_function_break(modp, pc, BREAK_IS_ERL, 0); + + clear_function_break(pc, ERTS_BPF_ALL); + erts_commit_staged_bp(); + *pc = (BeamInstr) 0; + consolidate_bp_data(modp, pc, 0); + ASSERT(pc[-4] == 0); } +BeamInstr +erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) +{ + GenericBp* g; + GenericBpData* bp; + Uint bp_flags; + ErtsBpIndex ix = erts_active_bp_ix(); + + g = (GenericBp *) I[-4]; + bp = &g->data[ix]; + bp_flags = bp->flags; + ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0); + if (bp_flags & (ERTS_BPF_LOCAL_TRACE| + ERTS_BPF_GLOBAL_TRACE| + ERTS_BPF_TIME_TRACE_ACTIVE) && + !IS_TRACED_FL(c_p, F_TRACE_CALLS)) { + bp_flags &= ~(ERTS_BPF_LOCAL_TRACE| + ERTS_BPF_GLOBAL_TRACE| + ERTS_BPF_TIME_TRACE| + ERTS_BPF_TIME_TRACE_ACTIVE); + if (bp_flags == 0) { /* Quick exit */ + return g->orig_instr; + } + } + + if (bp_flags & ERTS_BPF_LOCAL_TRACE) { + ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0); + (void) do_call_trace(c_p, I, reg, 1, bp->local_ms, am_true); + } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) { + (void) do_call_trace(c_p, I, reg, 0, bp->local_ms, am_true); + } + + if (bp_flags & ERTS_BPF_META_TRACE) { + Eterm old_pid; + Eterm new_pid; + + old_pid = (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid); + new_pid = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_pid); + if (new_pid != old_pid) { + erts_smp_atomic_set_nob(&bp->meta_pid->pid, new_pid); + } + } + + if (bp_flags & ERTS_BPF_COUNT_ACTIVE) { + erts_smp_atomic_inc_nob(&bp->count->acount); + } + + if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { + Eterm w; + erts_trace_time_call(c_p, I, bp->time); + w = (BeamInstr) *c_p->cp; + if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) || + w == (BeamInstr) BeamOp(op_return_trace) || + w == (BeamInstr) BeamOp(op_i_return_to_trace)) ) { + Eterm* E = c_p->stop; + ASSERT(c_p->htop <= E && E <= c_p->hend); + if (E - 2 < c_p->htop) { + (void) erts_garbage_collect(c_p, 2, reg, I[-1]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + } + E = c_p->stop; + ASSERT(c_p->htop <= E && E <= c_p->hend); + + E -= 2; + E[0] = make_cp(I); + E[1] = make_cp(c_p->cp); /* original return address */ + c_p->cp = beam_return_time_trace; + c_p->stop = E; + } + } + + if (bp_flags & ERTS_BPF_DEBUG) { + return (BeamInstr) BeamOp(op_i_debug_breakpoint); + } else { + return g->orig_instr; + } +} /* - * SMP NOTE: Process p may have become exiting on return! + * Entry point called by the trace wrap functions in erl_bif_wrap.c + * + * The trace wrap functions are themselves called through the export + * entries instead of the original BIF functions. */ -BeamInstr -erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, - Uint32 *ret_flags, Eterm *tracer_pid) { - Eterm tpid1, tpid2; - BpData **bds = (BpData **) (pc)[-4]; - BpDataTrace *bdt = NULL; +Eterm +erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) +{ + Eterm result; + Eterm (*func)(Process*, Eterm*, BeamInstr*); + Export* ep = bif_export[bif_index]; + Uint32 flags = 0, flags_meta = 0; + Eterm meta_tracer_pid = NIL; + int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif + * is actually in the + * export entry */ + BeamInstr *cp = p->cp; + GenericBp* g; + GenericBpData* bp = NULL; + Uint bp_flags = 0; + + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + + g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + if (g) { + bp = &g->data[erts_active_bp_ix()]; + bp_flags = bp->flags; + } - ASSERT(bds); - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - bdt = (BpDataTrace *) bds[bp_sched2ix_proc(p)]; - ASSERT(bdt); - bdt = (BpDataTrace *) bdt->next; - ASSERT(bdt); - ASSERT(ret_flags); - ASSERT(tracer_pid); - - ErtsSmpBPLock(bdt); - tpid1 = tpid2 = bdt->tracer_pid; - ErtsSmpBPUnlock(bdt); - - *ret_flags = erts_call_trace(p, pc-3/*mfa*/, bdt->match_spec, args, - 1, &tpid2); - *tracer_pid = tpid2; - if (tpid1 != tpid2) { - ErtsSmpBPLock(bdt); - bdt->tracer_pid = tpid2; - ErtsSmpBPUnlock(bdt); - } - bds[bp_sched2ix_proc(p)] = (BpData *) bdt; - return bdt->orig_instr; + /* + * Make continuation pointer OK, it is not during direct BIF calls, + * but it is correct during apply of bif. + */ + if (!applying) { + p->cp = I; + } + if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && + IS_TRACED_FL(p, F_TRACE_CALLS)) { + int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); + flags = erts_call_trace(p, ep->code, bp->local_ms, args, + local, &ERTS_TRACER_PROC(p)); + } + if (bp_flags & ERTS_BPF_META_TRACE) { + Eterm tpid1, tpid2; + + tpid1 = tpid2 = + (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid); + flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args, + 0, &tpid2); + meta_tracer_pid = tpid2; + if (tpid1 != tpid2) { + erts_smp_atomic_set_nob(&bp->meta_pid->pid, tpid2); + } + } + if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && + IS_TRACED_FL(p, F_TRACE_CALLS)) { + BeamInstr *pc = (BeamInstr *)ep->code+3; + erts_trace_time_call(p, pc, bp->time); + } + + /* Restore original continuation pointer (if changed). */ + p->cp = cp; + + func = bif_table[bif_index].f; + + result = func(p, args, I); + + if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_time_trace = beam_return_time_trace[0]; + Eterm *cpp; + /* Maybe advance cp to skip trace stack frames */ + for (cpp = p->stop; ; cp = cp_val(*cpp++)) { + if (*cp == i_return_trace) { + /* Skip stack frame variables */ + while (is_not_CP(*cpp)) cpp++; + cpp += 2; /* Skip return_trace parameters */ + } else if (*cp == i_return_time_trace) { + /* Skip stack frame variables */ + while (is_not_CP(*cpp)) cpp++; + cpp += 1; /* Skip return_time_trace parameters */ + } else if (*cp == i_return_to_trace) { + /* A return_to trace message is going to be generated + * by normal means, so we do not have to. + */ + cp = NULL; + break; + } else break; + } + } + + /* Try to get these in the order + * they usually appear in normal code... */ + if (is_non_value(result)) { + Uint reason = p->freason; + if (reason != TRAP) { + Eterm class; + Eterm value = p->fvalue; + DeclareTmpHeapNoproc(nocatch,3); + UseTmpHeapNoproc(3); + /* Expand error value like in handle_error() */ + if (reason & EXF_ARGLIST) { + Eterm *tp; + ASSERT(is_tuple(value)); + tp = tuple_val(value); + value = tp[1]; + } + if ((reason & EXF_THROWN) && (p->catches <= 0)) { + value = TUPLE2(nocatch, am_nocatch, value); + reason = EXC_ERROR; + } + /* Note: expand_error_value() could theoretically + * allocate on the heap, but not for any error + * returned by a BIF, and it would do no harm, + * just be annoying. + */ + value = expand_error_value(p, reason, value); + class = exception_tag[GET_EXC_CLASS(reason)]; + + if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { + erts_trace_exception(p, ep->code, class, value, + &meta_tracer_pid); + } + if (flags & MATCH_SET_EXCEPTION_TRACE) { + erts_trace_exception(p, ep->code, class, value, + &ERTS_TRACER_PROC(p)); + } + if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { + /* can only happen if(local)*/ + Eterm *ptr = p->stop; + ASSERT(is_CP(*ptr)); + ASSERT(ptr <= STACK_START(p)); + /* Search the nearest stack frame for a catch */ + while (++ptr < STACK_START(p)) { + if (is_CP(*ptr)) break; + if (is_catch(*ptr)) { + if (applying) { + /* Apply of BIF, cp is in calling function */ + if (cp) erts_trace_return_to(p, cp); + } else { + /* Direct bif call, I points into + * calling function */ + erts_trace_return_to(p, I); + } + } + } + } + UnUseTmpHeapNoproc(3); + if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + ERTS_TRACE_FLAGS(p) |= F_EXCEPTION_TRACE; + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + } + } + } else { + if (flags_meta & MATCH_SET_RX_TRACE) { + erts_trace_return(p, ep->code, result, &meta_tracer_pid); + } + /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ + if (flags & MATCH_SET_RX_TRACE) { + erts_trace_return(p, ep->code, result, &ERTS_TRACER_PROC(p)); + } + if (flags & MATCH_SET_RETURN_TO_TRACE) { + /* can only happen if(local)*/ + if (applying) { + /* Apply of BIF, cp is in calling function */ + if (cp) erts_trace_return_to(p, cp); + } else { + /* Direct bif call, I points into calling function */ + erts_trace_return_to(p, I); + } + } + } + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + return result; } +static Eterm +do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, + int local, Binary* ms, Eterm tracer_pid) +{ + Eterm* cpp; + int return_to_trace = 0; + BeamInstr w; + BeamInstr *cp_save; + Uint32 flags; + Uint need = 0; + Eterm* E = c_p->stop; + + w = *c_p->cp; + if (w == (BeamInstr) BeamOp(op_return_trace)) { + cpp = &E[2]; + } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) { + return_to_trace = 1; + cpp = &E[0]; + } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) { + cpp = &E[0]; + } else { + cpp = NULL; + } + if (cpp) { + for (;;) { + BeamInstr w = *cp_val(*cpp); + if (w == (BeamInstr) BeamOp(op_return_trace)) { + cpp += 3; + } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) { + return_to_trace = 1; + cpp += 1; + } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) { + cpp += 2; + } else { + break; + } + } + cp_save = c_p->cp; + c_p->cp = (BeamInstr *) cp_val(*cpp); + ASSERT(is_CP(*cpp)); + } + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer_pid); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + if (cpp) { + c_p->cp = cp_save; + } + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) { + need += 1; + } + if (flags & MATCH_SET_RX_TRACE) { + need += 3; + } + if (need) { + ASSERT(c_p->htop <= E && E <= c_p->hend); + if (E - need < c_p->htop) { + (void) erts_garbage_collect(c_p, need, reg, I[-1]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + E = c_p->stop; + } + } + if (flags & MATCH_SET_RETURN_TO_TRACE && !return_to_trace) { + E -= 1; + ASSERT(c_p->htop <= E && E <= c_p->hend); + E[0] = make_cp(c_p->cp); + c_p->cp = beam_return_to_trace; + } + if (flags & MATCH_SET_RX_TRACE) { + E -= 3; + ASSERT(c_p->htop <= E && E <= c_p->hend); + ASSERT(is_CP((Eterm) (UWord) (I - 3))); + ASSERT(am_true == tracer_pid || + is_internal_pid(tracer_pid) || is_internal_port(tracer_pid)); + E[2] = make_cp(c_p->cp); + E[1] = tracer_pid; + E[0] = make_cp(I - 3); /* We ARE at the beginning of an + instruction, + the funcinfo is above i. */ + c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? + beam_exception_trace : beam_return_trace; + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + ERTS_TRACE_FLAGS(c_p) |= F_EXCEPTION_TRACE; + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } + c_p->stop = E; + return tracer_pid; +} +void +erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) +{ + Uint ms,s,us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; -/* - * SMP NOTE: Process p may have become exiting on return! - */ -Uint32 -erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local, - Eterm *tracer_pid) { - BpData **bds = (BpData **) (pc)[-4]; - BpDataTrace *bdt = NULL; + ASSERT(c_p); + ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & ERTS_PSFLG_RUNNING); + /* get previous timestamp and breakpoint + * from the process psd */ - ASSERT(tracer_pid); - if (bds) { - Eterm tpid1, tpid2; - Uint32 flags; - bdt = (BpDataTrace *)bds[bp_sched2ix_proc(p)]; + pbt = ERTS_PROC_GET_CALL_TIME(c_p); + get_sys_now(&ms, &s, &us); - ErtsSmpBPLock(bdt); - tpid1 = tpid2 = bdt->tracer_pid; - ErtsSmpBPUnlock(bdt); + /* get pbt + * timestamp = t0 + * lookup bdt from code + * set ts0 to pbt + * add call count here? + */ + if (pbt == 0) { + /* First call of process to instrumented function */ + pbt = Alloc(sizeof(process_breakpoint_time_t)); + (void) ERTS_PROC_SET_CALL_TIME(c_p, ERTS_PROC_LOCK_MAIN, pbt); + } else { + ASSERT(pbt->pc); + /* add time to previous code */ + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = c_p->common.id; + sitem.count = 0; - flags = erts_call_trace(p, pc-3/*mfa*/, bdt->match_spec, args, - local, &tpid2); - *tracer_pid = tpid2; - if (tpid1 != tpid2) { - ErtsSmpBPLock(bdt); - bdt->tracer_pid = tpid2; - ErtsSmpBPUnlock(bdt); + /* previous breakpoint */ + pbdt = get_time_break(pbt->pc); + + /* if null then the breakpoint was removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(c_p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } } - return flags; } - *tracer_pid = NIL; - return 0; + + /* Add count to this code */ + sitem.pid = c_p->common.id; + sitem.count = 1; + sitem.s_time = 0; + sitem.us_time = 0; + + /* this breakpoint */ + ASSERT(bdt); + h = &(bdt->hash[bp_sched2ix_proc(c_p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + + pbt->pc = I; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; } +void +erts_trace_time_return(Process *p, BeamInstr *pc) +{ + Uint ms,s,us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; + + ASSERT(p); + ASSERT(erts_smp_atomic32_read_acqb(&p->state) & ERTS_PSFLG_RUNNING); + /* get previous timestamp and breakpoint + * from the process psd */ + + pbt = ERTS_PROC_GET_CALL_TIME(p); + get_sys_now(&ms,&s,&us); + + /* get pbt + * lookup bdt from code + * timestamp = t1 + * get ts0 from pbt + * get item from bdt->hash[bp_hash(p->id)] + * ack diff (t1, t0) to item + */ + + if (pbt) { + /* might have been removed due to + * trace_pattern(false) + */ + ASSERT(pbt->pc); + + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->common.id; + sitem.count = 0; + + /* previous breakpoint */ + pbdt = get_time_break(pbt->pc); + + /* beware, the trace_pattern might have been removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + + pbt->pc = pc; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + } +} int -erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { - BpDataTrace *bdt = - (BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_trace_breakpoint)); - - if (bdt) { +erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) +{ + Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; + GenericBpData* bp = check_break(pc, flags); + + if (bp) { if (match_spec_ret) { - *match_spec_ret = bdt->match_spec; + *match_spec_ret = bp->local_ms; } - if (tracer_pid_ret) { - ErtsSmpBPLock(bdt); - *tracer_pid_ret = bdt->tracer_pid; - ErtsSmpBPUnlock(bdt); - } - return !0; + return 1; } return 0; } int -erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { - BpDataTrace *bdt = - (BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); +erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, + Eterm *tracer_pid_ret) +{ + GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE); - if (bdt) { + if (bp) { if (match_spec_ret) { - *match_spec_ret = bdt->match_spec; + *match_spec_ret = bp->meta_ms; } if (tracer_pid_ret) { - ErtsSmpBPLock(bdt); - *tracer_pid_ret = bdt->tracer_pid; - ErtsSmpBPUnlock(bdt); + *tracer_pid_ret = + (Eterm) erts_smp_atomic_read_nob(&bp->meta_pid->pid); } - return !0; + return 1; } return 0; } @@ -402,15 +1129,15 @@ erts_is_native_break(BeamInstr *pc) { } int -erts_is_count_break(BeamInstr *pc, Sint *count_ret) { - BpDataCount *bdc = - (BpDataCount *) is_break(pc, (BeamInstr) BeamOp(op_i_count_breakpoint)); +erts_is_count_break(BeamInstr *pc, Uint *count_ret) +{ + GenericBpData* bp = check_break(pc, ERTS_BPF_COUNT); - if (bdc) { + if (bp) { if (count_ret) { - *count_ret = (Sint) erts_smp_atomic_read_nob(&bdc->acount); + *count_ret = (Uint) erts_smp_atomic_read_nob(&bp->count->acount); } - return !0; + return 1; } return 0; } @@ -421,7 +1148,7 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { Uint size; Eterm *hp, t; bp_data_time_item_t *item = NULL; - BpDataTime *bdt = (BpDataTime *) is_break(pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + BpDataTime *bdt = get_time_break(pc); if (bdt) { if (retval) { @@ -464,7 +1191,7 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { } bp_hash_delete(&hash); } - return !0; + return 1; } return 0; @@ -478,15 +1205,16 @@ erts_find_local_func(Eterm mfa[3]) { BeamInstr* code_ptr; Uint i,n; - if ((modp = erts_get_module(mfa[0])) == NULL) + if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) return NULL; - if ((code_base = (BeamInstr **) modp->code) == NULL) + if ((code_base = (BeamInstr **) modp->curr.code) == NULL) return NULL; n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; for (i = 0; i < n; ++i) { code_ptr = code_base[MI_FUNCTIONS+i]; ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); - ASSERT(mfa[0] == ((Eterm) code_ptr[2])); + ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || + is_nil((Eterm) code_ptr[2])); if (mfa[1] == ((Eterm) code_ptr[3]) && ((BeamInstr) mfa[2]) == code_ptr[4]) { return code_ptr + 5; @@ -654,11 +1382,11 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * the previous breakpoint. */ - pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + pbdt = get_time_break(pbt->pc); if (pbdt) { get_sys_now(&ms,&s,&us); bp_time_diff(&sitem, pbt, ms, s, us); - sitem.pid = p->id; + sitem.pid = p->common.id; sitem.count = 0; h = &(pbdt->hash[bp_sched2ix_proc(p)]); @@ -692,669 +1420,260 @@ void erts_schedule_time_break(Process *p, Uint schedule) { } /* pbt */ } -/* call_time breakpoint - * Accumulated times are added to the previous bp, - * not the current one. The current one is saved - * for future reference. - * The previous breakpoint is stored in the process it self, the psd. - * We do not need to store in a stack frame. - * There is no need for locking, each thread has its own - * area in each bp to save data. - * Since we need to diffrentiate between processes for each bp, - * every bp has a hash (per thread) to process-bp statistics. - * - egil - */ - -void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type) { - Uint ms,s,us; - process_breakpoint_time_t *pbt = NULL; - bp_data_time_item_t sitem, *item = NULL; - bp_time_hash_t *h = NULL; - BpDataTime *pbdt = NULL; - - ASSERT(p); - ASSERT(p->status == P_RUNNING); - - /* get previous timestamp and breakpoint - * from the process psd */ - - pbt = ERTS_PROC_GET_CALL_TIME(p); - get_sys_now(&ms,&s,&us); - - switch(type) { - /* get pbt - * timestamp = t0 - * lookup bdt from code - * set ts0 to pbt - * add call count here? - */ - case ERTS_BP_CALL_TIME_CALL: - case ERTS_BP_CALL_TIME_TAIL_CALL: - - if (pbt) { - ASSERT(pbt->pc); - /* add time to previous code */ - bp_time_diff(&sitem, pbt, ms, s, us); - sitem.pid = p->id; - sitem.count = 0; - - /* previous breakpoint */ - pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); - - /* if null then the breakpoint was removed */ - if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(p)]); - - ASSERT(h); - ASSERT(h->item); - - item = bp_hash_get(h, &sitem); - if (!item) { - item = bp_hash_put(h, &sitem); - } else { - BP_TIME_ADD(item, &sitem); - } - } - - } else { - /* first call of process to instrumented function */ - pbt = Alloc(sizeof(process_breakpoint_time_t)); - (void *) ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCK_MAIN, pbt); - } - /* add count to this code */ - sitem.pid = p->id; - sitem.count = 1; - sitem.s_time = 0; - sitem.us_time = 0; - - /* this breakpoint */ - ASSERT(bdt); - h = &(bdt->hash[bp_sched2ix_proc(p)]); - - ASSERT(h); - ASSERT(h->item); - - item = bp_hash_get(h, &sitem); - if (!item) { - item = bp_hash_put(h, &sitem); - } else { - BP_TIME_ADD(item, &sitem); - } - - pbt->pc = pc; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; - break; - - case ERTS_BP_CALL_TIME_RETURN: - /* get pbt - * lookup bdt from code - * timestamp = t1 - * get ts0 from pbt - * get item from bdt->hash[bp_hash(p->id)] - * ack diff (t1, t0) to item - */ - - if(pbt) { - /* might have been removed due to - * trace_pattern(false) - */ - ASSERT(pbt->pc); - - bp_time_diff(&sitem, pbt, ms, s, us); - sitem.pid = p->id; - sitem.count = 0; - - /* previous breakpoint */ - pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); - - /* beware, the trace_pattern might have been removed */ - if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(p)]); - - ASSERT(h); - ASSERT(h->item); - - item = bp_hash_get(h, &sitem); - if (!item) { - item = bp_hash_put(h, &sitem); - } else { - BP_TIME_ADD(item, &sitem); - } - } - - pbt->pc = pc; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; - } - break; - default : - ASSERT(0); - /* will never happen */ - break; - } -} - - /* ************************************************************************* ** Local helpers */ -static int set_break(Eterm mfa[3], int specified, - Binary *match_spec, BeamInstr break_op, - enum erts_break_op count_op, Eterm tracer_pid) +static void +set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, + enum erts_break_op count_op, Eterm tracer_pid) { - Module *modp; - int num_processed = 0; - if (!specified) { - /* Find and process all modules in the system... */ - int current; - int last = module_code_size(); - for (current = 0; current < last; current++) { - modp = module_code(current); - ASSERT(modp != NULL); - num_processed += - set_module_break(modp, mfa, specified, - match_spec, break_op, count_op, - tracer_pid); - } - } else { - /* Process a single module */ - if ((modp = erts_get_module(mfa[0])) != NULL) { - num_processed += - set_module_break(modp, mfa, specified, - match_spec, break_op, count_op, - tracer_pid); - } - } - return num_processed; -} - -static int set_module_break(Module *modp, Eterm mfa[3], int specified, - Binary *match_spec, BeamInstr break_op, - enum erts_break_op count_op, Eterm tracer_pid) { - BeamInstr** code_base; - BeamInstr* code_ptr; - int num_processed = 0; - Uint i,n; + Uint i; + Uint n; - ASSERT(break_op); - ASSERT(modp); - code_base = (BeamInstr **) modp->code; - if (code_base == NULL) { - return 0; - } - n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; - for (i = 0; i < n; ++i) { - code_ptr = code_base[MI_FUNCTIONS+i]; - ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) && - (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) { - BeamInstr *pc = code_ptr+5; - - num_processed += - set_function_break(modp, pc, BREAK_IS_ERL, match_spec, - break_op, count_op, tracer_pid); - } + n = f->matched; + for (i = 0; i < n; i++) { + BeamInstr* pc = f->matching[i].pc; + set_function_break(pc, match_spec, break_flags, + count_op, tracer_pid); } - return num_processed; } -static int set_function_break(Module *modp, BeamInstr *pc, int bif, - Binary *match_spec, BeamInstr break_op, - enum erts_break_op count_op, Eterm tracer_pid) { - - BeamInstr **code_base = NULL; - BpData *bd, **r, ***rs; - size_t size; - Uint ix = 0; - - if (bif == BREAK_IS_ERL) { - code_base = (BeamInstr **)modp->code; - ASSERT(code_base); - ASSERT(code_base <= (BeamInstr **)pc); - ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *))); - } else { - ASSERT(*pc == (BeamInstr) em_apply_bif); - ASSERT(modp == NULL); +static void +set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, + enum erts_break_op count_op, Eterm tracer_pid) +{ + GenericBp* g; + GenericBpData* bp; + Uint common; + ErtsBpIndex ix = erts_staging_bp_ix(); + + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + g = (GenericBp *) pc[-4]; + if (g == 0) { + int i; + if (count_op == erts_break_reset || count_op == erts_break_stop) { + /* Do not insert a new breakpoint */ + return; + } + g = Alloc(sizeof(GenericBp)); + g->orig_instr = *pc; + for (i = 0; i < ERTS_NUM_BP_IX; i++) { + g->data[i].flags = 0; + } + pc[-4] = (BeamInstr) g; } + bp = &g->data[ix]; /* - * Currently no trace support for native code. + * If we are changing an existing breakpoint, clean up old data. */ - if (erts_is_native_break(pc)) { - return 0; - } - /* Do not allow two breakpoints of the same kind */ - if ( (bd = is_break(pc, break_op))) { - if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) - || break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { - - BpDataTrace *bdt = (BpDataTrace *) bd; - Binary *old_match_spec; - - /* Update match spec and tracer */ - MatchSetRef(match_spec); - ErtsSmpBPLock(bdt); - old_match_spec = bdt->match_spec; - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - ErtsSmpBPUnlock(bdt); - MatchSetUnref(old_match_spec); - } else { - BpDataCount *bdc = (BpDataCount *) bd; - erts_aint_t count = 0; - erts_aint_t res = 0; - - ASSERT(! match_spec); - ASSERT(is_nil(tracer_pid)); - - if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { - if (count_op == erts_break_stop) { - count = erts_smp_atomic_read_nob(&bdc->acount); - if (count >= 0) { - while(1) { - res = erts_smp_atomic_cmpxchg_nob(&bdc->acount, -count - 1, count); - if ((res == count) || count < 0) break; - count = res; - } - } - } else { - /* Reset call counter */ - erts_smp_atomic_set_nob(&bdc->acount, 0); - } - - } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { - BpDataTime *bdt = (BpDataTime *) bd; - Uint i = 0; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - if (count_op == erts_break_stop) { - bdt->pause = 1; - } else { - bdt->pause = 0; - for (i = 0; i < bdt->n; i++) { - bp_hash_delete(&(bdt->hash[i])); - bp_hash_init(&(bdt->hash[i]), 32); - } - } - } else { - ASSERT (! count_op); - } - } - return 1; - } - if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || - break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { - size = sizeof(BpDataTrace); - } else { - ASSERT(! match_spec); - ASSERT(is_nil(tracer_pid)); - if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { - if (count_op == erts_break_reset || count_op == erts_break_stop) { - /* Do not insert a new breakpoint */ - return 1; - } - size = sizeof(BpDataCount); - } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { - if (count_op == erts_break_reset || count_op == erts_break_stop) { - /* Do not insert a new breakpoint */ - return 1; - } - size = sizeof(BpDataTime); + common = break_flags & bp->flags; + if (common & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE)) { + MatchSetUnref(bp->local_ms); + } else if (common & ERTS_BPF_META_TRACE) { + MatchSetUnref(bp->meta_ms); + bp_meta_unref(bp->meta_pid); + } else if (common & ERTS_BPF_COUNT) { + if (count_op == erts_break_stop) { + bp->flags &= ~ERTS_BPF_COUNT_ACTIVE; } else { - ASSERT(! count_op); - ASSERT(break_op == (BeamInstr) BeamOp(op_i_debug_breakpoint)); - size = sizeof(BpDataDebug); + bp->flags |= ERTS_BPF_COUNT_ACTIVE; + erts_smp_atomic_set_nob(&bp->count->acount, 0); } - } - rs = (BpData ***) (pc-4); - if (! *rs) { - size_t ssize = sizeof(BeamInstr) * erts_no_schedulers; - *rs = (BpData **) Alloc(ssize); - sys_memzero(*rs, ssize); - } - - r = &((*rs)[0]); - - if (! *r) { - ASSERT(*pc != (BeamInstr) BeamOp(op_i_trace_breakpoint)); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_debug_breakpoint)); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_count_breakpoint)); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_time_breakpoint)); - /* First breakpoint; create singleton ring */ - bd = Alloc(size); - BpInit(bd, *pc); - *r = bd; - if (bif == BREAK_IS_ERL) { - *pc = break_op; - } - } else { - ASSERT(*pc == (BeamInstr) BeamOp(op_i_trace_breakpoint) || - *pc == (BeamInstr) BeamOp(op_i_mtrace_breakpoint) || - *pc == (BeamInstr) BeamOp(op_i_debug_breakpoint) || - *pc == (BeamInstr) BeamOp(op_i_time_breakpoint) || - *pc == (BeamInstr) BeamOp(op_i_count_breakpoint) || - *pc == (BeamInstr) em_apply_bif); - if (*pc == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { - /* Debug bp must be last, so if it is also first; - * it must be singleton. */ - ASSERT(BpSingleton(*r)); - /* Insert new bp first in the ring, i.e second to last. */ - bd = Alloc(size); - BpInitAndSpliceNext(bd, *pc, *r); - if (bif == BREAK_IS_ERL) { - *pc = break_op; - } - } else if ((*r)->prev->orig_instr - == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { - /* Debug bp last in the ring; insert new second to last. */ - bd = Alloc(size); - BpInitAndSplicePrev(bd, (*r)->prev->orig_instr, *r); - (*r)->prev->orig_instr = break_op; + ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0); + return; + } else if (common & ERTS_BPF_TIME_TRACE) { + BpDataTime* bdt = bp->time; + Uint i = 0; + + if (count_op == erts_break_stop) { + bp->flags &= ~ERTS_BPF_TIME_TRACE_ACTIVE; } else { - /* Just insert last in the ring */ - bd = Alloc(size); - BpInitAndSpliceNext(bd, (*r)->orig_instr, *r); - (*r)->orig_instr = break_op; - *r = bd; + bp->flags |= ERTS_BPF_TIME_TRACE_ACTIVE; + for (i = 0; i < bdt->n; i++) { + bp_hash_delete(&(bdt->hash[i])); + bp_hash_init(&(bdt->hash[i]), 32); + } } - } - for (ix = 1; ix < erts_no_schedulers; ++ix) { - (*rs)[ix] = (*rs)[0]; + ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0); + return; } - bd->this_instr = break_op; - /* Init the bp type specific data */ - if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || - break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { - - BpDataTrace *bdt = (BpDataTrace *) bd; - - MatchSetRef(match_spec); - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { - BpDataTime *bdt = (BpDataTime *) bd; - Uint i = 0; - - bdt->pause = 0; - bdt->n = erts_no_schedulers; - bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); + /* + * Initialize the new breakpoint data. + */ + if (break_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE)) { + MatchSetRef(match_spec); + bp->local_ms = match_spec; + } else if (break_flags & ERTS_BPF_META_TRACE) { + BpMetaPid* bmp; + MatchSetRef(match_spec); + bp->meta_ms = match_spec; + bmp = Alloc(sizeof(BpMetaPid)); + erts_refc_init(&bmp->refc, 1); + erts_smp_atomic_init_nob(&bmp->pid, tracer_pid); + bp->meta_pid = bmp; + } else if (break_flags & ERTS_BPF_COUNT) { + BpCount* bcp; + + ASSERT((bp->flags & ERTS_BPF_COUNT) == 0); + bcp = Alloc(sizeof(BpCount)); + erts_refc_init(&bcp->refc, 1); + erts_smp_atomic_init_nob(&bcp->acount, 0); + bp->count = bcp; + } else if (break_flags & ERTS_BPF_TIME_TRACE) { + BpDataTime* bdt; + int i; + + ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); + bdt = Alloc(sizeof(BpDataTime)); + erts_refc_init(&bdt->refc, 1); + bdt->n = erts_no_schedulers; + bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); } - } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { - BpDataCount *bdc = (BpDataCount *) bd; - erts_smp_atomic_init_nob(&bdc->acount, 0); + bp->time = bdt; } - if (bif == BREAK_IS_ERL) { - ++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); + bp->flags |= break_flags; + ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0); +} + +static void +clear_break(BpFunctions* f, Uint break_flags) +{ + Uint i; + Uint n; + + n = f->matched; + for (i = 0; i < n; i++) { + BeamInstr* pc = f->matching[i].pc; + clear_function_break(pc, break_flags); } - return 1; } -static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op) +static int +clear_function_break(BeamInstr *pc, Uint break_flags) { - int num_processed = 0; - Module *modp; + GenericBp* g; + GenericBpData* bp; + Uint common; + ErtsBpIndex ix = erts_staging_bp_ix(); - if (!specified) { - /* Iterate over all modules */ - int current; - int last = module_code_size(); + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - for (current = 0; current < last; current++) { - modp = module_code(current); - ASSERT(modp != NULL); - num_processed += clear_module_break(modp, mfa, specified, break_op); - } - } else { - /* Process a single module */ - if ((modp = erts_get_module(mfa[0])) != NULL) { - num_processed += - clear_module_break(modp, mfa, specified, break_op); - } + if ((g = (GenericBp *) pc[-4]) == 0) { + return 1; } - return num_processed; -} -static int clear_module_break(Module *m, Eterm mfa[3], int specified, - BeamInstr break_op) { - BeamInstr** code_base; - BeamInstr* code_ptr; - int num_processed = 0; - Uint i; - BeamInstr n; - - ASSERT(m); - code_base = (BeamInstr **) m->code; - if (code_base == NULL) { - return 0; + bp = &g->data[ix]; + ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0); + common = bp->flags & break_flags; + bp->flags &= ~break_flags; + if (common & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE)) { + MatchSetUnref(bp->local_ms); } - n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; - for (i = 0; i < n; ++i) { - code_ptr = code_base[MI_FUNCTIONS+i]; - if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) && - (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) { - BeamInstr *pc = code_ptr + 5; - - num_processed += - clear_function_break(m, pc, BREAK_IS_ERL, break_op); - } + if (common & ERTS_BPF_META_TRACE) { + MatchSetUnref(bp->meta_ms); + bp_meta_unref(bp->meta_pid); + } + if (common & ERTS_BPF_COUNT) { + ASSERT((bp->flags & ERTS_BPF_COUNT_ACTIVE) == 0); + bp_count_unref(bp->count); + } + if (common & ERTS_BPF_TIME_TRACE) { + ASSERT((bp->flags & ERTS_BPF_TIME_TRACE_ACTIVE) == 0); + bp_time_unref(bp->time); } - return num_processed; -} -static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr break_op) { - BpData *bd; - Uint ix = 0; - BeamInstr **code_base = NULL; + ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0); + return 1; +} - if (bif == BREAK_IS_ERL) { - code_base = (BeamInstr **)m->code; - ASSERT(code_base); - ASSERT(code_base <= (BeamInstr **)pc); - ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *))); - } else { - ASSERT(*pc == (BeamInstr) em_apply_bif); - ASSERT(m == NULL); +static void +bp_meta_unref(BpMetaPid* bmp) +{ + if (erts_refc_dectest(&bmp->refc, 0) <= 0) { + Free(bmp); } +} - /* - * Currently no trace support for native code. - */ - if (erts_is_native_break(pc)) { - return 0; +static void +bp_count_unref(BpCount* bcp) +{ + if (erts_refc_dectest(&bcp->refc, 0) <= 0) { + Free(bcp); } +} - while ( (bd = is_break(pc, break_op))) { - /* Remove all breakpoints of this type. - * There should be only one of each type, - * but break_op may be 0 which matches any type. +static void +bp_time_unref(BpDataTime* bdt) +{ + if (erts_refc_dectest(&bdt->refc, 0) <= 0) { + Uint i = 0; + Uint j = 0; + Process *h_p = NULL; + bp_data_time_item_t* item = NULL; + process_breakpoint_time_t* pbt = NULL; + + /* remove all psd associated with the hash + * and then delete the hash. + * ... sigh ... */ - BeamInstr op; - BpData ***rs = (BpData ***) (pc - 4); - BpData **r = NULL; - -#ifdef DEBUG - for (ix = 1; ix < erts_no_schedulers; ++ix) { - ASSERT((*rs)[ix] == (*rs)[0]); - } -#endif - - r = &((*rs)[0]); - - ASSERT(*r); - /* Find opcode for this breakpoint */ - if (break_op) { - op = break_op; - } else { - if (bd == (*r)->next) { - /* First breakpoint in ring */ - op = *pc; - } else { - op = bd->prev->orig_instr; - } - } - if (BpSingleton(bd)) { - ASSERT(*r == bd); - /* Only one breakpoint to remove */ - if (bif == BREAK_IS_ERL) { - *pc = bd->orig_instr; - } - Free(*rs); - *rs = NULL; - } else { - BpData *bd_prev = bd->prev; - - BpSpliceNext(bd, bd_prev); - ASSERT(BpSingleton(bd)); - if (bd == *r) { - /* We removed the last breakpoint in the ring */ - *r = bd_prev; - bd_prev->orig_instr = bd->orig_instr; - } else if (bd_prev == *r) { - /* We removed the first breakpoint in the ring */ - if (bif == BREAK_IS_ERL) { - *pc = bd->orig_instr; - } - } else { - bd_prev->orig_instr = bd->orig_instr; - } - } - if (op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || - op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { - - BpDataTrace *bdt = (BpDataTrace *) bd; - MatchSetUnref(bdt->match_spec); - } - if (op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { - BpDataTime *bdt = (BpDataTime *) bd; - Uint i = 0; - Uint j = 0; - Process *h_p = NULL; - bp_data_time_item_t *item = NULL; - process_breakpoint_time_t *pbt = NULL; - - /* remove all psd associated with the hash - * and then delete the hash. - * ... sigh ... - */ - for( i = 0; i < bdt->n; ++i) { - if (bdt->hash[i].used) { - for (j = 0; j < bdt->hash[i].n; ++j) { - item = &(bdt->hash[i].item[j]); - if (item->pid != NIL) { - h_p = process_tab[internal_pid_index(item->pid)]; - if (h_p) { - pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL); - if (pbt) { - Free(pbt); - } + for (i = 0; i < bdt->n; ++i) { + if (bdt->hash[i].used) { + for (j = 0; j < bdt->hash[i].n; ++j) { + item = &(bdt->hash[i].item[j]); + if (item->pid != NIL) { + h_p = erts_pid2proc(NULL, 0, item->pid, + ERTS_PROC_LOCK_MAIN); + if (h_p) { + pbt = ERTS_PROC_SET_CALL_TIME(h_p, + ERTS_PROC_LOCK_MAIN, + NULL); + if (pbt) { + Free(pbt); } + erts_smp_proc_unlock(h_p, ERTS_PROC_LOCK_MAIN); } } } - bp_hash_delete(&(bdt->hash[i])); } - Free(bdt->hash); - bdt->hash = NULL; - bdt->n = 0; + bp_hash_delete(&(bdt->hash[i])); } - Free(bd); - if (bif == BREAK_IS_ERL) { - ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0); - --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); - } - if (*rs) { - for (ix = 1; ix < erts_no_schedulers; ++ix) { - (*rs)[ix] = (*rs)[0]; - } - } - } /* while bd != NULL */ - return 1; + Free(bdt->hash); + Free(bdt); + } } - - -/* -** Searches (linear forward) the breakpoint ring for a specified opcode -** and returns a pointer to the breakpoint data structure or NULL if -** not found. If the specified opcode is 0, the last breakpoint is -** returned. The program counter must point to the first executable -** (breakpoint) instruction of the function. -*/ - -BpData *erts_get_time_break(Process *p, BeamInstr *pc) { - return get_break(p, pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); +static BpDataTime* +get_time_break(BeamInstr *pc) +{ + GenericBpData* bp = check_break(pc, ERTS_BPF_TIME_TRACE); + return bp ? bp->time : 0; } -static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) { - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (! erts_is_native_break(pc)) { - BpData **rs = (BpData **) pc[-4]; - BpData *bd = NULL, *ebd = NULL; - - if (! rs) { - return NULL; - } - - bd = ebd = rs[bp_sched2ix_proc(p)]; - ASSERT(bd); - if (bd->this_instr == break_op) { - return bd; - } - - bd = bd->next; - while (bd != ebd) { - ASSERT(bd); - if (bd->this_instr == break_op) { - ASSERT(bd); - return bd; - } - bd = bd->next; - } - } - return NULL; -} +static GenericBpData* +check_break(BeamInstr *pc, Uint break_flags) +{ + GenericBp* g = (GenericBp *) pc[-4]; -static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { - BpData **rs; - BpData *bd = NULL, *ebd = NULL; ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (erts_is_native_break(pc)) { - return NULL; - } - rs = (BpData **) pc[-4]; - if (! rs) { - return NULL; - } - - bd = ebd = rs[erts_bp_sched2ix()]; - ASSERT(bd); - if ( (break_op == 0) || (bd->this_instr == break_op)) { - return bd; + return 0; } - - bd = bd->next; - while (bd != ebd) { - ASSERT(bd); - if (bd->this_instr == break_op) { - ASSERT(bd); - return bd; + if (g) { + GenericBpData* bp = &g->data[erts_active_bp_ix()]; + ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0); + if (bp->flags & break_flags) { + return bp; } - bd = bd->next; } - return NULL; + return 0; } diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 167069552f..b061401863 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,77 +25,6 @@ #include "erl_vm.h" #include "global.h" - - -/* A couple of gotchas: - * - * The breakpoint structure from BeamInstr, - * In beam_emu where the instruction counter pointer, I (or pc), - * points to the *current* instruction. At that time, if the instruction - * is a breakpoint instruction the pc looks like the following, - * - * I[-5] | op_i_func_info_IaaI | scheduler specific entries - * I[-4] | BpData** bpa | --> | BpData * bdas1 | ... | BpData * bdasN | - * I[-3] | Tagged Module | | | - * I[-2] | Tagged Function | V V - * I[-1] | Arity | BpData -> BpData -> BpData -> BpData - * I[0] | The bp instruction | ^ * the bp wheel * | - * |------------------------------ - * - * Common struct to all bp_data_* - * - * 1) The type of bp_data structure in the ring is deduced from the - * orig_instr field of the structure _before_ in the ring, except for - * the first structure in the ring that has its instruction in - * pc[0] of the code to execute. - * This is valid as long as you don't search for the function while it is - * being executed by something else. Or is in the middle of its rotation for - * any other reason. - * A key, the bp beam instruction, is included for this reason. - * - * 2) pc[-4][sched_id - 1] points to the _last_ structure in the ring before the - * breakpoints are being executed. - * - * So, as an example, when a breakpointed function starts to execute, - * the first instruction that is a breakpoint instruction at pc[0] finds - * its data at ((BpData **) pc[-4][sched_id - 1])->next and has to cast that pointer - * to the correct bp_data type. -*/ - -typedef struct bp_data { - struct bp_data *next; /* Doubly linked ring pointers */ - struct bp_data *prev; /* -"- */ - BeamInstr orig_instr; /* The original instruction to execute */ - BeamInstr this_instr; /* key */ -} BpData; -/* -** All the following bp_data_.. structs must begin the same way -*/ - -typedef struct bp_data_trace { - struct bp_data *next; - struct bp_data *prev; - BeamInstr orig_instr; - BeamInstr this_instr; /* key */ - Binary *match_spec; - Eterm tracer_pid; -} BpDataTrace; - -typedef struct bp_data_debug { - struct bp_data *next; - struct bp_data *prev; - BeamInstr orig_instr; - BeamInstr this_instr; /* key */ -} BpDataDebug; - -typedef struct bp_data_count { /* Call count */ - struct bp_data *next; - struct bp_data *prev; - BeamInstr orig_instr; - BeamInstr this_instr; /* key */ - erts_smp_atomic_t acount; -} BpDataCount; - typedef struct { Eterm pid; Sint count; @@ -110,13 +39,9 @@ typedef struct { } bp_time_hash_t; typedef struct bp_data_time { /* Call time */ - struct bp_data *next; - struct bp_data *prev; - BeamInstr orig_instr; - BeamInstr this_instr; /* key */ - Uint pause; - Uint n; - bp_time_hash_t *hash; + Uint n; + bp_time_hash_t *hash; + erts_refc_t refc; } BpDataTime; typedef struct { @@ -126,64 +51,42 @@ typedef struct { BeamInstr *pc; } process_breakpoint_time_t; /* used within psd */ -extern erts_smp_spinlock_t erts_bp_lock; +typedef struct { + erts_smp_atomic_t acount; + erts_refc_t refc; +} BpCount; + +typedef struct { + erts_smp_atomic_t pid; + erts_refc_t refc; +} BpMetaPid; + +typedef struct generic_bp_data { + Uint flags; + Binary* local_ms; /* Match spec for local call trace */ + Binary* meta_ms; /* Match spec for meta trace */ + BpMetaPid* meta_pid; /* Meta trace pid */ + BpCount* count; /* For call count */ + BpDataTime* time; /* For time trace */ +} GenericBpData; + +#define ERTS_NUM_BP_IX 2 + +typedef struct generic_bp { + BeamInstr orig_instr; + GenericBpData data[ERTS_NUM_BP_IX]; +} GenericBp; #define ERTS_BP_CALL_TIME_SCHEDULE_IN (0) #define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) -#define ERTS_BP_CALL_TIME_CALL (0) -#define ERTS_BP_CALL_TIME_RETURN (1) -#define ERTS_BP_CALL_TIME_TAIL_CALL (2) - -#ifdef ERTS_SMP -#define ErtsSmpBPLock(BDC) erts_smp_spin_lock(&erts_bp_lock) -#define ErtsSmpBPUnlock(BDC) erts_smp_spin_unlock(&erts_bp_lock) -#else -#define ErtsSmpBPLock(BDC) -#define ErtsSmpBPUnlock(BDC) -#endif - #ifdef ERTS_SMP #define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1) #else #define bp_sched2ix_proc(p) (0) #endif -#define ErtsCountBreak(p, pc,instr_result) \ -do { \ - BpData **bds = (BpData **) (pc)[-4]; \ - BpDataCount *bdc = NULL; \ - Uint ix = bp_sched2ix_proc( (p) ); \ - erts_aint_t count = 0; \ - \ - ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bds); \ - bdc = (BpDataCount *) bds[ix]; \ - bdc = (BpDataCount *) bdc->next; \ - ASSERT(bdc); \ - bds[ix] = (BpData *) bdc; \ - count = erts_smp_atomic_read_nob(&bdc->acount); \ - if (count >= 0) erts_smp_atomic_inc_nob(&bdc->acount); \ - *(instr_result) = bdc->orig_instr; \ -} while (0) - -#define ErtsBreakSkip(p, pc,instr_result) \ -do { \ - BpData **bds = (BpData **) (pc)[-4]; \ - BpData *bd = NULL; \ - Uint ix = bp_sched2ix_proc( (p) ); \ - \ - ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bds); \ - bd = bds[ix]; \ - ASSERT(bd); \ - bd = bd->next; \ - ASSERT(bd); \ - bds[ix] = bd; \ - *(instr_result) = bd->orig_instr; \ -} while (0) - enum erts_break_op{ erts_break_nop = 0, /* Must be false */ erts_break_set = !0, /* Must be true */ @@ -191,7 +94,17 @@ enum erts_break_op{ erts_break_stop }; +typedef Uint32 ErtsBpIndex; +typedef struct { + BeamInstr* pc; + Module* mod; +} BpFunction; + +typedef struct { + Uint matched; /* Number matched */ + BpFunction* matching; /* Matching functions */ +} BpFunctions; /* ** Function interface exported from beam_bp.c @@ -199,49 +112,66 @@ enum erts_break_op{ void erts_bp_init(void); -int erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec, - Eterm tracer_pid); -int erts_clear_trace_break(Eterm mfa[3], int specified); -int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, +void erts_prepare_bp_staging(void); +void erts_commit_staged_bp(void); + +ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void); +ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void); + +void erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified); +void erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified); +void erts_bp_free_matched_functions(BpFunctions* f); + +void erts_install_breakpoints(BpFunctions* f); +void erts_uninstall_breakpoints(BpFunctions* f); +void erts_consolidate_bp_data(BpFunctions* f, int local); +void erts_consolidate_bif_bp_data(void); + +void erts_set_trace_break(BpFunctions *f, Binary *match_spec); +void erts_clear_trace_break(BpFunctions *f); + +void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local); +void erts_clear_call_trace_bif(BeamInstr *pc, int local); + +void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, Eterm tracer_pid); -int erts_clear_mtrace_break(Eterm mfa[3], int specified); +void erts_clear_mtrace_break(BpFunctions *f); void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid); void erts_clear_mtrace_bif(BeamInstr *pc); -int erts_set_debug_break(Eterm mfa[3], int specified); -int erts_clear_debug_break(Eterm mfa[3], int specified); -int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op); -int erts_clear_count_break(Eterm mfa[3], int specified); +void erts_set_debug_break(BpFunctions *f); +void erts_clear_debug_break(BpFunctions *f); +void erts_set_count_break(BpFunctions *f, enum erts_break_op); +void erts_clear_count_break(BpFunctions *f); -int erts_clear_break(Eterm mfa[3], int specified); + +void erts_clear_all_breaks(BpFunctions* f); int erts_clear_module_break(Module *modp); -int erts_clear_function_break(Module *modp, BeamInstr *pc); +void erts_clear_export_break(Module *modp, BeamInstr* pc); +BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg); BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, Uint32 *ret_flags, Eterm *tracer_pid); -Uint32 erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, - int local, Eterm *tracer_pid); -int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, - Eterm *tracer_pid_ret); +int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local); int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_rte); int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret); int erts_is_native_break(BeamInstr *pc); -int erts_is_count_break(BeamInstr *pc, Sint *count_ret); +int erts_is_count_break(BeamInstr *pc, Uint *count_ret); int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); -void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type); +void erts_trace_time_call(Process* c_p, BeamInstr* pc, BpDataTime* bdt); +void erts_trace_time_return(Process* c_p, BeamInstr* pc); void erts_schedule_time_break(Process *p, Uint out); -int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op); -int erts_clear_time_break(Eterm mfa[3], int specified); +void erts_set_time_break(BpFunctions *f, enum erts_break_op); +void erts_clear_time_break(BpFunctions *f); int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); void erts_clear_time_trace_bif(BeamInstr *pc); -BpData *erts_get_time_break(Process *p, BeamInstr *pc); BeamInstr *erts_find_local_func(Eterm mfa[3]); @@ -258,6 +188,19 @@ ERTS_GLB_INLINE Uint erts_bp_sched2ix(void) return 0; #endif } + +extern erts_smp_atomic32_t erts_active_bp_index; +extern erts_smp_atomic32_t erts_staging_bp_index; + +ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void) +{ + return erts_smp_atomic32_read_nob(&erts_active_bp_index); +} + +ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void) +{ + return erts_smp_atomic32_read_nob(&erts_staging_bp_index); +} #endif #endif /* _BEAM_BP_H */ diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index 406ef1db5f..d374d0469e 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,78 +31,143 @@ typedef struct { unsigned cdr; } beam_catch_t; -static int free_list; -static unsigned high_mark; -static unsigned tabsize; -static beam_catch_t *beam_catches; +#ifdef DEBUG +# define IF_DEBUG(x) x +#else +# define IF_DEBUG(x) +#endif + +struct bc_pool { + int free_list; + unsigned high_mark; + unsigned tabsize; + beam_catch_t *beam_catches; + /* + * Note that the 'beam_catches' area is shared by pools. Used slots + * are readonly as long as the module is not purgable. The free-list is + * protected by the code_ix lock. + */ + + IF_DEBUG(int is_staging;) +}; + +static struct bc_pool bccix[ERTS_NUM_CODE_IX]; void beam_catches_init(void) { - tabsize = DEFAULT_TABSIZE; - free_list = -1; - high_mark = 0; + int i; + + bccix[0].tabsize = DEFAULT_TABSIZE; + bccix[0].free_list = -1; + bccix[0].high_mark = 0; + bccix[0].beam_catches = erts_alloc(ERTS_ALC_T_CODE, + sizeof(beam_catch_t)*DEFAULT_TABSIZE); + IF_DEBUG(bccix[0].is_staging = 0); + for (i=1; i<ERTS_NUM_CODE_IX; i++) { + bccix[i] = bccix[i-1]; + } + /* For initial load: */ + IF_DEBUG(bccix[erts_staging_code_ix()].is_staging = 1); +} - beam_catches = erts_alloc(ERTS_ALC_T_CODE, sizeof(beam_catch_t)*DEFAULT_TABSIZE); + +static void gc_old_vec(beam_catch_t* vec) +{ + int i; + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + if (bccix[i].beam_catches == vec) { + return; + } + } + erts_free(ERTS_ALC_T_CODE, vec); +} + + +void beam_catches_start_staging(void) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + ErtsCodeIndex src = erts_active_code_ix(); + beam_catch_t* prev_vec = bccix[dst].beam_catches; + + ASSERT(!bccix[src].is_staging && !bccix[dst].is_staging); + + bccix[dst] = bccix[src]; + gc_old_vec(prev_vec); + IF_DEBUG(bccix[dst].is_staging = 1); +} + +void beam_catches_end_staging(int commit) +{ + IF_DEBUG(bccix[erts_staging_code_ix()].is_staging = 0); } unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr) { int i; + struct bc_pool* p = &bccix[erts_staging_code_ix()]; + ASSERT(p->is_staging); /* * Allocate from free_list while it is non-empty. * If free_list is empty, allocate at high_mark. - * - * This avoids the need to initialise the free list in - * beam_catches_init(), which would cost O(TABSIZ) time. */ - if( free_list >= 0 ) { - i = free_list; - free_list = beam_catches[i].cdr; - } else if( high_mark < tabsize ) { - i = high_mark; - high_mark++; - } else { - /* No free slots and table is full: realloc table */ - tabsize = 2*tabsize; - beam_catches = erts_realloc(ERTS_ALC_T_CODE, beam_catches, sizeof(beam_catch_t)*tabsize); - i = high_mark; - high_mark++; + if (p->free_list >= 0) { + i = p->free_list; + p->free_list = p->beam_catches[i].cdr; + } + else { + if (p->high_mark >= p->tabsize) { + /* No free slots and table is full: realloc table */ + beam_catch_t* prev_vec = p->beam_catches; + unsigned newsize = p->tabsize*2; + + p->beam_catches = erts_alloc(ERTS_ALC_T_CODE, + newsize*sizeof(beam_catch_t)); + sys_memcpy(p->beam_catches, prev_vec, + p->tabsize*sizeof(beam_catch_t)); + gc_old_vec(prev_vec); + p->tabsize = newsize; + } + i = p->high_mark++; } - beam_catches[i].cp = cp; - beam_catches[i].cdr = cdr; + p->beam_catches[i].cp = cp; + p->beam_catches[i].cdr = cdr; return i; } BeamInstr *beam_catches_car(unsigned i) { - if( i >= tabsize ) { + struct bc_pool* p = &bccix[erts_active_code_ix()]; + + if (i >= p->tabsize ) { erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); } - return beam_catches[i].cp; + return p->beam_catches[i].cp; } -void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes) +void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes, + ErtsCodeIndex code_ix) { + struct bc_pool* p = &bccix[code_ix]; unsigned i, cdr; + ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging); for(i = head; i != (unsigned)-1;) { - if( i >= tabsize ) { + if (i >= p->tabsize) { erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); } - if( (char*)beam_catches[i].cp - (char*)code >= code_bytes ) { + if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) { erl_exit(1, - "beam_catches_delmod: item %#x has cp %#lx which is not " - "in module's range [%#lx,%#lx[\r\n", - i, (long)beam_catches[i].cp, - (long)code, (long)((char*)code + code_bytes)); + "beam_catches_delmod: item %#x has cp %p which is not " + "in module's range [%p,%p[\r\n", + i, p->beam_catches[i].cp, code, ((char*)code + code_bytes)); } - beam_catches[i].cp = 0; - cdr = beam_catches[i].cdr; - beam_catches[i].cdr = free_list; - free_list = i; + p->beam_catches[i].cp = 0; + cdr = p->beam_catches[i].cdr; + p->beam_catches[i].cdr = p->free_list; + p->free_list = i; i = cdr; } } diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h index 6223427f0d..51ef463b2f 100644 --- a/erts/emulator/beam/beam_catches.h +++ b/erts/emulator/beam/beam_catches.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -20,12 +20,21 @@ #ifndef __BEAM_CATCHES_H #define __BEAM_CATCHES_H +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include "sys.h" +#include "code_ix.h" + #define BEAM_CATCHES_NIL (-1) void beam_catches_init(void); +void beam_catches_start_staging(void); +void beam_catches_end_staging(int commit); unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr); BeamInstr *beam_catches_car(unsigned i); -void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes); +void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes, + ErtsCodeIndex); #define catch_pc(x) beam_catches_car(catch_val((x))) diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8041c92162..e36ec2a93e 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -84,6 +84,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) int i; int specified = 0; Eterm res; + BpFunctions f; if (bool != am_true && bool != am_false) goto error; @@ -114,18 +115,30 @@ erts_debug_breakpoint_2(BIF_ALIST_2) mfa[2] = signed_val(mfa[2]); } + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_breakpoint_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); + erts_bp_match_functions(&f, mfa, specified); if (bool == am_true) { - res = make_small(erts_set_debug_break(mfa, specified)); + erts_set_debug_break(&f); + erts_install_breakpoints(&f); + erts_commit_staged_bp(); } else { - res = make_small(erts_clear_debug_break(mfa, specified)); + erts_clear_debug_break(&f); + erts_commit_staged_bp(); + erts_uninstall_breakpoints(&f); } + erts_consolidate_bp_data(&f, 1); + res = make_small(f.matched); + erts_bp_free_matched_functions(&f); erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - + erts_release_code_write_permission(); return res; error: @@ -207,6 +220,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_false); } } else if (is_tuple(addr)) { + ErtsCodeIndex code_ix; Module* modp; Eterm mod; Eterm name; @@ -225,14 +239,14 @@ erts_debug_disassemble_1(BIF_ALIST_1) goto error; } arity = signed_val(tp[3]); - modp = erts_get_module(mod); + code_ix = erts_active_code_ix(); + modp = erts_get_module(mod, code_ix); /* * Try the export entry first to allow disassembly of special functions * such as erts_debug:apply/4. Then search for it in the module. */ - - if ((ep = erts_find_function(mod, name, arity)) != NULL) { + if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) { /* XXX: add "&& ep->address != ep->code+3" condition? * Consider a traced function. * Its ep will have ep->address == ep->code+3. @@ -241,9 +255,9 @@ erts_debug_disassemble_1(BIF_ALIST_1) * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((BeamInstr *) ep->address) - 5; + code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; funcinfo = code_ptr+2; - } else if (modp == NULL || (code_base = modp->code) == NULL) { + } else if (modp == NULL || (code_base = modp->curr.code) == NULL) { BIF_RET(am_undef); } else { n = code_base[MI_NUM_FUNCTIONS]; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index c5c57c1125..da36c4437e 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -63,11 +63,7 @@ # define PROCESS_MAIN_CHK_LOCKS(P) \ do { \ if ((P)) { \ - erts_pix_lock_t *pix_lock__ = ERTS_PIX2PIXLOCK(internal_pid_index((P)->id));\ erts_proc_lc_chk_only_proc_main((P)); \ - erts_pix_lock(pix_lock__); \ - ASSERT(0 < (P)->lock.refc && (P)->lock.refc < erts_no_schedulers*5);\ - erts_pix_unlock(pix_lock__); \ } \ else \ erts_lc_check_exact(NULL, 0); \ @@ -221,7 +217,6 @@ BeamInstr beam_continue_exit[1]; BeamInstr* em_call_error_handler; BeamInstr* em_apply_bif; -BeamInstr* em_call_traced_function; /* NOTE These should be the only variables containing trace instructions. @@ -236,11 +231,6 @@ BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* - * We should warn only once for tuple funs. - */ -static erts_smp_atomic_t warned_for_tuple_funs; - -/* * All Beam instructions in numerical order. */ @@ -495,7 +485,7 @@ extern int count_instructions; do { \ if (FCALLS > 0) { \ Eterm* dis_next; \ - SET_I(((Export *) Arg(0))->address); \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ dis_next = (Eterm *) *I; \ FCALLS--; \ CHECK_ARGS(I); \ @@ -504,7 +494,7 @@ extern int count_instructions; && FCALLS > neg_o_reds) { \ goto save_calls1; \ } else { \ - SET_I(((Export *) Arg(0))->address); \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ CHECK_ARGS(I); \ goto context_switch; \ } \ @@ -526,7 +516,7 @@ extern int count_instructions; # define Dispatchfun() DispatchMacroFun() #endif -#define Self(R) R = c_p->id +#define Self(R) R = c_p->common.id #define Node(R) R = erts_this_node->sysname #define Arg(N) I[(N)+1] @@ -966,17 +956,9 @@ static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); -#if defined(VXWORKS) -static int init_done; -#endif - void init_emulator(void) { -#if defined(VXWORKS) - init_done = 0; -#endif - erts_smp_atomic_init_nob(&warned_for_tuple_funs, (erts_aint_t) 0); process_main(); } @@ -1089,17 +1071,6 @@ init_emulator(void) #endif /* USE_VM_PROBES */ -#ifdef USE_VM_PROBES -void -dtrace_drvport_str(ErlDrvPort drvport, char *port_buf) -{ - Port *port = erts_drvport2port(drvport); - - erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>", - port_channel_no(port->id), - port_number(port->id)); -} -#endif /* * process_main() is called twice: * The first call performs some initialisation, including exporting @@ -1108,9 +1079,7 @@ dtrace_drvport_str(ErlDrvPort drvport, char *port_buf) */ void process_main(void) { -#if !defined(VXWORKS) static int init_done = 0; -#endif Process* c_p = NULL; int reds_used; #ifdef DEBUG @@ -1230,7 +1199,7 @@ void process_main(void) ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; #ifdef DEBUG - pid = c_p->id; /* Save for debugging purpouses */ + pid = c_p->common.id; /* Save for debugging purpouses */ #endif ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -1268,7 +1237,7 @@ void process_main(void) reds = c_p->fcalls; if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) - && (c_p->trace_flags & F_SENSITIVE) == 0) { + && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) { neg_o_reds = -reds; FCALLS = REDS_IN(c_p) = 0; } else { @@ -1528,7 +1497,7 @@ void process_main(void) */ #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); } #endif @@ -1543,7 +1512,7 @@ void process_main(void) SET_CP(c_p, I+2); #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); } #endif @@ -1556,7 +1525,7 @@ void process_main(void) OpCase(i_call_ext_only_e): #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address); + BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); } #endif @@ -1632,6 +1601,7 @@ void process_main(void) reg[0] = r(0); result = erl_send(c_p, r(0), x(1)); PreFetch(0, next); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { @@ -1847,13 +1817,12 @@ void process_main(void) msgp = PEEK_MESSAGE(c_p); if (msgp) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - else { + else #endif + { SET_I((BeamInstr *) Arg(0)); Goto(*I); /* Jump to a wait or wait_timeout instruction */ -#ifdef ERTS_SMP } -#endif } ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS, { @@ -1908,14 +1877,14 @@ void process_main(void) erts_fprintf(stderr, "Dtrace -> (%T) stop spreading " "tag %T with message %T\r\n", - c_p->id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); + c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); #endif } else { #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) kill tag %T with " "message %T\r\n", - c_p->id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); + c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); #endif DT_UTAG(c_p) = NIL; SEQ_TRACE_TOKEN(c_p) = NIL; @@ -1940,7 +1909,7 @@ void process_main(void) erts_fprintf(stderr, "Dtrace -> (%T) receive tag (%T) " "with message %T\r\n", - c_p->id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp)); + c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp)); #endif } else { #endif @@ -1956,7 +1925,7 @@ void process_main(void) } msg = ERL_MESSAGE_TERM(msgp); seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, - c_p->id, c_p); + c_p->common.id, c_p); #ifdef USE_VM_PROBES } #endif @@ -2082,11 +2051,11 @@ void process_main(void) OpCase(wait_f): wait2: { - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; - c_p->status = P_WAITING; + erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); c_p->current = NULL; goto do_schedule; @@ -2609,6 +2578,7 @@ void process_main(void) reg[0] = r(0); result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -3165,10 +3135,6 @@ void process_main(void) c_p->arg_reg[0] = r(0); SWAPOUT; c_p->i = I; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - if (c_p->status != P_SUSPENDED) - erts_add_to_runq(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); goto do_schedule1; } @@ -3347,7 +3313,6 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = I[-1]; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); { @@ -3392,7 +3357,6 @@ void process_main(void) bif_nif_arity = I[-1]; ASSERT(bif_nif_arity <= 3); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); reg[0] = r(0); { Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; @@ -4601,64 +4565,6 @@ void process_main(void) * Trace and debugging support. */ - /* - * At this point, I points to the code[3] in the export entry for - * a trace-enabled function. - * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&call_traced_function - * code[4]: Address of function. - */ - OpCase(call_traced_function): { - if (IS_TRACED_FL(c_p, F_TRACE_CALLS)) { - unsigned offset = offsetof(Export, code) + 3*sizeof(BeamInstr); - Export* ep = (Export *) (((char *)I)-offset); - Uint32 flags; - - SWAPOUT; - reg[0] = r(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - flags = erts_call_trace(c_p, ep->code, ep->match_prog_set, reg, - 0, &c_p->tracer_proc); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - SWAPIN; - - if (flags & MATCH_SET_RX_TRACE) { - ASSERT(c_p->htop <= E && E <= c_p->hend); - if (E - 3 < HTOP) { - /* SWAPOUT, SWAPIN was done and r(0) was saved above */ - PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect(c_p, 3, reg, ep->code[2]); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - r(0) = reg[0]; - SWAPIN; - } - E -= 3; - ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((BeamInstr)(ep->code))); - ASSERT(is_internal_pid(c_p->tracer_proc) || - is_internal_port(c_p->tracer_proc)); - E[2] = make_cp(c_p->cp); /* Code in lower range on halfword */ - E[1] = am_true; /* Process tracer */ - E[0] = make_cp(ep->code); - c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) - ? beam_exception_trace : beam_return_trace; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - c_p->trace_flags |= F_EXCEPTION_TRACE; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - SET_I((BeamInstr *)Arg(0)); - Dispatch(); - } - OpCase(return_trace): { BeamInstr* code = (BeamInstr *) (UWord) E[0]; @@ -4673,80 +4579,22 @@ void process_main(void) Goto(*I); } - OpCase(i_count_breakpoint): { - BeamInstr real_I; - - ErtsCountBreak(c_p, (BeamInstr *) I, &real_I); - ASSERT(VALID_INSTR(real_I)); - Goto(real_I); - } - - /* need to send mfa instead of bdt pointer - * the pointer might be deallocated. - */ - - OpCase(i_time_breakpoint): { + OpCase(i_generic_breakpoint): { BeamInstr real_I; - BpData **bds = (BpData **) (I)[-4]; - BpDataTime *bdt = NULL; - Uint ix = 0; -#ifdef ERTS_SMP - ix = c_p->scheduler_data->no - 1; -#else - ix = 0; -#endif - bdt = (BpDataTime *)bds[ix]; - - ASSERT((I)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - ASSERT(bdt); - bdt = (BpDataTime *) bdt->next; - ASSERT(bdt); - bds[ix] = (BpData *) bdt; - real_I = bdt->orig_instr; + ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + SWAPOUT; + reg[0] = r(0); + real_I = erts_generic_breakpoint(c_p, I, reg); + r(0) = reg[0]; + SWAPIN; ASSERT(VALID_INSTR(real_I)); - - if (IS_TRACED_FL(c_p, F_TRACE_CALLS) && !(bdt->pause)) { - if ( (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) || - (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) || - (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace))) { - /* This _IS_ a tail recursive call */ - SWAPOUT; - erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_TAIL_CALL); - SWAPIN; - } else { - SWAPOUT; - erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_CALL); - - /* r register needs to be copied to the array - * for the garbage collector - */ - ASSERT(c_p->htop <= E && E <= c_p->hend); - if (E - 2 < HTOP) { - reg[0] = r(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - r(0) = reg[0]; - } - SWAPIN; - - ASSERT(c_p->htop <= E && E <= c_p->hend); - - E -= 2; - E[0] = make_cp(I); - E[1] = make_cp(c_p->cp); /* original return address */ - c_p->cp = beam_return_time_trace; - } - } - Goto(real_I); } OpCase(i_return_time_trace): { BeamInstr *pc = (BeamInstr *) (UWord) E[0]; SWAPOUT; - erts_trace_time_break(c_p, pc, NULL, ERTS_BP_CALL_TIME_RETURN); + erts_trace_time_return(c_p, pc); SWAPIN; c_p->cp = NULL; SET_I((BeamInstr *) cp_val(E[1])); @@ -4754,114 +4602,6 @@ void process_main(void) Goto(*I); } - OpCase(i_trace_breakpoint): - if (! IS_TRACED_FL(c_p, F_TRACE_CALLS)) { - BeamInstr real_I; - - ErtsBreakSkip(c_p, (BeamInstr *) I, &real_I); - Goto(real_I); - } - /* Fall through to next case */ - OpCase(i_mtrace_breakpoint): { - BeamInstr real_I; - Uint32 flags; - Eterm tracer_pid; - Uint* cpp; - int return_to_trace = 0, need = 0; - flags = 0; - SWAPOUT; - reg[0] = r(0); - - if (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) { - cpp = &E[2]; - } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace)) { - return_to_trace = !0; - cpp = &E[0]; - } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) { - return_to_trace = !0; - cpp = &E[0]; - } else { - cpp = NULL; - } - if (cpp) { - /* This _IS_ a tail recursive call, if there are - * return_trace and/or i_return_to_trace stackframes - * on the stack, they are not intermixed with y registers - */ - BeamInstr *cp_save = c_p->cp; - for (;;) { - ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { - cpp += 3; - } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { - return_to_trace = !0; - cpp += 1; - } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_time_trace)) { - cpp += 2; - } else - break; - } - c_p->cp = (BeamInstr *) cp_val(*cpp); - ASSERT(is_CP(*cpp)); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - SWAPIN; /* Needed by shared heap. */ - c_p->cp = cp_save; - } else { - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - SWAPIN; /* Needed by shared heap. */ - } - - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - - if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) { - need += 1; - } - if (flags & MATCH_SET_RX_TRACE) { - need += 3; - } - if (need) { - ASSERT(c_p->htop <= E && E <= c_p->hend); - if (E - need < HTOP) { - /* SWAPOUT was done and r(0) was saved above */ - PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect(c_p, need, reg, I[-1]); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - r(0) = reg[0]; - SWAPIN; - } - } - if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) { - E -= 1; - ASSERT(c_p->htop <= E && E <= c_p->hend); - E[0] = make_cp(c_p->cp); - c_p->cp = (BeamInstr *) beam_return_to_trace; - } - if (flags & MATCH_SET_RX_TRACE) { - E -= 3; - ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm) (UWord) (I - 3))); - ASSERT(am_true == tracer_pid || - is_internal_pid(tracer_pid) || is_internal_port(tracer_pid)); - E[2] = make_cp(c_p->cp); - E[1] = tracer_pid; - E[0] = make_cp(I - 3); /* We ARE at the beginning of an - instruction, - the funcinfo is above i. */ - c_p->cp = - (flags & MATCH_SET_EXCEPTION_TRACE) - ? beam_exception_trace : beam_return_trace; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - c_p->trace_flags |= F_EXCEPTION_TRACE; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - } - Goto(real_I); - } - OpCase(i_return_to_trace): { if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) { Uint *cpp = (Uint*) E; @@ -5131,9 +4871,6 @@ void process_main(void) c_p->arity = 1; /* One living register (the 'true' return value) */ SWAPOUT; c_p->i = I + 1; /* Next instruction */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - erts_add_to_runq(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); c_p->current = NULL; goto do_schedule; } @@ -5214,7 +4951,6 @@ void process_main(void) #endif /* NO_JUMP_TABLE */ em_call_error_handler = OpCode(call_error_handler); - em_call_traced_function = OpCode(call_traced_function); em_apply_bif = OpCode(apply_bif); beam_apply[0] = (BeamInstr) OpCode(i_apply); @@ -5255,7 +4991,7 @@ void process_main(void) save_calls(c_p, (Export *) Arg(0)); - SET_I(((Export *) Arg(0))->address); + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); dis_next = (Eterm *) *I; FCALLS--; @@ -5531,7 +5267,7 @@ terminate_proc(Process* c_p, Eterm Value) /* EXF_LOG is a primary exception flag */ if (c_p->freason & EXF_LOG) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Error in process %T ", c_p->id); + erts_dsprintf(dsbufp, "Error in process %T ", c_p->common.id); if (erts_is_alive) erts_dsprintf(dsbufp, "on node %T ", erts_this_node->sysname); erts_dsprintf(dsbufp,"with exit value: %0.*T\n", display_items, Value); @@ -5932,7 +5668,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) /* * Search for the error_handler module. */ - ep = erts_find_function(erts_proc_get_error_handler(p), func, 3); + ep = erts_find_function(erts_proc_get_error_handler(p), func, 3, + erts_active_code_ix()); if (ep == NULL) { /* No error handler */ p->current = fi; p->freason = EXC_UNDEF; @@ -5962,7 +5699,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) reg[0] = fi[0]; reg[1] = fi[1]; reg[2] = args; - return ep->address; + return ep->addressv[erts_active_code_ix()]; } @@ -5976,7 +5713,7 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, * there is no error handler module. */ - if ((ep = erts_find_export_entry(erts_proc_get_error_handler(p), + if ((ep = erts_active_export_entry(erts_proc_get_error_handler(p), am_undefined_function, 3)) == NULL) { return NULL; } else { @@ -6083,7 +5820,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) * Note: All BIFs have export entries; thus, no special case is needed. */ - if ((ep = erts_find_export_entry(module, function, arity)) == NULL) { + if ((ep = erts_active_export_entry(module, function, arity)) == NULL) { if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL) goto error; } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); @@ -6091,11 +5828,11 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->address; + BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); } #endif - return ep->address; + return ep->addressv[erts_active_code_ix()]; } static BeamInstr* @@ -6137,7 +5874,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) * Note: All BIFs have export entries; thus, no special case is needed. */ - if ((ep = erts_find_export_entry(module, function, arity)) == NULL) { + if ((ep = erts_active_export_entry(module, function, arity)) == NULL) { if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL) goto error; } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { @@ -6146,11 +5883,11 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) #ifdef USE_VM_CALL_PROBES if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->address; + BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); } #endif - return ep->address; + return ep->addressv[erts_active_code_ix()]; } int @@ -6227,9 +5964,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re */ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); - if (c_p->msg.len > 0) { - erts_add_to_runq(c_p); - } else { + if (!c_p->msg.len) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->fvalue = NIL; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -6237,14 +5972,12 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); #ifdef ERTS_SMP ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); - if (c_p->msg.len > 0) - erts_add_to_runq(c_p); - else + if (!c_p->msg.len) #endif - c_p->status = P_WAITING; + erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->current = bif_export[BIF_hibernate_3]->code; @@ -6261,7 +5994,6 @@ call_fun(Process* p, /* Current process. */ Eterm fun = reg[arity]; Eterm hdr; int i; - Eterm function; Eterm* hp; if (!is_boxed(fun)) { @@ -6332,7 +6064,7 @@ call_fun(Process* p, /* Current process. */ Export* ep; Module* modp; Eterm module; - + ErtsCodeIndex code_ix = erts_active_code_ix(); /* * No arity. There is no module loaded that defines the fun, @@ -6340,9 +6072,9 @@ call_fun(Process* p, /* Current process. */ * representation (the module has never been loaded), * or the module defining the fun has been unloaded. */ - module = fe->module; - if ((modp = erts_get_module(module)) != NULL && modp->code != NULL) { + if ((modp = erts_get_module(module, code_ix)) != NULL + && modp->curr.code != NULL) { /* * There is a module loaded, but obviously the fun is not * defined in it. We must not call the error_handler @@ -6357,7 +6089,7 @@ call_fun(Process* p, /* Current process. */ */ ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_lambda, 3); + am_undefined_lambda, 3, code_ix); if (ep == NULL) { /* No error handler */ p->current = NULL; p->freason = EXC_UNDEF; @@ -6367,7 +6099,7 @@ call_fun(Process* p, /* Current process. */ reg[1] = fun; reg[2] = args; reg[3] = NIL; - return ep->address; + return ep->addressv[erts_active_code_ix()]; } } } else if (is_export_header(hdr)) { @@ -6379,7 +6111,7 @@ call_fun(Process* p, /* Current process. */ if (arity == actual_arity) { DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); - return ep->address; + return ep->addressv[erts_active_code_ix()]; } else { /* * Wrong arity. First build a list of the arguments. @@ -6399,63 +6131,6 @@ call_fun(Process* p, /* Current process. */ p->fvalue = TUPLE2(hp, fun, args); return NULL; } - } else if (hdr == make_arityval(2)) { - Eterm* tp; - Export* ep; - Eterm module; - - tp = tuple_val(fun); - module = tp[1]; - function = tp[2]; - if (!is_atom(module) || !is_atom(function)) { - goto badfun; - } - - /* - * If this is the first time a tuple fun is used, - * send a warning to the logger. - */ - if (erts_smp_atomic_xchg_nob(&warned_for_tuple_funs, - (erts_aint_t) 1) == 0) { - erts_dsprintf_buf_t* dsbufp; - - dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Call to tuple fun {%T,%T}.\n\n" - "Tuple funs are deprecated and will be removed " - "in R16. Use \"fun M:F/A\" instead, for example " - "\"fun %T:%T/%d\".\n\n" - "(This warning will only be shown the first time " - "a tuple fun is called.)\n", - module, function, module, function, arity); - erts_send_warning_to_logger(p->group_leader, dsbufp); - } - - if ((ep = erts_find_export_entry(module, function, arity)) == NULL) { - ep = erts_find_export_entry(erts_proc_get_error_handler(p), - am_undefined_function, 3); - if (ep == NULL) { - p->freason = EXC_UNDEF; - return 0; - } - if (is_non_value(args)) { - Uint sz = 2 * arity; - if (HeapWordsLeft(p) < sz) { - erts_garbage_collect(p, sz, reg, arity); - } - hp = HEAP_TOP(p); - HEAP_TOP(p) += sz; - args = NIL; - while (arity-- > 0) { - args = CONS(hp, reg[arity], args); - hp += 2; - } - } - reg[0] = module; - reg[1] = function; - reg[2] = args; - } - DTRACE_GLOBAL_CALL(p, module, function, arity); - return ep->address; } else { badfun: p->current = NULL; @@ -6522,7 +6197,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) MSO(p).first = (struct erl_off_heap_header*) funp; funp->fe = fe; funp->num_free = num_free; - funp->creator = p->id; + funp->creator = p->common.id; #ifdef HIPE funp->native_address = fe->native_address; #endif @@ -6559,7 +6234,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->address == ep->code+3 && (ep->code[3] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->code+3 + && (ep->code[3] == (BeamInstr) em_apply_bif); } @@ -6580,3 +6256,12 @@ erts_current_reductions(Process *current, Process *p) } } +int +erts_beam_jump_table(void) +{ +#if defined(NO_JUMP_TABLE) + return 0; +#else + return 1; +#endif +} diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index dd788df6e4..4193eb4f3f 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -205,7 +205,7 @@ typedef struct { typedef struct { Eterm term; /* The tagged term (in the heap). */ Uint heap_size; /* (Exact) size on the heap. */ - Uint offset; /* Offset from temporary location to final. */ + SWord offset; /* Offset from temporary location to final. */ ErlOffHeap off_heap; /* Start of linked list of ProcBins. */ Eterm* heap; /* Heap for term. */ } Literal; @@ -352,27 +352,6 @@ typedef struct LoaderState { int loc_size; /* Size of location info in bytes (2/4) */ } LoaderState; -/* - * Layout of the line table. - */ - -#define MI_LINE_FNAME_PTR 0 -#define MI_LINE_LOC_TAB 1 -#define MI_LINE_LOC_SIZE 2 -#define MI_LINE_FUNC_TAB 3 - -#define LINE_INVALID_LOCATION (0) - -/* - * Macros for manipulating locations. - */ - -#define IS_VALID_LOCATION(File, Line) \ - ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1)) -#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line)) -#define LOC_FILE(Loc) ((Loc) >> 24) -#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) - #define GetTagAndValue(Stp, Tag, Val) \ do { \ BeamInstr __w; \ @@ -496,7 +475,8 @@ typedef struct LoaderState { } while (0) -static void free_state(LoaderState* stp); +static void free_loader_state(Binary* magic); +static void loader_state_dtor(Binary* magic); static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, BeamInstr* code, Uint size); @@ -507,6 +487,7 @@ static int verify_chunks(LoaderState* stp); static int load_atom_table(LoaderState* stp); static int load_import_table(LoaderState* stp); static int read_export_table(LoaderState* stp); +static int is_bif(Eterm mod, Eterm func, unsigned arity); static int read_lambda_table(LoaderState* stp); static int read_literal_table(LoaderState* stp); static int read_line_table(LoaderState* stp); @@ -548,22 +529,9 @@ static Eterm native_addresses(Process* p, Eterm mod); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); -static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, - BeamInstr* modp, int idx); - static int must_swap_floats; -/* - * The following variables keep a sorted list of address ranges for - * each module. It allows us to quickly find a function given an - * instruction pointer. - */ -Range* modules = NULL; /* Sorted lists of module addresses. */ -int num_loaded_modules; /* Number of loaded modules. */ -int allocated_modules; /* Number of slots allocated. */ -Range* mid_module = NULL; /* Cached search start point */ - Uint erts_total_code_size; /**********************************************************************/ @@ -579,11 +547,7 @@ void init_load(void) f.fd = 1.0; must_swap_floats = (f.fw[0] == 0); - allocated_modules = 128; - modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS, - allocated_modules*sizeof(Range)); - mid_module = modules; - num_loaded_modules = 0; + erts_init_ranges(); } static void @@ -595,7 +559,7 @@ define_file(LoaderState* stp, char* name, int idx) } Eterm -erts_load_module(Process *c_p, +erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, /* Group leader or NIL if none. */ Eterm* modp, /* @@ -605,15 +569,16 @@ erts_load_module(Process *c_p, byte* code, /* Points to the code to load */ Uint size) /* Size of code to load. */ { - LoaderState* stp = erts_alloc_loader_state(); + Binary* magic = erts_alloc_loader_state(); Eterm retval; - retval = erts_prepare_loading(stp, c_p, group_leader, modp, + ASSERT(!erts_initialized); + retval = erts_prepare_loading(magic, c_p, group_leader, modp, code, size); if (retval != NIL) { return retval; } - return erts_finish_loading(stp, c_p, c_p_locks, modp); + return erts_finish_loading(magic, c_p, c_p_locks, modp); } /* #define LOAD_MEMORY_HARD_DEBUG 1*/ @@ -629,11 +594,13 @@ extern void check_allocated_block(Uint type, void *blk); #endif Eterm -erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, +erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint unloaded_size) { Eterm retval = am_badfile; + LoaderState* stp; + stp = ERTS_MAGIC_BIN_DATA(magic); stp->module = *modp; stp->group_leader = group_leader; @@ -666,7 +633,7 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, /* * Initialize code area. */ - stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0); + stp->code_buffer_size = 2048 + stp->num_functions; stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, sizeof(BeamInstr) * stp->code_buffer_size); @@ -679,8 +646,6 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, stp->code[MI_COMPILE_PTR] = 0; stp->code[MI_COMPILE_SIZE] = 0; stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; - stp->code[MI_NUM_BREAKPOINTS] = 0; - /* * Read the atom table. @@ -774,23 +739,24 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader, load_error: if (retval != NIL) { - free_state(stp); + free_loader_state(magic); } return retval; } Eterm -erts_finish_loading(LoaderState* stp, Process* c_p, +erts_finish_loading(Binary* magic, Process* c_p, ErtsProcLocks c_p_locks, Eterm* modp) { Eterm retval; + LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); /* * No other process may run since we will update the export * table which is not protected by any locks. */ - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || + ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() || erts_smp_thr_progress_is_blocking()); /* @@ -811,7 +777,6 @@ erts_finish_loading(LoaderState* stp, Process* c_p, * exported and imported functions. This can't fail. */ - erts_export_consolidate(); CHKBLK(ERTS_ALC_T_CODE,stp->code); final_touch(stp); @@ -837,16 +802,20 @@ erts_finish_loading(LoaderState* stp, Process* c_p, } load_error: - free_state(stp); + free_loader_state(magic); return retval; } -LoaderState* +Binary* erts_alloc_loader_state(void) { LoaderState* stp; + Binary* magic; - stp = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LoaderState)); + magic = erts_create_magic_binary(sizeof(LoaderState), + loader_state_dtor); + erts_refc_inc(&magic->refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); stp->bin = NULL; stp->function = THE_NON_VALUE; /* Function not known yet */ stp->arity = 0; @@ -875,76 +844,123 @@ erts_alloc_loader_state(void) stp->line_instr = 0; stp->func_line = 0; stp->fname = 0; - return stp; + return magic; +} + +/* + * Return the module name (a tagged atom) for the prepared code + * in the magic binary, or NIL if the binary does not contain + * prepared code. + */ +Eterm +erts_module_for_prepared_code(Binary* magic) +{ + LoaderState* stp; + + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { + return NIL; + } + stp = ERTS_MAGIC_BIN_DATA(magic); + if (stp->code != 0) { + return stp->module; + } else { + return NIL; + } } static void -free_state(LoaderState* stp) +free_loader_state(Binary* magic) { + loader_state_dtor(magic); + if (erts_refc_dectest(&magic->refc, 0) == 0) { + erts_bin_free(magic); + } +} + +/* + * This destructor function can safely be called multiple times. + */ +static void +loader_state_dtor(Binary* magic) +{ + LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + if (stp->bin != 0) { driver_free_binary(stp->bin); + stp->bin = 0; } if (stp->code != 0) { erts_free(ERTS_ALC_T_CODE, stp->code); + stp->code = 0; } - if (stp->labels != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->labels); + if (stp->labels != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels); + stp->labels = 0; } - if (stp->atom != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->atom); + if (stp->atom != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->atom); + stp->atom = 0; } - if (stp->import != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->import); + if (stp->import != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->import); + stp->import = 0; } - if (stp->export != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->export); + if (stp->export != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->export); + stp->export = 0; } if (stp->lambdas != stp->def_lambdas) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->lambdas); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->lambdas); + stp->lambdas = stp->def_lambdas; } - if (stp->literals != NULL) { + if (stp->literals != 0) { int i; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap != NULL) { - erts_free(ERTS_ALC_T_LOADER_TMP, + if (stp->literals[i].heap != 0) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals[i].heap); + stp->literals[i].heap = 0; } } - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literals); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals); + stp->literals = 0; } - while (stp->literal_patches != NULL) { + while (stp->literal_patches != 0) { LiteralPatch* next = stp->literal_patches->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literal_patches); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literal_patches); stp->literal_patches = next; } - while (stp->string_patches != NULL) { + while (stp->string_patches != 0) { StringPatch* next = stp->string_patches->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->string_patches); + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->string_patches); stp->string_patches = next; } - while (stp->genop_blocks) { - GenOpBlock* next = stp->genop_blocks->next; - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks); - stp->genop_blocks = next; - } if (stp->line_item != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_item); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->line_item); + stp->line_item = 0; } if (stp->line_instr != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_instr); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->line_instr); + stp->line_instr = 0; } if (stp->func_line != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->func_line); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->func_line); + stp->func_line = 0; } if (stp->fname != 0) { - erts_free(ERTS_ALC_T_LOADER_TMP, stp->fname); + erts_free(ERTS_ALC_T_PREPARED_CODE, stp->fname); + stp->fname = 0; } - erts_free(ERTS_ALC_T_LOADER_TMP, stp); + /* + * The following data items should have been freed earlier. + */ + + ASSERT(stp->genop_blocks == 0); } static Eterm @@ -954,7 +970,6 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, { Module* modp; Eterm retval; - int i; if ((retval = beam_make_current_old(c_p, c_p_locks, module)) != NIL) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); @@ -971,30 +986,15 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_total_code_size += size; modp = erts_put_module(module); - modp->code = code; - modp->code_length = size; - modp->catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ + modp->curr.code = code; + modp->curr.code_length = size; + modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ /* - * Update address table (used for finding a function from a PC value). + * Update ranges (used for finding a function from a PC value). */ - if (num_loaded_modules == allocated_modules) { - allocated_modules *= 2; - modules = (Range *) erts_realloc(ERTS_ALC_T_MODULE_REFS, - (void *) modules, - allocated_modules * sizeof(Range)); - } - for (i = num_loaded_modules; i > 0; i--) { - if (code > modules[i-1].start) { - break; - } - modules[i] = modules[i-1]; - } - modules[i].start = code; - modules[i].end = (BeamInstr *) (((byte *)code) + size); - num_loaded_modules++; - mid_module = &modules[num_loaded_modules/2]; + erts_update_ranges(code, size); return NIL; } @@ -1217,9 +1217,8 @@ load_atom_table(LoaderState* stp) GetInt(stp, 4, stp->num_atoms); stp->num_atoms++; - stp->atom = erts_alloc(ERTS_ALC_T_LOADER_TMP, - erts_next_heap_size((stp->num_atoms*sizeof(Eterm)), - 0)); + stp->atom = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_atoms*sizeof(Eterm)); /* * Read all atoms. @@ -1231,7 +1230,7 @@ load_atom_table(LoaderState* stp) GetByte(stp, n); GetString(stp, atom, n); - stp->atom[i] = am_atom_put((char*)atom, n); + stp->atom[i] = erts_atom_put(atom, n, ERTS_ATOM_ENC_LATIN1, 1); } /* @@ -1241,7 +1240,7 @@ load_atom_table(LoaderState* stp) if (is_nil(stp->module)) { stp->module = stp->atom[1]; } else if (stp->atom[1] != stp->module) { - char sbuf[256]; + char sbuf[MAX_ATOM_SZ_FROM_LATIN1]; Atom* ap; ap = atom_tab(atom_val(stp->atom[1])); @@ -1263,10 +1262,8 @@ load_import_table(LoaderState* stp) int i; GetInt(stp, 4, stp->num_imports); - stp->import = erts_alloc(ERTS_ALC_T_LOADER_TMP, - erts_next_heap_size((stp->num_imports * - sizeof(ImportEntry)), - 0)); + stp->import = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_imports * sizeof(ImportEntry)); for (i = 0; i < stp->num_imports; i++) { int n; Eterm mod; @@ -1296,7 +1293,7 @@ load_import_table(LoaderState* stp) * If the export entry refers to a BIF, get the pointer to * the BIF function. */ - if ((e = erts_find_export_entry(mod, func, arity)) != NULL) { + if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { if (e->code[3] == (BeamInstr) em_apply_bif) { stp->import[i].bf = (BifFunction) e->code[4]; if (func == am_load_nif && mod == am_erlang && arity == 2) { @@ -1315,16 +1312,8 @@ load_import_table(LoaderState* stp) static int read_export_table(LoaderState* stp) { - static struct { - Eterm mod; - Eterm func; - int arity; - } allow_redef[] = { - /* The BIFs that are allowed to be redefined by Erlang code */ - {am_erlang,am_apply,2}, - {am_erlang,am_apply,3}, - }; int i; + BeamInstr* address; GetInt(stp, 4, stp->num_exps); if (stp->num_exps > stp->num_functions) { @@ -1332,7 +1321,7 @@ read_export_table(LoaderState* stp) stp->num_exps, stp->num_functions); } stp->export - = (ExportEntry *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + = (ExportEntry *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, (stp->num_exps * sizeof(ExportEntry))); for (i = 0; i < stp->num_exps; i++) { @@ -1340,7 +1329,6 @@ read_export_table(LoaderState* stp) Uint value; Eterm func; Uint arity; - Export* e; GetInt(stp, 4, n); GetAtom(stp, n, func); @@ -1358,29 +1346,34 @@ read_export_table(LoaderState* stp) if (value == 0) { LoadError2(stp, "export table entry %d: label %d not resolved", i, n); } - stp->export[i].address = stp->code + value; + stp->export[i].address = address = stp->code + value; /* - * Check that we are not redefining a BIF (except the ones allowed to - * redefine). + * Find out if there is a BIF with the same name. */ - if ((e = erts_find_export_entry(stp->module, func, arity)) != NULL) { - if (e->code[3] == (BeamInstr) em_apply_bif) { - int j; - for (j = 0; j < sizeof(allow_redef)/sizeof(allow_redef[0]); j++) { - if (stp->module == allow_redef[j].mod && - func == allow_redef[j].func && - arity == allow_redef[j].arity) { - break; - } - } - if (j == sizeof(allow_redef)/sizeof(allow_redef[0])) { - LoadError2(stp, "exported function %T/%d redefines BIF", - func, arity); - } - } + if (!is_bif(stp->module, func, arity)) { + continue; + } + + /* + * This is a stub for a BIF. + * + * It should not be exported, and the information in its + * func_info instruction should be invalidated so that it + * can be filtered out by module_info(functions) and by + * any other functions that walk through all local functions. + */ + + if (stp->labels[n].patches) { + LoadError3(stp, "there are local calls to the stub for " + "the BIF %T:%T/%d", + stp->module, func, arity); } + stp->export[i].address = NULL; + address[-1] = 0; + address[-2] = NIL; + address[-3] = NIL; } return 1; @@ -1388,15 +1381,39 @@ read_export_table(LoaderState* stp) return 0; } + +static int +is_bif(Eterm mod, Eterm func, unsigned arity) +{ + Export* e = erts_active_export_entry(mod, func, arity); + if (e == NULL) { + return 0; + } + if (e->code[3] != (BeamInstr) em_apply_bif) { + return 0; + } + if (mod == am_erlang && func == am_apply && arity == 3) { + /* + * erlang:apply/3 is a special case -- it is implemented + * as an instruction and it is OK to redefine it. + */ + return 0; + } + return 1; +} + static int read_lambda_table(LoaderState* stp) { int i; GetInt(stp, 4, stp->num_lambdas); - stp->lambdas_allocated = stp->num_lambdas; - stp->lambdas = (Lambda *) erts_alloc(ERTS_ALC_T_LOADER_TMP, - stp->num_lambdas * sizeof(Lambda)); + if (stp->num_lambdas > stp->lambdas_allocated) { + ASSERT(stp->lambdas == stp->def_lambdas); + stp->lambdas_allocated = stp->num_lambdas; + stp->lambdas = (Lambda *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_lambdas * sizeof(Lambda)); + } for (i = 0; i < stp->num_lambdas; i++) { Uint n; Uint32 Index; @@ -1446,7 +1463,7 @@ read_literal_table(LoaderState* stp) stp->file_p = uncompressed; stp->file_left = (unsigned) uncompressed_sz; GetInt(stp, 4, stp->num_literals); - stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_literals * sizeof(Literal)); stp->allocated_literals = stp->num_literals; @@ -1466,7 +1483,7 @@ read_literal_table(LoaderState* stp) if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { LoadError1(stp, "literal %d: bad external format", i); } - hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, + hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); stp->literals[i].off_heap.first = 0; stp->literals[i].off_heap.overhead = 0; @@ -1538,7 +1555,7 @@ read_line_table(LoaderState* stp) */ num_line_items++; - lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, num_line_items * sizeof(BeamInstr)); stp->line_item = lp; stp->num_line_items = num_line_items; @@ -1594,7 +1611,7 @@ read_line_table(LoaderState* stp) */ if (stp->num_fnames != 0) { - stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_fnames * sizeof(Eterm)); for (i = 0; i < stp->num_fnames; i++) { @@ -1603,18 +1620,18 @@ read_line_table(LoaderState* stp) GetInt(stp, 2, n); GetString(stp, fname, n); - stp->fname[i] = am_atom_put((char*)fname, n); + stp->fname[i] = erts_atom_put(fname, n, ERTS_ATOM_ENC_LATIN1, 1); } } /* * Allocate the arrays to be filled while code is being loaded. */ - stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_line_instrs * sizeof(LineInstr)); stp->current_li = 0; - stp->func_line = (int *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->func_line = (int *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_functions * sizeof(int)); @@ -1677,7 +1694,7 @@ read_code_header(LoaderState* stp) * Initialize label table. */ - stp->labels = (Label *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->labels = (Label *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_labels * sizeof(Label)); for (i = 0; i < stp->num_labels; i++) { stp->labels[i].value = 0; @@ -1703,9 +1720,9 @@ read_code_header(LoaderState* stp) #define CodeNeed(w) do { \ ASSERT(ci <= code_buffer_size); \ if (code_buffer_size < ci+(w)) { \ - code_buffer_size = erts_next_heap_size(ci+(w), 0); \ - stp->code = code \ - = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ + code_buffer_size = 2*ci+(w); \ + stp->code = code = \ + (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ (void *) code, \ code_buffer_size * sizeof(BeamInstr)); \ } \ @@ -1731,6 +1748,7 @@ load_code(LoaderState* stp) GenOp* last_op = NULL; GenOp** last_op_next = NULL; int arity; + int retval = 1; /* * The size of the loaded func_info instruction is needed @@ -1835,7 +1853,10 @@ load_code(LoaderState* stp) unsigned tag; switch (last_op->a[arg].val) { - case 0: /* Floating point number */ + case 0: + /* Floating point number. + * Not generated by the compiler in R16B and later. + */ { Eterm* hp; /* XXX:PaN - Halfword should use ARCH_64 variant instead */ @@ -2457,7 +2478,11 @@ load_code(LoaderState* stp) case op_int_code_end: stp->code_buffer_size = code_buffer_size; stp->ci = ci; - return 1; + stp->function = THE_NON_VALUE; + stp->genop = NULL; + stp->specific_op = -1; + retval = 1; + goto cleanup; } /* @@ -2471,9 +2496,20 @@ load_code(LoaderState* stp) } } - load_error: - return 0; + retval = 0; + + cleanup: + /* + * Clean up everything that is not needed any longer. + */ + + while (stp->genop_blocks) { + GenOpBlock* next = stp->genop_blocks->next; + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks); + stp->genop_blocks = next; + } + return retval; } @@ -2928,19 +2964,19 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, NEW_GENOP(stp, op); NATIVE_ENDIAN(Flags); - if (Size.type == TAG_i && Size.val < 0) { - error: /* Negative size must fail */ - op->op = genop_badarg_1; - op->arity = 1; - op->a[0] = Fail; - } else if (Size.type == TAG_i) { + if (Size.type == TAG_i) { op->op = genop_i_new_bs_put_integer_imm_4; op->arity = 4; op->a[0] = Fail; op->a[1].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) { - goto error; + error: + op->op = genop_badarg_1; + op->arity = 1; + op->a[0] = Fail; + op->next = NULL; + return op; } op->a[1].val = Size.val * Unit.val; op->a[2].type = Flags.type; @@ -4009,7 +4045,7 @@ freeze_code(LoaderState* stp) code[MI_LITERALS_END] = (BeamInstr) high; ptr = low; for (i = 0; i < stp->num_literals; i++) { - Uint offset; + SWord offset; struct erl_off_heap_header* t_off_heap; sys_memcpy(ptr, stp->literals[i].heap, @@ -4265,24 +4301,31 @@ final_touch(LoaderState* stp) index = next; } modp = erts_put_module(stp->module); - modp->catches = catches; + modp->curr.catches = catches; /* * Export functions. */ for (i = 0; i < stp->num_exps; i++) { - Export* ep = erts_export_put(stp->module, stp->export[i].function, - stp->export[i].arity); + Export* ep; + BeamInstr* address = stp->export[i].address; + + if (address == NULL) { + /* Skip stub for a BIF */ + continue; + } + ep = erts_export_put(stp->module, stp->export[i].function, + stp->export[i].arity); if (!on_load) { - ep->address = stp->export[i].address; + ep->addressv[erts_staging_code_ix()] = address; } else { /* * Don't make any of the exported functions * callable yet. */ - ep->address = ep->code+3; - ep->code[4] = (BeamInstr) stp->export[i].address; + ep->addressv[erts_staging_code_ix()] = ep->code+3; + ep->code[4] = (BeamInstr) address; } } @@ -4926,7 +4969,7 @@ new_label(LoaderState* stp) int num = stp->num_labels; stp->num_labels++; - stp->labels = (Label *) erts_realloc(ERTS_ALC_T_LOADER_TMP, + stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels, stp->num_labels * sizeof(Label)); stp->labels[num].value = 0; @@ -4937,7 +4980,8 @@ new_label(LoaderState* stp) static void new_literal_patch(LoaderState* stp, int pos) { - LiteralPatch* p = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LiteralPatch)); + LiteralPatch* p = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + sizeof(LiteralPatch)); p->pos = pos; p->next = stp->literal_patches; stp->literal_patches = p; @@ -4946,7 +4990,7 @@ new_literal_patch(LoaderState* stp, int pos) static void new_string_patch(LoaderState* stp, int pos) { - StringPatch* p = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(StringPatch)); + StringPatch* p = erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(StringPatch)); p->pos = pos; p->next = stp->string_patches; stp->string_patches = p; @@ -4964,14 +5008,14 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) ASSERT(stp->num_literals == 0); stp->allocated_literals = 8; need = stp->allocated_literals * sizeof(Literal); - stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, need); } else if (stp->allocated_literals <= stp->num_literals) { Uint need; stp->allocated_literals *= 2; need = stp->allocated_literals * sizeof(Literal); - stp->literals = (Literal *) erts_realloc(ERTS_ALC_T_LOADER_TMP, + stp->literals = (Literal *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals, need); } @@ -4980,7 +5024,7 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) lit = stp->literals + stp->num_literals; lit->offset = 0; lit->heap_size = heap_size; - lit->heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm)); + lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); lit->term = make_boxed(lit->heap); lit->off_heap.first = 0; lit->off_heap.overhead = 0; @@ -4999,7 +5043,7 @@ erts_module_info_0(Process* p, Eterm module) return THE_NON_VALUE; } - if (erts_get_module(module) == NULL) { + if (erts_get_module(module, erts_active_code_ix()) == NULL) { return THE_NON_VALUE; } @@ -5054,32 +5098,43 @@ functions_in_module(Process* p, /* Process whose heap to use. */ BeamInstr* code; int i; Uint num_functions; + Uint need; Eterm* hp; + Eterm* hp_end; Eterm result = NIL; if (is_not_atom(mod)) { return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; - hp = HAlloc(p, 5*num_functions); + need = 5*num_functions; + hp = HAlloc(p, need); + hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; Eterm name = (Eterm) func_info[3]; int arity = (int) func_info[4]; Eterm tuple; - ASSERT(is_atom(name)); - tuple = TUPLE2(hp, name, make_small(arity)); - hp += 3; - result = CONS(hp, tuple, result); - hp += 2; + /* + * If the function name is [], this entry is a stub for + * a BIF that should be ignored. + */ + ASSERT(is_atom(name) || is_nil(name)); + if (is_atom(name)) { + tuple = TUPLE2(hp, name, make_small(arity)); + hp += 3; + result = CONS(hp, tuple, result); + hp += 2; + } } + HRelease(p, hp_end, hp); return result; } @@ -5106,12 +5161,12 @@ native_addresses(Process* p, Eterm mod) return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; hp = HAlloc(p, need); @@ -5122,9 +5177,11 @@ native_addresses(Process* p, Eterm mod) int arity = (int) func_info[4]; Eterm tuple; - ASSERT(is_atom(name)); + ASSERT(is_atom(name) || is_nil(name)); /* [] if BIF stub */ if (func_info[1] != 0) { - Eterm addr = erts_bld_uint(&hp, NULL, func_info[1]); + Eterm addr; + ASSERT(is_atom(name)); + addr = erts_bld_uint(&hp, NULL, func_info[1]); tuple = erts_bld_tuple(&hp, NULL, 3, name, make_small(arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } @@ -5149,18 +5206,20 @@ exported_from_module(Process* p, /* Process whose heap to use. */ Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; + ErtsCodeIndex code_ix; if (is_not_atom(mod)) { return THE_NON_VALUE; } - for (i = 0; i < export_list_size(); i++) { - Export* ep = export_list(i); + code_ix = erts_active_code_ix(); + for (i = 0; i < export_list_size(code_ix); i++) { + Export* ep = export_list(i,code_ix); if (ep->code[0] == mod) { Eterm tuple; - if (ep->address == ep->code+3 && + if (ep->addressv[code_ix] == ep->code+3 && ep->code[3] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; @@ -5204,11 +5263,11 @@ attributes_for_module(Process* p, /* Process whose heap to use. */ return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; ext = (byte *) code[MI_ATTR_PTR]; if (ext != NULL) { hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]); @@ -5244,11 +5303,11 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ return THE_NON_VALUE; } - modp = erts_get_module(mod); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp == NULL) { return THE_NON_VALUE; } - code = modp->code; + code = modp->curr.code; ext = (byte *) code[MI_COMPILE_PTR]; if (ext != NULL) { hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]); @@ -5263,113 +5322,6 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */ } /* - * Find a function from the given pc and fill information in - * the FunctionInfo struct. If the full_info is non-zero, fill - * in all available information (including location in the - * source code). If no function is found, the 'current' field - * will be set to NULL. - */ - -void -erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) -{ - Range* low = modules; - Range* high = low + num_loaded_modules; - Range* mid = mid_module; - - fi->current = NULL; - fi->needed = 5; - fi->loc = LINE_INVALID_LOCATION; - while (low < high) { - if (pc < mid->start) { - high = mid; - } else if (pc > mid->end) { - low = mid + 1; - } else { - BeamInstr** low1 = (BeamInstr **) (mid->start + MI_FUNCTIONS); - BeamInstr** high1 = low1 + mid->start[MI_NUM_FUNCTIONS]; - BeamInstr** mid1; - - while (low1 < high1) { - mid1 = low1 + (high1-low1) / 2; - if (pc < mid1[0]) { - high1 = mid1; - } else if (pc < mid1[1]) { - mid_module = mid; - fi->current = mid1[0]+2; - if (full_info) { - BeamInstr** fp = (BeamInstr **) (mid->start + - MI_FUNCTIONS); - int idx = mid1 - fp; - lookup_loc(fi, pc, mid->start, idx); - } - return; - } else { - low1 = mid1 + 1; - } - } - return; - } - mid = low + (high-low) / 2; - } -} - -static void -lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) -{ - Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; - Eterm* low; - Eterm* high; - Eterm* mid; - Eterm pc; - - if (line == 0) { - return; - } - - pc = (Eterm) (BeamInstr) orig_pc; - fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; - low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; - high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; - while (high > low) { - mid = low + (high-low) / 2; - if (pc < mid[0]) { - high = mid; - } else if (pc < mid[1]) { - int file; - int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; - - if (line[MI_LINE_LOC_SIZE] == 2) { - Uint16* loc_table = - (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - fi->loc = loc_table[index]; - } else { - Uint32* loc_table = - (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - ASSERT(line[MI_LINE_LOC_SIZE] == 4); - fi->loc = loc_table[index]; - } - if (fi->loc == LINE_INVALID_LOCATION) { - return; - } - fi->needed += 3+2+3+2; - file = LOC_FILE(fi->loc); - if (file == 0) { - /* Special case: Module name with ".erl" appended */ - Atom* mod_atom = atom_tab(atom_val(fi->current[0])); - fi->needed += 2*(mod_atom->len+4); - } else { - Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); - fi->needed += 2*ap->len; - } - return; - } else { - low = mid + 1; - } - } -} - -/* * Build a single {M,F,A,Loction} item to be part of * a stack trace. */ @@ -5449,6 +5401,7 @@ code_get_chunk_2(BIF_ALIST_2) Process* p = BIF_P; Eterm Bin = BIF_ARG_1; Eterm Chunk = BIF_ARG_2; + Binary* magic = 0; LoaderState* stp; Uint chunk = 0; ErlSubBin* sb; @@ -5461,12 +5414,13 @@ code_get_chunk_2(BIF_ALIST_2) Eterm real_bin; byte* temp_alloc = NULL; - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); + stp = ERTS_MAGIC_BIN_DATA(magic); if ((start = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) { error: erts_free_aligned_binary_bytes(temp_alloc); - if (stp) { - free_state(stp); + if (magic) { + free_loader_state(magic); } BIF_ERROR(p, BADARG); } @@ -5511,7 +5465,7 @@ code_get_chunk_2(BIF_ALIST_2) done: erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); return res; } @@ -5524,14 +5478,16 @@ code_module_md5_1(BIF_ALIST_1) { Process* p = BIF_P; Eterm Bin = BIF_ARG_1; + Binary* magic; LoaderState* stp; byte* bytes; byte* temp_alloc = NULL; Eterm res; - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); + stp = ERTS_MAGIC_BIN_DATA(magic); if ((bytes = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) { - free_state(stp); + free_loader_state(magic); BIF_ERROR(p, BADARG); } stp->module = THE_NON_VALUE; /* Suppress diagnostiscs */ @@ -5545,7 +5501,7 @@ code_module_md5_1(BIF_ALIST_1) done: erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); return res; } @@ -5574,7 +5530,8 @@ stub_copy_info(LoaderState* stp, int chunk, /* Chunk: ATTR_CHUNK or COMPILE_CHUNK */ byte* info, /* Where to store info. */ BeamInstr* ptr_word, /* Where to store pointer into info. */ - BeamInstr* size_word) /* Where to store size of info. */ + BeamInstr* size_word, /* Where to store size into info. */ + BeamInstr* size_on_heap_word) /* Where to store size on heap. */ { Sint decoded_size; Uint size = stp->chunks[chunk].size; @@ -5585,7 +5542,8 @@ stub_copy_info(LoaderState* stp, if (decoded_size < 0) { return 0; } - *size_word = decoded_size; + *size_word = (BeamInstr) size; + *size_on_heap_word = decoded_size; } return info + size; } @@ -5601,7 +5559,7 @@ stub_read_export_table(LoaderState* stp) stp->num_exps, stp->num_functions); } stp->export - = (ExportEntry *) erts_alloc(ERTS_ALC_T_LOADER_TMP, + = (ExportEntry *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_exps * sizeof(ExportEntry)); for (i = 0; i < stp->num_exps; i++) { @@ -5627,20 +5585,29 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) { int i; int n = stp->num_exps; + Eterm mod = fp[2]; Eterm function = fp[3]; int arity = fp[4]; #ifdef HIPE Lambda* lp; #endif + if (is_bif(mod, function, arity)) { + fp[1] = 0; + fp[2] = 0; + fp[3] = 0; + fp[4] = 0; + return; + } + /* * Test if the function should be exported. */ for (i = 0; i < n; i++) { if (stp->export[i].function == function && stp->export[i].arity == arity) { - Export* ep = erts_export_put(fp[2], function, arity); - ep->address = fp+5; + Export* ep = erts_export_put(mod, function, arity); + ep->addressv[erts_staging_code_ix()] = fp+5; return; } } @@ -5819,6 +5786,7 @@ patch_funentries(Eterm Patchlist) Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) { + Binary* magic; LoaderState* stp; BeamInstr Funcs; BeamInstr Patchlist; @@ -5840,7 +5808,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Must initialize stp->lambdas here because the error handling code * at label 'error' uses it. */ - stp = erts_alloc_loader_state(); + magic = erts_alloc_loader_state(); + stp = ERTS_MAGIC_BIN_DATA(magic); if (is_not_atom(Mod)) { goto error; @@ -5920,7 +5889,6 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_COMPILE_PTR] = 0; code[MI_COMPILE_SIZE] = 0; code[MI_COMPILE_SIZE_ON_HEAP] = 0; - code[MI_NUM_BREAKPOINTS] = 0; code[MI_LITERALS_START] = 0; code[MI_LITERALS_END] = 0; code[MI_LITERALS_OFF_HEAP] = 0; @@ -5939,7 +5907,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) Eterm* tp; Eterm func; Eterm arity_term; - Uint arity; + Sint arity; Uint native_address; Eterm op; @@ -5997,12 +5965,16 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) info = (byte *) fp; info = stub_copy_info(stp, ATTR_CHUNK, info, - code+MI_ATTR_PTR, code+MI_ATTR_SIZE_ON_HEAP); + code+MI_ATTR_PTR, + code+MI_ATTR_SIZE, + code+MI_ATTR_SIZE_ON_HEAP); if (info == NULL) { goto error; } info = stub_copy_info(stp, COMPILE_CHUNK, info, - code+MI_COMPILE_PTR, code+MI_COMPILE_SIZE_ON_HEAP); + code+MI_COMPILE_PTR, + code+MI_COMPILE_SIZE, + code+MI_COMPILE_SIZE_ON_HEAP); if (info == NULL) { goto error; } @@ -6028,13 +6000,13 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (patch_funentries(Patchlist)) { erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); return Mod; } error: erts_free_aligned_binary_bytes(temp_alloc); - free_state(stp); + free_loader_state(magic); BIF_ERROR(p, BADARG); } @@ -6051,3 +6023,4 @@ static int safe_mul(UWord a, UWord b, UWord* resp) return (res / b) == a; } } + diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 997ba197db..65a8f26d7c 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2011. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -49,11 +49,6 @@ extern void** beam_ops; extern BeamInstr beam_debug_apply[]; extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; -extern BeamInstr* em_call_traced_function; -typedef struct { - BeamInstr* start; /* Pointer to start of module. */ - BeamInstr* end; /* Points one word beyond last function in module. */ -} Range; /* * The following variables keep a sorted list of address ranges for @@ -61,11 +56,6 @@ typedef struct { * instruction pointer. */ -extern Range* modules; -extern int num_loaded_modules; -extern int allocated_modules; -extern Range* mid_module; - /* Total code size in bytes */ extern Uint erts_total_code_size; /* @@ -94,27 +84,22 @@ extern Uint erts_total_code_size; #define MI_COMPILE_SIZE_ON_HEAP 6 /* - * Number of breakpoints in module is stored in this word - */ -#define MI_NUM_BREAKPOINTS 7 - -/* * Literal area (constant pool). */ -#define MI_LITERALS_START 8 -#define MI_LITERALS_END 9 -#define MI_LITERALS_OFF_HEAP 10 +#define MI_LITERALS_START 7 +#define MI_LITERALS_END 8 +#define MI_LITERALS_OFF_HEAP 9 /* * Pointer to the on_load function (or NULL if none). */ -#define MI_ON_LOAD_FUNCTION_PTR 11 +#define MI_ON_LOAD_FUNCTION_PTR 10 /* * Pointer to the line table (or NULL if none). */ -#define MI_LINE_TABLE 12 +#define MI_LINE_TABLE 11 /* * Start of function pointer table. This table contains pointers to @@ -125,5 +110,27 @@ extern Uint erts_total_code_size; * this table. */ -#define MI_FUNCTIONS 13 +#define MI_FUNCTIONS 12 + +/* + * Layout of the line table. + */ + +#define MI_LINE_FNAME_PTR 0 +#define MI_LINE_LOC_TAB 1 +#define MI_LINE_LOC_SIZE 2 +#define MI_LINE_FUNC_TAB 3 + +#define LINE_INVALID_LOCATION (0) + +/* + * Macros for manipulating locations. + */ + +#define IS_VALID_LOCATION(File, Line) \ + ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1)) +#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line)) +#define LOC_FILE(Loc) ((Loc) >> 24) +#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c new file mode 100644 index 0000000000..0f2d5d0c2a --- /dev/null +++ b/erts/emulator/beam/beam_ranges.c @@ -0,0 +1,349 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "beam_load.h" + +typedef struct { + BeamInstr* start; /* Pointer to start of module. */ + erts_smp_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */ +} Range; + +/* Range 'end' needs to be atomic as we purge module + by setting end=start in active code_ix */ +#define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end)) + +static Range* find_range(BeamInstr* pc); +static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, + BeamInstr* modp, int idx); + +/* + * The following variables keep a sorted list of address ranges for + * each module. It allows us to quickly find a function given an + * instruction pointer. + */ +struct ranges { + Range* modules; /* Sorted lists of module addresses. */ + Sint n; /* Number of range entries. */ + Sint allocated; /* Number of allocated entries. */ + erts_smp_atomic_t mid; /* Cached search start point */ +}; +static struct ranges r[ERTS_NUM_CODE_IX]; +static erts_smp_atomic_t mem_used; + +#ifdef HARD_DEBUG +static void check_consistency(struct ranges* p) +{ + int i; + + ASSERT(p->n <= p->allocated); + ASSERT((Uint)(p->mid - p->modules) < p->n || + (p->mid == p->modules && p->n == 0)); + for (i = 0; i < p->n; i++) { + ASSERT(p->modules[i].start <= RANGE_END(&p->modules[i])); + ASSERT(!i || RANGE_END(&p->modules[i-1]) < p->modules[i].start); + } +} +# define CHECK(r) check_consistency(r) +#else +# define CHECK(r) +#endif /* HARD_DEBUG */ + + +void +erts_init_ranges(void) +{ + Sint i; + + erts_smp_atomic_init_nob(&mem_used, 0); + for (i = 0; i < ERTS_NUM_CODE_IX; i++) { + r[i].modules = 0; + r[i].n = 0; + r[i].allocated = 0; + erts_smp_atomic_init_nob(&r[i].mid, 0); + } +} + +void +erts_start_staging_ranges(void) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + + if (r[dst].modules) { + erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated); + erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules); + r[dst].modules = NULL; + } +} + +void +erts_end_staging_ranges(int commit) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + + if (commit && r[dst].modules == NULL) { + Sint i; + Sint n; + + /* No modules added, just clone src and remove purged code. */ + ErtsCodeIndex src = erts_active_code_ix(); + + erts_smp_atomic_add_nob(&mem_used, r[src].n); + r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, + r[src].n * sizeof(Range)); + r[dst].allocated = r[src].n; + n = 0; + for (i = 0; i < r[src].n; i++) { + Range* rp = r[src].modules+i; + if (rp->start < RANGE_END(rp)) { + /* Only insert a module that has not been purged. */ + r[dst].modules[n] = *rp; + n++; + } + } + r[dst].n = n; + erts_smp_atomic_set_nob(&r[dst].mid, + (erts_aint_t) (r[dst].modules + n / 2)); + } +} + +void +erts_update_ranges(BeamInstr* code, Uint size) +{ + ErtsCodeIndex dst = erts_staging_code_ix(); + ErtsCodeIndex src = erts_active_code_ix(); + Sint i; + Sint n; + Sint need; + + if (src == dst) { + ASSERT(!erts_initialized); + + /* + * During start-up of system, the indices are the same. + * Handle this by faking a source area. + */ + src = (src+1) % ERTS_NUM_CODE_IX; + if (r[src].modules) { + erts_smp_atomic_add_nob(&mem_used, -r[src].allocated); + erts_free(ERTS_ALC_T_MODULE_REFS, r[src].modules); + } + r[src] = r[dst]; + r[dst].modules = 0; + } + + CHECK(&r[src]); + + ASSERT(r[dst].modules == NULL); + need = r[dst].allocated = r[src].n + 1; + erts_smp_atomic_add_nob(&mem_used, need); + r[dst].modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS, + need * sizeof(Range)); + n = 0; + for (i = 0; i < r[src].n; i++) { + Range* rp = r[src].modules+i; + if (code < rp->start) { + r[dst].modules[n].start = code; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(((byte *)code) + size)); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); + n++; + break; + } + if (rp->start < RANGE_END(rp)) { + /* Only insert a module that has not been purged. */ + r[dst].modules[n].start = rp->start; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(RANGE_END(rp))); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); + n++; + } + } + + while (i < r[src].n) { + Range* rp = r[src].modules+i; + if (rp->start < RANGE_END(rp)) { + /* Only insert a module that has not been purged. */ + r[dst].modules[n].start = rp->start; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(RANGE_END(rp))); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); + n++; + } + i++; + } + + if (n == 0 || code > r[dst].modules[n-1].start) { + r[dst].modules[n].start = code; + erts_smp_atomic_init_nob(&r[dst].modules[n].end, + (erts_aint_t)(((byte *)code) + size)); + ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); + n++; + } + + ASSERT(n <= r[src].n+1); + r[dst].n = n; + erts_smp_atomic_set_nob(&r[dst].mid, + (erts_aint_t) (r[dst].modules + n / 2)); + + CHECK(&r[dst]); + CHECK(&r[src]); +} + +void +erts_remove_from_ranges(BeamInstr* code) +{ + Range* rp = find_range(code); + erts_smp_atomic_set_nob(&rp->end, (erts_aint_t)rp->start); +} + +UWord +erts_ranges_sz(void) +{ + return erts_smp_atomic_read_nob(&mem_used) * sizeof(Range); +} + +/* + * Find a function from the given pc and fill information in + * the FunctionInfo struct. If the full_info is non-zero, fill + * in all available information (including location in the + * source code). If no function is found, the 'current' field + * will be set to NULL. + */ + +void +erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) +{ + BeamInstr** low; + BeamInstr** high; + BeamInstr** mid; + Range* rp; + + fi->current = NULL; + fi->needed = 5; + fi->loc = LINE_INVALID_LOCATION; + rp = find_range(pc); + if (rp == 0) { + return; + } + + low = (BeamInstr **) (rp->start + MI_FUNCTIONS); + high = low + rp->start[MI_NUM_FUNCTIONS]; + while (low < high) { + mid = low + (high-low) / 2; + if (pc < mid[0]) { + high = mid; + } else if (pc < mid[1]) { + fi->current = mid[0]+2; + if (full_info) { + BeamInstr** fp = (BeamInstr **) (rp->start + + MI_FUNCTIONS); + int idx = mid - fp; + lookup_loc(fi, pc, rp->start, idx); + } + return; + } else { + low = mid + 1; + } + } +} + +static Range* +find_range(BeamInstr* pc) +{ + ErtsCodeIndex active = erts_active_code_ix(); + Range* low = r[active].modules; + Range* high = low + r[active].n; + Range* mid = (Range *) erts_smp_atomic_read_nob(&r[active].mid); + + CHECK(&r[active]); + while (low < high) { + if (pc < mid->start) { + high = mid; + } else if (pc > RANGE_END(mid)) { + low = mid + 1; + } else { + erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid); + return mid; + } + mid = low + (high-low) / 2; + } + return 0; +} + +static void +lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) +{ + Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; + Eterm* low; + Eterm* high; + Eterm* mid; + Eterm pc; + + if (line == 0) { + return; + } + + pc = (Eterm) (BeamInstr) orig_pc; + fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; + low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; + high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; + while (high > low) { + mid = low + (high-low) / 2; + if (pc < mid[0]) { + high = mid; + } else if (pc < mid[1]) { + int file; + int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; + + if (line[MI_LINE_LOC_SIZE] == 2) { + Uint16* loc_table = + (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + fi->loc = loc_table[index]; + } else { + Uint32* loc_table = + (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; + ASSERT(line[MI_LINE_LOC_SIZE] == 4); + fi->loc = loc_table[index]; + } + if (fi->loc == LINE_INVALID_LOCATION) { + return; + } + fi->needed += 3+2+3+2; + file = LOC_FILE(fi->loc); + if (file == 0) { + /* Special case: Module name with ".erl" appended */ + Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + fi->needed += 2*(mod_atom->len+4); + } else { + Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); + fi->needed += 2*ap->len; + } + return; + } else { + low = mid + 1; + } + } +} diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index fc00b42454..755c5e6882 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -37,10 +37,14 @@ #include "erl_db_util.h" #include "register.h" #include "erl_thr_progress.h" +#define ERTS_PTAB_WANT_BIF_IMPL__ +#include "erl_ptab.h" +#include "erl_bits.h" static Export* flush_monitor_message_trap = NULL; static Export* set_cpu_topology_trap = NULL; static Export* await_proc_exit_trap = NULL; +static Export* await_port_send_result_trap = NULL; Export* erts_format_cpu_topology_trap = NULL; static Export *await_sched_wall_time_mod_trap; @@ -83,8 +87,10 @@ static int insert_internal_link(Process* p, Eterm rpid) ASSERT(is_internal_pid(rpid)); #ifdef ERTS_SMP - if (IS_TRACED(p) && (p->trace_flags & (F_TRACE_SOL|F_TRACE_SOL1))) + if (IS_TRACED(p) + && (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1))) { rp_locks = ERTS_PROC_LOCKS_ALL; + } erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK); #endif @@ -100,27 +106,27 @@ static int insert_internal_link(Process* p, Eterm rpid) } if (p != rp) { - erts_add_link(&(p->nlinks), LINK_PID, rp->id); - erts_add_link(&(rp->nlinks), LINK_PID, p->id); + erts_add_link(&ERTS_P_LINKS(p), LINK_PID, rp->common.id); + erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, p->common.id); - ASSERT(is_nil(p->tracer_proc) - || is_internal_pid(p->tracer_proc) - || is_internal_port(p->tracer_proc)); + ASSERT(is_nil(ERTS_TRACER_PROC(p)) + || is_internal_pid(ERTS_TRACER_PROC(p)) + || is_internal_port(ERTS_TRACER_PROC(p))); if (IS_TRACED(p)) { - if (p->trace_flags & (F_TRACE_SOL|F_TRACE_SOL1)) { - rp->trace_flags |= (p->trace_flags & TRACEE_FLAGS); - rp->tracer_proc = p->tracer_proc; /* maybe steal */ + if (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1)) { + ERTS_TRACE_FLAGS(rp) |= (ERTS_TRACE_FLAGS(p) & TRACEE_FLAGS); + ERTS_TRACER_PROC(rp) = ERTS_TRACER_PROC(p); /* maybe steal */ - if (p->trace_flags & F_TRACE_SOL1) { /* maybe override */ - rp->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - p->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + if (ERTS_TRACE_FLAGS(p) & F_TRACE_SOL1) { /* maybe override */ + ERTS_TRACE_FLAGS(rp) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); } } } } if (IS_TRACED_FL(rp, F_TRACE_PROCS)) - trace_proc(p, rp, am_getting_linked, p->id); + trace_proc(p, rp, am_getting_linked, p->common.id); if (p == rp) erts_smp_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN); @@ -144,10 +150,6 @@ BIF_RETTYPE link_1(BIF_ALIST_1) /* check that the pid or port which is our argument is OK */ if (is_internal_pid(BIF_ARG_1)) { - if (internal_pid_index(BIF_ARG_1) >= erts_max_processes) { - BIF_ERROR(BIF_P, BADARG); - } - if (insert_internal_link(BIF_P, BIF_ARG_1)) { BIF_RET(am_true); } @@ -157,19 +159,37 @@ BIF_RETTYPE link_1(BIF_ALIST_1) } if (is_internal_port(BIF_ARG_1)) { - Port *pt = erts_id2port(BIF_ARG_1, BIF_P, ERTS_PROC_LOCK_MAIN); - if (!pt) { + int send_link_signal = 0; + Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (!prt) { goto res_no_proc; } erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); - if (erts_add_link(&(BIF_P->nlinks), LINK_PID, BIF_ARG_1) >= 0) - erts_add_link(&(pt->nlinks), LINK_PID, BIF_P->id); + if (erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1) >= 0) + send_link_signal = 1; /* else: already linked */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - erts_smp_port_unlock(pt); + + if (send_link_signal) { + Eterm ref; + Eterm *refp = erts_port_synchronous_ops ? &ref : NULL; + + switch (erts_port_link(BIF_P, prt, BIF_P->common.id, refp)) { + case ERTS_PORT_OP_DROPPED: + case ERTS_PORT_OP_BADARG: + goto res_no_proc; + case ERTS_PORT_OP_SCHEDULED: + if (refp) { + ASSERT(is_internal_ref(ref)); + BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); + } + default: + break; + } + } BIF_RET(am_true); } else if (is_external_port(BIF_ARG_1) @@ -182,7 +202,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); /* We may earn time by checking first that we're not linked already */ - if (erts_lookup_link(BIF_P->nlinks, BIF_ARG_1) != NULL) { + if (erts_lookup_link(ERTS_P_LINKS(BIF_P), BIF_ARG_1) != NULL) { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); BIF_RET(am_true); } @@ -209,10 +229,10 @@ BIF_RETTYPE link_1(BIF_ALIST_1) erts_smp_de_links_lock(dep); - erts_add_link(&(BIF_P->nlinks), LINK_PID, BIF_ARG_1); + erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1); lnk = erts_add_or_lookup_link(&(dep->nlinks), LINK_PID, - BIF_P->id); + BIF_P->common.id); ASSERT(lnk != NULL); erts_add_link(&ERTS_LINK_ROOT(lnk), LINK_PID, BIF_ARG_1); @@ -220,7 +240,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1) erts_smp_de_runlock(dep); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - code = erts_dsig_send_link(&dsd, BIF_P->id, BIF_ARG_1); + code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); BIF_RET(am_true); @@ -233,15 +253,17 @@ BIF_RETTYPE link_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); - res_no_proc: - if (BIF_P->flags & F_TRAPEXIT) { - ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; - erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL); - erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); - BIF_RET(am_true); +res_no_proc: { + erts_aint32_t state = erts_smp_atomic32_read_nob(&BIF_P->state); + if (state & ERTS_PSFLG_TRAP_EXIT) { + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; + erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL); + erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); + BIF_RET(am_true); + } + else + BIF_ERROR(BIF_P, EXC_NOPROC); } - else - BIF_ERROR(BIF_P, EXC_NOPROC); } #define ERTS_DEMONITOR_FALSE 2 @@ -287,7 +309,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) if (dmon) erts_destroy_monitor(dmon); } - mon = erts_remove_monitor(&c_p->monitors, ref); + mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); res = ERTS_DEMONITOR_TRUE; @@ -296,7 +318,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) case ERTS_DSIG_PREP_CONNECTED: erts_smp_de_links_lock(dep); - mon = erts_remove_monitor(&c_p->monitors, ref); + mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); dmon = erts_remove_monitor(&dep->monitors, ref); erts_smp_de_links_unlock(dep); erts_smp_de_runlock(dep); @@ -323,7 +345,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) * the atom is stored there. Yield if necessary. */ code = erts_dsig_send_demonitor(&dsd, - c_p->id, + c_p->common.id, (mon->name != NIL ? mon->name : mon->pid), @@ -338,8 +360,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) break; default: ASSERT(! "Invalid dsig prepare result"); - res = ERTS_DEMONITOR_INTERNAL_ERROR; - break; + return ERTS_DEMONITOR_INTERNAL_ERROR; } #ifndef ERTS_SMP @@ -385,7 +406,7 @@ static int demonitor(Process *c_p, Eterm ref) goto done; /* Cannot be this monitor's ref */ } - mon = erts_lookup_monitor(c_p->monitors, ref); + mon = erts_lookup_monitor(ERTS_P_MONITORS(c_p), ref); if (!mon) { res = ERTS_DEMONITOR_FALSE; goto done; @@ -424,7 +445,7 @@ static int demonitor(Process *c_p, Eterm ref) to, ERTS_PROC_LOCK_LINK, ERTS_P2P_FLG_ALLOW_OTHER_X); - mon = erts_remove_monitor(&c_p->monitors, ref); + mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); #ifndef ERTS_SMP ASSERT(mon); #else @@ -438,7 +459,7 @@ static int demonitor(Process *c_p, Eterm ref) } if (rp) { ErtsMonitor *rmon; - rmon = erts_remove_monitor(&(rp->monitors), ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); if (rp != c_p) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon != NULL) @@ -580,7 +601,7 @@ local_pid_monitor(Process *p, Eterm target) mon_ref = erts_make_ref(p); ERTS_BIF_PREP_RET(ret, mon_ref); - if (target == p->id) { + if (target == p->common.id) { return ret; } @@ -597,8 +618,8 @@ local_pid_monitor(Process *p, Eterm target) else { ASSERT(rp != p); - erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, target, NIL); - erts_add_monitor(&(rp->monitors), MON_TARGET, mon_ref, p->id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); + erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } @@ -633,9 +654,9 @@ local_name_monitor(Process *p, Eterm target_name) UnUseTmpHeap(3,p); } else if (rp != p) { - erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, rp->id, + erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, rp->common.id, target_name); - erts_add_monitor(&(rp->monitors), MON_TARGET, mon_ref, p->id, + erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, target_name); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } @@ -687,16 +708,16 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2, erts_smp_de_links_lock(dep); - erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, p_trgt, + erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, p_trgt, p_name); - erts_add_monitor(&(dep->monitors), MON_TARGET, mon_ref, p->id, + erts_add_monitor(&(dep->monitors), MON_TARGET, mon_ref, p->common.id, d_name); erts_smp_de_links_unlock(dep); erts_smp_de_runlock(dep); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); - code = erts_dsig_send_monitor(&dsd, p->id, target, mon_ref); + code = erts_dsig_send_monitor(&dsd, p->common.id, target, mon_ref); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_PREP_YIELD_RETURN(ret, p, mon_ref); else @@ -939,36 +960,39 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) } if (is_internal_port(BIF_ARG_1)) { - Port *pt = erts_id2port_sflgs(BIF_ARG_1, - BIF_P, - ERTS_PROC_LOCK_MAIN, - ERTS_PORT_SFLGS_DEAD); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); #ifdef ERTS_SMP - if (ERTS_PROC_PENDING_EXIT(BIF_P)) { - if (pt) - erts_smp_port_unlock(pt); + if (ERTS_PROC_PENDING_EXIT(BIF_P)) goto handle_pending_exit; - } #endif - l = erts_remove_link(&BIF_P->nlinks, BIF_ARG_1); - - ASSERT(pt || !l); - - if (pt) { - rl = erts_remove_link(&pt->nlinks, BIF_P->id); - erts_smp_port_unlock(pt); - if (rl) - erts_destroy_link(rl); - } + l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - if (l) + if (l) { + Port *prt; + erts_destroy_link(l); + /* Send unlink signal */ + prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_DEAD); + if (prt) { + ErtsPortOpResult res; + Eterm ref; + Eterm *refp = erts_port_synchronous_ops ? &ref : NULL; +#ifdef DEBUG + ref = NIL; +#endif + res = erts_port_unlink(BIF_P, prt, BIF_P->common.id, refp); + + if (refp && res == ERTS_PORT_OP_SCHEDULED) { + ASSERT(is_internal_ref(ref)); + BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); + } + } + } + BIF_RET(am_true); } else if (is_external_port(BIF_ARG_1) @@ -991,7 +1015,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) if (ERTS_PROC_PENDING_EXIT(BIF_P)) goto handle_pending_exit; #endif - l = erts_remove_link(&BIF_P->nlinks,BIF_ARG_1); + l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); @@ -1020,8 +1044,8 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) #endif case ERTS_DSIG_PREP_CONNECTED: - erts_remove_dist_link(&dld, BIF_P->id, BIF_ARG_1, dep); - code = erts_dsig_send_unlink(&dsd, BIF_P->id, BIF_ARG_1); + erts_remove_dist_link(&dld, BIF_P->common.id, BIF_ARG_1, dep); + code = erts_dsig_send_unlink(&dsd, BIF_P->common.id, BIF_ARG_1); erts_destroy_dist_link(&dld); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); @@ -1035,10 +1059,6 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) /* Internal pid... */ - /* process ok ? */ - if (internal_pid_index(BIF_ARG_1) >= erts_max_processes) - BIF_ERROR(BIF_P, BADARG); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); /* get process struct */ @@ -1057,7 +1077,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) #endif /* unlink and ignore errors */ - l = erts_remove_link(&BIF_P->nlinks,BIF_ARG_1); + l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); if (l != NULL) erts_destroy_link(l); @@ -1065,12 +1085,12 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P); } else { - rl = erts_remove_link(&(rp->nlinks),BIF_P->id); + rl = erts_remove_link(&ERTS_P_LINKS(rp), BIF_P->common.id); if (rl != NULL) erts_destroy_link(rl); if (IS_TRACED_FL(rp, F_TRACE_PROCS) && rl != NULL) { - trace_proc(BIF_P, rp, am_getting_unlinked, BIF_P->id); + trace_proc(BIF_P, rp, am_getting_unlinked, BIF_P->common.id); } if (rp != BIF_P) @@ -1103,8 +1123,9 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3) if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) { /* - * If hibernate succeeded, TRAP. The process will be suspended - * if status is P_WAITING or continue (if any message was in the queue). + * If hibernate succeeded, TRAP. The process will be wait in a + * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE); + * otherwise, continue executing (if any message was in the queue). */ BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i); } @@ -1342,15 +1363,28 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) */ if (is_internal_port(BIF_ARG_1)) { - Port *prt; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - prt = erts_id2port(BIF_ARG_1, NULL, 0); + Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (prt) { - erts_do_exit_port(prt, BIF_P->id, BIF_ARG_2); - erts_port_release(prt); + Eterm ref; + Eterm *refp = erts_port_synchronous_ops ? &ref : NULL; + ErtsPortOpResult res; + +#ifdef DEBUG + ref = NIL; +#endif + + res = erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, refp); + + ERTS_BIF_CHK_EXITED(BIF_P); + + if (refp && res == ERTS_PORT_OP_SCHEDULED) { + ASSERT(is_internal_ref(ref)); + BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true); + } + } - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - ERTS_BIF_CHK_EXITED(BIF_P); + BIF_RET(am_true); } else if(is_external_port(BIF_ARG_1) @@ -1376,7 +1410,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) case ERTS_DSIG_PREP_NOT_CONNECTED: BIF_TRAP2(dexit_trap, BIF_P, BIF_ARG_1, BIF_ARG_2); case ERTS_DSIG_PREP_CONNECTED: - code = erts_dsig_send_exit2(&dsd, BIF_P->id, BIF_ARG_1, BIF_ARG_2); + code = erts_dsig_send_exit2(&dsd, BIF_P->common.id, BIF_ARG_1, BIF_ARG_2); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); BIF_RET(am_true); @@ -1394,18 +1428,15 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) */ ErtsProcLocks rp_locks; - if (internal_pid_index(BIF_ARG_1) >= erts_max_processes) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_1 == BIF_P->id) { + if (BIF_ARG_1 == BIF_P->common.id) { rp_locks = ERTS_PROC_LOCKS_ALL; rp = BIF_P; erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_ALL_MINOR); } else { rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, rp_locks); if (!rp) { BIF_RET(am_true); } @@ -1415,7 +1446,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) * Send an exit signal. */ erts_send_exit_signal(BIF_P, - BIF_P->id, + BIF_P->common.id, rp, &rp_locks, BIF_ARG_2, @@ -1427,8 +1458,6 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - if (rp != BIF_P) - erts_smp_proc_dec_refc(rp); #endif /* * We may have exited ourselves and may have to take action. @@ -1502,14 +1531,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) BIF_RET(old_value); } else if (BIF_ARG_1 == am_priority) { - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS); old_value = erts_set_process_priority(BIF_P, BIF_ARG_2); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); if (old_value == THE_NON_VALUE) goto error; BIF_RET(old_value); } else if (BIF_ARG_1 == am_trap_exit) { + erts_aint32_t state; Uint trap_exit; if (BIF_ARG_2 == am_true) { trap_exit = 1; @@ -1520,63 +1548,58 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } /* * NOTE: It is important that we check for pending exit signals - * and handle them before flag trap_exit is set to true. - * For more info, see implementation of erts_send_exit_signal(). + * and handle them before returning if trap_exit is set to + * true. For more info, see implementation of + * erts_send_exit_signal(). */ - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS); - ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - old_value = ERTS_PROC_IS_TRAPPING_EXITS(BIF_P) ? am_true : am_false; - if (trap_exit) { - ERTS_PROC_SET_TRAP_EXIT(BIF_P); - } else { - ERTS_PROC_UNSET_TRAP_EXIT(BIF_P); + if (trap_exit) + state = erts_smp_atomic32_read_bor_mb(&BIF_P->state, + ERTS_PSFLG_TRAP_EXIT); + else + state = erts_smp_atomic32_read_band_mb(&BIF_P->state, + ~ERTS_PSFLG_TRAP_EXIT); +#ifdef ERTS_SMP + if (ERTS_PROC_PENDING_EXIT(BIF_P)) { + erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); + ERTS_BIF_EXITED(BIF_P); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); +#endif + + old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false; BIF_RET(old_value); } else if (BIF_ARG_1 == am_scheduler) { - int yield; - ErtsRunQueue *old; - ErtsRunQueue *new; + ErtsRunQueue *old, *new, *curr; Sint sched; + erts_aint32_t state; + if (!is_small(BIF_ARG_2)) goto error; sched = signed_val(BIF_ARG_2); if (sched < 0 || erts_no_schedulers < sched) goto error; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS); - old = BIF_P->bound_runq; -#ifdef ERTS_SMP - ASSERT(!old || old == BIF_P->run_queue); -#endif - new = !sched ? NULL : erts_schedid2runq(sched); -#ifndef ERTS_SMP - yield = 0; -#else - if (new == old) - yield = 0; + + if (sched == 0) { + new = NULL; + state = erts_smp_atomic32_read_band_mb(&BIF_P->state, + ~ERTS_PSFLG_BOUND); + } else { - ErtsRunQueue *curr = BIF_P->run_queue; - if (!new) - erts_smp_runq_lock(curr); - else - erts_smp_runqs_lock(curr, new); - yield = new && BIF_P->run_queue != new; -#endif - BIF_P->bound_runq = new; + new = erts_schedid2runq(sched); #ifdef ERTS_SMP - if (new) - BIF_P->run_queue = new; - if (!new) - erts_smp_runq_unlock(curr); - else - erts_smp_runqs_unlock(curr, new); - } + erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new); #endif - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_bor_mb(&BIF_P->state, + ERTS_PSFLG_BOUND); + } + + curr = ERTS_GET_SCHEDULER_DATA_FROM_PROC(BIF_P)->run_queue; + old = (ERTS_PSFLG_BOUND & state) ? curr : NULL; + + ASSERT(!old || old == curr); + old_value = old ? make_small(old->ix+1) : make_small(0); - if (yield) + if (new && new != curr) ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler); else BIF_RET(old_value); @@ -1625,11 +1648,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) goto error; } erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); - old_value = BIF_P->trace_flags & F_SENSITIVE ? am_true : am_false; + old_value = (ERTS_TRACE_FLAGS(BIF_P) & F_SENSITIVE + ? am_true + : am_false); if (is_sensitive) { - BIF_P->trace_flags |= F_SENSITIVE; + ERTS_TRACE_FLAGS(BIF_P) |= F_SENSITIVE; } else { - BIF_P->trace_flags &= ~F_SENSITIVE; + ERTS_TRACE_FLAGS(BIF_P) &= ~F_SENSITIVE; } erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); BIF_RET(old_value); @@ -1755,8 +1780,9 @@ ebif_bang_2(BIF_ALIST_2) #define SEND_BADARG (-4) #define SEND_USER_ERROR (-5) #define SEND_INTERNAL_ERROR (-6) +#define SEND_AWAIT_RESULT (-7) -Sint do_send(Process *p, Eterm to, Eterm msg, int suspend); +Sint do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp); static Sint remote_send(Process *p, DistEntry *dep, Eterm to, Eterm full_to, Eterm msg, int suspend) @@ -1810,7 +1836,7 @@ static Sint remote_send(Process *p, DistEntry *dep, } Sint -do_send(Process *p, Eterm to, Eterm msg, int suspend) { +do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { Eterm portid; Port *pt; Process* rp; @@ -1822,17 +1848,10 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - - if (internal_pid_index(to) >= erts_max_processes) - return SEND_BADARG; - rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN, - to, 0, ERTS_P2P_FLG_SMP_INC_REFC); - - if (!rp) { - ERTS_SMP_ASSERT_IS_NOT_EXITING(p); + rp = erts_proc_lookup_raw(to); + if (!rp) return 0; - } } else if (is_external_pid(to)) { dep = external_pid_dist_entry(to); if(dep == erts_this_dist_entry) { @@ -1841,7 +1860,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { "Discarding message %T from %T to %T in an old " "incarnation (%d) of this node (%d)\n", msg, - p->id, + p->common.id, to, external_pid_creation(to), erts_this_node->creation); @@ -1850,45 +1869,24 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { } return remote_send(p, dep, to, to, msg, suspend); } else if (is_atom(to)) { - - /* Need to virtual schedule out sending process - * because of lock wait. This is only necessary - * for internal port calling but the lock is bundled - * with name lookup. - */ - - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_out); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(p, am_inactive); - } - erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, - to, - &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC, - &pt); + Eterm id = erts_whereis_name_to_id(p, to); + rp = erts_proc_lookup(id); + if (rp) + goto send_message; + + pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP); if (pt) { - portid = pt->id; + portid = id; goto port_common; } - - /* Not a port virtually schedule the process back in */ - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_in); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(p, am_active); - } if (IS_TRACED(p)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - if (!rp) { - return SEND_BADARG; - } + return SEND_BADARG; } else if (is_external_port(to) && (external_port_dist_entry(to) == erts_this_dist_entry)) { @@ -1897,50 +1895,56 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { "Discarding message %T from %T to %T in an old " "incarnation (%d) of this node (%d)\n", msg, - p->id, + p->common.id, to, external_port_creation(to), erts_this_node->creation); erts_send_error_to_logger(p->group_leader, dsbufp); return 0; } else if (is_internal_port(to)) { + int ret_val; portid = to; - /* schedule out calling process, waiting for lock*/ - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_out); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(p, am_inactive); - } - pt = erts_id2port(to, p, ERTS_PROC_LOCK_MAIN); + + pt = erts_port_lookup(portid, ERTS_PORT_SFLGS_INVALID_LOOKUP); + port_common: - ERTS_SMP_LC_ASSERT(!pt || erts_lc_is_port_locked(pt)); + ret_val = 0; - /* We have waited for locks, trace schedule ports */ - if (pt && IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(pt, am_in, am_command); - } - if (pt && erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) { - profile_runnable_port(pt, am_active); - } - - /* XXX let port_command handle the busy stuff !!! */ - if (pt && (pt->status & ERTS_PORT_SFLG_PORT_BUSY)) { - if (suspend) { - erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); - if (erts_system_monitor_flags.busy_port) { - monitor_generic(p, am_busy_port, portid); + if (pt) { + int ps_flags = suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; + *refp = NIL; + + switch (erts_port_command(p, ps_flags, pt, msg, refp)) { + case ERTS_PORT_OP_CALLER_EXIT: + /* We are exiting... */ + return SEND_USER_ERROR; + case ERTS_PORT_OP_BUSY: + /* Nothing has been sent */ + if (suspend) + erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); + return SEND_YIELD; + case ERTS_PORT_OP_BUSY_SCHEDULED: + /* Message was sent */ + if (suspend) { + erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); + ret_val = SEND_YIELD_RETURN; + break; } + /* Fall through */ + case ERTS_PORT_OP_SCHEDULED: + if (is_not_nil(*refp)) { + ASSERT(is_internal_ref(*refp)); + ret_val = SEND_AWAIT_RESULT; + } + break; + case ERTS_PORT_OP_DROPPED: + case ERTS_PORT_OP_BADARG: + case ERTS_PORT_OP_DONE: + break; + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_command() result"); + break; } - /* Virtually schedule out the port before releasing */ - if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(pt, am_out, am_command); - } - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) { - profile_runnable_port(pt, am_inactive); - } - erts_port_release(pt); - return SEND_YIELD; } if (IS_TRACED(p)) /* trace once only !! */ @@ -1958,30 +1962,11 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { SEQ_TRACE_SEND, portid, p); } - /* XXX NO GC in port command */ - erts_port_command(p, p->id, pt, msg); - if (pt) { - /* Virtually schedule out the port before releasing */ - if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(pt, am_out, am_command); - } - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) { - profile_runnable_port(pt, am_inactive); - } - erts_port_release(pt); - } - /* Virtually schedule in process */ - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_in); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(p, am_active); - } if (ERTS_PROC_IS_EXITING(p)) { KILL_CATCHES(p); /* Must exit */ return SEND_USER_ERROR; } - return 0; + return ret_val; } else if (is_tuple(to)) { /* Remote send */ int ret; tp = tuple_val(to); @@ -1997,47 +1982,24 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { dep = erts_sysname_to_connected_dist_entry(tp[2]); if (dep == erts_this_dist_entry) { + Eterm id; erts_deref_dist_entry(dep); if (IS_TRACED(p)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - - /* Need to virtual schedule out sending process - * because of lock wait. This is only necessary - * for internal port calling but the lock is bundled. - */ - - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_out); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(p, am_inactive); - } - erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, - tp[1], - &rp, 0, ERTS_P2P_FLG_SMP_INC_REFC, - &pt); + id = erts_whereis_name_to_id(p, tp[1]); + + rp = erts_proc_lookup_raw(id); + if (rp) + goto send_message; + pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP); if (pt) { - portid = pt->id; + portid = id; goto port_common; } - /* Port lookup failed, virtually schedule the process - * back in. - */ - - if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(p, am_in); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(p, am_active); - } - - if (!rp) { - return 0; - } - goto send_message; + return 0; } ret = remote_send(p, dep, tp[1], to, msg, suspend); @@ -2060,23 +2022,15 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { rp_locks |= ERTS_PROC_LOCK_MAIN; #endif /* send to local process */ - erts_send_message(p, rp, &rp_locks, msg, 0); - if (!erts_use_sender_punish) + res = erts_send_message(p, rp, &rp_locks, msg, 0); + if (erts_use_sender_punish) + res *= 4; + else res = 0; - else { -#ifdef ERTS_SMP - res = rp->msg_inq.len*4; - if (ERTS_PROC_LOCK_MAIN & rp_locks) - res += rp->msg.len*4; -#else - res = rp->msg.len*4; -#endif - } erts_smp_proc_unlock(rp, p == rp ? (rp_locks & ~ERTS_PROC_LOCK_MAIN) : rp_locks); - erts_smp_proc_dec_refc(rp); return res; } } @@ -2084,6 +2038,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { BIF_RETTYPE send_3(BIF_ALIST_3) { + Eterm ref; Process *p = BIF_P; Eterm to = BIF_ARG_1; Eterm msg = BIF_ARG_2; @@ -2107,13 +2062,24 @@ BIF_RETTYPE send_3(BIF_ALIST_3) if(!is_nil(l)) { BIF_ERROR(p, BADARG); } - - result = do_send(p, to, msg, suspend); + +#ifdef DEBUG + ref = NIL; +#endif + + result = do_send(p, to, msg, suspend, &ref); if (result > 0) { ERTS_VBUMP_REDS(p, result); + if (ERTS_IS_PROC_OUT_OF_REDS(p)) + goto yield_return; BIF_RET(am_ok); - } else switch (result) { + } + + switch (result) { case 0: + /* May need to yield even though we do not bump reds here... */ + if (ERTS_IS_PROC_OUT_OF_REDS(p)) + goto yield_return; BIF_RET(am_ok); break; case SEND_TRAP: @@ -2131,10 +2097,13 @@ BIF_RETTYPE send_3(BIF_ALIST_3) } break; case SEND_YIELD_RETURN: - if (suspend) - ERTS_BIF_YIELD_RETURN(p, am_ok); - else + if (!suspend) BIF_RET(am_nosuspend); + yield_return: + ERTS_BIF_YIELD_RETURN(p, am_ok); + case SEND_AWAIT_RESULT: + ASSERT(is_internal_ref(ref)); + BIF_TRAP3(await_port_send_result_trap, p, ref, am_nosuspend, am_ok); case SEND_BADARG: BIF_ERROR(p, BADARG); break; @@ -2159,13 +2128,27 @@ BIF_RETTYPE send_2(BIF_ALIST_2) Eterm erl_send(Process *p, Eterm to, Eterm msg) { - Sint result = do_send(p, to, msg, !0); + Eterm ref; + Sint result; + +#ifdef DEBUG + ref = NIL; +#endif + + result = do_send(p, to, msg, !0, &ref); if (result > 0) { ERTS_VBUMP_REDS(p, result); + if (ERTS_IS_PROC_OUT_OF_REDS(p)) + goto yield_return; BIF_RET(msg); - } else switch (result) { + } + + switch (result) { case 0: + /* May need to yield even though we do not bump reds here... */ + if (ERTS_IS_PROC_OUT_OF_REDS(p)) + goto yield_return; BIF_RET(msg); break; case SEND_TRAP: @@ -2175,7 +2158,11 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) ERTS_BIF_YIELD2(bif_export[BIF_send_2], p, to, msg); break; case SEND_YIELD_RETURN: + yield_return: ERTS_BIF_YIELD_RETURN(p, msg); + case SEND_AWAIT_RESULT: + ASSERT(is_internal_ref(ref)); + BIF_TRAP3(await_port_send_result_trap, p, ref, msg, msg); case SEND_BADARG: BIF_ERROR(p, BADARG); break; @@ -2445,9 +2432,7 @@ BIF_RETTYPE setelement_3(BIF_ALIST_3) /* copy the tuple */ resp = hp; - while (size--) { /* XXX use memcpy? */ - *hp++ = *ptr++; - } + sys_memcpy(hp, ptr, sizeof(Eterm)*size); resp[ix] = BIF_ARG_3; BIF_RET(make_tuple(resp)); } @@ -2460,7 +2445,7 @@ BIF_RETTYPE make_tuple_2(BIF_ALIST_2) Eterm* hp; Eterm res; - if (is_not_small(BIF_ARG_1) || (n = signed_val(BIF_ARG_1)) < 0) { + if (is_not_small(BIF_ARG_1) || (n = signed_val(BIF_ARG_1)) < 0 || n > ERTS_MAX_TUPLE_SIZE) { BIF_ERROR(BIF_P, BADARG); } hp = HAlloc(BIF_P, n+1); @@ -2481,7 +2466,7 @@ BIF_RETTYPE make_tuple_3(BIF_ALIST_3) Eterm list = BIF_ARG_3; Eterm* tup; - if (is_not_small(BIF_ARG_1) || (n = signed_val(BIF_ARG_1)) < 0) { + if (is_not_small(BIF_ARG_1) || (n = signed_val(BIF_ARG_1)) < 0 || n > ERTS_MAX_TUPLE_SIZE) { error: BIF_ERROR(BIF_P, BADARG); } @@ -2533,11 +2518,16 @@ BIF_RETTYPE append_element_2(BIF_ALIST_2) Eterm res; if (is_not_tuple(BIF_ARG_1)) { + error: BIF_ERROR(BIF_P, BADARG); } - ptr = tuple_val(BIF_ARG_1); + ptr = tuple_val(BIF_ARG_1); arity = arityval(*ptr); - hp = HAlloc(BIF_P, arity + 2); + + if (arity + 1 > ERTS_MAX_TUPLE_SIZE) + goto error; + + hp = HAlloc(BIF_P, arity + 2); res = make_tuple(hp); *hp = make_arityval(arity+1); while (arity--) { @@ -2547,15 +2537,87 @@ BIF_RETTYPE append_element_2(BIF_ALIST_2) BIF_RET(res); } +BIF_RETTYPE insert_element_3(BIF_ALIST_3) +{ + Eterm* ptr; + Eterm* hp; + Uint arity; + Eterm res; + Sint ix, c1, c2; + + if (is_not_tuple(BIF_ARG_2) || is_not_small(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + ptr = tuple_val(BIF_ARG_2); + arity = arityval(*ptr); + ix = signed_val(BIF_ARG_1); + + if ((ix < 1) || (ix > (arity + 1))) { + BIF_ERROR(BIF_P, BADARG); + } + + hp = HAlloc(BIF_P, arity + 1 + 1); + res = make_tuple(hp); + *hp = make_arityval(arity + 1); + + c1 = ix - 1; + c2 = arity - ix + 1; + + while (c1--) { *++hp = *++ptr; } + *++hp = BIF_ARG_3; + while (c2--) { *++hp = *++ptr; } + + BIF_RET(res); +} + +BIF_RETTYPE delete_element_2(BIF_ALIST_3) +{ + Eterm* ptr; + Eterm* hp; + Uint arity; + Eterm res; + Sint ix, c1, c2; + + if (is_not_tuple(BIF_ARG_2) || is_not_small(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + ptr = tuple_val(BIF_ARG_2); + arity = arityval(*ptr); + ix = signed_val(BIF_ARG_1); + + if ((ix < 1) || (ix > arity) || (arity == 0)) { + BIF_ERROR(BIF_P, BADARG); + } + + hp = HAlloc(BIF_P, arity + 1 - 1); + res = make_tuple(hp); + *hp = make_arityval(arity - 1); + + c1 = ix - 1; + c2 = arity - ix; + + while (c1--) { *++hp = *++ptr; } + ++ptr; + while (c2--) { *++hp = *++ptr; } + + BIF_RET(res); +} + /**********************************************************************/ /* convert an atom to a list of ascii integer */ BIF_RETTYPE atom_to_list_1(BIF_ALIST_1) { - Uint need; - Eterm* hp; Atom* ap; + Uint num_chars, num_built, num_eaten; + byte* err_pos; + Eterm res; +#ifdef DEBUG + int ares; +#endif if (is_not_atom(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); @@ -2564,9 +2626,18 @@ BIF_RETTYPE atom_to_list_1(BIF_ALIST_1) ap = atom_tab(atom_val(BIF_ARG_1)); if (ap->len == 0) BIF_RET(NIL); /* the empty atom */ - need = ap->len*2; - hp = HAlloc(BIF_P, need); - BIF_RET(buf_to_intlist(&hp,(char*)ap->name,ap->len, NIL)); + +#ifdef DEBUG + ares = +#endif + erts_analyze_utf8(ap->name, ap->len, &err_pos, &num_chars, NULL); + ASSERT(ares == ERTS_UTF8_OK); + + res = erts_utf8_to_list(BIF_P, num_chars, ap->name, ap->len, ap->len, + &num_built, &num_eaten, NIL); + ASSERT(num_built == num_chars); + ASSERT(num_eaten == ap->len); + BIF_RET(res); } /**********************************************************************/ @@ -2576,18 +2647,19 @@ BIF_RETTYPE atom_to_list_1(BIF_ALIST_1) BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) { Eterm res; - char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_LENGTH); - int i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_LENGTH); + char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); + int i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); if (i < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); i = list_length(BIF_ARG_1); - if (i > MAX_ATOM_LENGTH) { + if (i > MAX_ATOM_CHARACTERS) { BIF_ERROR(BIF_P, SYSTEM_LIMIT); } BIF_ERROR(BIF_P, BADARG); } - res = am_atom_put(buf, i); + res = erts_atom_put((byte *) buf, i, ERTS_ATOM_ENC_LATIN1, 1); + ASSERT(is_atom(res)); erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(res); } @@ -2597,16 +2669,16 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) { int i; - char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_LENGTH); + char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); - if ((i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_LENGTH)) < 0) { + if ((i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS)) < 0) { error: erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_ERROR(BIF_P, BADARG); } else { Eterm a; - if (erts_atom_get(buf, i, &a)) { + if (erts_atom_get(buf, i, &a, ERTS_ATOM_ENC_LATIN1)) { erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(a); } else { @@ -2831,42 +2903,168 @@ BIF_RETTYPE string_to_integer_1(BIF_ALIST_1) BIF_RET(TUPLE2(hp, res, tail)); } } - BIF_RETTYPE list_to_integer_1(BIF_ALIST_1) -{ + { + /* Using do_list_to_integer is about twice as fast as using + erts_chars_to_integer because we do not have to copy the + entire list */ Eterm res; Eterm dummy; /* must be a list */ - if (do_list_to_integer(BIF_P,BIF_ARG_1,&res,&dummy) != LTI_ALL_INTEGER) { BIF_ERROR(BIF_P,BADARG); } BIF_RET(res); } +BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) +{ + + /* Bif implementation is about 50% faster than pure erlang, + and since we have erts_chars_to_integer now it is simpler + as well. This could be optmized further if we did not have to + copy the list to buf. */ + int i; + Eterm res; + char *buf = NULL; + int base; + + i = list_length(BIF_ARG_1); + if (i < 0) + BIF_ERROR(BIF_P, BADARG); + + base = signed_val(BIF_ARG_2); + + if (base < 2 || base > 36) + BIF_ERROR(BIF_P, BADARG); + + /* Take fast path if base it 10 */ + if (base == 10) + return list_to_integer_1(BIF_P,&BIF_ARG_1); + + buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); + + if (intlist_to_buf(BIF_ARG_1, buf, i) < 0) + goto list_to_integer_1_error; + buf[i] = '\0'; /* null terminal */ + + if ((res = erts_chars_to_integer(BIF_P,buf,i,base)) == THE_NON_VALUE) + goto list_to_integer_1_error; + + erts_free(ERTS_ALC_T_TMP, (void *) buf); + BIF_RET(res); + + list_to_integer_1_error: + erts_free(ERTS_ALC_T_TMP, (void *) buf); + BIF_ERROR(BIF_P, BADARG); + + } + /**********************************************************************/ +static int do_float_to_charbuf(Process *p, Eterm efloat, Eterm list, + char *fbuf, int sizeof_fbuf) { + + const static int arity_two = make_arityval(2); + int decimals = SYS_DEFAULT_FLOAT_DECIMALS; + int compact = 0; + enum fmt_type_ { + FMT_LEGACY, + FMT_FIXED, + FMT_SCIENTIFIC + } fmt_type = FMT_LEGACY; + Eterm arg; + FloatDef f; + + /* check the arguments */ + if (is_not_float(efloat)) + goto badarg; + + for(; is_list(list); list = CDR(list_val(list))) { + arg = CAR(list_val(list)); + if (arg == am_compact) { + compact = 1; + continue; + } else if (is_tuple(arg)) { + Eterm* tp = tuple_val(arg); + if (*tp == arity_two && is_small(tp[2])) { + decimals = signed_val(tp[2]); + switch (tp[1]) { + case am_decimals: + fmt_type = FMT_FIXED; + continue; + case am_scientific: + fmt_type = FMT_SCIENTIFIC; + continue; + } + } + } + goto badarg; + } + if (is_not_nil(list)) { + goto badarg; + } + + GET_DOUBLE(efloat, f); + + if (fmt_type == FMT_FIXED) { + return sys_double_to_chars_fast(f.fd, fbuf, sizeof_fbuf, + decimals, compact); + } else { + return sys_double_to_chars_ext(f.fd, fbuf, sizeof_fbuf, decimals); + } + +badarg: + return -1; +} + /* convert a float to a list of ascii characters */ +static BIF_RETTYPE do_float_to_list(Process *BIF_P, Eterm arg, Eterm opts) { + int used; + Eterm* hp; + char fbuf[256]; + + if ((used = do_float_to_charbuf(BIF_P,arg,opts,fbuf,sizeof(fbuf))) <= 0) { + BIF_ERROR(BIF_P, BADARG); + } + hp = HAlloc(BIF_P, (Uint)used*2); + BIF_RET(buf_to_intlist(&hp, fbuf, (Uint)used, NIL)); +} + + BIF_RETTYPE float_to_list_1(BIF_ALIST_1) { - int i; - Uint need; - Eterm* hp; - FloatDef f; - char fbuf[30]; - - /* check the arguments */ - if (is_not_float(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); - GET_DOUBLE(BIF_ARG_1, f); - if ((i = sys_double_to_chars(f.fd, fbuf)) <= 0) - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); - need = i*2; - hp = HAlloc(BIF_P, need); - BIF_RET(buf_to_intlist(&hp, fbuf, i, NIL)); - } + return do_float_to_list(BIF_P,BIF_ARG_1,NIL); +} + +BIF_RETTYPE float_to_list_2(BIF_ALIST_2) +{ + return do_float_to_list(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +/* convert a float to a binary of ascii characters */ + +static BIF_RETTYPE do_float_to_binary(Process *BIF_P, Eterm arg, Eterm opts) { + int used; + char fbuf[256]; + + if ((used = do_float_to_charbuf(BIF_P,arg,opts,fbuf,sizeof(fbuf))) <= 0) { + BIF_ERROR(BIF_P, BADARG); + } + BIF_RET(new_binary(BIF_P, (byte*)fbuf, (Uint)used)); +} + +BIF_RETTYPE float_to_binary_1(BIF_ALIST_1) +{ + return do_float_to_binary(BIF_P,BIF_ARG_1,NIL); +} + +BIF_RETTYPE float_to_binary_2(BIF_ALIST_2) +{ + return do_float_to_binary(BIF_P,BIF_ARG_1,BIF_ARG_2); +} /**********************************************************************/ @@ -3050,36 +3248,101 @@ BIF_RETTYPE string_to_float_1(BIF_ALIST_1) BIF_RET(tup); } +static BIF_RETTYPE do_charbuf_to_float(Process *BIF_P,char *buf) { + FloatDef f; + Eterm res; + Eterm* hp; + + if (sys_chars_to_double(buf, &f.fd) != 0) + BIF_ERROR(BIF_P, BADARG); + + hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT); + res = make_float(hp); + PUT_DOUBLE(f, hp); + BIF_RET(res); + +} BIF_RETTYPE list_to_float_1(BIF_ALIST_1) { int i; - FloatDef f; Eterm res; - Eterm* hp; char *buf = NULL; i = list_length(BIF_ARG_1); - if (i < 0) { - badarg: - if (buf) - erts_free(ERTS_ALC_T_TMP, (void *) buf); - BIF_ERROR(BIF_P, BADARG); - } - + if (i < 0) + BIF_ERROR(BIF_P, BADARG); + buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); if (intlist_to_buf(BIF_ARG_1, buf, i) < 0) - goto badarg; + goto list_to_float_1_error; buf[i] = '\0'; /* null terminal */ + + if ((res = do_charbuf_to_float(BIF_P,buf)) == THE_NON_VALUE) + goto list_to_float_1_error; + + erts_free(ERTS_ALC_T_TMP, (void *) buf); + BIF_RET(res); + + list_to_float_1_error: + erts_free(ERTS_ALC_T_TMP, (void *) buf); + BIF_ERROR(BIF_P, BADARG); + +} + +BIF_RETTYPE binary_to_float_1(BIF_ALIST_1) +{ + Eterm res; + Eterm binary = BIF_ARG_1; + Sint size; + byte* bytes, *buf; + Eterm* real_bin; + Uint offs = 0; + Uint bit_offs = 0; + + if (is_not_binary(binary) || (size = binary_size(binary)) == 0) + BIF_ERROR(BIF_P, BADARG); + + /* + * Unfortunately we have to copy the binary because we have to insert + * the '\0' at the end of the binary for strtod to work + * (there is no nstrtod :( ) + */ + + buf = erts_alloc(ERTS_ALC_T_TMP, size + 1); + + real_bin = binary_val(binary); + if (*real_bin == HEADER_SUB_BIN) { + ErlSubBin* sb = (ErlSubBin *) real_bin; + if (sb->bitsize) { + goto binary_to_float_1_error; + } + offs = sb->offs; + bit_offs = sb->bitoffs; + real_bin = binary_val(sb->orig); + } + if (*real_bin == HEADER_PROC_BIN) { + bytes = ((ProcBin *) real_bin)->bytes + offs; + } else { + bytes = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + offs; + } + if (bit_offs) + erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, size*8); + else + memcpy(buf, bytes, size); + + buf[size] = '\0'; + + if ((res = do_charbuf_to_float(BIF_P,(char*)buf)) == THE_NON_VALUE) + goto binary_to_float_1_error; - if (sys_chars_to_double(buf, &f.fd) != 0) - goto badarg; - hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT); - res = make_float(hp); - PUT_DOUBLE(f, hp); erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(res); + + binary_to_float_1_error: + erts_free(ERTS_ALC_T_TMP, (void *) buf); + BIF_ERROR(BIF_P, BADARG); } /**********************************************************************/ @@ -3121,7 +3384,7 @@ BIF_RETTYPE list_to_tuple_1(BIF_ALIST_1) Eterm* hp; int len; - if ((len = list_length(list)) < 0) { + if ((len = list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) { BIF_ERROR(BIF_P, BADARG); } @@ -3143,7 +3406,7 @@ BIF_RETTYPE list_to_tuple_1(BIF_ALIST_1) BIF_RETTYPE self_0(BIF_ALIST_0) { - BIF_RET(BIF_P->id); + BIF_RET(BIF_P->common.id); } /**********************************************************************/ @@ -3180,11 +3443,9 @@ static erts_smp_spinlock_t make_ref_lock; static erts_smp_mtx_t ports_snapshot_mtx; erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) +void +erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) { - Eterm* hp = buffer; - Uint32 ref0, ref1, ref2; - erts_smp_spin_lock(&make_ref_lock); reference0++; @@ -3196,24 +3457,36 @@ Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) } } - ref0 = reference0; - ref1 = reference1; - ref2 = reference2; + ref[0] = reference0; + ref[1] = reference1; + ref[2] = reference2; erts_smp_spin_unlock(&make_ref_lock); +} + +Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) +{ + Eterm* hp = buffer; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; - write_ref_thing(hp, ref0, ref1, ref2); + erts_make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); return make_internal_ref(hp); } Eterm erts_make_ref(Process *p) { Eterm* hp; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); hp = HAlloc(p, REF_THING_SIZE); - return erts_make_ref_in_buffer(hp); + + erts_make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + + return make_internal_ref(hp); } BIF_RETTYPE make_ref_0(BIF_ALIST_0) @@ -3477,7 +3750,7 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } - if (BIF_P->id == BIF_ARG_1) + if (BIF_P->common.id == BIF_ARG_1) rp = BIF_P; else { #ifdef ERTS_SMP @@ -3486,7 +3759,7 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1) if (rp == ERTS_PROC_LOCK_BUSY) ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); #else - rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); + rp = erts_proc_lookup(BIF_ARG_1); #endif if (!rp) BIF_RET(am_false); @@ -3517,71 +3790,23 @@ BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) } /**********************************************************************/ -/* Return a list of active ports */ +/* + * The erlang:processes/0 BIF. + */ -BIF_RETTYPE ports_0(BIF_ALIST_0) +BIF_RETTYPE processes_0(BIF_ALIST_0) { - Eterm res = NIL; - Eterm* port_buf = erts_alloc(ERTS_ALC_T_TMP, - sizeof(Eterm)*erts_max_ports); - Eterm* pp = port_buf; - Eterm* dead_ports; - int alive, dead; - Uint32 next_ss; - int i; - - /* To get a consistent snapshot... - * We add alive ports from start of the buffer - * while dying ports are added from the other end by the killing threads. - */ - - erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */ - - erts_smp_atomic_set_nob(&erts_dead_ports_ptr, - (erts_aint_t) (port_buf + erts_max_ports)); - - next_ss = erts_smp_atomic32_inc_read_relb(&erts_ports_snapshot); - - for (i = erts_max_ports-1; i >= 0; i--) { - Port* prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD) - && prt->snapshot != next_ss) { - ASSERT(prt->snapshot == next_ss - 1); - *pp++ = prt->id; - prt->snapshot = next_ss; /* Consumed by this snapshot */ - } - erts_smp_port_state_unlock(prt); - } - - dead_ports = (Eterm*)erts_smp_atomic_xchg_nob(&erts_dead_ports_ptr, - (erts_aint_t) NULL); - erts_smp_mtx_unlock(&ports_snapshot_mtx); - - ASSERT(pp <= dead_ports); - - alive = pp - port_buf; - dead = port_buf + erts_max_ports - dead_ports; - - ASSERT((alive+dead) <= erts_max_ports); - - if (alive+dead > 0) { - erts_aint_t i; - Eterm *hp = HAlloc(BIF_P, (alive+dead)*2); - - for (i = 0; i < alive; i++) { - res = CONS(hp, port_buf[i], res); - hp += 2; - } - for (i = 0; i < dead; i++) { - res = CONS(hp, dead_ports[i], res); - hp += 2; - } - } + return erts_ptab_list(BIF_P, &erts_proc); +} - erts_free(ERTS_ALC_T_TMP, port_buf); +/**********************************************************************/ +/* + * The erlang:ports/0 BIF. + */ - BIF_RET(res); +BIF_RETTYPE ports_0(BIF_ALIST_0) +{ + return erts_ptab_list(BIF_P, &erts_port); } /**********************************************************************/ @@ -3709,7 +3934,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) { Sint code; Eterm optlist = BIF_ARG_2; - int flush = 0; + int flush = 1; for (optlist = BIF_ARG_2; is_list(optlist); @@ -3780,7 +4005,8 @@ BIF_RETTYPE function_exported_3(BIF_ALIST_3) is_not_small(BIF_ARG_3)) { BIF_ERROR(BIF_P, BADARG); } - if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3)) == NULL) { + if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3), + erts_active_code_ix()) == NULL) { BIF_RET(am_false); } BIF_RET(am_true); @@ -4210,14 +4436,15 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_RET(old_value); } } else if (BIF_ARG_1 == make_small(1)) { - Uint i; + int i, max; ErlMessage* mp; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - for (i = 0; i < erts_max_processes; i++) { - if (process_tab[i] != (Process*) 0) { - Process* p = process_tab[i]; + max = erts_ptab_max(&erts_proc); + for (i = 0; i < max; i++) { + Process *p = erts_pix2proc(i); + if (p) { #ifdef USE_VM_PROBES p->seq_trace_token = (p->dt_utag != NIL) ? am_have_dt_utag : NIL; #else @@ -4513,6 +4740,21 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Export bif_return_trap_export; +void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, + Eterm (*bif)(BIF_ALIST_0)) +{ + int i; + sys_memset((void *) ep, 0, sizeof(Export)); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + ep->addressv[i] = &ep->code[3]; + } + ep->code[0] = m; + ep->code[1] = f; + ep->code[2] = a; + ep->code[3] = (BeamInstr) em_apply_bif; + ep->code[4] = (BeamInstr) bif; +} + void erts_init_bif(void) { reference0 = 0; @@ -4528,17 +4770,13 @@ void erts_init_bif(void) * yield the calling process traps to. The only thing it does: * return the value passed as argument. */ - sys_memset((void *) &bif_return_trap_export, 0, sizeof(Export)); - bif_return_trap_export.address = &bif_return_trap_export.code[3]; - bif_return_trap_export.code[0] = am_erlang; - bif_return_trap_export.code[1] = am_bif_return_trap; + erts_init_trap_export(&bif_return_trap_export, am_erlang, am_bif_return_trap, #ifdef DEBUG - bif_return_trap_export.code[2] = 2; + 2 #else - bif_return_trap_export.code[2] = 1; + 1 #endif - bif_return_trap_export.code[3] = (BeamInstr) em_apply_bif; - bif_return_trap_export.code[4] = (BeamInstr) &bif_return_trap; + , &bif_return_trap); flush_monitor_message_trap = erts_export_put(am_erlang, am_flush_monitor_message, @@ -4551,6 +4789,8 @@ void erts_init_bif(void) am_format_cpu_topology, 1); await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); + await_port_send_result_trap + = erts_export_put(am_erts_internal, am_await_port_send_result, 3); await_sched_wall_time_mod_trap = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); erts_smp_atomic32_init_nob(&sched_wall_time, 0); @@ -4566,19 +4806,18 @@ bif erlang:send_to_logger/2 BIF_RETTYPE send_to_logger_2(BIF_ALIST_2) { byte *buf; - int len; + ErlDrvSizeT len; if (!is_atom(BIF_ARG_1) || !(is_list(BIF_ARG_2) || is_nil(BIF_ARG_1))) { BIF_ERROR(BIF_P,BADARG); } - len = io_list_len(BIF_ARG_2); - if (len < 0) + if (erts_iolist_size(BIF_ARG_2, &len) != 0) BIF_ERROR(BIF_P,BADARG); else if (len == 0) buf = ""; else { #ifdef DEBUG - int len2; + ErlDrvSizeT len2; #endif buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, len+1); #ifdef DEBUG @@ -4586,7 +4825,7 @@ BIF_RETTYPE send_to_logger_2(BIF_ALIST_2) #else (void) #endif - io_list_to_buf(BIF_ARG_2, buf, len); + erts_iolist_to_buf(BIF_ARG_2, buf, len); ASSERT(len2 == len); buf[len] = '\0'; switch (BIF_ARG_1) { @@ -4680,7 +4919,6 @@ BIF_RETTYPE dt_prepend_vm_tag_data_1(BIF_ALIST_1) #ifdef USE_VM_PROBES Eterm b; Eterm *hp; - hp = HAlloc(BIF_P,2); if (is_binary((DT_UTAG(BIF_P)))) { Uint sz = binary_size(DT_UTAG(BIF_P)); int i; @@ -4697,6 +4935,7 @@ BIF_RETTYPE dt_prepend_vm_tag_data_1(BIF_ALIST_1) } else { b = new_binary(BIF_P,(byte *)"\0",1); } + hp = HAlloc(BIF_P,2); BIF_RET(CONS(hp,b,BIF_ARG_1)); #else BIF_RET(BIF_ARG_1); @@ -4707,7 +4946,6 @@ BIF_RETTYPE dt_append_vm_tag_data_1(BIF_ALIST_1) #ifdef USE_VM_PROBES Eterm b; Eterm *hp; - hp = HAlloc(BIF_P,2); if (is_binary((DT_UTAG(BIF_P)))) { Uint sz = binary_size(DT_UTAG(BIF_P)); int i; @@ -4724,6 +4962,7 @@ BIF_RETTYPE dt_append_vm_tag_data_1(BIF_ALIST_1) } else { b = new_binary(BIF_P,(byte *)"\0",1); } + hp = HAlloc(BIF_P,2); BIF_RET(CONS(hp,BIF_ARG_1,b)); #else BIF_RET(BIF_ARG_1); @@ -4747,14 +4986,14 @@ BIF_RETTYPE dt_spread_tag_1(BIF_ALIST_1) #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) start spreading tag %T\r\n", - BIF_P->id,DT_UTAG(BIF_P)); + BIF_P->common.id,DT_UTAG(BIF_P)); #endif } else { DT_UTAG_FLAGS(BIF_P) &= ~DT_UTAG_SPREADING; #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) stop spreading tag %T\r\n", - BIF_P->id,DT_UTAG(BIF_P)); + BIF_P->common.id,DT_UTAG(BIF_P)); #endif } } @@ -4780,7 +5019,7 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1) #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) restore Killing tag!\r\n", - BIF_P->id); + BIF_P->common.id); #endif } DT_UTAG(BIF_P) = NIL; @@ -4797,12 +5036,12 @@ BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1) erts_fprintf(stderr, "Dtrace -> (%T) restore stop spreading " "tag %T\r\n", - BIF_P->id, tpl[2]); + BIF_P->common.id, tpl[2]); } else if ((x & DT_UTAG_SPREADING) && !(DT_UTAG_FLAGS(BIF_P) & DT_UTAG_SPREADING)) { erts_fprintf(stderr, "Dtrace -> (%T) restore start spreading " - "tag %T\r\n",BIF_P->id,tpl[2]); + "tag %T\r\n",BIF_P->common.id,tpl[2]); } #endif DT_UTAG_FLAGS(BIF_P) = x; diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index d20089a9fb..51b77a95ed 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -35,6 +35,13 @@ extern Export* erts_format_cpu_topology_trap; #define BIF_ARG_2 (BIF__ARGS[1]) #define BIF_ARG_3 (BIF__ARGS[2]) +#define ERTS_IS_PROC_OUT_OF_REDS(p) \ + ((p)->fcalls > 0 \ + ? 0 \ + : (!ERTS_PROC_GET_SAVED_CALLS_BUF((p)) \ + ? (p)->fcalls == 0 \ + : ((p)->fcalls == -CONTEXT_REDS))) + #define BUMP_ALL_REDS(p) do { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) \ (p)->fcalls = 0; \ @@ -59,6 +66,8 @@ do { \ } while(0) #define BUMP_REDS(p, gc) do { \ + ASSERT(p); \ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\ (p)->fcalls -= (gc); \ if ((p)->fcalls < 0) { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) \ @@ -125,7 +134,7 @@ do { \ #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ do { \ (Proc)->arity = 0; \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -135,7 +144,7 @@ do { \ Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ (Proc)->arity = 1; \ reg[0] = (Eterm) (A0); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -146,7 +155,7 @@ do { \ (Proc)->arity = 2; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -158,7 +167,7 @@ do { \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -170,13 +179,13 @@ do { \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ - (Proc)->i = (BeamInstr*) ((Trap)->address); \ + (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ (Proc)->freason = TRAP; \ } while (0) #define BIF_TRAP0(p, Trap_) do { \ (p)->arity = 0; \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -185,7 +194,7 @@ do { \ Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ (p)->arity = 1; \ reg[0] = (A0); \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -195,7 +204,7 @@ do { \ (p)->arity = 2; \ reg[0] = (A0); \ reg[1] = (A1); \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -206,7 +215,7 @@ do { \ reg[0] = (A0); \ reg[1] = (A1); \ reg[2] = (A2); \ - (p)->i = (BeamInstr*) ((Trap_)->address); \ + (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ (p)->freason = TRAP; \ return THE_NON_VALUE; \ } while(0) @@ -322,27 +331,6 @@ do { \ ERTS_BIF_EXITED((PROC)); \ } while (0) -#ifdef ERTS_SMP -#define ERTS_SMP_BIF_CHK_PENDING_EXIT(P, L) \ -do { \ - ERTS_SMP_LC_ASSERT((L) == erts_proc_lc_my_proc_locks((P))); \ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & (L)); \ - if (!((L) & ERTS_PROC_LOCK_STATUS)) \ - erts_smp_proc_lock((P), ERTS_PROC_LOCK_STATUS); \ - if (ERTS_PROC_PENDING_EXIT((P))) { \ - erts_handle_pending_exit((P), (L)|ERTS_PROC_LOCK_STATUS); \ - erts_smp_proc_unlock((P), \ - (((L)|ERTS_PROC_LOCK_STATUS) \ - & ~ERTS_PROC_LOCK_MAIN)); \ - ERTS_BIF_EXITED((P)); \ - } \ - if (!((L) & ERTS_PROC_LOCK_STATUS)) \ - erts_smp_proc_unlock((P), ERTS_PROC_LOCK_STATUS); \ -} while (0) -#else -#define ERTS_SMP_BIF_CHK_PENDING_EXIT(P, L) -#endif - /* * The ERTS_BIF_*_AWAIT_X_*_TRAP makros either exits the caller, or * sets up a trap to erlang:await_proc_exit/3. diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 797bce43ab..dc8e9101de 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2012. All Rights Reserved. +# Copyright Ericsson AB 1996-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -31,434 +31,236 @@ # # Important: Use "ubif" for guard BIFs and operators; use "bif" for ordinary BIFs. # -# Add new BIFs to the end of the file. Do not bother adding a "packaged BIF name" -# (such as 'erl.lang.number'); if/when packages will be supported we will add -# all those names. +# Add new BIFs to the end of the file. # # Note: Guards BIFs require special support in the compiler (to be able to actually # call them from within a guard). # ubif erlang:abs/1 -ubif 'erl.lang.number':abs/1 ebif_abs_1 bif erlang:adler32/1 -bif 'erl.util.crypt.adler32':sum/1 ebif_adler32_1 bif erlang:adler32/2 -bif 'erl.util.crypt.adler32':sum/2 ebif_adler32_2 bif erlang:adler32_combine/3 -bif 'erl.util.crypt.adler32':combine/3 ebif_adler32_combine_3 bif erlang:apply/3 -bif 'erl.lang':apply/3 ebif_apply_3 bif erlang:atom_to_list/1 -bif 'erl.lang.atom':to_string/1 ebif_atom_to_string_1 atom_to_list_1 bif erlang:binary_to_list/1 -bif 'erl.lang.binary':to_list/1 ebif_binary_to_list_1 bif erlang:binary_to_list/3 -bif 'erl.lang.binary':to_list/3 ebif_binary_to_list_3 bif erlang:binary_to_term/1 -bif 'erl.lang.binary':to_term/1 ebif_binary_to_term_1 bif erlang:check_process_code/2 -bif 'erl.system.code':check_process/2 ebif_check_process_code_2 bif erlang:crc32/1 -bif 'erl.util.crypt.crc32':sum/1 ebif_crc32_1 bif erlang:crc32/2 -bif 'erl.util.crypt.crc32':sum/2 ebif_crc32_2 bif erlang:crc32_combine/3 -bif 'erl.util.crypt.crc32':combine/3 ebif_crc32_combine_3 bif erlang:date/0 -bif 'erl.util.date':today/0 ebif_date_0 bif erlang:delete_module/1 -bif 'erl.system.code':delete/1 ebif_delete_module_1 bif erlang:display/1 -bif 'erl.system.debug':display/1 ebif_display_1 bif erlang:display_string/1 -bif 'erl.system.debug':display_string/1 ebif_display_string_1 bif erlang:display_nl/0 -bif 'erl.system.debug':display_nl/0 ebif_display_nl_0 ubif erlang:element/2 -ubif 'erl.lang.tuple':element/2 ebif_element_2 bif erlang:erase/0 -bif 'erl.lang.proc.pdict':erase/0 ebif_erase_0 bif erlang:erase/1 -bif 'erl.lang.proc.pdict':erase/1 ebif_erase_1 bif erlang:exit/1 -bif 'erl.lang':exit/1 ebif_exit_1 bif erlang:exit/2 -bif 'erl.lang.proc':signal/2 ebif_signal_2 exit_2 bif erlang:external_size/1 -bif 'erl.lang.term':external_size/1 ebif_external_size_1 bif erlang:external_size/2 -bif 'erl.lang.term':external_size/2 ebif_external_size_2 ubif erlang:float/1 -ubif 'erl.lang.number':to_float/1 ebif_to_float_1 float_1 bif erlang:float_to_list/1 -bif 'erl.lang.float':to_string/1 ebif_float_to_string_1 float_to_list_1 +bif erlang:float_to_list/2 bif erlang:fun_info/2 -bif 'erl.lang.function':info/2 ebif_fun_info_2 bif erlang:garbage_collect/0 -bif 'erl.system':garbage_collect/0 ebif_garbage_collect_0 bif erlang:garbage_collect/1 -bif 'erl.system':garbage_collect/1 ebif_garbage_collect_1 bif erlang:get/0 -bif 'erl.lang.proc.pdict':get/0 ebif_get_0 bif erlang:get/1 -bif 'erl.lang.proc.pdict':get/1 ebif_get_1 bif erlang:get_keys/1 -bif 'erl.lang.proc.pdict':get_keys/1 ebif_get_keys_1 bif erlang:group_leader/0 -bif 'erl.lang.proc':group_leader/0 ebif_group_leader_0 bif erlang:group_leader/2 -bif 'erl.lang.proc':set_group_leader/2 ebif_group_leader_2 bif erlang:halt/0 -bif 'erl.lang.system':halt/0 ebif_halt_0 bif erlang:halt/1 -bif 'erl.lang.system':halt/1 ebif_halt_1 bif erlang:halt/2 -bif 'erl.lang.system':halt/2 ebif_halt_2 bif erlang:phash/2 bif erlang:phash2/1 bif erlang:phash2/2 -bif 'erl.lang.term':hash/1 ebif_phash2_1 -bif 'erl.lang.term':hash/2 ebif_phash2_2 ubif erlang:hd/1 -ubif 'erl.lang.list':hd/1 ebif_hd_1 bif erlang:integer_to_list/1 -bif 'erl.lang.integer':to_string/1 ebif_integer_to_string_1 integer_to_list_1 bif erlang:is_alive/0 -bif 'erl.lang.node':is_alive/0 ebif_is_alive_0 ubif erlang:length/1 -ubif 'erl.lang.list':length/1 ebif_length_1 bif erlang:link/1 -bif 'erl.lang.proc':link/1 ebif_link_1 bif erlang:list_to_atom/1 -bif 'erl.lang.atom':from_string/1 ebif_string_to_atom_1 list_to_atom_1 bif erlang:list_to_binary/1 -bif 'erl.lang.binary':from_list/1 ebif_list_to_binary_1 bif erlang:list_to_float/1 -bif 'erl.lang.float':from_string/1 ebif_string_to_float_1 list_to_float_1 bif erlang:list_to_integer/1 -bif 'erl.lang.integer':from_string/1 ebif_string_to_integer_1 list_to_integer_1 bif erlang:list_to_pid/1 -bif 'erl.lang.proc':string_to_pid/1 ebif_string_to_pid_1 list_to_pid_1 bif erlang:list_to_tuple/1 -bif 'erl.lang.tuple':from_list/1 ebif_list_to_tuple_1 -bif erlang:load_module/2 -bif 'erl.system.code':load/2 ebif_load_module_2 bif erlang:loaded/0 -bif 'erl.system.code':loaded/0 ebif_loaded_0 bif erlang:localtime/0 -bif 'erl.util.date':local/0 ebif_localtime_0 bif erlang:localtime_to_universaltime/2 -bif 'erl.util.date':local_to_utc/2 ebif_localtime_to_universaltime_2 bif erlang:make_ref/0 -bif 'erl.lang.ref':new/0 ebif_make_ref_0 bif erlang:md5/1 -bif 'erl.util.crypt.md5':digest/1 ebif_md5_1 bif erlang:md5_init/0 -bif 'erl.util.crypt.md5':init/0 ebif_md5_init_0 bif erlang:md5_update/2 -bif 'erl.util.crypt.md5':update/2 ebif_md5_update_2 bif erlang:md5_final/1 -bif 'erl.util.crypt.md5':final/1 ebif_md5_final_1 bif erlang:module_loaded/1 -bif 'erl.system.code':is_loaded/1 ebif_is_loaded_1 module_loaded_1 bif erlang:function_exported/3 -bif 'erl.system.code':is_loaded/3 ebif_is_loaded_3 function_exported_3 bif erlang:monitor_node/2 -bif 'erl.lang.node':monitor/2 ebif_monitor_node_2 bif erlang:monitor_node/3 -bif 'erl.lang.node':monitor/3 ebif_monitor_node_3 ubif erlang:node/1 -ubif 'erl.lang.node':node/1 ebif_node_1 ubif erlang:node/0 -ubif 'erl.lang.node':node/0 ebif_node_0 bif erlang:nodes/1 -bif 'erl.lang.node':nodes/1 ebif_nodes_1 bif erlang:now/0 -bif 'erl.system':now/0 ebif_now_0 bif erlang:open_port/2 -bif 'erl.lang.port':open/2 ebif_open_port_2 open_port_2 bif erlang:pid_to_list/1 -bif 'erl.lang.proc':pid_to_string/1 ebif_pid_to_string_1 pid_to_list_1 -bif erlang:port_info/1 -bif 'erl.lang.port':info/1 ebif_port_info_1 -bif erlang:port_info/2 -bif 'erl.lang.port':info/2 ebif_port_info_2 bif erlang:ports/0 -bif 'erl.lang.node':ports/0 ebif_ports_0 bif erlang:pre_loaded/0 -bif 'erl.system.code':preloaded/0 ebif_pre_loaded_0 bif erlang:process_flag/2 -bif 'erl.lang.proc':set_flag/2 ebif_process_flag_2 bif erlang:process_flag/3 -bif 'erl.lang.proc':set_flag/3 ebif_process_flag_3 bif erlang:process_info/1 -bif 'erl.lang.proc':info/1 ebif_process_info_1 bif erlang:process_info/2 -bif 'erl.lang.proc':info/2 ebif_process_info_2 bif erlang:processes/0 -bif 'erl.lang.node':processes/0 ebif_processes_0 bif erlang:purge_module/1 -bif 'erl.system.code':purge/1 ebif_purge_module_1 bif erlang:put/2 -bif 'erl.lang.proc.pdict':put/2 ebif_put_2 bif erlang:register/2 -bif 'erl.lang.node':register/2 ebif_register_2 bif erlang:registered/0 -bif 'erl.lang.node':registered/0 ebif_registered_0 ubif erlang:round/1 -ubif 'erl.lang.number':round/1 ebif_round_1 ubif erlang:self/0 -ubif 'erl.lang.proc':self/0 ebif_self_0 bif erlang:setelement/3 -bif 'erl.lang.tuple':setelement/3 ebif_setelement_3 ubif erlang:size/1 -ubif 'erl.lang.term':size/1 ebif_size_1 bif erlang:spawn/3 -bif 'erl.lang.proc':spawn/3 ebif_spawn_3 bif erlang:spawn_link/3 -bif 'erl.lang.proc':spawn_link/3 ebif_spawn_link_3 bif erlang:split_binary/2 -bif 'erl.lang.binary':split/2 ebif_split_binary_2 bif erlang:statistics/1 -bif 'erl.system':statistics/1 ebif_statistics_1 bif erlang:term_to_binary/1 -bif 'erl.lang.binary':from_term/1 ebif_term_to_binary_1 bif erlang:term_to_binary/2 -bif 'erl.lang.binary':from_term/2 ebif_term_to_binary_2 bif erlang:throw/1 -bif 'erl.lang':throw/1 ebif_throw_1 bif erlang:time/0 -bif 'erl.util.date':time_of_day/0 ebif_time_0 ubif erlang:tl/1 -ubif 'erl.lang.list':tl/1 ebif_tl_1 ubif erlang:trunc/1 -ubif 'erl.lang.number':trunc/1 ebif_trunc_1 bif erlang:tuple_to_list/1 -bif 'erl.lang.tuple':to_list/1 ebif_tuple_to_list_1 bif erlang:universaltime/0 -bif 'erl.util.date':utc/0 ebif_universaltime_0 bif erlang:universaltime_to_localtime/1 -bif 'erl.util.date':utc_to_local/1 ebif_universaltime_to_localtime_1 bif erlang:unlink/1 -bif 'erl.lang.proc':unlink/1 ebif_unlink_1 bif erlang:unregister/1 -bif 'erl.lang.node':unregister/1 ebif_unregister_1 bif erlang:whereis/1 -bif 'erl.lang.node':whereis/1 ebif_whereis_1 bif erlang:spawn_opt/1 -bif 'erl.lang.proc':spawn_opt/1 ebif_spawn_opt_1 bif erlang:setnode/2 bif erlang:setnode/3 bif erlang:dist_exit/3 -bif erlang:port_call/2 -bif 'erl.lang.port':call/2 ebif_port_call_2 -bif erlang:port_call/3 -bif 'erl.lang.port':call/3 ebif_port_call_3 -bif erlang:port_command/2 -bif 'erl.lang.port':command/2 ebif_port_command_2 -bif erlang:port_command/3 -bif 'erl.lang.port':command/3 ebif_port_command_3 -bif erlang:port_control/3 -bif 'erl.lang.port':control/3 ebif_port_control_3 -bif erlang:port_close/1 -bif 'erl.lang.port':close/1 ebif_port_close_1 -bif erlang:port_connect/2 -bif 'erl.lang.port':connect/2 ebif_port_connect_2 +# Static native functions in erts_internal +bif erts_internal:port_info/1 +bif erts_internal:port_info/2 +bif erts_internal:port_call/3 +bif erts_internal:port_command/3 +bif erts_internal:port_control/3 +bif erts_internal:port_close/1 +bif erts_internal:port_connect/2 + +# inet_db support bif erlang:port_set_data/2 -bif 'erl.lang.port':set_data/2 ebif_port_set_data_2 bif erlang:port_get_data/1 -bif 'erl.lang.port':get_data/1 ebif_port_get_data_1 # Tracing & debugging. bif erlang:trace_pattern/2 -bif 'erl.system.debug':trace_pattern/2 ebif_trace_pattern_2 bif erlang:trace_pattern/3 -bif 'erl.system.debug':trace_pattern/3 ebif_trace_pattern_3 bif erlang:trace/3 -bif 'erl.system.debug':trace/3 ebif_trace_3 bif erlang:trace_info/2 -bif 'erl.system.debug':trace_info/2 ebif_trace_info_2 bif erlang:trace_delivered/1 -bif 'erl.system.debug':trace_delivered/1 ebif_trace_delivered_1 bif erlang:seq_trace/2 -bif 'erl.system.debug':seq_trace/2 ebif_seq_trace_2 bif erlang:seq_trace_info/1 -bif 'erl.system.debug':seq_trace_info/1 ebif_seq_trace_info_1 bif erlang:seq_trace_print/1 -bif 'erl.system.debug':seq_trace_print/1 ebif_seq_trace_print_1 bif erlang:seq_trace_print/2 -bif 'erl.system.debug':seq_trace_print/2 ebif_seq_trace_print_2 bif erlang:suspend_process/2 -bif 'erl.system.debug':suspend_process/2 ebif_suspend_process_2 bif erlang:resume_process/1 -bif 'erl.system.debug':resume_process/1 ebif_resume_process_1 bif erlang:process_display/2 -bif 'erl.system.debug':process_display/2 ebif_process_display_2 bif erlang:bump_reductions/1 -bif 'erl.lang.proc':bump_reductions/1 ebif_bump_reductions_1 bif math:cos/1 -bif 'erl.lang.math':cos/1 ebif_math_cos_1 bif math:cosh/1 -bif 'erl.lang.math':cosh/1 ebif_math_cosh_1 bif math:sin/1 -bif 'erl.lang.math':sin/1 ebif_math_sin_1 bif math:sinh/1 -bif 'erl.lang.math':sinh/1 ebif_math_sinh_1 bif math:tan/1 -bif 'erl.lang.math':tan/1 ebif_math_tan_1 bif math:tanh/1 -bif 'erl.lang.math':tanh/1 ebif_math_tanh_1 bif math:acos/1 -bif 'erl.lang.math':acos/1 ebif_math_acos_1 bif math:acosh/1 -bif 'erl.lang.math':acosh/1 ebif_math_acosh_1 bif math:asin/1 -bif 'erl.lang.math':asin/1 ebif_math_asin_1 bif math:asinh/1 -bif 'erl.lang.math':asinh/1 ebif_math_asinh_1 bif math:atan/1 -bif 'erl.lang.math':atan/1 ebif_math_atan_1 bif math:atanh/1 -bif 'erl.lang.math':atanh/1 ebif_math_atanh_1 bif math:erf/1 -bif 'erl.lang.math':erf/1 ebif_math_erf_1 bif math:erfc/1 -bif 'erl.lang.math':erfc/1 ebif_math_erfc_1 bif math:exp/1 -bif 'erl.lang.math':exp/1 ebif_math_exp_1 bif math:log/1 -bif 'erl.lang.math':log/1 ebif_math_log_1 bif math:log10/1 -bif 'erl.lang.math':log10/1 ebif_math_log10_1 bif math:sqrt/1 -bif 'erl.lang.math':sqrt/1 ebif_math_sqrt_1 bif math:atan2/2 -bif 'erl.lang.math':atan2/2 ebif_math_atan2_2 bif math:pow/2 -bif 'erl.lang.math':pow/2 ebif_math_pow_2 bif erlang:start_timer/3 -bif 'erl.lang.timer':start/3 ebif_start_timer_3 bif erlang:send_after/3 -bif 'erl.lang.timer':send_after/3 ebif_send_after_3 bif erlang:cancel_timer/1 -bif 'erl.lang.timer':cancel/1 ebif_cancel_timer_1 bif erlang:read_timer/1 -bif 'erl.lang.timer':read/1 ebif_read_timer_1 bif erlang:make_tuple/2 -bif 'erl.lang.tuple':make/2 ebif_make_tuple_2 bif erlang:append_element/2 -bif 'erl.lang.tuple':append_element/2 ebif_append_element_2 bif erlang:make_tuple/3 bif erlang:system_flag/2 -bif 'erl.system':set_flag/2 ebif_system_flag_2 bif erlang:system_info/1 -bif 'erl.system':info/1 ebif_system_info_1 # New in R9C bif erlang:system_monitor/0 -bif 'erl.system':monitor/0 ebif_system_monitor_0 bif erlang:system_monitor/1 -bif 'erl.system':monitor/1 ebif_system_monitor_1 bif erlang:system_monitor/2 -bif 'erl.system':monitor/2 ebif_system_monitor_2 # Added 2006-11-07 bif erlang:system_profile/2 -bif 'erl.system':profile/2 ebif_system_profile_2 # End Added 2006-11-07 # Added 2007-01-17 bif erlang:system_profile/0 -bif 'erl.system':profile/0 ebif_system_profile_0 # End Added 2007-01-17 bif erlang:ref_to_list/1 -bif 'erl.lang.ref':to_string/1 ebif_ref_to_string_1 ref_to_list_1 bif erlang:port_to_list/1 -bif 'erl.lang.port':to_string/1 ebif_port_to_string_1 port_to_list_1 bif erlang:fun_to_list/1 -bif 'erl.lang.function':to_string/1 ebif_fun_to_string_1 fun_to_list_1 bif erlang:monitor/2 -bif 'erl.lang.proc':monitor/2 ebif_monitor_2 bif erlang:demonitor/1 -bif 'erl.lang.proc':demonitor/1 ebif_demonitor_1 bif erlang:demonitor/2 -bif 'erl.lang.proc':demonitor/2 ebif_demonitor_2 bif erlang:is_process_alive/1 -bif 'erl.lang.proc':is_alive/1 ebif_proc_is_alive_1 is_process_alive_1 bif erlang:error/1 error_1 -bif 'erl.lang':error/1 ebif_error_1 error_1 bif erlang:error/2 error_2 -bif 'erl.lang':error/2 ebif_error_2 error_2 bif erlang:raise/3 raise_3 -bif 'erl.lang':raise/3 ebif_raise_3 raise_3 bif erlang:get_stacktrace/0 -bif 'erl.lang.proc':get_stacktrace/0 ebif_get_stacktrace_0 bif erlang:is_builtin/3 -bif 'erl.system.code':is_builtin/3 ebif_is_builtin_3 ubif erlang:'and'/2 -ubif 'erl.lang.bool':'and'/2 ebif_and_2 ubif erlang:'or'/2 -ubif 'erl.lang.bool':'or'/2 ebif_or_2 ubif erlang:'xor'/2 -ubif 'erl.lang.bool':'xor'/2 ebif_xor_2 ubif erlang:'not'/1 -ubif 'erl.lang.bool':'not'/1 ebif_not_1 ubif erlang:'>'/2 sgt_2 -ubif 'erl.lang.term':greater/2 ebif_gt_2 sgt_2 ubif erlang:'>='/2 sge_2 -ubif 'erl.lang.term':greater_or_equal/2 ebif_ge_2 sge_2 ubif erlang:'<'/2 slt_2 -ubif 'erl.lang.term':less/2 ebif_lt_2 slt_2 ubif erlang:'=<'/2 sle_2 -ubif 'erl.lang.term':less_or_equal/2 ebif_le_2 sle_2 ubif erlang:'=:='/2 seq_2 -ubif 'erl.lang.term':equal/2 ebif_eq_2 seq_2 ubif erlang:'=='/2 seqeq_2 -ubif 'erl.lang.term':arith_equal/2 ebif_areq_2 seqeq_2 ubif erlang:'=/='/2 sneq_2 -ubif 'erl.lang.term':not_equal/2 ebif_neq_2 sneq_2 ubif erlang:'/='/2 sneqeq_2 -ubif 'erl.lang.term':not_arith_equal/2 ebif_nareq_2 sneqeq_2 ubif erlang:'+'/2 splus_2 -ubif 'erl.lang.number':plus/2 ebif_plus_2 splus_2 ubif erlang:'-'/2 sminus_2 -ubif 'erl.lang.number':minus/2 ebif_minus_2 sminus_2 ubif erlang:'*'/2 stimes_2 -ubif 'erl.lang.number':multiply/2 ebif_multiply_2 stimes_2 ubif erlang:'/'/2 div_2 -ubif 'erl.lang.number':divide/2 ebif_divide_2 div_2 ubif erlang:'div'/2 intdiv_2 -ubif 'erl.lang.integer':'div'/2 ebif_intdiv_2 ubif erlang:'rem'/2 -ubif 'erl.lang.integer':'rem'/2 ebif_rem_2 ubif erlang:'bor'/2 -ubif 'erl.lang.integer':'bor'/2 ebif_bor_2 ubif erlang:'band'/2 -ubif 'erl.lang.integer':'band'/2 ebif_band_2 ubif erlang:'bxor'/2 -ubif 'erl.lang.integer':'bxor'/2 ebif_bxor_2 ubif erlang:'bsl'/2 -ubif 'erl.lang.integer':'bsl'/2 ebif_bsl_2 ubif erlang:'bsr'/2 -ubif 'erl.lang.integer':'bsr'/2 ebif_bsr_2 ubif erlang:'bnot'/1 -ubif 'erl.lang.integer':'bnot'/1 ebif_bnot_1 ubif erlang:'-'/1 sminus_1 -ubif 'erl.lang.number':minus/1 ebif_minus_1 sminus_1 ubif erlang:'+'/1 splus_1 -ubif 'erl.lang.number':plus/1 ebif_plus_1 splus_1 # New operators in R8. These were the only operators missing. # erlang:send/2, erlang:append/2 and erlang:subtract/2 are now also @@ -466,45 +268,27 @@ ubif 'erl.lang.number':plus/1 ebif_plus_1 splus_1 # internal references have been updated to the new ebif_... entries. bif erlang:'!'/2 ebif_bang_2 -bif 'erl.lang.proc':send/2 ebif_send_2 send_2 bif erlang:send/2 -bif 'erl.lang':send/3 ebif_send_3 send_3 bif erlang:send/3 bif erlang:'++'/2 ebif_plusplus_2 -bif 'erl.lang.list':append/2 ebif_append_2 ebif_plusplus_2 bif erlang:append/2 bif erlang:'--'/2 ebif_minusminus_2 -bif 'erl.lang.list':subtract/2 ebif_list_subtract_2 ebif_minusminus_2 bif erlang:subtract/2 ubif erlang:is_atom/1 -ubif 'erl.lang.term':is_atom/1 ebif_is_atom_1 ubif erlang:is_list/1 -ubif 'erl.lang.term':is_list/1 ebif_is_list_1 ubif erlang:is_tuple/1 -ubif 'erl.lang.term':is_tuple/1 ebif_is_tuple_1 ubif erlang:is_float/1 -ubif 'erl.lang.term':is_float/1 ebif_is_float_1 ubif erlang:is_integer/1 -ubif 'erl.lang.term':is_integer/1 ebif_is_integer_1 ubif erlang:is_number/1 -ubif 'erl.lang.term':is_number/1 ebif_is_number_1 ubif erlang:is_pid/1 -ubif 'erl.lang.term':is_pid/1 ebif_is_pid_1 ubif erlang:is_port/1 -ubif 'erl.lang.term':is_port/1 ebif_is_port_1 ubif erlang:is_reference/1 -ubif 'erl.lang.term':is_reference/1 ebif_is_reference_1 ubif erlang:is_binary/1 -ubif 'erl.lang.term':is_binary/1 ebif_is_binary_1 ubif erlang:is_function/1 -ubif 'erl.lang.term':is_function/1 ebif_is_function_1 ubif erlang:is_function/2 -ubif 'erl.lang.term':is_function/2 ebif_is_function_2 ubif erlang:is_record/2 -ubif 'erl.lang.term':is_record/2 ebif_is_record_2 ubif erlang:is_record/3 -ubif 'erl.lang.term':is_record/3 ebif_is_record_3 bif erlang:match_spec_test/3 @@ -513,96 +297,53 @@ bif erlang:match_spec_test/3 # bif ets:all/0 -bif 'erl.lang.ets':all/0 ebif_ets_all_0 bif ets:new/2 -bif 'erl.lang.ets':new/2 ebif_ets_new_2 bif ets:delete/1 -bif 'erl.lang.ets':delete/1 ebif_ets_delete_1 bif ets:delete/2 -bif 'erl.lang.ets':delete/2 ebif_ets_delete_2 bif ets:delete_all_objects/1 -bif 'erl.lang.ets':delete_all_objects/1 ebif_ets_delete_all_objects_1 bif ets:delete_object/2 -bif 'erl.lang.ets':delete_object/2 ebif_ets_delete_object_2 bif ets:first/1 -bif 'erl.lang.ets':first/1 ebif_ets_first_1 bif ets:is_compiled_ms/1 -bif 'erl.lang.ets':is_compiled_ms/1 ebif_ets_is_compiled_ms_1 bif ets:lookup/2 -bif 'erl.lang.ets':lookup/2 ebif_ets_lookup_2 bif ets:lookup_element/3 -bif 'erl.lang.ets':lookup_element/3 ebif_ets_lookup_element_3 bif ets:info/1 -bif 'erl.lang.ets':info/1 ebif_ets_info_1 bif ets:info/2 -bif 'erl.lang.ets':info/2 ebif_ets_info_2 bif ets:last/1 -bif 'erl.lang.ets':last/1 ebif_ets_last_1 bif ets:match/1 -bif 'erl.lang.ets':match/1 ebif_ets_match_1 bif ets:match/2 -bif 'erl.lang.ets':match/2 ebif_ets_match_2 bif ets:match/3 -bif 'erl.lang.ets':match/3 ebif_ets_match_3 bif ets:match_object/1 -bif 'erl.lang.ets':match_object/1 ebif_ets_match_object_1 bif ets:match_object/2 -bif 'erl.lang.ets':match_object/2 ebif_ets_match_object_2 bif ets:match_object/3 -bif 'erl.lang.ets':match_object/3 ebif_ets_match_object_3 bif ets:member/2 -bif 'erl.lang.ets':is_key/2 ebif_ets_member_2 bif ets:next/2 -bif 'erl.lang.ets':next/2 ebif_ets_next_2 bif ets:prev/2 -bif 'erl.lang.ets':prev/2 ebif_ets_prev_2 bif ets:insert/2 -bif 'erl.lang.ets':insert/2 ebif_ets_insert_2 bif ets:insert_new/2 -bif 'erl.lang.ets':insert_new/2 ebif_ets_insert_new_2 bif ets:rename/2 -bif 'erl.lang.ets':rename/2 ebif_ets_rename_2 bif ets:safe_fixtable/2 -bif 'erl.lang.ets':fixtable/2 ebif_ets_safe_fixtable_2 bif ets:slot/2 -bif 'erl.lang.ets':slot/2 ebif_ets_slot_2 bif ets:update_counter/3 -bif 'erl.lang.ets':update_counter/3 ebif_ets_update_counter_3 bif ets:select/1 -bif 'erl.lang.ets':select/1 ebif_ets_select_1 bif ets:select/2 -bif 'erl.lang.ets':select/2 ebif_ets_select_2 bif ets:select/3 -bif 'erl.lang.ets':select/3 ebif_ets_select_3 bif ets:select_count/2 -bif 'erl.lang.ets':select/2 ebif_ets_select_count_2 bif ets:select_reverse/1 -bif 'erl.lang.ets':select_reverse/1 ebif_ets_select_reverse_1 bif ets:select_reverse/2 -bif 'erl.lang.ets':select_reverse/2 ebif_ets_select_reverse_2 bif ets:select_reverse/3 -bif 'erl.lang.ets':select_reverse/3 ebif_ets_select_reverse_3 bif ets:select_delete/2 -bif 'erl.lang.ets':select_delete/2 ebif_ets_select_delete_2 bif ets:match_spec_compile/1 -bif 'erl.lang.ets':match_spec_compile/1 ebif_ets_match_spec_compile_1 bif ets:match_spec_run_r/3 -bif 'erl.lang.ets':match_spec_run_r/3 ebif_ets_match_spec_run_r_3 # # Bifs in os module. # bif os:putenv/2 -bif 'erl.system.os':setenv/2 ebif_os_setenv_2 os_putenv_2 bif os:getenv/0 -bif 'erl.system.os':getenv/0 ebif_os_getenv_0 bif os:getenv/1 -bif 'erl.system.os':getenv/1 ebif_os_getenv_1 bif os:getpid/0 -bif 'erl.system.os':pid/0 ebif_os_pid_0 os_getpid_0 bif os:timestamp/0 -bif 'erl.system.os':timestamp/0 ebif_os_timestamp_0 os_timestamp_0 # # Bifs in the erl_ddll module (the module actually does not exist) @@ -629,13 +370,9 @@ bif re:run/3 # bif lists:member/2 -bif 'erl.lang.list':is_element/2 ebif_list_is_element_2 lists_member_2 bif lists:reverse/2 -bif 'erl.lang.list':reverse/2 ebif_list_reverse_2 lists_reverse_2 bif lists:keymember/3 -bif 'erl.lang.list.keylist':is_element/3 ebif_keylist_is_element_3 lists_keymember_3 bif lists:keysearch/3 -bif 'erl.lang.list.keylist':search/3 ebif_keylist_search_3 lists_keysearch_3 bif lists:keyfind/3 # @@ -643,21 +380,13 @@ bif lists:keyfind/3 # bif erts_debug:disassemble/1 -bif 'erl.system.debug':disassemble/1 ebif_erts_debug_disassemble_1 bif erts_debug:breakpoint/2 -bif 'erl.system.debug':breakpoint/2 ebif_erts_debug_breakpoint_2 bif erts_debug:same/2 -bif 'erl.system.debug':same/2 ebif_erts_debug_same_2 bif erts_debug:flat_size/1 -bif 'erl.system.debug':flat_size/1 ebif_erts_debug_flat_size_1 bif erts_debug:get_internal_state/1 -bif 'erl.system.debug':get_internal_state/1 ebif_erts_debug_get_internal_state_1 bif erts_debug:set_internal_state/2 -bif 'erl.system.debug':set_internal_state/2 ebif_erts_debug_set_internal_state_2 bif erts_debug:display/1 -bif 'erl.system.debug':display/1 ebif_erts_debug_display_1 bif erts_debug:dist_ext_to_term/2 -bif 'erl.system.debug':dist_ext_to_term/2 ebif_erts_debug_dist_ext_to_term_2 bif erts_debug:instructions/0 # @@ -677,13 +406,9 @@ bif erts_debug:lock_counters/1 # bif code:get_chunk/2 -bif 'erl.system.code':get_chunk/2 ebif_code_get_chunk_2 bif code:module_md5/1 -bif 'erl.system.code':module_md5/1 ebif_code_module_md5_1 bif code:make_stub_module/3 -bif 'erl.system.code':make_stub_module/3 ebif_code_make_stub_module_3 bif code:is_module_native/1 -bif 'erl.system.code':is_native/1 ebif_code_is_native_1 code_is_module_native_1 # # New Bifs in R9C. @@ -799,6 +524,7 @@ bif erlang:nif_error/2 bif prim_file:internal_name2native/1 bif prim_file:internal_native2name/1 bif prim_file:internal_normalize_utf8/1 +bif prim_file:is_translatable/1 bif file:native_name_encoding/0 # @@ -829,6 +555,24 @@ bif erlang:dt_restore_tag/1 bif erlang:dt_prepend_vm_tag_data/1 bif erlang:dt_append_vm_tag_data/1 + +# +# New in R16B. +# +bif erlang:prepare_loading/2 +bif erlang:finish_loading/1 +bif erlang:insert_element/3 +bif erlang:delete_element/2 +bif erlang:binary_to_integer/1 +bif erlang:binary_to_integer/2 +bif erlang:integer_to_binary/1 +bif erlang:list_to_integer/2 +bif erlang:float_to_binary/1 +bif erlang:float_to_binary/2 +bif erlang:binary_to_float/1 + +bif io:printable_range/0 + # # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 5a5b162b9c..6b43c53985 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -150,14 +150,14 @@ #define D2GTE(a1,a0,b1,b0) (!D2LT(a1,a0,b1,b0)) #define D2LTE(a1,a0,b1,b0) (!D2GT(a1,a0,b1,b0)) -// Add (A+B), A=(a1B+a0) B=(b1B+b0) +/* Add (A+B), A=(a1B+a0) B=(b1B+b0) */ #define D2ADD(a1,a0,b1,b0,c1,c0) do { \ ErtsDigit __ci = 0; \ DSUM(a0,b0,__ci,c0); \ DSUMc(a1,b1,__ci,c1); \ } while(0) -// Subtract (A-B), A=(a1B+a0), B=(b1B+b0) (A>=B) +/* Subtract (A-B), A=(a1B+a0), B=(b1B+b0) (A>=B) */ #define D2SUB(a1,a0,b1,b0,c1,c0) do { \ ErtsDigit __bi; \ DSUB(a0,b0,__bi,c0); \ @@ -1337,7 +1337,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, while(bw--) *r++ = 0; - if (sw) { // NOTE! x >> 32 is not = 0! + if (sw) { /* NOTE! x >> 32 is not = 0! */ while(xl--) { a0 = (*x << sw) | a1; a1 = (*x >> (D_EXP - sw)); @@ -1384,7 +1384,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, x += (xl-1); r += (rl-1); xl -= bw; - if (sw) { // NOTE! x >> 32 is not = 0! + if (sw) { /* NOTE! x >> 32 is not = 0! */ while(xl--) { a1 = (*x >> sw) | a0; a0 = (*x << (D_EXP-sw)); @@ -1674,26 +1674,26 @@ int big_decimal_estimate(Wterm x) ** Convert a bignum into a string of decimal numbers */ -static void write_big(Wterm x, void (*write_func)(void *, char), void *arg) +static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg) { Eterm* xp = big_val(x); ErtsDigit* dx = BIG_V(xp); dsize_t xl = BIG_SIZE(xp); short sign = BIG_SIGN(xp); ErtsDigit rem; + Uint n = 0; if (xl == 1 && *dx < D_DECIMAL_BASE) { rem = *dx; - if (rem == 0) - (*write_func)(arg, '0'); - else { + if (rem == 0) { + (*write_func)(arg, '0'); n++; + } else { while(rem) { - (*write_func)(arg, (rem % 10) + '0'); + (*write_func)(arg, (rem % 10) + '0'); n++; rem /= 10; } } - } - else { + } else { ErtsDigit* tmp = (ErtsDigit*) erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsDigit)*xl); dsize_t tmpl = xl; @@ -1704,15 +1704,14 @@ static void write_big(Wterm x, void (*write_func)(void *, char), void *arg) tmpl = D_div(tmp, tmpl, D_DECIMAL_BASE, tmp, &rem); if (tmpl == 1 && *tmp == 0) { while(rem) { - (*write_func)(arg, (rem % 10)+'0'); + (*write_func)(arg, (rem % 10)+'0'); n++; rem /= 10; } break; - } - else { + } else { int i = D_DECIMAL_EXP; while(i--) { - (*write_func)(arg, (rem % 10)+'0'); + (*write_func)(arg, (rem % 10)+'0'); n++; rem /= 10; } } @@ -1720,8 +1719,10 @@ static void write_big(Wterm x, void (*write_func)(void *, char), void *arg) erts_free(ERTS_ALC_T_TMP, (void *) tmp); } - if (sign) - (*write_func)(arg, '-'); + if (sign) { + (*write_func)(arg, '-'); n++; + } + return n; } struct big_list__ { @@ -1762,6 +1763,20 @@ char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz) return big_str; } +/* Bignum to binary bytes + * e.g. 1 bsl 64 -> "18446744073709551616" + */ + +Uint erts_big_to_binary_bytes(Eterm x, char *buf, Uint buf_sz) +{ + char *big_str = buf + buf_sz; + Uint n; + n = write_big(x, write_string, (void *) &big_str); + ASSERT(buf <= big_str && big_str <= buf + buf_sz); + return n; +} + + /* ** Normalize a bignum given thing pointer length in digits and a sign ** patch zero if odd length @@ -2453,7 +2468,7 @@ int term_equals_2pow32(Eterm x) if (!is_big(x)) return 0; bp = big_val(x); -#if D_EXP == 16 // 16 bit platfrom not really supported!!! +#if D_EXP == 16 /* 16 bit platfrom not really supported!!! */ return (BIG_SIZE(bp) == 3) && !BIG_DIGIT(bp,0) && !BIG_DIGIT(bp,1) && BIG_DIGIT(bp,2) == 1; #elif D_EXP == 32 @@ -2467,3 +2482,210 @@ int term_equals_2pow32(Eterm x) return 0; } } + + +#define IS_VALID_CHARACTER(CHAR,BASE) \ + (CHAR < '0' \ + || (CHAR > ('0' + BASE - 1) \ + && !(BASE > 10 \ + && ((CHAR >= 'a' && CHAR < ('a' + BASE - 10)) \ + || (CHAR >= 'A' && CHAR < ('A' + BASE - 10)))))) +#define CHARACTER_FROM_BASE(CHAR) \ + ((CHAR <= '9') ? CHAR - '0' : 10 + ((CHAR <= 'Z') ? CHAR - 'A' : CHAR - 'a')) +#define D_BASE_EXP(BASE) (d_base_exp_lookup[BASE-2]) +#define D_BASE_BASE(BASE) (d_base_base_lookup[BASE-2]) +#define LG2_LOOKUP(BASE) (lg2_lookup[base-2]) + +/* + * for i in 2..64 do + * lg2_lookup[i-2] = log2(i) + * end + * How many bits are needed to store string of size n + */ +const double lg2_lookup[] = { 1.0, 1.58496, 2, 2.32193, 2.58496, 2.80735, 3.0, + 3.16993, 3.32193, 3.45943, 3.58496, 3.70044, 3.80735, 3.90689, 4.0, + 4.08746, 4.16993, 4.24793, 4.32193, 4.39232, 4.45943, 4.52356, 4.58496, + 4.64386, 4.70044, 4.75489, 4.80735, 4.85798, 4.90689, 4.9542, 5.0, + 5.04439, 5.08746, 5.12928, 5.16993, 5.20945, 5.24793, 5.2854, 5.32193, + 5.35755, 5.39232, 5.42626, 5.45943, 5.49185, 5.52356, 5.55459, 5.58496, + 5.61471, 5.64386, 5.67243, 5.70044, 5.72792, 5.75489, 5.78136, 5.80735, + 5.83289, 5.85798, 5.88264, 5.90689, 5.93074, 5.9542, 5.97728, 6.0 }; + +/* + * for i in 2..64 do + * d_base_exp_lookup[i-2] = 31 / lg2_lookup[i-2]; + * end + * How many characters can fit in 31 bits + */ +const byte d_base_exp_lookup[] = { 31, 19, 15, 13, 11, 11, 10, 9, 9, 8, 8, 8, 8, + 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5 }; + +/* + * for i in 2..64 do + * d_base_base_lookup[i-2] = pow(i,d_base_exp_lookup[i-2]); + * end + * How much can the characters which fit in 31 bit represent + */ +const Uint d_base_base_lookup[] = { 2147483648u, 1162261467u, 1073741824u, + 1220703125u, 362797056u, 1977326743u, 1073741824u, 387420489u, + 1000000000u, 214358881u, 429981696u, 815730721u, 1475789056u, + 170859375u, 268435456u, 410338673u, 612220032u, 893871739u, 1280000000u, + 1801088541u, 113379904u, 148035889u, 191102976u, 244140625u, 308915776u, + 387420489u, 481890304u, 594823321u, 729000000u, 887503681u, 1073741824u, + 1291467969u, 1544804416u, 1838265625u, 60466176u, 69343957u, 79235168u, + 90224199u, 102400000u, 115856201u, 130691232u, 147008443u, 164916224u, + 184528125u, 205962976u, 229345007u, 254803968u, 282475249u, 312500000u, + 345025251u, 380204032u, 418195493u, 459165024u, 503284375u, 550731776u, + 601692057u, 656356768u, 714924299u, 777600000u, 844596301u, 916132832u, + 992436543u, 1073741824u }; + +Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, + Uint size, const int base) { + Eterm res; + Sint i = 0; + int n = 0; + int neg = 0; + byte b; + Eterm *hp, *hp_end; + int m; + int lg2; + + if (size == 0) + goto bytebuf_to_integer_1_error; + + if (bytes[0] == '-') { + neg = 1; + bytes++; + size--; + + } else if (bytes[0] == '+') { + bytes++; + size--; + } + + if (size < SMALL_DIGITS && base <= 10) { + /* * + * Take shortcut if we know that all chars are '0' < b < '9' and + * fit in a small. This improves speed by about 10% over the generic + * small case. + * */ + while (size--) { + b = *bytes++; + + if (b < '0' || b > ('0'+base-1)) + goto bytebuf_to_integer_1_error; + + i = i * base + b - '0'; + } + + if (neg) + i = -i; + res = make_small(i); + goto bytebuf_to_integer_1_done; + } + + /* + * Calculate the maximum number of bits which will + * be needed to represent the binary + */ + lg2 = ((size+2)*LG2_LOOKUP(base)+1); + + if (lg2 < SMALL_BITS) { + /* Take shortcut if we know it will fit in a small. + * This improves speed by about 30%. + */ + while (size) { + b = *bytes++; + size--; + + if (IS_VALID_CHARACTER(b,base)) + goto bytebuf_to_integer_1_error; + + i = i * base + CHARACTER_FROM_BASE(b); + + } + + if (neg) + i = -i; + res = make_small(i); + goto bytebuf_to_integer_1_done; + + } + + /* Start calculating bignum */ + m = (lg2 + D_EXP-1)/D_EXP; + m = BIG_NEED_SIZE(m); + + hp = HAlloc(BIF_P, m); + hp_end = hp + m; + + if ((i = (size % D_BASE_EXP(base))) == 0) + i = D_BASE_EXP(base); + + n = size - i; + m = 0; + + while (i--) { + b = *bytes++; + + if (IS_VALID_CHARACTER(b,base)) { + HRelease(BIF_P, hp_end, hp); + goto bytebuf_to_integer_1_error; + } + + m = base * m + CHARACTER_FROM_BASE(b); + } + + res = small_to_big(m, hp); + + while (n) { + i = D_BASE_EXP(base); + n -= D_BASE_EXP(base); + m = 0; + while (i--) { + b = *bytes++; + + if (IS_VALID_CHARACTER(b,base)) { + HRelease(BIF_P, hp_end, hp); + goto bytebuf_to_integer_1_error; + } + + m = base * m + CHARACTER_FROM_BASE(b); + } + if (is_small(res)) { + res = small_to_big(signed_val(res), hp); + } + res = big_times_small(res, D_BASE_BASE(base), hp); + if (is_small(res)) { + res = small_to_big(signed_val(res), hp); + } + res = big_plus_small(res, m, hp); + } + + if (is_big(res)) /* check if small */ + res = big_plus_small(res, 0, hp); /* includes conversion to small */ + + if (neg) { + if (is_small(res)) + res = make_small(-signed_val(res)); + else { + Uint *big = big_val(res); /* point to thing */ + *big = bignum_header_neg(*big); + } + } + + if (is_big(res)) { + hp += (big_arity(res) + 1); + } + HRelease(BIF_P, hp_end, hp); + goto bytebuf_to_integer_1_done; + +bytebuf_to_integer_1_error: + return THE_NON_VALUE; + +bytebuf_to_integer_1_done: + return res; + +} diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 7eb1e5afe2..1a7b14170f 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -117,6 +117,7 @@ typedef Uint dsize_t; /* Vector size type */ int big_decimal_estimate(Wterm); Eterm erts_big_to_list(Eterm, Eterm**); char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz); +Uint erts_big_to_binary_bytes(Eterm x, char *buf, Uint buf_sz); Eterm small_times(Sint, Sint, Eterm*); @@ -165,5 +166,7 @@ int term_equals_2pow32(Eterm); Eterm erts_uint64_to_big(Uint64, Eterm **); Eterm erts_sint64_to_big(Sint64, Eterm **); +Eterm erts_chars_to_integer(Process *, char*, Uint, const int); + #endif diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 3d2725e239..33abac2f3d 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -240,6 +240,98 @@ erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint b return previous; } +BIF_RETTYPE binary_to_integer_1(BIF_ALIST_1) +{ + byte *temp_alloc = NULL; + char *bytes; + Uint size; + Eterm res; + + if ((bytes = (char*)erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) + == NULL ) + goto binary_to_integer_1_error; + + size = binary_size(BIF_ARG_1); + + if ((res = erts_chars_to_integer(BIF_P,bytes,size,10)) != THE_NON_VALUE) { + erts_free_aligned_binary_bytes(temp_alloc); + return res; + } + + binary_to_integer_1_error: + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P, BADARG); +} + +BIF_RETTYPE binary_to_integer_2(BIF_ALIST_2) +{ + byte *temp_alloc = NULL; + char *bytes; + Uint size; + int base; + Eterm res; + + if (!is_small(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + + base = signed_val(BIF_ARG_2); + + if (base < 2 || base > 36) + BIF_ERROR(BIF_P, BADARG); + + if ((bytes = (char*)erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) + == NULL ) + goto binary_to_integer_2_error; + + size = binary_size(BIF_ARG_1); + + if ((res = erts_chars_to_integer(BIF_P,bytes,size,base)) != THE_NON_VALUE) { + erts_free_aligned_binary_bytes(temp_alloc); + return res; + } + + binary_to_integer_2_error: + + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P, BADARG); + +} + +BIF_RETTYPE integer_to_binary_1(BIF_ALIST_1) +{ + Uint size; + Eterm res; + + if (is_not_integer(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + if (is_small(BIF_ARG_1)) { + char *c; + struct Sint_buf ibuf; + + /* Enhancement: If we can calculate the buffer size exactly + * we could avoid an unnecessary copy of buffers. + * Useful if size determination is faster than a copy. + */ + c = Sint_to_buf(signed_val(BIF_ARG_1), &ibuf); + size = sys_strlen(c); + res = new_binary(BIF_P, (byte *)c, size); + } else { + byte* bytes; + Uint n = 0; + + /* Here we also have multiple copies of buffers + * due to new_binary interface + */ + size = big_decimal_estimate(BIF_ARG_1) - 1; /* remove null */ + bytes = (byte*) erts_alloc(ERTS_ALC_T_TMP, sizeof(byte)*size); + n = erts_big_to_binary_bytes(BIF_ARG_1, (char *)bytes, size); + res = new_binary(BIF_P, bytes + size - n, n); + erts_free(ERTS_ALC_T_TMP, (void *) bytes); + } + BIF_RET(res); +} BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) { @@ -355,10 +447,10 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; - Uint size; + ErlDrvSizeT size; byte* bytes; #ifdef DEBUG - int offset; + ErlDrvSizeT offset; #endif if (is_nil(arg)) { @@ -377,7 +469,7 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) #ifdef DEBUG offset = #endif - io_list_to_buf(arg, (char*) bytes, size); + erts_iolist_to_buf(arg, (char*) bytes, size); ASSERT(offset == 0); BIF_RET(bin); diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index f7e9f15655..ad9a89b642 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -61,19 +61,23 @@ extern char* erts_system_version[]; static void port_info(int to, void *to_arg) { - int i; - for (i = 0; i < erts_max_ports; i++) - print_port_info(to, to_arg, i); + int i, max = erts_ptab_max(&erts_port); + for (i = 0; i < max; i++) { + Port *p = erts_pix2port(i); + if (p) + print_port_info(p, to, to_arg); + } } void process_info(int to, void *to_arg) { - int i; - for (i = 0; i < erts_max_processes; i++) { - if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) { - if (process_tab[i]->status != P_EXITING) - print_process_info(to, to_arg, process_tab[i]); + int i, max = erts_ptab_max(&erts_proc); + for (i = 0; i < max; i++) { + Process *p = erts_pix2proc(i); + if (p && p->i != ENULL) { + if (!ERTS_PROC_IS_EXITING(p)) + print_process_info(to, to_arg, p); } } @@ -83,13 +87,14 @@ process_info(int to, void *to_arg) static void process_killer(void) { - int i, j; + int i, j, max = erts_ptab_max(&erts_proc); Process* rp; erts_printf("\n\nProcess Information\n\n"); erts_printf("--------------------------------------------------\n"); - for (i = erts_max_processes-1; i >= 0; i--) { - if (((rp = process_tab[i]) != NULL) && rp->i != ENULL) { + for (i = max-1; i >= 0; i--) { + rp = erts_pix2proc(i); + if (rp && rp->i != ENULL) { int br; print_process_info(ERTS_PRINT_STDOUT, NULL, rp); erts_printf("(k)ill (n)ext (r)eturn:\n"); @@ -97,11 +102,20 @@ process_killer(void) if ((j = sys_get_key(0)) <= 0) erl_exit(0, ""); switch(j) { - case 'k': - if (rp->status == P_WAITING) { - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - erts_smp_proc_inc_refc(rp); - erts_smp_proc_lock(rp, rp_locks); + case 'k': { + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + erts_aint32_t state; + erts_smp_proc_inc_refc(rp); + erts_smp_proc_lock(rp, rp_locks); + state = erts_smp_atomic32_read_acqb(&rp->state); + if (state & (ERTS_PSFLG_FREE + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_IN_RUNQ + | ERTS_PSFLG_RUNNING)) { + erts_printf("Can only kill WAITING processes this way\n"); + } + else { (void) erts_send_exit_signal(NULL, NIL, rp, @@ -110,12 +124,10 @@ process_killer(void) NIL, NULL, 0); - erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } - else - erts_printf("Can only kill WAITING processes this way\n"); - + erts_smp_proc_unlock(rp, rp_locks); + erts_smp_proc_dec_refc(rp); + } case 'n': br = 1; break; case 'r': return; default: return; @@ -180,49 +192,45 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) void print_process_info(int to, void *to_arg, Process *p) { + time_t approx_started; int garbing = 0; int running = 0; - time_t tmp_t; struct saved_calls *scb; + erts_aint32_t state; /* display the PID */ - erts_print(to, to_arg, "=proc:%T\n", p->id); + erts_print(to, to_arg, "=proc:%T\n", p->common.id); /* Display the state */ erts_print(to, to_arg, "State: "); - switch (p->status) { - case P_FREE: + + state = erts_smp_atomic32_read_acqb(&p->state); + if (state & ERTS_PSFLG_FREE) erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */ - break; - case P_RUNABLE: - erts_print(to, to_arg, "Scheduled\n"); - break; - case P_WAITING: - erts_print(to, to_arg, "Waiting\n"); - break; - case P_SUSPENDED: - erts_print(to, to_arg, "Suspended\n"); - break; - case P_RUNNING: - erts_print(to, to_arg, "Running\n"); - running = 1; - break; - case P_EXITING: + else if (state & ERTS_PSFLG_EXITING) erts_print(to, to_arg, "Exiting\n"); - break; - case P_GARBING: - erts_print(to, to_arg, "Garbing\n"); + else if (state & ERTS_PSFLG_GC) { garbing = 1; running = 1; - break; + erts_print(to, to_arg, "Garbing\n"); + } + else if (state & ERTS_PSFLG_SUSPENDED) + erts_print(to, to_arg, "Suspended\n"); + else if (state & ERTS_PSFLG_RUNNING) { + running = 1; + erts_print(to, to_arg, "Running\n"); } + else if (state & ERTS_PSFLG_ACTIVE) + erts_print(to, to_arg, "Scheduled\n"); + else + erts_print(to, to_arg, "Waiting\n"); /* * If the process is registered as a global process, display the * registered name */ - if (p->reg != NULL) - erts_print(to, to_arg, "Name: %T\n", p->reg->name); + if (p->common.u.alive.reg) + erts_print(to, to_arg, "Name: %T\n", p->common.u.alive.reg->name); /* * Display the initial function name @@ -245,8 +253,8 @@ print_process_info(int to, void *to_arg, Process *p) } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); - tmp_t = p->started.tv_sec; - erts_print(to, to_arg, "Started: %s", ctime(&tmp_t)); + approx_started = (time_t) p->approx_started; + erts_print(to, to_arg, "Started: %s", ctime(&approx_started)); ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len); @@ -296,11 +304,11 @@ print_process_info(int to, void *to_arg, Process *p) } /* display the links only if there are any*/ - if (p->nlinks != NULL || p->monitors != NULL) { + if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p)) { PrintMonitorContext context = {1,to}; erts_print(to, to_arg,"Link list: ["); - erts_doforall_links(p->nlinks, &doit_print_link, &context); - erts_doforall_monitors(p->monitors, &doit_print_monitor, &context); + erts_doforall_links(ERTS_P_LINKS(p), &doit_print_link, &context); + erts_doforall_monitors(ERTS_P_MONITORS(p), &doit_print_monitor, &context); erts_print(to, to_arg,"]\n"); } @@ -323,6 +331,7 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p)); if (garbing) { print_garb_info(to, to_arg, p); @@ -377,17 +386,22 @@ loaded(int to, void *to_arg) int old = 0; int cur = 0; BeamInstr* code; + Module* modp; + ErtsCodeIndex code_ix; + + code_ix = erts_active_code_ix(); + erts_rlock_old_code(code_ix); /* * Calculate and print totals. */ - for (i = 0; i < module_code_size(); i++) { - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { - cur += module_code(i)->code_length; - if (module_code(i)->old_code_length != 0) { - old += module_code(i)->old_code_length; + for (i = 0; i < module_code_size(code_ix); i++) { + if ((modp = module_code(i, code_ix)) != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { + cur += modp->curr.code_length; + if (modp->old.code_length != 0) { + old += modp->old.code_length; } } } @@ -398,21 +412,22 @@ loaded(int to, void *to_arg) * Print one line per module. */ - for (i = 0; i < module_code_size(); i++) { + for (i = 0; i < module_code_size(code_ix); i++) { + modp = module_code(i, code_ix); if (!ERTS_IS_CRASH_DUMPING) { /* * Interactive dump; keep it brief. */ - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { - erts_print(to, to_arg, "%T", make_atom(module_code(i)->module)); - cur += module_code(i)->code_length; - erts_print(to, to_arg, " %d", module_code(i)->code_length ); - if (module_code(i)->old_code_length != 0) { + if (modp != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { + erts_print(to, to_arg, "%T", make_atom(modp->module)); + cur += modp->curr.code_length; + erts_print(to, to_arg, " %d", modp->curr.code_length ); + if (modp->old.code_length != 0) { erts_print(to, to_arg, " (%d old)", - module_code(i)->old_code_length ); - old += module_code(i)->old_code_length; + modp->old.code_length ); + old += modp->old.code_length; } erts_print(to, to_arg, "\n"); } @@ -420,15 +435,15 @@ loaded(int to, void *to_arg) /* * To crash dump; make it parseable. */ - if (module_code(i) != NULL && - ((module_code(i)->code_length != 0) || - (module_code(i)->old_code_length != 0))) { + if (modp != NULL && + ((modp->curr.code_length != 0) || + (modp->old.code_length != 0))) { erts_print(to, to_arg, "=mod:"); - erts_print(to, to_arg, "%T", make_atom(module_code(i)->module)); + erts_print(to, to_arg, "%T", make_atom(modp->module)); erts_print(to, to_arg, "\n"); erts_print(to, to_arg, "Current size: %d\n", - module_code(i)->code_length); - code = module_code(i)->code; + modp->curr.code_length); + code = modp->curr.code; if (code != NULL && code[MI_ATTR_PTR]) { erts_print(to, to_arg, "Current attributes: "); dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], @@ -440,9 +455,9 @@ loaded(int to, void *to_arg) code[MI_COMPILE_SIZE]); } - if (module_code(i)->old_code_length != 0) { - erts_print(to, to_arg, "Old size: %d\n", module_code(i)->old_code_length); - code = module_code(i)->old_code; + if (modp->old.code_length != 0) { + erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length); + code = modp->old.code; if (code[MI_ATTR_PTR]) { erts_print(to, to_arg, "Old attributes: "); dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], @@ -457,6 +472,7 @@ loaded(int to, void *to_arg) } } } + erts_runlock_old_code(code_ix); } @@ -613,16 +629,17 @@ bin_check(void) { Process *rp; struct erl_off_heap_header* hdr; - int i, printed = 0; + int i, printed = 0, max = erts_ptab_max(&erts_proc); - for (i=0; i < erts_max_processes; i++) { - if ((rp = process_tab[i]) == NULL) + for (i=0; i < max; i++) { + rp = erts_pix2proc(i); + if (!rp) continue; for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) { if (hdr->thing_word == HEADER_PROC_BIN) { ProcBin *bp = (ProcBin*) hdr; if (!printed) { - erts_printf("Process %T holding binary data \n", rp->id); + erts_printf("Process %T holding binary data \n", rp->common.id); printed = 1; } erts_printf("%p orig_size: %bpd, norefs = %bpd\n", @@ -737,7 +754,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) return; /* Can't create the crash dump, skip it */ time(&now); - erts_fdprintf(fd, "=erl_crash_dump:0.1\n%s", ctime(&now)); + erts_fdprintf(fd, "=erl_crash_dump:0.2\n%s", ctime(&now)); if (file != NULL) erts_fdprintf(fd, "The error occurred in file %s, line %d\n", file, line); @@ -753,7 +770,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_nif_taints(fd, NULL); erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); info(fd, NULL); /* General system info */ - if (process_tab != NULL) /* XXX true at init */ + if (erts_ptab_initialized(&erts_proc)) process_info(fd, NULL); /* Info about each process and port */ db_info(fd, NULL, 0); erts_print_bif_timer_info(fd, NULL); diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c new file mode 100644 index 0000000000..c66d5a2f05 --- /dev/null +++ b/erts/emulator/beam/code_ix.c @@ -0,0 +1,168 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "code_ix.h" +#include "global.h" +#include "beam_catches.h" + + + +#if 0 +# define CIX_TRACE(text) erts_fprintf(stderr, "CIX_TRACE: " text " act=%u load=%u\r\n", erts_active_code_ix(), erts_staging_code_ix()) +#else +# define CIX_TRACE(text) +#endif + +erts_smp_atomic32_t the_active_code_index; +erts_smp_atomic32_t the_staging_code_index; + +static Process* code_writing_process = NULL; +struct code_write_queue_item { + Process *p; + struct code_write_queue_item* next; +}; +static struct code_write_queue_item* code_write_queue = NULL; +static erts_smp_mtx_t code_write_permission_mtx; + +#ifdef ERTS_ENABLE_LOCK_CHECK +static erts_tsd_key_t has_code_write_permission; +#endif + +void erts_code_ix_init(void) +{ + /* We start emulator by initializing preloaded modules + * single threaded with active and staging set both to zero. + * Preloading is finished by a commit that will set things straight. + */ + erts_smp_atomic32_init_nob(&the_active_code_index, 0); + erts_smp_atomic32_init_nob(&the_staging_code_index, 0); + erts_smp_mtx_init(&code_write_permission_mtx, "code_write_permission"); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_tsd_key_create(&has_code_write_permission); +#endif + CIX_TRACE("init"); +} + +void erts_start_staging_code_ix(void) +{ + beam_catches_start_staging(); + export_start_staging(); + module_start_staging(); + erts_start_staging_ranges(); + CIX_TRACE("start"); +} + + +void erts_end_staging_code_ix(void) +{ + beam_catches_end_staging(1); + export_end_staging(1); + module_end_staging(1); + erts_end_staging_ranges(1); + CIX_TRACE("end"); +} + +void erts_commit_staging_code_ix(void) +{ + ErtsCodeIndex ix; + /* We need to this lock as we are now making the staging export table active */ + export_staging_lock(); + ix = erts_staging_code_ix(); + erts_smp_atomic32_set_nob(&the_active_code_index, ix); + ix = (ix + 1) % ERTS_NUM_CODE_IX; + erts_smp_atomic32_set_nob(&the_staging_code_index, ix); + export_staging_unlock(); + CIX_TRACE("activate"); +} + +void erts_abort_staging_code_ix(void) +{ + beam_catches_end_staging(0); + export_end_staging(0); + module_end_staging(0); + erts_end_staging_ranges(0); + CIX_TRACE("abort"); +} + + +/* + * Calller _must_ yield if we return 0 + */ +int erts_try_seize_code_write_permission(Process* c_p) +{ + int success; +#ifdef ERTS_SMP + ASSERT(!erts_smp_thr_progress_is_blocking()); /* to avoid deadlock */ +#endif + ASSERT(c_p != NULL); + + erts_smp_mtx_lock(&code_write_permission_mtx); + success = (code_writing_process == NULL); + if (success) { + code_writing_process = c_p; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_tsd_set(has_code_write_permission, (void *) 1); +#endif + } + else { /* Already locked */ + struct code_write_queue_item* qitem; + ASSERT(code_writing_process != c_p); + qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem)); + qitem->p = c_p; + erts_smp_proc_inc_refc(c_p); + qitem->next = code_write_queue; + code_write_queue = qitem; + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + } + erts_smp_mtx_unlock(&code_write_permission_mtx); + return success; +} + +void erts_release_code_write_permission(void) +{ + erts_smp_mtx_lock(&code_write_permission_mtx); + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + while (code_write_queue != NULL) { /* unleash the entire herd */ + struct code_write_queue_item* qitem = code_write_queue; + erts_smp_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(qitem->p)) { + erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); + code_write_queue = qitem->next; + erts_smp_proc_dec_refc(qitem->p); + erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem); + } + code_writing_process = NULL; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_tsd_set(has_code_write_permission, (void *) 0); +#endif + erts_smp_mtx_unlock(&code_write_permission_mtx); +} + +#ifdef ERTS_ENABLE_LOCK_CHECK +int erts_has_code_write_permission(void) +{ + return (code_writing_process != NULL) && erts_tsd_get(has_code_write_permission); +} +#endif diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h new file mode 100644 index 0000000000..16ad900228 --- /dev/null +++ b/erts/emulator/beam/code_ix.h @@ -0,0 +1,142 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* Description: + * This is the interface that facilitates changing the beam code + * (load,upgrade,delete) while allowing executing Erlang processes to + * access the code without any locks or other expensive memory barriers. + * + * The basic idea is to maintain several "logical copies" of the code. These + * copies are identified by a global 'code index', an integer of 0, 1 or 2. + * The code index is used as argument to code access structures like + * export, module, beam_catches, beam_ranges. + * + * The current 'active' code index is used to access the current running + * code. The 'staging' code index is used by the process that performs + * a code change operation. When a code change operation completes + * succesfully, the staging code index becomes the new active code index. + * + * The third code index is not explicitly used. It can be thought of as + * the "previous active" or the "next staging" index. It is needed to make + * sure that we do not reuse a code index for staging until we are sure + * that no executing BIFs are still referencing it. + * We could get by with only two (0 and 1), but that would require that we + * must wait for all schedulers to re-schedule before each code change + * operation can start staging. + * + * Note that the 'code index' is very loosely coupled to the concept of + * 'current' and 'old' module versions. You can almost say that they are + * orthogonal to each other. Code index is an emulator global concept while + * 'current' and 'old' is specific for each module. + */ + +#ifndef __CODE_IX_H__ +#define __CODE_IX_H__ + +#ifndef __SYS_H__ +# ifdef HAVE_CONFIG_H +# include "config.h" +# endif +# include "sys.h" +#endif +struct process; + + +#define ERTS_NUM_CODE_IX 3 +typedef unsigned ErtsCodeIndex; + + +/* Called once at emulator initialization. + */ +void erts_code_ix_init(void); + +/* Return active code index. + * Is guaranteed to be valid until the calling BIF returns. + * To get a consistent view of the code, only one call to erts_active_code_ix() + * should be made and the returned ix reused within the same BIF call. + */ +ERTS_GLB_INLINE +ErtsCodeIndex erts_active_code_ix(void); + +/* Return staging code ix. + * Only used by a process performing code loading/upgrading/deleting/purging. + * Code write permission must be seized. + */ +ERTS_GLB_INLINE +ErtsCodeIndex erts_staging_code_ix(void); + +/* Try seize exclusive code write permission. Needed for code staging. + * Main process lock (only) must be held. + * System thread progress must not be blocked. + * Caller must not already hold the code write permission. + * Caller is suspended and *must* yield if 0 is returned. + */ +int erts_try_seize_code_write_permission(struct process* c_p); + +/* Release code write permission. + * Will resume any suspended waiters. + */ +void erts_release_code_write_permission(void); + +/* Prepare the "staging area" to be a complete copy of the active code. + * Code write permission must have been seized. + * Must be followed by calls to either "end" and "commit" or "abort" before + * code write permission can be released. + */ +void erts_start_staging_code_ix(void); + +/* End the staging. + * Preceded by "start" and must be followed by "commit". + */ +void erts_end_staging_code_ix(void); + +/* Set staging code index as new active code index. + * Preceded by "end". + */ +void erts_commit_staging_code_ix(void); + +/* Abort the staging. + * Preceded by "start". + */ +void erts_abort_staging_code_ix(void); + +#ifdef ERTS_ENABLE_LOCK_CHECK +int erts_has_code_write_permission(void); +#endif + + + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +extern erts_smp_atomic32_t the_active_code_index; +extern erts_smp_atomic32_t the_staging_code_index; + +ERTS_GLB_INLINE ErtsCodeIndex erts_active_code_ix(void) +{ + return erts_smp_atomic32_read_nob(&the_active_code_index); +} +ERTS_GLB_INLINE ErtsCodeIndex erts_staging_code_ix(void) +{ + return erts_smp_atomic32_read_nob(&the_staging_code_index); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* !__CODE_IX_H__ */ + diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 36eda04de2..23c0fca6aa 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -47,7 +47,7 @@ copy_object(Eterm obj, Process* to) if (DTRACE_ENABLED(copy_object)) { DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(proc_name), "%T", to->id); + erts_snprintf(proc_name, sizeof(proc_name), "%T", to->common.id); DTRACE2(copy_object, proc_name, size); } #endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 025258e8de..44f4eb9d43 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -67,7 +67,7 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) { byte *extp = edep->extp; Eterm msg; - Sint size = erts_decode_dist_ext_size(edep, 0); + Sint size = erts_decode_dist_ext_size(edep); if (size < 0) { erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", @@ -124,6 +124,13 @@ static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm); static void init_nodes_monitors(void); static erts_smp_atomic_t no_caches; +static erts_smp_atomic_t no_nodes; + +struct { + Eterm reason; + ErlHeapFragment *bp; +} nodedown; + static void delete_cache(ErtsAtomCache *cache) @@ -144,7 +151,7 @@ create_cache(DistEntry *dep) ERTS_SMP_LC_ASSERT( is_internal_port(dep->cid) - && erts_lc_is_port_locked(&erts_port[internal_port_index(dep->cid)])); + && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); ASSERT(!dep->cache); dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE, @@ -171,11 +178,10 @@ get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs) return NULL; } else { - ErtsProcList *plp; - plp = dep->suspended.first; - dep->suspended.first = NULL; - dep->suspended.last = NULL; - return plp; + ErtsProcList *suspended = dep->suspended; + dep->suspended = NULL; + erts_proclist_fetch(&suspended, NULL); + return suspended; } } @@ -252,7 +258,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) if (mon->type == MON_ORIGIN) { /* local pid is beeing monitored */ - rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); /* ASSERT(rmon != NULL); nope, can happen during process exit */ if (rmon != NULL) { erts_destroy_monitor(rmon); @@ -262,7 +268,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) Eterm watched; UseTmpHeapNoproc(3); ASSERT(mon->type == MON_TARGET); - rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); /* ASSERT(rmon != NULL); can happen during process exit */ if (rmon != NULL) { ASSERT(is_atom(rmon->name) || is_nil(rmon->name)); @@ -311,7 +317,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp) goto done; } - rlnk = erts_remove_link(&(rp->nlinks), sublnk->pid); + rlnk = erts_remove_link(&ERTS_P_LINKS(rp), sublnk->pid); xres = erts_send_exit_signal(NULL, sublnk->pid, rp, @@ -370,7 +376,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) if (!rp) { goto done; } - rlnk = erts_remove_link(&(rp->nlinks), name); + rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name); if (rlnk != NULL) { ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE)); erts_destroy_link(rlnk); @@ -394,6 +400,47 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) erts_destroy_link(lnk); } +static void +set_node_not_alive(void *unused) +{ + ErlHeapFragment *bp; + Eterm nodename = erts_this_dist_entry->sysname; + + ASSERT(erts_smp_atomic_read_nob(&no_nodes) == 0); + + erts_smp_thr_progress_block(); + erts_set_this_node(am_Noname, 0); + erts_is_alive = 0; + send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nodedown.reason); + nodedown.reason = NIL; + bp = nodedown.bp; + nodedown.bp = NULL; + erts_smp_thr_progress_unblock(); + if (bp) + free_message_buffer(bp); +} + +static ERTS_INLINE void +dec_no_nodes(void) +{ + erts_aint_t no = erts_smp_atomic_dec_read_mb(&no_nodes); + ASSERT(no >= 0); + ASSERT(erts_get_scheduler_id()); /* Need to be a scheduler */ + if (no == 0) + erts_schedule_misc_aux_work(erts_get_scheduler_id(), + set_node_not_alive, + NULL); +} + +static ERTS_INLINE void +inc_no_nodes(void) +{ +#ifdef DEBUG + erts_aint_t no = erts_smp_atomic_read_nob(&no_nodes); + ASSERT(erts_is_alive ? no > 0 : no == 0); +#endif + erts_smp_atomic_inc_mb(&no_nodes); +} /* * proc is currently running or exiting process. @@ -403,47 +450,76 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) Eterm nodename; if (dep == erts_this_dist_entry) { /* Net kernel has died (clean up!!) */ + DistEntry *tdep; + int no_dist_port = 0; Eterm nd_reason = (reason == am_no_network ? am_no_network : am_net_kernel_terminated); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); + + for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) + no_dist_port++; + for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) + no_dist_port++; /* KILL all port controllers */ - while(erts_visible_dist_entries || erts_hidden_dist_entries) { - DistEntry *tdep; - Eterm prt_id; - Port *prt; - if(erts_hidden_dist_entries) - tdep = erts_hidden_dist_entries; + if (no_dist_port == 0) + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + else { + Eterm def_buf[128]; + int i = 0; + Eterm *dist_port; + + if (no_dist_port <= sizeof(def_buf)/sizeof(def_buf[0])) + dist_port = &def_buf[0]; else - tdep = erts_visible_dist_entries; - prt_id = tdep->cid; - ASSERT(is_internal_port(prt_id)); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - - prt = erts_id2port(prt_id, NULL, 0); - if (prt) { - ASSERT(prt->status & ERTS_PORT_SFLG_DISTRIBUTION); - ASSERT(prt->dist_entry); - /* will call do_net_exists !!! */ - erts_do_exit_port(prt, prt_id, nd_reason); - erts_port_release(prt); + dist_port = erts_alloc(ERTS_ALC_T_TMP, + sizeof(Eterm)*no_dist_port); + for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) { + ASSERT(is_internal_port(tdep->cid)); + dist_port[i++] = tdep->cid; + } + for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) { + ASSERT(is_internal_port(tdep->cid)); + dist_port[i++] = tdep->cid; } + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); - } + for (i = 0; i < no_dist_port; i++) { + Port *prt = erts_port_lookup(dist_port[i], + ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (!prt) + continue; + ASSERT(erts_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLG_DISTRIBUTION); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED, + prt, dist_port[i], nd_reason, NULL); + } - nodename = erts_this_dist_entry->sysname; - erts_smp_thr_progress_block(); - erts_set_this_node(am_Noname, 0); - erts_is_alive = 0; - send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nd_reason); - erts_smp_thr_progress_unblock(); + if (dist_port != &def_buf[0]) + erts_free(ERTS_ALC_T_TMP, dist_port); + } + /* + * When last dist port exits, node will be taken + * from alive to not alive. + */ + ASSERT(is_nil(nodedown.reason) && !nodedown.bp); + if (is_immed(nd_reason)) + nodedown.reason = nd_reason; + else { + Eterm *hp; + Uint sz = size_object(nd_reason); + nodedown.bp = new_message_buffer(sz); + hp = nodedown.bp->mem; + nodedown.reason = copy_struct(nd_reason, + sz, + &hp, + &nodedown.bp->off_heap); + } } - else { /* recursive call via erts_do_exit_port() will end up here */ + else { /* Call from distribution port */ NetExitsContext nec = {dep}; ErtsLink *nlinks; ErtsLink *node_links; @@ -454,10 +530,10 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_smp_de_rwlock(dep); ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid) - && erts_lc_is_port_locked(&erts_port[internal_port_index(dep->cid)])); + && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); if (erts_port_task_is_scheduled(&dep->dist_cmd)) - erts_port_task_abort(dep->cid, &dep->dist_cmd); + erts_port_task_abort(&dep->dist_cmd); if (dep->status & ERTS_DE_SFLG_EXITING) { #ifdef DEBUG @@ -503,6 +579,9 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) clear_dist_entry(dep); } + + dec_no_nodes(); + return 1; } @@ -516,6 +595,10 @@ void init_dist(void) { init_nodes_monitors(); + nodedown.reason = NIL; + nodedown.bp = NULL; + + erts_smp_atomic_init_nob(&no_nodes, 0); erts_smp_atomic_init_nob(&no_caches, 0); /* Lookup/Install all references to trap functions */ @@ -769,7 +852,7 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id); + erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(receiver_name), "%T", remote); msize = size_object(message); if (token != NIL && token != am_have_dt_utag) { @@ -826,7 +909,7 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id); + erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(receiver_name), "{%T,%s}", remote_name, node_name); msize = size_object(message); @@ -840,10 +923,10 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) if (token != NIL) ctl = TUPLE5(&ctl_heap[0], make_small(DOP_REG_SEND_TT), - sender->id, am_Cookie, remote_name, token); + sender->common.id, am_Cookie, remote_name, token); else ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND), - sender->id, am_Cookie, remote_name); + sender->common.id, am_Cookie, remote_name); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, @@ -889,7 +972,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, *node_name = *sender_name = *remote_name = '\0'; if (DTRACE_ENABLED(process_exit_signal_remote)) { erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id); + erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); erts_snprintf(remote_name, sizeof(remote_name), "{%T,%s}", remote, node_name); erts_snprintf(reason_str, sizeof(reason), "%T", reason); @@ -1141,7 +1224,7 @@ int erts_net_message(Port *prt, } erts_smp_de_links_lock(dep); - res = erts_add_link(&(rp->nlinks), LINK_PID, from); + res = erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, from); if (res < 0) { /* It was already there! Lets skip the rest... */ @@ -1149,7 +1232,7 @@ int erts_net_message(Port *prt, erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); break; } - lnk = erts_add_or_lookup_link(&(dep->nlinks), LINK_PID, rp->id); + lnk = erts_add_or_lookup_link(&(dep->nlinks), LINK_PID, rp->common.id); erts_add_link(&(ERTS_LINK_ROOT(lnk)), LINK_PID, from); erts_smp_de_links_unlock(dep); @@ -1176,7 +1259,7 @@ int erts_net_message(Port *prt, if (!rp) break; - lnk = erts_remove_link(&(rp->nlinks), from); + lnk = erts_remove_link(&ERTS_P_LINKS(rp), from); if (IS_TRACED_FL(rp, F_TRACE_PROCS) && lnk != NULL) { trace_proc(NULL, rp, am_getting_unlinked, from); @@ -1233,10 +1316,10 @@ int erts_net_message(Port *prt, } else { if (is_atom(watched)) - watched = rp->id; + watched = rp->common.id; erts_smp_de_links_lock(dep); erts_add_monitor(&(dep->monitors), MON_ORIGIN, ref, watched, name); - erts_add_monitor(&(rp->monitors), MON_TARGET, ref, watcher, name); + erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, watcher, name); erts_smp_de_links_unlock(dep); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } @@ -1275,7 +1358,7 @@ int erts_net_message(Port *prt, if (!rp) { break; } - mon = erts_remove_monitor(&(rp->monitors),ref); + mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); ASSERT(mon != NULL); if (mon == NULL) { @@ -1312,7 +1395,7 @@ int erts_net_message(Port *prt, if (is_not_pid(from) || is_not_atom(to)){ goto invalid_message; } - rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_whereis_process(NULL, 0, to, 0, 0); if (rp) { Uint xsize = (type == DOP_REG_SEND ? 0 @@ -1338,7 +1421,6 @@ int erts_net_message(Port *prt, erts_queue_dist_message(rp, &locks, ede_copy, token); if (locks) erts_smp_proc_unlock(rp, locks); - erts_smp_proc_dec_refc(rp); } break; @@ -1364,7 +1446,7 @@ int erts_net_message(Port *prt, if (is_not_pid(to)) { goto invalid_message; } - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_proc_lookup(to); if (rp) { Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); ErtsProcLocks locks = 0; @@ -1388,7 +1470,6 @@ int erts_net_message(Port *prt, erts_queue_dist_message(rp, &locks, ede_copy, token); if (locks) erts_smp_proc_unlock(rp, locks); - erts_smp_proc_dec_refc(rp); } break; @@ -1434,7 +1515,7 @@ int erts_net_message(Port *prt, erts_destroy_monitor(mon); - mon = erts_remove_monitor(&(rp->monitors),ref); + mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); if (mon == NULL) { erts_smp_proc_unlock(rp, rp_locks); @@ -1485,7 +1566,7 @@ int erts_net_message(Port *prt, if (!rp) lnk = NULL; else { - lnk = erts_remove_link(&(rp->nlinks), from); + lnk = erts_remove_link(&ERTS_P_LINKS(rp), from); /* If lnk == NULL, we have unlinked on this side, i.e. * ignore exit. @@ -1544,8 +1625,7 @@ int erts_net_message(Port *prt, if (is_not_pid(from) || is_not_internal_pid(to)) { goto invalid_message; } - rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_pid2proc(NULL, 0, to, rp_locks); if (rp) { (void) erts_send_exit_signal(NULL, from, @@ -1556,7 +1636,6 @@ int erts_net_message(Port *prt, NULL, 0); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } break; } @@ -1601,7 +1680,7 @@ int erts_net_message(Port *prt, erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - erts_do_exit_port(prt, dep->cid, am_killed); + erts_deliver_port_exit(prt, dep->cid, am_killed, 0); ERTS_SMP_CHK_NO_PROC_LOCKS; return -1; } @@ -1650,7 +1729,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) data_size += erts_encode_dist_ext_size(ctl, flags, acmp); if (is_value(msg)) data_size += erts_encode_dist_ext_size(msg, flags, acmp); - erts_finalize_atom_cache_map(acmp); + erts_finalize_atom_cache_map(acmp, flags); dhdr_ext_size = erts_encode_ext_dist_header_size(acmp); data_size += dhdr_ext_size; @@ -1697,7 +1776,6 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) erts_smp_mtx_unlock(&dep->qlock); plp = erts_proclist_create(c_p); - plp->next = NULL; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); suspended = 1; erts_smp_mtx_lock(&dep->qlock); @@ -1730,11 +1808,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) else { /* Enqueue suspended process on dist entry */ ASSERT(plp); - if (dep->suspended.last) - dep->suspended.last->next = plp; - else - dep->suspended.first = plp; - dep->suspended.last = plp; + erts_proclist_store_last(&dep->suspended, plp); } } @@ -1783,7 +1857,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) erts_snprintf(port_str, sizeof(port_str), "%T", cid); erts_snprintf(remote_str, sizeof(remote_str), "%T", dep->sysname); - erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->id); + erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->common.id); DTRACE4(dist_port_busy, erts_this_node_sysname, port_str, remote_str, pid_str); } @@ -1816,7 +1890,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); + erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); erts_snprintf(remote_str, sizeof(remote_str), "%T", prt->dist_entry->sysname); DTRACE4(dist_output, erts_this_node_sysname, port_str, @@ -1870,7 +1944,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); + erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); erts_snprintf(remote_str, sizeof(remote_str), "%T", prt->dist_entry->sysname); DTRACE4(dist_outputv, erts_this_node_sysname, port_str, @@ -1907,13 +1981,13 @@ int erts_dist_command(Port *prt, int reds_limit) { Sint reds = ERTS_PORT_REDS_DIST_CMD_START; - int prt_busy; Uint32 status; Uint32 flags; Sint obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = prt->dist_entry; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); + erts_aint32_t sched_flags; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -1929,7 +2003,7 @@ erts_dist_command(Port *prt, int reds_limit) erts_smp_de_runlock(dep); if (status & ERTS_DE_SFLG_EXITING) { - erts_do_exit_port(prt, prt->id, am_killed); + erts_deliver_port_exit(prt, prt->common.id, am_killed, 0); erts_deref_dist_entry(dep); return reds + ERTS_PORT_REDS_DIST_CMD_EXIT; } @@ -1956,12 +2030,12 @@ erts_dist_command(Port *prt, int reds_limit) dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; + sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + if (reds > reds_limit) goto preempted; - prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY); - - if (!prt_busy && foq.first) { + if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) && foq.first) { int preempt = 0; do { Uint size; @@ -1973,15 +2047,15 @@ erts_dist_command(Port *prt, int reds_limit) bw(foq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); + erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = foq.first; obufsize += size_obuf(fob); foq.first = foq.first->next; free_dist_obuf(fob); - preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); - if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - prt_busy = 1; + sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); + if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) break; - } } while (foq.first && !preempt); if (!foq.first) foq.last = NULL; @@ -1989,7 +2063,7 @@ erts_dist_command(Port *prt, int reds_limit) goto preempted; } - if (prt_busy) { + if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) { if (oq.first) { ErtsDistOutputBuf *ob; int preempt; @@ -1999,7 +2073,8 @@ erts_dist_command(Port *prt, int reds_limit) ASSERT(ob); do { ob->extp = erts_encode_ext_dist_header_finalize(ob->extp, - dep->cache); + dep->cache, + flags); if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) *--ob->extp = PASS_THROUGH; /* Old node; 'pass through' needed */ @@ -2043,7 +2118,8 @@ erts_dist_command(Port *prt, int reds_limit) Uint size; oq.first->extp = erts_encode_ext_dist_header_finalize(oq.first->extp, - dep->cache); + dep->cache, + flags); reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through' @@ -2056,16 +2132,15 @@ erts_dist_command(Port *prt, int reds_limit) bw(oq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); + erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = oq.first; obufsize += size_obuf(fob); oq.first = oq.first->next; free_dist_obuf(fob); - preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); - if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - prt_busy = 1; - if (oq.first && !preempt) - goto finalize_only; - } + sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); + if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt) + goto finalize_only; } ASSERT(!oq.first || preempt); @@ -2093,7 +2168,7 @@ erts_dist_command(Port *prt, int reds_limit) ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; obufsize = 0; - if (!prt_busy + if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) && (dep->qflgs & ERTS_DE_QFLG_BUSY) && dep->qsize < erts_dist_buf_busy_limit) { ErtsProcList *suspendees; @@ -2139,11 +2214,15 @@ erts_dist_command(Port *prt, int reds_limit) return reds; preempted: + /* + * Here we assume that state has been read + * since last call to driver. + */ ASSERT(oq.first || !oq.last); ASSERT(!oq.first || oq.last); - if (prt->status & ERTS_PORT_SFLGS_DEAD) { + if (sched_flags & ERTS_PTS_FLG_EXIT) { /* * Port died during port command; clean up 'oq' * and 'foq'. Things buffered in dist entry after @@ -2201,7 +2280,7 @@ erts_dist_port_not_busy(Port *prt) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); + erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); erts_snprintf(remote_str, sizeof(remote_str), "%T", prt->dist_entry->sysname); DTRACE3(dist_port_not_busy, erts_this_node_sysname, @@ -2242,8 +2321,8 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) void *arg = ((struct print_to_data *) vptdp)->arg; Process *rp; ErtsMonitor *rmon; - rp = erts_pid2proc_unlocked(mon->pid); - if (!rp || (rmon = erts_lookup_monitor(rp->monitors, mon->ref)) == NULL) { + rp = erts_proc_lookup(mon->pid); + if (!rp || (rmon = erts_lookup_monitor(ERTS_P_MONITORS(rp), mon->ref)) == NULL) { erts_print(to, arg, "Warning, stray monitor for: %T\n", mon->pid); } else if (mon->type == MON_ORIGIN) { /* Local pid is being monitored */ @@ -2281,7 +2360,7 @@ static void doit_print_link_info2(ErtsLink *lnk, void *vpplc) static void doit_print_link_info(ErtsLink *lnk, void *vptdp) { - if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) { + if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) { PrintLinkContext plc = {(struct print_to_data *) vptdp, lnk->pid}; erts_doforall_links(ERTS_LINK_ROOT(lnk), &doit_print_link_info2, &plc); } @@ -2303,7 +2382,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext) { PrintNodeLinkContext *pcontext = vpcontext; - if (is_internal_pid(lnk->pid) && erts_pid2proc_unlocked(lnk->pid)) + if (is_internal_pid(lnk->pid) && erts_proc_lookup(lnk->pid)) erts_print(pcontext->ptd.to, pcontext->ptd.arg, "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname); } @@ -2451,15 +2530,15 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) goto error; /* Check that all trap functions are defined !! */ - if (dsend2_trap->address == NULL || - dsend3_trap->address == NULL || + if (dsend2_trap->addressv[0] == NULL || + dsend3_trap->addressv[0] == NULL || /* dsend_nosuspend_trap->address == NULL ||*/ - dlink_trap->address == NULL || - dunlink_trap->address == NULL || - dmonitor_node_trap->address == NULL || - dgroup_leader_trap->address == NULL || - dmonitor_p_trap->address == NULL || - dexit_trap->address == NULL) { + dlink_trap->addressv[0] == NULL || + dunlink_trap->addressv[0] == NULL || + dmonitor_node_trap->addressv[0] == NULL || + dgroup_leader_trap->addressv[0] == NULL || + dmonitor_p_trap->addressv[0] == NULL || + dexit_trap->addressv[0] == NULL) { goto error; } @@ -2471,9 +2550,9 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) /* By setting dist_entry==erts_this_dist_entry and DISTRIBUTION on net_kernel do_net_exist will be called when net_kernel is terminated !! */ - (void *) ERTS_PROC_SET_DIST_ENTRY(net_kernel, - ERTS_PROC_LOCK_MAIN, - erts_this_dist_entry); + (void) ERTS_PROC_SET_DIST_ENTRY(net_kernel, + ERTS_PROC_LOCK_MAIN, + erts_this_dist_entry); erts_refc_inc(&erts_this_dist_entry->refc, 2); net_kernel->flags |= F_DISTRIBUTION; @@ -2481,13 +2560,14 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) erts_smp_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN); #ifdef DEBUG - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); ASSERT(!erts_visible_dist_entries && !erts_hidden_dist_entries); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); #endif erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); + inc_no_nodes(); erts_set_this_node(BIF_ARG_1, (Uint32) creation); erts_is_alive = 1; send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL); @@ -2556,9 +2636,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) /* DFLAG_EXTENDED_REFERENCES is compulsory from R9 and forward */ if (!(DFLAG_EXTENDED_REFERENCES & flags)) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "%T", BIF_P->id); - if (BIF_P->reg) - erts_dsprintf(dsbufp, " (%T)", BIF_P->reg->name); + erts_dsprintf(dsbufp, "%T", BIF_P->common.id); + if (BIF_P->common.u.alive.reg) + erts_dsprintf(dsbufp, " (%T)", BIF_P->common.u.alive.reg->name); erts_dsprintf(dsbufp, " attempted to enable connection to node %T " "which is not able to handle extended references.\n", @@ -2578,10 +2658,14 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) else if (!dep) goto system_limit; /* Should never happen!!! */ - pp = erts_id2port(BIF_ARG_2, BIF_P, ERTS_PROC_LOCK_MAIN); + pp = erts_id2port_sflgs(BIF_ARG_2, + BIF_P, + ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_LOOKUP); erts_smp_de_rwlock(dep); - if (!pp || (pp->status & ERTS_PORT_SFLG_EXITING)) + if (!pp || (erts_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLG_EXITING)) goto badarg; if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0) @@ -2596,11 +2680,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) plp->next = NULL; erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); erts_smp_mtx_lock(&dep->qlock); - if (dep->suspended.last) - dep->suspended.last->next = plp; - else - dep->suspended.first = plp; - dep->suspended.last = plp; + erts_proclist_store_last(&dep->suspended, plp); erts_smp_mtx_unlock(&dep->qlock); goto yield; } @@ -2610,7 +2690,16 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) if (pp->dist_entry || is_not_nil(dep->cid)) goto badarg; - erts_port_status_bor_set(pp, ERTS_PORT_SFLG_DISTRIBUTION); + erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); + + /* + * Dist-ports do not use the "busy port message queue" functionality, but + * instead use "busy dist entry" functionality. + */ + { + ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED; + erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL); + } pp->dist_entry = dep; @@ -2642,6 +2731,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) erts_smp_de_rwunlock(dep); dep = NULL; /* inc of refc transferred to port (dist_entry field) */ + inc_no_nodes(); + send_nodes_mon_msgs(BIF_P, am_nodeup, BIF_ARG_1, @@ -2655,7 +2746,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) } if (pp) - erts_smp_port_unlock(pp); + erts_port_release(pp); return ret; @@ -2699,16 +2790,15 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) if (is_internal_pid(local)) { Process *lp; ErtsProcLocks lp_locks; - if (BIF_P->id == local) { + if (BIF_P->common.id == local) { lp_locks = ERTS_PROC_LOCKS_ALL; lp = BIF_P; erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); } else { lp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - lp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - local, lp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + lp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + local, lp_locks); if (!lp) { BIF_RET(am_true); /* ignore */ } @@ -2727,14 +2817,18 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) lp_locks &= ~ERTS_PROC_LOCK_MAIN; #endif erts_smp_proc_unlock(lp, lp_locks); - if (lp != BIF_P) - erts_smp_proc_dec_refc(lp); - else { + if (lp == BIF_P) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&BIF_P->state); /* * We may have exited current process and may have to take action. */ - ERTS_BIF_CHK_EXITED(BIF_P); - ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN); + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { +#ifdef ERTS_SMP + if (state & ERTS_PSFLG_PENDING_EXIT) + erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); +#endif + ERTS_BIF_EXITED(BIF_P); + } } } else if (is_external_pid(local) @@ -2818,7 +2912,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) length = 0; - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); ASSERT(erts_no_of_not_connected_dist_entries >= 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); @@ -2835,7 +2929,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) result = NIL; if (length == 0) { - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); goto done; } @@ -2864,7 +2958,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) hp += 2; } ASSERT(endp == hp); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); done: UnUseTmpHeap(2,BIF_P); @@ -2932,23 +3026,23 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) if (Bool == am_true) { ASSERT(dep->cid != NIL); lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, - p->id); + p->common.id); ++ERTS_LINK_REFC(lnk); - lnk = erts_add_or_lookup_link(&(p->nlinks), LINK_NODE, Node); + lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node); ++ERTS_LINK_REFC(lnk); } else { - lnk = erts_lookup_link(dep->node_links, p->id); + lnk = erts_lookup_link(dep->node_links, p->common.id); if (lnk != NULL) { if ((--ERTS_LINK_REFC(lnk)) == 0) { erts_destroy_link(erts_remove_link(&(dep->node_links), - p->id)); + p->common.id)); } } - lnk = erts_lookup_link(p->nlinks, Node); + lnk = erts_lookup_link(ERTS_P_LINKS(p), Node); if (lnk != NULL) { if ((--ERTS_LINK_REFC(lnk)) == 0) { - erts_destroy_link(erts_remove_link(&(p->nlinks), + erts_destroy_link(erts_remove_link(&ERTS_P_LINKS(p), Node)); } } @@ -3510,7 +3604,7 @@ erts_processes_monitoring_nodes(Process *c_p) olist = erts_bld_cons(hpp, szp, am_nodedown_reason, olist); res = erts_bld_cons(hpp, szp, erts_bld_tuple(hpp, szp, 2, - nmp->proc->id, + nmp->proc->common.id, olist), res); } diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 845151c895..ff8f5e106f 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -38,7 +38,8 @@ #define DFLAG_UNICODE_IO 0x1000 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000 #define DFLAG_SMALL_ATOM_TAGS 0x4000 -#define DFLAGS_INTERNAL_TAGS 0x8000 +#define DFLAG_INTERNAL_TAGS 0x8000 +#define DFLAG_UTF8_ATOMS 0x10000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -187,11 +188,12 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) if (prt) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - ASSERT((erts_port_status_get(prt) & ERTS_PORT_SFLGS_DEAD) == 0); + ASSERT((erts_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLGS_DEAD) == 0); ASSERT(prt->dist_entry); dep = prt->dist_entry; - id = prt->id; + id = prt->common.id; } else { ASSERT(dist_entry); @@ -203,13 +205,8 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) id = dep->cid; } - if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) { - (void) erts_port_task_schedule(id, - &dep->dist_cmd, - ERTS_PORT_TASK_DIST_CMD, - (ErlDrvEvent) -1, - NULL); - } + if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) + erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD); } #endif diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 570cc59be2..eca4e3b3bb 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -37,15 +37,20 @@ #define GET_ERL_AF_ALLOC_IMPL #include "erl_afit_alloc.h" +struct AFFreeBlock_t_ { + Block_t block_head; + AFFreeBlock_t *prev; + AFFreeBlock_t *next; +}; +#define AF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->block_head) #define MIN_MBC_SZ (16*1024) #define MIN_MBC_FIRST_FREE_SZ (4*1024) /* Prototypes of callback functions */ -static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void link_free_block (Allctr_t *, Block_t *, Uint32); -static void unlink_free_block (Allctr_t *, Block_t *, Uint32); +static Block_t * get_free_block (Allctr_t *, Uint, Block_t *, Uint); +static void link_free_block (Allctr_t *, Block_t *); +static void unlink_free_block (Allctr_t *, Block_t *); static Eterm info_options (Allctr_t *, char *, int *, @@ -78,8 +83,6 @@ erts_afalc_start(AFAllctr_t *afallctr, sys_memcpy((void *) afallctr, (void *) &zero.allctr, sizeof(AFAllctr_t)); - init->sbmbct = 0; /* Small mbc not supported by afit */ - allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; @@ -95,6 +98,9 @@ erts_afalc_start(AFAllctr_t *afallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = NULL; allctr->destroying_mbc = NULL; + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -111,14 +117,13 @@ erts_afalc_start(AFAllctr_t *afallctr, } static Block_t * -get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, - Uint32 flags) +get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) { AFAllctr_t *afallctr = (AFAllctr_t *) allctr; ASSERT(!cand_blk || cand_size >= size); - if (afallctr->free_list && BLK_SZ(afallctr->free_list) >= size) { + if (afallctr->free_list && AF_BLK_SZ(afallctr->free_list) >= size) { AFFreeBlock_t *res = afallctr->free_list; afallctr->free_list = res->next; if (res->next) @@ -130,12 +135,12 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, } static void -link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +link_free_block(Allctr_t *allctr, Block_t *block) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; - if (afallctr->free_list && BLK_SZ(afallctr->free_list) > BLK_SZ(blk)) { + if (afallctr->free_list && AF_BLK_SZ(afallctr->free_list) > AF_BLK_SZ(blk)) { blk->next = afallctr->free_list->next; blk->prev = afallctr->free_list; afallctr->free_list->next = blk; @@ -151,7 +156,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +unlink_free_block(Allctr_t *allctr, Block_t *block) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -254,10 +259,10 @@ info_options(Allctr_t *allctr, * to erts_afalc_test() * \* */ -unsigned long -erts_afalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_afalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - default: ASSERT(0); return ~((unsigned long) 0); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h index ea408a7194..b90ac8f7c5 100644 --- a/erts/emulator/beam/erl_afit_alloc.h +++ b/erts/emulator/beam/erl_afit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -49,11 +49,6 @@ Allctr_t *erts_afalc_start(AFAllctr_t *, AFAllctrInit_t *, AllctrInit_t *); #include "erl_alloc_util.h" typedef struct AFFreeBlock_t_ AFFreeBlock_t; -struct AFFreeBlock_t_ { - Block_t block_head; - AFFreeBlock_t *prev; - AFFreeBlock_t *next; -}; struct AFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ @@ -61,7 +56,7 @@ struct AFAllctr_t_ { AFFreeBlock_t * free_list; }; -unsigned long erts_afalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_afalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_AF_ALLOC_IMPL) && !defined(ERL_AF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 6fce032f9d..6dec383cee 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -71,6 +71,23 @@ #define AU_ALLOC_DEFAULT_ENABLE(X) (X) #endif +#define ERTS_ALC_DEFAULT_ENABLED_ACUL 60 +#define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45 +#define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85 + +#define ERTS_ALC_DEFAULT_ACUL 0 +#define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0 +#define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0 + +#ifndef ERTS_SMP +# undef ERTS_ALC_DEFAULT_ACUL +# define ERTS_ALC_DEFAULT_ACUL 0 +# undef ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC +# define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0 +# undef ERTS_ALC_DEFAULT_ACUL_LL_ALLOC +# define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0 +#endif + #ifdef DEBUG static Uint install_debug_functions(void); #if 0 @@ -101,11 +118,9 @@ typedef union { char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))]; } ErtsAllocatorState_t; -static ErtsAllocatorState_t sbmbc_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; #if HALFWORD_HEAP -static ErtsAllocatorState_t sbmbc_low_alloc_state; static ErtsAllocatorState_t std_low_alloc_state; static ErtsAllocatorState_t ll_low_alloc_state; #endif @@ -120,6 +135,7 @@ static ErtsAllocatorState_t fix_alloc_state; typedef struct { erts_smp_atomic32_t refc; int only_sz; + int internal; Uint req_sched; Process *proc; Eterm ref; @@ -162,6 +178,7 @@ enum allctr_type { struct au_init { int enable; int thr_spec; + int carrier_migration_allowed; enum allctr_type atype; struct { AllctrInit_t util; @@ -200,7 +217,6 @@ typedef struct { char *mtrace; char *nodename; } instr; - struct au_init sbmbc_alloc; struct au_init sl_alloc; struct au_init std_alloc; struct au_init ll_alloc; @@ -211,13 +227,12 @@ typedef struct { struct au_init driver_alloc; struct au_init fix_alloc; #if HALFWORD_HEAP - struct au_init sbmbc_low_alloc; struct au_init std_low_alloc; struct au_init ll_low_alloc; #endif } erts_alc_hndl_args_init_t; -#define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} +#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ @@ -225,43 +240,33 @@ do { \ sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \ } while (0) -static void -set_default_sbmbc_alloc_opts(struct au_init *ip) +#if ERTS_ALC_DEFAULT_ACUL \ + || ERTS_ALC_DEFAULT_ACUL_LL_ALLOC \ + || ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC + +static ERTS_INLINE void +set_default_acul(struct au_init *ip, int acul) { - SET_DEFAULT_ALLOC_OPTS(ip); - ip->enable = 0; - ip->thr_spec = 0; - ip->atype = BESTFIT; - ip->init.bf.ao = 1; - ip->init.util.ramv = 0; - ip->init.util.mmsbc = 0; - ip->init.util.mmmbc = 500; - ip->init.util.sbct = ~((UWord) 0); - ip->init.util.name_prefix = "sbmbc_"; - ip->init.util.alloc_no = ERTS_ALC_A_SBMBC; -#ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ -#else - ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ -#endif - ip->init.util.ts = ERTS_ALC_MTA_SBMBC; - ip->init.util.asbcst = 0; - ip->init.util.rsbcst = 0; - ip->init.util.rsbcmt = 0; - ip->init.util.rmbcmt = 0; - ip->init.util.sbmbct = 0; - ip->init.util.sbmbcs = 0; + ip->thr_spec = 1; + ip->atype = AOFIRSTFIT; + ip->init.aoff.flavor = AOFF_BF; + ip->init.util.acul = acul; } +#endif + static void set_default_sl_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = GOODFIT; +#endif ip->init.util.name_prefix = "sl_"; - ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ @@ -282,10 +287,13 @@ set_default_std_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.util.name_prefix = "std_"; - ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_STANDARD; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ @@ -300,27 +308,28 @@ set_default_ll_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL_LL_ALLOC + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_LL_ALLOC); +#else ip->thr_spec = 0; ip->atype = BESTFIT; ip->init.bf.ao = 1; +#endif ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; - ip->init.util.mmmbc = 0; ip->init.util.sbct = ~((UWord) 0); ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 2*1024*1024 - 40; /* Main carrier size */ #else - ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 1*1024*1024 - 40; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED; ip->init.util.asbcst = 0; ip->init.util.rsbcst = 0; ip->init.util.rsbcmt = 0; ip->init.util.rmbcmt = 0; - ip->init.util.sbmbct = 0; - ip->init.util.sbmbcs = 0; } static void @@ -329,6 +338,7 @@ set_default_temp_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; + ip->carrier_migration_allowed = 0; ip->atype = AFIT; ip->init.util.name_prefix = "temp_"; ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY; @@ -351,9 +361,12 @@ set_default_eheap_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC); +#else ip->thr_spec = 1; ip->atype = GOODFIT; - ip->init.util.mmmbc = 100; +#endif ip->init.util.name_prefix = "eheap_"; ip->init.util.alloc_no = ERTS_ALC_A_EHEAP; #ifndef SMALL_MEMORY @@ -374,9 +387,12 @@ set_default_binary_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; - ip->init.util.mmmbc = 50; +#endif ip->init.util.name_prefix = "binary_"; ip->init.util.alloc_no = ERTS_ALC_A_BINARY; #ifndef SMALL_MEMORY @@ -392,9 +408,12 @@ set_default_ets_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; - ip->init.util.mmmbc = 100; +#endif ip->init.util.name_prefix = "ets_"; ip->init.util.alloc_no = ERTS_ALC_A_ETS; #ifndef SMALL_MEMORY @@ -410,8 +429,12 @@ set_default_driver_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.util.name_prefix = "driver_"; ip->init.util.alloc_no = ERTS_ALC_A_DRIVER; #ifndef SMALL_MEMORY @@ -428,8 +451,12 @@ set_default_fix_alloc_opts(struct au_init *ip, { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.bf.ao = 1; ip->init.util.name_prefix = "fix_"; ip->init.util.fix_type_size = fix_type_sizes; @@ -462,13 +489,6 @@ adjust_tpref(struct au_init *ip, int no_sched) /* ... shrink smallest multi-block carrier size */ if (ip->default_.smbcs) ip->init.util.smbcs /= ERTS_MIN(4, no_sched); - /* ... and more than three allocators shrink - max mseg multi-block carriers */ - if (ip->default_.mmmbc && no_sched > 2) { - ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1); - if (ip->init.util.mmmbc < 3) - ip->init.util.mmmbc = 3; - } } } @@ -495,6 +515,71 @@ refuse_af_strategy(struct au_init *init) static void hdbg_init(void); #endif +static void adjust_fix_alloc_sizes(UWord extra_block_size) +{ + + if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) { + int j; + +#ifdef ERTS_SMP + if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) { + int i; + ErtsAllocatorThrSpec_t* tspec; + + tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; + ASSERT(tspec->enabled); + + for (i=0; i < tspec->size; i++) { + Allctr_t* allctr = tspec->allctr[i]; + for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { + allctr->fix[j].type_size += extra_block_size; + } + } + } + else +#endif + { + Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra; + for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { + allctr->fix[j].type_size += extra_block_size; + } + } + } +} + +static ERTS_INLINE int +strategy_support_carrier_migration(struct au_init *auip) +{ + /* + * Currently only aoff, aoffcbf and aoffcaobf support carrier + * migration, i.e, type AOFIRSTFIT. + */ + return auip->atype == AOFIRSTFIT; +} + +static ERTS_INLINE void +check_disable_carrier_migration(struct au_init *auip) +{ + if (!strategy_support_carrier_migration(auip) || !auip->thr_spec) + auip->init.util.acul = 0; +} + +static ERTS_INLINE void +ensure_carrier_migration_support(struct au_init *auip) +{ + auip->thr_spec = 1; /* Need thread preferred */ + + /* + * If strategy cannot handle carrier migration, + * default to a strategy that can... + */ + if (!strategy_support_carrier_migration(auip)) { + /* Default to aoffcbf */ + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_BF; + } +} + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { @@ -515,9 +600,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = sizeof(Process); #if !HALFWORD_HEAP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] - = ERTS_MONITOR_SH_SIZE; + = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] - = ERTS_LINK_SH_SIZE; + = ERTS_LINK_SH_SIZE * sizeof(Uint); #endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] = sizeof(ErtsDrvEventDataState); @@ -533,15 +618,15 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) hdbg_init(); #endif - erts_have_sbmbc_alloc = 0; ncpu = eaiop->ncpu; if (ncpu < 1) ncpu = 1; + erts_tsd_key_create(&erts_allctr_prelock_tsd_key); + erts_sys_alloc_init(); erts_init_utils_mem(); - set_default_sbmbc_alloc_opts(&init.sbmbc_alloc); set_default_sl_alloc_opts(&init.sl_alloc); set_default_std_alloc_opts(&init.std_alloc); set_default_ll_alloc_opts(&init.ll_alloc); @@ -557,7 +642,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) handle_args(argc, argv, &init); #ifndef ERTS_SMP - init.sbmbc_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; init.ll_alloc.thr_spec = 0; @@ -570,7 +654,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ - init.sbmbc_alloc.thr_spec = 0; init.temp_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; @@ -582,13 +665,22 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.fix_alloc.thr_spec = 0; } + check_disable_carrier_migration(&init.sl_alloc); + check_disable_carrier_migration(&init.std_alloc); + check_disable_carrier_migration(&init.ll_alloc); + check_disable_carrier_migration(&init.eheap_alloc); + check_disable_carrier_migration(&init.binary_alloc); + check_disable_carrier_migration(&init.ets_alloc); + check_disable_carrier_migration(&init.driver_alloc); + check_disable_carrier_migration(&init.fix_alloc); + + #ifdef ERTS_SMP /* Only temp_alloc can use thread specific interface */ if (init.temp_alloc.thr_spec) init.temp_alloc.thr_spec = erts_no_schedulers; /* Others must use thread preferred interface */ - adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers); adjust_tpref(&init.sl_alloc, erts_no_schedulers); adjust_tpref(&init.std_alloc, erts_no_schedulers); adjust_tpref(&init.ll_alloc, erts_no_schedulers); @@ -607,7 +699,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) * The following allocators cannot be run with afit strategy. * Make sure they don't... */ - refuse_af_strategy(&init.sbmbc_alloc); refuse_af_strategy(&init.sl_alloc); refuse_af_strategy(&init.std_alloc); refuse_af_strategy(&init.ll_alloc); @@ -651,11 +742,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #if HALFWORD_HEAP /* Init low memory variants by cloning */ - init.sbmbc_low_alloc = init.sbmbc_alloc; - init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_"; - init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW; - init.sbmbc_low_alloc.init.util.low_mem = 1; - init.std_low_alloc = init.std_alloc; init.std_low_alloc.init.util.name_prefix = "std_low_"; init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; @@ -668,13 +754,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ll_low_alloc.init.util.force = 1; init.ll_low_alloc.init.util.low_mem = 1; - set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); #endif /* HALFWORD */ set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); - set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu); @@ -701,20 +785,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_init(init.instr.mtrace, init.instr.nodename); - /* sbmbc_alloc() needs to be started first */ - start_au_allocator(ERTS_ALC_A_SBMBC, - &init.sbmbc_alloc, - &sbmbc_alloc_state); -#if HALFWORD_HEAP - start_au_allocator(ERTS_ALC_A_SBMBC_LOW, - &init.sbmbc_low_alloc, - &sbmbc_low_alloc_state); - erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable - && init.sbmbc_low_alloc.enable); -#else - erts_have_sbmbc_alloc = init.sbmbc_alloc.enable; -#endif - start_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, &temp_alloc_state); @@ -768,7 +838,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #ifdef DEBUG extra_block_size += install_debug_functions(); #endif - + adjust_fix_alloc_sizes(extra_block_size); } void @@ -1104,6 +1174,7 @@ get_kb_value(char *param_end, char** argv, int* ip) return ((Uint) tmp)*1024; } +#if 0 static Uint get_byte_value(char *param_end, char** argv, int* ip) { @@ -1117,6 +1188,7 @@ get_byte_value(char *param_end, char** argv, int* ip) bad_value(param, param_end, value); return (Uint) tmp; } +#endif static Uint get_amount_value(char *param_end, char** argv, int* ip) @@ -1132,17 +1204,59 @@ get_amount_value(char *param_end, char** argv, int* ip) return (Uint) tmp; } +static Uint +get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip) +{ + Sint tmp; + char *rest; + char *param = argv[*ip]+1; + char *value = get_value(param_end, argv, ip); + if (sys_strcmp(value, "de") == 0) { + switch (auip->init.util.alloc_no) { + case ERTS_ALC_A_LONG_LIVED: +#if HALFWORD_HEAP + case ERTS_ALC_A_LONG_LIVED_LOW: +#endif + return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC; + case ERTS_ALC_A_EHEAP: + return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC; + default: + return ERTS_ALC_DEFAULT_ENABLED_ACUL; + } + } + errno = 0; + tmp = (Sint) ErtsStrToSint(value, &rest, 10); + if (errno != 0 || rest == value || tmp < 0 || 100 < tmp) + bad_value(param, param_end, value); + return (Uint) tmp; +} + static void handle_au_arg(struct au_init *auip, char* sub_param, char** argv, - int* ip) + int* ip, + int u_switch) { char *param = argv[*ip]+1; switch (sub_param[0]) { case 'a': - if(has_prefix("asbcst", sub_param)) { + if (has_prefix("acul", sub_param)) { + if (!auip->carrier_migration_allowed) { + if (!u_switch) + goto bad_switch; + else { + /* ignore */ + (void) get_acul_value(auip, sub_param + 4, argv, ip); + break; + } + } + ensure_carrier_migration_support(auip); + + auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip); + } + else if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } else if(has_prefix("as", sub_param)) { @@ -1163,10 +1277,20 @@ handle_au_arg(struct au_init *auip, } else if (strcmp("aoff", alg) == 0) { auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_AOFF; + } + else if (strcmp("aoffcbf", alg) == 0) { + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_BF; + } + else if (strcmp("aoffcaobf", alg) == 0) { + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_AOBF; } else { bad_value(param, sub_param + 1, alg); } + check_disable_carrier_migration(auip); } else goto bad_switch; @@ -1232,12 +1356,6 @@ handle_au_arg(struct au_init *auip, if(has_prefix("sbct", sub_param)) { auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip); } - else if (has_prefix("sbmbcs", sub_param)) { - auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip); - } - else if (has_prefix("sbmbct", sub_param)) { - auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip); - } else if (has_prefix("smbcs", sub_param)) { auip->default_.smbcs = 0; auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip); @@ -1253,6 +1371,7 @@ handle_au_arg(struct au_init *auip, } else if (res == 0) { auip->thr_spec = 0; + check_disable_carrier_migration(auip); break; } goto bad_switch; @@ -1267,7 +1386,6 @@ static void handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) { struct au_init *aui[] = { - &init->sbmbc_alloc, &init->binary_alloc, &init->std_alloc, &init->ets_alloc, @@ -1294,25 +1412,22 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'M': switch (argv[i][2]) { case 'B': - handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i); - break; - case 'C': - handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0); break; case 'D': - handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0); break; case 'E': - handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0); break; case 'F': - handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0); break; case 'H': - handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0); break; case 'L': - handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0); break; case 'M': if (has_prefix("amcbf", argv[i]+3)) { @@ -1338,13 +1453,13 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) } break; case 'R': - handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0); break; case 'S': - handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0); break; case 'T': - handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0); break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { @@ -1430,8 +1545,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) for (a = 0; a < aui_sz; a++) { aui[a]->thr_spec = 0; + check_disable_carrier_migration(aui[a]); aui[a]->init.util.ramv = 0; - aui[a]->init.util.mmmbc = 10; aui[a]->init.util.lmbcs = 5*1024*1024; } } @@ -1494,7 +1609,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[start + 1] = val; i = start; } - handle_au_arg(aui[a], &argv[i][3], argv, &i); + handle_au_arg(aui[a], &argv[i][3], argv, &i, 1); } } break; @@ -1675,7 +1790,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) t_str = type_no_str(n); if (!t_str) { - sprintf(buf, "%d", (int) n); + erts_snprintf(buf, sizeof(buf), "%d", (int) n); t_str = buf; } @@ -2040,15 +2155,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */ + /* All alloc_util allocators *have* to be enabled */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_A_SBMBC: -#if HALFWORD_HEAP - case ERTS_ALC_A_SBMBC_LOW: -#endif break; default: if (!erts_allctrs_info[ai].enabled @@ -2088,12 +2199,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * Often not thread safe and usually never * contain any allocated memory. */ - case ERTS_ALC_A_SBMBC: - /* Included in other allocators */ -#if HALFWORD_HEAP - case ERTS_ALC_A_SBMBC_LOW: - /* Included in other allocators */ -#endif continue; case ERTS_ALC_A_EHEAP: save = &size.processes; @@ -2137,7 +2242,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) fi, ERTS_ALC_NO_FIXED_SIZES); tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0); } - tmp += erts_max_processes*sizeof(Process*); + tmp += erts_ptab_mem_size(&erts_proc); tmp += erts_bif_timer_memory_size(); tmp += erts_tot_link_lh_size(); @@ -2182,9 +2287,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (want.code) { size.code = module_table_sz(); size.code += export_table_sz(); - size.code += export_list_size() * sizeof(Export); + size.code += export_entries_sz(); size.code += erts_fun_table_sz(); - size.code += allocated_modules*sizeof(Range); + size.code += erts_ranges_sz(); size.code += erts_total_code_size; } @@ -2263,7 +2368,7 @@ struct aa_values { Eterm erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) { -#define MAX_AA_VALUES (23) +#define MAX_AA_VALUES (24) struct aa_values values[MAX_AA_VALUES]; Eterm res = THE_NON_VALUE; int i, length; @@ -2298,13 +2403,9 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "static"; - values[i].ui[0] = - erts_max_ports*sizeof(Port) /* Port table */ - + erts_timer_wheel_memory_size() /* Timer wheel */ -#ifdef SYS_TMP_BUF_SIZE - + SYS_TMP_BUF_SIZE /* tmp_buf in sys on vxworks & ose */ -#endif - ; + values[i].ui[0] = + sizeof(ErtsPTab)*2 /* proc & port tables */ + + erts_timer_wheel_memory_size(); /* Timer wheel */ i++; erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space); @@ -2332,7 +2433,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "export_list"; - values[i].ui[0] = export_list_size() * sizeof(Export); + values[i].ui[0] = export_entries_sz(); i++; values[i].arity = 2; @@ -2347,7 +2448,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "module_refs"; - values[i].ui[0] = allocated_modules*sizeof(Range); + values[i].ui[0] = erts_ranges_sz(); i++; values[i].arity = 2; @@ -2382,7 +2483,12 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "process_table"; - values[i].ui[0] = erts_max_processes*sizeof(Process*); + values[i].ui[0] = erts_ptab_mem_size(&erts_proc); + i++; + + values[i].arity = 2; + values[i].name = "port_table"; + values[i].ui[0] = erts_ptab_mem_size(&erts_port); i++; values[i].arity = 2; @@ -2536,7 +2642,7 @@ erts_allocator_info(int to, void *arg) as = erts_allctr_thr_spec[a].allctr[ai]; } /* Binary alloc has its own thread safety... */ - erts_alcu_info(as, 0, &to, arg, NULL, NULL); + erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL); } else { switch (a) { @@ -2596,8 +2702,8 @@ erts_allocator_options(void *proc) #endif Uint sz, *szp, *hp, **hpp; Eterm res, features, settings; - Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; - Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; + Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6]; + Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+6]; int a, length; SysAllocStat sas; Uint *endp = NULL; @@ -2710,6 +2816,9 @@ erts_allocator_options(void *proc) if (use_mseg) terms[length++] = am_atom_put("mseg_alloc", 10); #endif +#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + terms[length++] = am_atom_put("sys_aligned_alloc", 17); +#endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -2798,6 +2907,7 @@ reply_alloc_info(void *vair) int i; Eterm (*info_func)(Allctr_t *, int, + int, int *, void *, Uint **, @@ -2926,8 +3036,8 @@ reply_alloc_info(void *vair) allctr = erts_allctr_thr_spec[ai].allctr[0]; else allctr = erts_allctrs_info[ai].extra; - ainfo = info_func(allctr, hpp != NULL, NULL, - NULL, hpp, szp); + ainfo = info_func(allctr, air->internal, hpp != NULL, + NULL, NULL, hpp, szp); ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom, make_small(0), ainfo); } @@ -2962,7 +3072,7 @@ reply_alloc_info(void *vair) alloc_atom = erts_bld_atom(hpp, szp, (char *) ERTS_ALC_A2AD(ai)); allctr = erts_allctr_thr_spec[ai].allctr[sched_id]; - ainfo = info_func(allctr, hpp != NULL, NULL, + ainfo = info_func(allctr, air->internal, hpp != NULL, NULL, NULL, hpp, szp); ai_list = erts_bld_cons(hpp, szp, erts_bld_tuple( @@ -3018,7 +3128,8 @@ int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, - int only_sz) + int only_sz, + int internal) { ErtsAllocInfoReq *air = aireq_alloc(); Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0}; @@ -3030,6 +3141,8 @@ erts_request_alloc_info(struct process *c_p, air->only_sz = only_sz; + air->internal = internal; + air->proc = c_p; if (is_not_internal_ref(ref)) @@ -3049,13 +3162,13 @@ erts_request_alloc_info(struct process *c_p, Eterm alloc = CAR(consp); for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) - if (erts_is_atom_str((char *) erts_alc_a2ad[ai], alloc)) + if (erts_is_atom_str(erts_alc_a2ad[ai], alloc, 0)) goto save_alloc; - if (erts_is_atom_str("mseg_alloc", alloc)) { + if (erts_is_atom_str("mseg_alloc", alloc, 0)) { ai = ERTS_ALC_INFO_A_MSEG_ALLOC; goto save_alloc; } - if (erts_is_atom_str("alloc_util", alloc)) { + if (erts_is_atom_str("alloc_util", alloc, 0)) { ai = ERTS_ALC_INFO_A_ALLOC_UTIL; save_alloc: if (req_ai[ai]) @@ -3094,6 +3207,55 @@ erts_request_alloc_info(struct process *c_p, return 1; } +/* + * The allocator wrapper prelocking stuff below is about the locking order. + * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks + * during alloc/realloc/free. + * + * Some query functions in erl_alloc_util.c lock the allocator mutex and then + * use erts_printf that in turn may call the sys allocator through the wrappers. + * To avoid breaking locking order these query functions first "pre-locks" all + * allocator wrappers. + */ +ErtsAllocatorWrapper_t *erts_allctr_wrappers; +int erts_allctr_wrapper_prelocked = 0; +erts_tsd_key_t erts_allctr_prelock_tsd_key; + +void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper) +{ + ASSERT(wrapper->lock && wrapper->unlock); + wrapper->next = erts_allctr_wrappers; + erts_allctr_wrappers = wrapper; +} + +void erts_allctr_wrapper_pre_lock(void) +{ + if (erts_allctr_wrappers) { + ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers; + for ( ; wrapper; wrapper = wrapper->next) { + wrapper->lock(); + } + ASSERT(!erts_allctr_wrapper_prelocked); + erts_allctr_wrapper_prelocked = 1; + erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)1); + } +} + +void erts_allctr_wrapper_pre_unlock(void) +{ + if (erts_allctr_wrappers) { + ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers; + + erts_allctr_wrapper_prelocked = 0; + erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)0); + for ( ; wrapper; wrapper = wrapper->next) { + wrapper->unlock(); + } + } +} + + + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Deprecated functions * * * @@ -3123,10 +3285,7 @@ void *safe_realloc(void *ptr, Uint sz) \* */ #define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") -UWord erts_alc_test(UWord op, - UWord a1, - UWord a2, - UWord a3) +UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) { switch (op >> 8) { case 0x0: return erts_alcu_test(op, a1, a2); @@ -3178,14 +3337,13 @@ UWord erts_alc_test(UWord op, init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; init.init.util.ts = a2 ? 1 : 0; - init.init.util.sbmbct = 0; if ((char **) a3) { char **argv = (char **) a3; int i = 0; while (argv[i]) { if (argv[i][0] == '-' && argv[i][1] == 't') - handle_au_arg(&init, &argv[i][2], argv, &i); + handle_au_arg(&init, &argv[i][2], argv, &i, 0); else return (UWord) NULL; i++; @@ -3305,6 +3463,11 @@ UWord erts_alc_test(UWord op, ERTS_ALC_TEST_ABORT; break; #endif /* #ifdef USE_THREADS */ +#ifdef ERTS_SMP + case 0xf13: return (UWord) 1; +#else + case 0xf13: return (UWord) 0; +#endif default: break; } @@ -3576,12 +3739,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) ftype = type_no_str(found_type); if (!ftype) { - sprintf(fbuf, "%d", (int) found_type); + erts_snprintf(fbuf, sizeof(fbuf), "%d", (int) found_type); ftype = fbuf; } otype = type_no_str(n); if (!otype) { - sprintf(obuf, "%d", (int) n); + erts_snprintf(obuf, sizeof(obuf), "%d", (int) n); otype = obuf; } diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index e475f9d8a2..f83f6b39cf 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,6 +54,16 @@ void erts_sys_alloc_init(void); void *erts_sys_alloc(ErtsAlcType_t, void *, Uint); void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint); void erts_sys_free(ErtsAlcType_t, void *, void *); +#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +/* + * Note 'alignment' must remain the same in calls to + * 'erts_sys_aligned_realloc()' and 'erts_sys_aligned_free()' + * as in the initial call to 'erts_sys_aligned_alloc()'. + */ +void *erts_sys_aligned_alloc(UWord alignment, UWord size); +void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size); +void erts_sys_aligned_free(UWord alignment, void *ptr); +#endif Eterm erts_memory(int *, void *, void *, Eterm); Eterm erts_allocated_areas(int *, void *, void *); @@ -65,7 +75,7 @@ Eterm erts_allocator_options(void *proc); struct process; int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, - int only_sz); + int only_sz, int internal); #define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0} typedef struct { @@ -100,14 +110,6 @@ UWord erts_alc_test(UWord, #define ERTS_ALC_MIN_LONG_LIVED_TIME (10*60*1000) -#if HALFWORD_HEAP -#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ - ((NO) == ERTS_ALC_A_SBMBC || (NO) == ERTS_ALC_A_SBMBC_LOW) -#else -#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ - ((NO) == ERTS_ALC_A_SBMBC) -#endif - typedef struct { int alloc_util; int enabled; @@ -135,6 +137,18 @@ typedef struct { extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; +typedef struct ErtsAllocatorWrapper_t_ { + void (*lock)(void); + void (*unlock)(void); + struct ErtsAllocatorWrapper_t_* next; +}ErtsAllocatorWrapper_t; +ErtsAllocatorWrapper_t *erts_allctr_wrappers; +extern int erts_allctr_wrapper_prelocked; +extern erts_tsd_key_t erts_allctr_prelock_tsd_key; +void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper); +void erts_allctr_wrapper_pre_lock(void); +void erts_allctr_wrapper_pre_unlock(void); + void erts_alloc_register_scheduler(void *vesdp); #ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, @@ -188,6 +202,7 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size); void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); +int erts_is_allctr_wrapper_prelocked(void); #endif /* #if !ERTS_ALC_DO_INLINE */ @@ -258,6 +273,13 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) size); } +ERTS_ALC_INLINE +int erts_is_allctr_wrapper_prelocked(void) +{ + return erts_allctr_wrapper_prelocked /* locked */ + && !!erts_tsd_get(erts_allctr_prelock_tsd_key); /* by me */ +} + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) @@ -267,6 +289,8 @@ typedef void (*erts_alloc_verify_func_t)(Allctr_t *); erts_alloc_verify_func_t erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr); +#define ERTS_ALC_DATA_ALIGN_SIZE(SZ) \ + (((((SZ) - 1) / 8) + 1) * 8) #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \ (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 4aa8fa82fb..bb5eba80be 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2012. All Rights Reserved. +# Copyright Ericsson AB 2003-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -49,6 +49,8 @@ # true after a "+enable X" statement or if it has been passed as a # command line argument to make_alloc_types. The variable X is false # after a "+disable X" statement or if it has never been mentioned. +# +# IMPORTANT! Only use 7-bit ascii text in this file! +if smp +disable threads_no_smp @@ -74,11 +76,6 @@ allocator SYSTEM true sys_alloc -allocator SBMBC true sbmbc_alloc -+if halfword -allocator SBMBC_LOW true sbmbc_low_alloc -+endif - +if smp allocator TEMPORARY true temp_alloc @@ -144,8 +141,8 @@ class SYSTEM system_data # # <TYPE> <ALLOCATOR> <CLASS> <DESCRIPTION> -type SBMBC SBMBC SYSTEM small_block_mbc type PROC FIXED_SIZE PROCESSES proc +type PORT DRIVER SYSTEM port type ATOM LONG_LIVED ATOM atom_entry type MODULE LONG_LIVED CODE module_entry type REG_PROC STANDARD PROCESSES reg_proc @@ -153,6 +150,7 @@ type LINK_LH STANDARD PROCESSES link_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list +type EXTRA_ROOT SHORT_LIVED PROCESSES extra_root type FUN_ENTRY LONG_LIVED CODE fun_entry type ATOM_TXT LONG_LIVED ATOM atom_text type BEAM_REGISTER EHEAP PROCESSES beam_register @@ -164,6 +162,7 @@ type MSG_REF FIXED_SIZE PROCESSES msg_ref type MSG_ROOTS TEMPORARY PROCESSES msg_roots type ROOTSET TEMPORARY PROCESSES root_set type LOADER_TMP TEMPORARY CODE loader_tmp +type PREPARED_CODE SHORT_LIVED CODE prepared_code type BIF_TIMER_TABLE LONG_LIVED SYSTEM bif_timer_table type SL_BIF_TIMER SHORT_LIVED PROCESSES bif_timer_sl type LL_BIF_TIMER STANDARD PROCESSES bif_timer_ll @@ -188,7 +187,10 @@ type PORT_TABLE LONG_LIVED SYSTEM port_tab type TIMER_WHEEL LONG_LIVED SYSTEM timer_wheel type DRV DRIVER SYSTEM drv_internal type DRV_BINARY BINARY BINARIES drv_binary -type DRIVER STANDARD SYSTEM driver +type DRIVER DRIVER SYSTEM driver +type DRV_CMD_DATA DRIVER SYSTEM driver_command_data +type DRV_CTRL_DATA DRIVER SYSTEM driver_control_data +type DRV_CALL_DATA DRIVER SYSTEM driver_call_data type NIF DRIVER SYSTEM nif_internal type BINARY BINARY BINARIES binary type NBIF_TABLE SYSTEM SYSTEM nbif_tab @@ -196,14 +198,12 @@ type ARG_REG STANDARD PROCESSES arg_reg type PROC_DICT STANDARD PROCESSES proc_dict type CALLS_BUF STANDARD PROCESSES calls_buf type BPD STANDARD SYSTEM bpd -type PORT_NAME STANDARD SYSTEM port_name type LINEBUF STANDARD SYSTEM line_buf type IOQ STANDARD SYSTEM io_queue type BITS_BUF STANDARD SYSTEM bits_buf type TMP_DIST_BUF TEMPORARY SYSTEM tmp_dist_buf type ASYNC_DATA LONG_LIVED SYSTEM internal_async_data type ESTACK TEMPORARY SYSTEM estack -type PORT_CALL_BUF TEMPORARY SYSTEM port_call_buf type DB_TABLE ETS ETS db_tab type DB_FIXATION SHORT_LIVED ETS db_fixation type DB_FIX_DEL SHORT_LIVED ETS fixed_del @@ -233,14 +233,14 @@ type DDLL_HANDLE STANDARD SYSTEM ddll_handle type DDLL_ERRCODES LONG_LIVED SYSTEM ddll_errcodes type DDLL_TMP_BUF TEMPORARY SYSTEM ddll_tmp_buf type PORT_TASK SHORT_LIVED SYSTEM port_task -type PORT_TASKQ SHORT_LIVED SYSTEM port_task_queue +type PT_HNDL_LIST SHORT_LIVED SYSTEM port_task_handle_list type MISC_OP_LIST SHORT_LIVED SYSTEM misc_op_list type PORT_NAMES SHORT_LIVED SYSTEM port_names -type PORT_DATA_LOCK STANDARD SYSTEM port_data_lock +type PORT_DATA_LOCK DRIVER SYSTEM port_data_lock type NODES_MON STANDARD PROCESSES nodes_monitor -type PROCS_TPROC_EL SHORT_LIVED PROCESSES processes_term_proc_el -type PROCS_CNKINF SHORT_LIVED PROCESSES processes_chunk_info -type PROCS_PIDS SHORT_LIVED PROCESSES processes_pids +type PTAB_LIST_DEL SHORT_LIVED PROCESSES ptab_list_deleted_el +type PTAB_LIST_CNKI SHORT_LIVED PROCESSES ptab_list_chunk_info +type PTAB_LIST_PIDS SHORT_LIVED PROCESSES ptab_list_pids type RE_TMP_BUF TEMPORARY SYSTEM re_tmp_buf type RE_SUBJECT SHORT_LIVED SYSTEM re_subject type RE_HEAP STANDARD SYSTEM re_heap @@ -263,6 +263,11 @@ type ZLIB STANDARD SYSTEM zlib type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q +type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q +type PROC_INTERVAL LONG_LIVED SYSTEM process_interval +type BUSY_CALLER_TAB SHORT_LIVED SYSTEM busy_caller_table +type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller +type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -321,6 +326,8 @@ type SL_PTIMER SHORT_LIVED SYSTEM ptimer_sl type LL_PTIMER STANDARD SYSTEM ptimer_ll type SYS_MSG_Q SHORT_LIVED PROCESSES system_messages_queue type FP_EXCEPTION LONG_LIVED SYSTEM fp_exception +type LL_MPATHS LONG_LIVED SYSTEM ll_migration_paths +type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths +endif +if hipe @@ -339,7 +346,6 @@ type SSB SHORT_LIVED PROCESSES ssb +if halfword -type SBMBC_LOW SBMBC_LOW SYSTEM small_block_mbc_low type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh type NLINK_LH STANDARD_LOW PROCESSES nlink_lh @@ -354,6 +360,7 @@ type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh type NLINK_SH STANDARD_LOW PROCESSES nlink_sh type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request +type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request +else # "fullword" @@ -371,6 +378,7 @@ type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request +type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request +endif @@ -390,6 +398,7 @@ type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds type POLL_RES_EVS LONG_LIVED SYSTEM poll_result_events type FD_STATUS LONG_LIVED SYSTEM fd_status +type SELECT_FDS LONG_LIVED SYSTEM select_fds +if unix @@ -414,28 +423,4 @@ type CON_VPRINTF_BUF TEMPORARY SYSTEM con_vprintf_buf +endif -+if vxworks - -type SYS_TMP_BUF LONG_LIVED SYSTEM sys_tmp_buf -type PEND_DATA SYSTEM SYSTEM pending_data -type FD_TAB LONG_LIVED SYSTEM fd_tab -type FD_ENTRY_BUF SYSTEM SYSTEM fd_entry_buf - -+endif - -+if ose - -type SYS_TMP_BUF LONG_LIVED SYSTEM sys_tmp_buf -type PUTENV_STR SYSTEM SYSTEM putenv_string -type GETENV_STR SYSTEM SYSTEM getenv_string -type GETENV_STATE SYSTEM SYSTEM getenv_state -type SIG_ENTRY SYSTEM SYSTEM sig_entry -type DRIVER_DATA SYSTEM SYSTEM driver_data -type PGM_TAB SYSTEM SYSTEM pgm_tab -type PGM_ENTRY SYSTEM SYSTEM pgm_entry -type PRT_TAB SYSTEM SYSTEM prt_tab -type PRT_ENTRY SYSTEM SYSTEM prt_entry - -+endif - # ---------------------------------------------------------------------------- diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 97ba306a79..825b68bb85 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -71,20 +71,16 @@ #define ALLOC_ZERO_EQ_NULL 0 +#ifndef ERTS_MSEG_FLG_2POW +# define ERTS_MSEG_FLG_2POW 0 +#endif +#ifndef ERTS_MSEG_FLG_NONE +# define ERTS_MSEG_FLG_NONE 0 +#endif + static int atoms_initialized = 0; static int initialized = 0; -int erts_have_sbmbc_alloc; - -#if HAVE_ERTS_MSEG - -#define INV_MSEG_UNIT_MASK ((UWord) (mseg_unit_size - 1)) -#define MSEG_UNIT_MASK (~INV_MSEG_UNIT_MASK) -#define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK) -#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + INV_MSEG_UNIT_MASK) - -#endif - #define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1)) #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_FLOOR(X) ((X) & SYS_ALLOC_CARRIER_MASK) @@ -104,29 +100,58 @@ int erts_have_sbmbc_alloc; static Uint sys_alloc_carrier_size; #if HAVE_ERTS_MSEG static Uint max_mseg_carriers; -static Uint mseg_unit_size; #endif #define ONE_GIGA (1000000000) -#define INC_CC(CC) ((CC).no == ONE_GIGA - 1 \ - ? ((CC).giga_no++, (CC).no = 0) \ - : (CC).no++) +#define ERTS_ALC_CC_GIGA_VAL(CC) ((CC) / ONE_GIGA) +#define ERTS_ALC_CC_VAL(CC) ((CC) % ONE_GIGA) + +#define INC_CC(CC) ((CC)++) + +#define DEC_CC(CC) ((CC)--) + +/* Multi block carrier (MBC) memory layout in R16: + +Empty MBC: +[Carrier_t|pad|Block_t L0T|fhdr| free... ] + +MBC after allocating first block: +[Carrier_t|pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] + +MBC after allocating second block: +[Carrier_t|pad|Block_t 000| udata |pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] + +MBC after deallocating first block: +[Carrier_t|pad|Block_t 00T|fhdr| free |FreeBlkFtr_t|Block_t 0P0| udata |pad|Block_t L0T|fhdr| free... ] + + + udata = Allocated user data + pad = Padding to ensure correct alignment for user data + fhdr = Allocator specific header to keep track of free block + free = Unused free memory + T = This block is free (THIS_FREE_BLK_HDR_FLG) + P = Previous block is free (PREV_FREE_BLK_HDR_FLG) + L = Last block in carrier (LAST_BLK_HDR_FLG) +*/ -#define DEC_CC(CC) ((CC).no == 0 \ - ? ((CC).giga_no--, (CC).no = ONE_GIGA - 1) \ - : (CC).no--) +/* Single block carrier (SBC): +[Carrier_t|pad|Block_t 111| udata... ] +*/ -/* ... */ /* Blocks ... */ -#define SBC_BLK_FTR_FLG (((UWord) 1) << 0) +#define UNUSED0_BLK_FTR_FLG (((UWord) 1) << 0) #define UNUSED1_BLK_FTR_FLG (((UWord) 1) << 1) #define UNUSED2_BLK_FTR_FLG (((UWord) 1) << 2) -#define ABLK_HDR_SZ (sizeof(Block_t)) -#define FBLK_FTR_SZ (sizeof(UWord)) +#if MBC_ABLK_OFFSET_BITS +# define ABLK_HDR_SZ (offsetof(Block_t,u)) +#else +# define ABLK_HDR_SZ (sizeof(Block_t)) +#endif +#define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t)) #define UMEMSZ2BLKSZ(AP, SZ) \ (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size \ @@ -136,88 +161,236 @@ static Uint mseg_unit_size; #define UMEM2BLK(P) ((Block_t *) (((char *) (P)) - ABLK_HDR_SZ)) #define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ)) -#define PREV_BLK_SZ(B) \ - ((UWord) (*(((UWord *) (B)) - 1) & SZ_MASK)) +#define PREV_BLK_SZ(B) ((UWord) (((FreeBlkFtr_t *)(B))[-1])) #define SET_BLK_SZ_FTR(B, SZ) \ - (*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ)) - -#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) -#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) -#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) + (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) -#define SET_BLK_SZ(B, SZ) \ +#define SET_MBC_ABLK_SZ(B, SZ) \ + (ASSERT(((SZ) & FLG_MASK) == 0), \ + (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ)) +#define SET_MBC_FBLK_SZ(B, SZ) \ + (ASSERT(((SZ) & FLG_MASK) == 0), \ + (B)->bhdr = (((B)->bhdr) & ~MBC_FBLK_SZ_MASK) | (SZ)) +#define SET_SBC_BLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ - (*((Block_t *) (B)) = ((*((Block_t *) (B)) & FLG_MASK) | (SZ)))) -#define SET_BLK_FREE(B) \ - (*((Block_t *) (B)) |= THIS_FREE_BLK_HDR_FLG) -#define SET_BLK_ALLOCED(B) \ - (*((Block_t *) (B)) &= ~THIS_FREE_BLK_HDR_FLG) -#define SET_PREV_BLK_FREE(B) \ - (*((Block_t *) (B)) |= PREV_FREE_BLK_HDR_FLG) + (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ)) +#define SET_PREV_BLK_FREE(AP,B) \ + (ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \ + ASSERT(!IS_FREE_BLK(B)), \ + (B)->bhdr |= PREV_FREE_BLK_HDR_FLG) #define SET_PREV_BLK_ALLOCED(B) \ - (*((Block_t *) (B)) &= ~PREV_FREE_BLK_HDR_FLG) + ((B)->bhdr &= ~PREV_FREE_BLK_HDR_FLG) #define SET_LAST_BLK(B) \ - (*((Block_t *) (B)) |= LAST_BLK_HDR_FLG) + ((B)->bhdr |= LAST_BLK_HDR_FLG) #define SET_NOT_LAST_BLK(B) \ - (*((Block_t *) (B)) &= ~LAST_BLK_HDR_FLG) + ((B)->bhdr &= ~LAST_BLK_HDR_FLG) #define SBH_THIS_FREE THIS_FREE_BLK_HDR_FLG -#define SBH_THIS_ALLOCED ((UWord) 0) #define SBH_PREV_FREE PREV_FREE_BLK_HDR_FLG -#define SBH_PREV_ALLOCED ((UWord) 0) #define SBH_LAST_BLK LAST_BLK_HDR_FLG -#define SBH_NOT_LAST_BLK ((UWord) 0) -#define SET_BLK_HDR(B, Sz, F) \ - (ASSERT(((Sz) & FLG_MASK) == 0), *((Block_t *) (B)) = ((Sz) | (F))) + +#if MBC_ABLK_OFFSET_BITS + +# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << ERTS_SUPER_ALIGN_BITS) + +# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> ERTS_SACRR_UNIT_SHIFT) + +# define SET_MBC_ABLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ + ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ + (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << MBC_ABLK_OFFSET_SHIFT))) + +# define SET_MBC_FBLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \ + ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (B)->bhdr = ((Sz) | (F)), \ + (B)->u.carrier = (C)) + +# define IS_MBC_FIRST_ABLK(AP,B) \ + ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ + && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) + +# define IS_MBC_FIRST_FBLK(AP,B) \ + ((char*)(B) == (char*)((B)->u.carrier) + MBC_HEADER_SIZE(AP)) + +# define IS_MBC_FIRST_BLK(AP,B) \ + (IS_FREE_BLK(B) ? IS_MBC_FIRST_FBLK(AP,B) : IS_MBC_FIRST_ABLK(AP,B)) + +# define SET_BLK_FREE(B) \ + (ASSERT(!IS_PREV_BLK_FREE(B)), \ + (B)->u.carrier = ABLK_TO_MBC(B), \ + (B)->bhdr |= THIS_FREE_BLK_HDR_FLG, \ + (B)->bhdr &= (MBC_ABLK_SZ_MASK|FLG_MASK)) + +# define SET_BLK_ALLOCED(B) \ + (ASSERT(((B)->bhdr & (MBC_ABLK_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG, \ + (B)->bhdr |= (BLK_CARRIER_OFFSET(B,(B)->u.carrier) << MBC_ABLK_OFFSET_SHIFT)) + +#else /* !MBC_ABLK_OFFSET_BITS */ + +# define MBC_SZ_MAX_LIMIT ((UWord)~0) + +# define SET_MBC_ABLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & FLG_MASK) == 0), \ + ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ + ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ + (B)->bhdr = ((Sz) | (F)), \ + (B)->carrier = (C)) + +# define SET_MBC_FBLK_HDR(B, Sz, F, C) \ + (ASSERT(((Sz) & FLG_MASK) == 0), \ + ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (B)->bhdr = ((Sz) | (F)), \ + (B)->carrier = (C)) + +# define IS_MBC_FIRST_BLK(AP,B) \ + ((char*)(B) == (char*)((B)->carrier) + MBC_HEADER_SIZE(AP)) +# define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B) +# define IS_MBC_FIRST_FBLK(AP,B) IS_MBC_FIRST_BLK(AP,B) + +# define SET_BLK_FREE(B) \ + (ASSERT(!IS_PREV_BLK_FREE(B)), \ + (B)->bhdr |= THIS_FREE_BLK_HDR_FLG) + +# define SET_BLK_ALLOCED(B) \ + ((B)->bhdr &= ~THIS_FREE_BLK_HDR_FLG) + +#endif /* !MBC_ABLK_OFFSET_BITS */ + +#define SET_SBC_BLK_HDR(B, Sz) \ + (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG))) + #define BLK_UMEM_SZ(B) \ (BLK_SZ(B) - (ABLK_HDR_SZ)) #define IS_PREV_BLK_FREE(B) \ - (*((Block_t *) (B)) & PREV_FREE_BLK_HDR_FLG) + ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define IS_PREV_BLK_ALLOCED(B) \ (!IS_PREV_BLK_FREE((B))) -#define IS_FREE_BLK(B) \ - (*((Block_t *) (B)) & THIS_FREE_BLK_HDR_FLG) #define IS_ALLOCED_BLK(B) \ (!IS_FREE_BLK((B))) #define IS_LAST_BLK(B) \ - (*((Block_t *) (B)) & LAST_BLK_HDR_FLG) + ((B)->bhdr & LAST_BLK_HDR_FLG) #define IS_NOT_LAST_BLK(B) \ (!IS_LAST_BLK((B))) #define GET_LAST_BLK_HDR_FLG(B) \ - (*((Block_t*) (B)) & LAST_BLK_HDR_FLG) + ((B)->bhdr & LAST_BLK_HDR_FLG) #define GET_THIS_FREE_BLK_HDR_FLG(B) \ - (*((Block_t*) (B)) & THIS_FREE_BLK_HDR_FLG) + ((B)->bhdr & THIS_FREE_BLK_HDR_FLG) #define GET_PREV_FREE_BLK_HDR_FLG(B) \ - (*((Block_t*) (B)) & PREV_FREE_BLK_HDR_FLG) + ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define GET_BLK_HDR_FLGS(B) \ - (*((Block_t*) (B)) & FLG_MASK) - -#define IS_FIRST_BLK(B) \ - (IS_PREV_BLK_FREE((B)) && (PREV_BLK_SZ((B)) == 0)) -#define IS_NOT_FIRST_BLK(B) \ - (!IS_FIRST_BLK((B))) - -#define SET_SBC_BLK_FTR(FTR) \ - ((FTR) = (0 | SBC_BLK_FTR_FLG)) -#define SET_MBC_BLK_FTR(FTR) \ - ((FTR) = 0) - -#define IS_SBC_BLK(B) \ - (IS_PREV_BLK_FREE((B)) && (((UWord *) (B))[-1] & SBC_BLK_FTR_FLG)) -#define IS_MBC_BLK(B) \ - (!IS_SBC_BLK((B))) + ((B)->bhdr & FLG_MASK) #define NXT_BLK(B) \ - ((Block_t *) (((char *) (B)) + BLK_SZ((B)))) + (ASSERT(IS_MBC_BLK(B)), \ + (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B)))) #define PREV_BLK(B) \ ((Block_t *) (((char *) (B)) - PREV_BLK_SZ((B)))) +#define BLK_AFTER(B,Sz) \ + ((Block_t *) (((char *) (B)) + (Sz))) + +#define BLK_SZ(B) ((B)->bhdr & (((B)->bhdr & THIS_FREE_BLK_HDR_FLG) ? MBC_FBLK_SZ_MASK : MBC_ABLK_SZ_MASK)) + /* Carriers ... */ +/* #define ERTS_ALC_CPOOL_DEBUG */ + +#if defined(DEBUG) && !defined(ERTS_ALC_CPOOL_DEBUG) +# define ERTS_ALC_CPOOL_DEBUG +#endif + +#ifndef ERTS_SMP +# undef ERTS_ALC_CPOOL_DEBUG +#endif + +#ifdef ERTS_ALC_CPOOL_DEBUG +# define ERTS_ALC_CPOOL_ASSERT(A) \ + ((void) ((A) \ + ? 1 \ + : (erts_alcu_assert_failed(#A, \ + (char *) __FILE__, \ + __LINE__, \ + (char *) __func__), \ + 0))) +#else +# define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1) +#endif + +#ifdef ERTS_SMP +#define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit) +#else +#define ERTS_ALC_IS_CPOOL_ENABLED(A) (0) +#endif + +#ifdef ERTS_SMP + +#define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000 +#define ERTS_ALC_CPOOL_ALLOC_OP_INC 8 +#define ERTS_ALC_CPOOL_FREE_OP_DEC 10 + +#define ERTS_ALC_CPOOL_ALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon < ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) { \ + (A)->cpool.disable_abandon += ERTS_ALC_CPOOL_ALLOC_OP_INC; \ + if ((A)->cpool.disable_abandon > ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) \ + (A)->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; \ + } \ +} while (0) + + +#if ERTS_ALC_CPOOL_ALLOC_OP_INC >= ERTS_ALC_CPOOL_FREE_OP_DEC +# error "Implementation assume ERTS_ALC_CPOOL_ALLOC_OP_INC < ERTS_ALC_CPOOL_FREE_OP_DEC" +#endif + +#define ERTS_ALC_CPOOL_REALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= (ERTS_ALC_CPOOL_FREE_OP_DEC \ + - ERTS_ALC_CPOOL_ALLOC_OP_INC); \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#define ERTS_ALC_CPOOL_FREE_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= ERTS_ALC_CPOOL_FREE_OP_DEC; \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#else +#define ERTS_ALC_CPOOL_ALLOC_OP(A) +#define ERTS_ALC_CPOOL_REALLOC_OP(A) +#define ERTS_ALC_CPOOL_FREE_OP(A) +#endif + +#define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) +#define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) + +#ifdef ERTS_SMP +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(sizeof(Carrier_t) \ + - sizeof(ErtsAlcCPoolData_t) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#else +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(sizeof(Carrier_t) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#endif +#define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size) + + #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) #define SBC_CARRIER_HDR_FLAG (((UWord) 1) << 1) @@ -226,20 +399,21 @@ static Uint mseg_unit_size; #define SCH_MBC 0 #define SCH_SBC SBC_CARRIER_HDR_FLAG -#define SET_CARRIER_HDR(C, Sz, F) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F))) +#define SET_CARRIER_HDR(C, Sz, F, AP) \ + (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ + erts_smp_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) -#define BLK2SBC(AP, B) \ - ((Carrier_t *) (((char *) (B)) - (AP)->sbc_header_size)) -#define FBLK2MBC(AP, B) \ - ((Carrier_t *) (((char *) (B)) - (AP)->mbc_header_size)) +#define BLK_TO_SBC(B) \ + ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE)) +#define FIRST_BLK_TO_MBC(AP, B) \ + ((Carrier_t *) (((char *) (B)) - MBC_HEADER_SIZE(AP))) -#define MBC2FBLK(AP, P) \ - ((Block_t *) (((char *) (P)) + (AP)->mbc_header_size)) +#define MBC_TO_FIRST_BLK(AP, P) \ + ((Block_t *) (((char *) (P)) + MBC_HEADER_SIZE(AP))) #define SBC2BLK(AP, P) \ - ((Block_t *) (((char *) (P)) + (AP)->sbc_header_size)) + ((Block_t *) (((char *) (P)) + SBC_HEADER_SIZE)) #define SBC2UMEM(AP, P) \ - ((void *) (((char *) (P)) + ((AP)->sbc_header_size + ABLK_HDR_SZ))) + ((void *) (((char *) (P)) + (SBC_HEADER_SIZE + ABLK_HDR_SZ))) #define IS_MSEG_CARRIER(C) \ ((C)->chdr & MSEG_CARRIER_HDR_FLAG) @@ -250,15 +424,6 @@ static Uint mseg_unit_size; #define IS_MB_CARRIER(C) \ (!IS_SB_CARRIER((C))) -#define SET_MSEG_CARRIER(C) \ - ((C)->chdr |= MSEG_CARRIER_HDR_FLAG) -#define SET_SYS_ALLOC_CARRIER(C) \ - ((C)->chdr &= ~MSEG_CARRIER_HDR_FLAG) -#define SET_SB_CARRIER(C) \ - ((C)->chdr |= SBC_CARRIER_HDR_FLAG) -#define SET_MB_CARRIER(C) \ - ((C)->chdr &= ~SBC_CARRIER_HDR_FLAG) - #define SET_CARRIER_SZ(C, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ ((C)->chdr = ((C)->chdr & FLG_MASK) | (SZ))) @@ -269,6 +434,7 @@ static Uint mseg_unit_size; #define CFLG_FORCE_SYS_ALLOC (1 << 3) #define CFLG_FORCE_SIZE (1 << 4) #define CFLG_MAIN_CARRIER (1 << 5) +#define CFLG_NO_CPOOL (1 << 6) #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void check_blk_carrier(Allctr_t *, Block_t *); @@ -296,11 +462,7 @@ static void check_blk_carrier(Allctr_t *, Block_t *); ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \ && (AP)->mbcs.curr.norm.sys_alloc.size) \ || (!(AP)->mbcs.curr.norm.sys_alloc.no \ - && !(AP)->mbcs.curr.norm.sys_alloc.size)); \ - ASSERT(((AP)->sbmbcs.curr.small_block.no \ - && (AP)->sbmbcs.curr.small_block.size) \ - || (!(AP)->sbmbcs.curr.small_block.no \ - && !(AP)->sbmbcs.curr.small_block.size)) + && !(AP)->mbcs.curr.norm.sys_alloc.size)); #else #define DEBUG_CHECK_CARRIER_NO_SZ(AP) @@ -371,17 +533,6 @@ do { \ + (AP)->mbcs.curr.norm.sys_alloc.size) -#define STAT_SBMBC_ALLOC(AP, CSZ) \ -do { \ - (AP)->sbmbcs.curr.small_block.no++; \ - (AP)->sbmbcs.curr.small_block.size += (CSZ); \ - if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \ - (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \ - if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \ - (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - #define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ do { \ (AP)->mbcs.curr.norm.mseg.no++; \ @@ -398,13 +549,19 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_SBMBC_FREE(AP, CSZ) \ +#define STAT_MBC_CPOOL_FETCH(AP, CRR) \ do { \ - ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \ - (AP)->sbmbcs.curr.small_block.no--; \ - ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \ - (AP)->sbmbcs.curr.small_block.size -= (CSZ); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_ALLOC((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ + (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \ + if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ + (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ + (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \ + if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ + (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ } while (0) #define STAT_MSEG_MBC_FREE(AP, CSZ) \ @@ -425,28 +582,88 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \ +#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_FREE((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ + >= (CRR)->cpool.blocks); \ + (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ + >= (CRR)->cpool.blocks_size); \ + (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \ +} while (0) + +#ifdef ERTS_SMP +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \ +do { \ + (CRR)->cpool.blocks++; \ + (CRR)->cpool.blocks_size += (BSZ); \ +} while (0) +#else +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */ +#endif + +#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ +do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ cstats__->blocks.curr.no++; \ if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \ cstats__->blocks.max.no = cstats__->blocks.curr.no; \ cstats__->blocks.curr.size += (BSZ); \ if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ cstats__->blocks.max.size = cstats__->blocks.curr.size; \ + STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \ } while (0) -#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \ +static ERTS_INLINE int +stat_cpool_mbc_blk_free(Allctr_t *allctr, + Carrier_t *crr, + Carrier_t **busy_pcrr_pp, + UWord blksz) +{ +#ifdef ERTS_SMP + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0); + crr->cpool.blocks--; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz); + crr->cpool.blocks_size -= blksz; + + if (!busy_pcrr_pp || !*busy_pcrr_pp) + return 0; + + ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp); + +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0); + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)) >= 0); +#else + erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks); + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)); +#endif + + return 1; +#else + return 0; +#endif +} + +#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ - ASSERT(cstats__->blocks.curr.no > 0); \ - cstats__->blocks.curr.no--; \ - ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ - cstats__->blocks.curr.size -= (BSZ); \ + if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ + } \ } while (0) /* Debug stuff... */ @@ -472,17 +689,21 @@ do { \ #ifdef DEBUG #ifdef USE_THREADS +# ifdef ERTS_SMP +# define IS_ACTUALLY_BLOCKING (erts_thr_progress_is_blocking()) +# else +# define IS_ACTUALLY_BLOCKING 0 +# endif #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) \ do { \ - if (!(A)->thread_safe) { \ - if (!(A)->debug.saved_tid) { \ + if (!(A)->thread_safe && !IS_ACTUALLY_BLOCKING) { \ + if (!(A)->debug.saved_tid) { \ (A)->debug.tid = erts_thr_self(); \ (A)->debug.saved_tid = 1; \ } \ else { \ ERTS_SMP_LC_ASSERT( \ - ethr_equal_tids((A)->debug.tid, erts_thr_self()) \ - || erts_thr_progress_is_blocking()); \ + ethr_equal_tids((A)->debug.tid, erts_thr_self())); \ } \ } \ } while (0) @@ -493,24 +714,53 @@ do { \ #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif - static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); -static void mbc_free(Allctr_t *allctr, void *p); +static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); +static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); +static void dealloc_block(Allctr_t *, void *, int); + +/* internal data... */ + +#if 0 + +static ERTS_INLINE void * +internal_alloc(UWord size) +{ + void *res = erts_sys_alloc(0, NULL, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} +static ERTS_INLINE void * +internal_realloc(void *ptr, UWord size) +{ + void *res = erts_sys_realloc(0, NULL, ptr, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} + +static ERTS_INLINE void +internal_free(void *ptr) +{ + erts_sys_free(0, NULL, ptr); +} + +#endif /* mseg ... */ #if HAVE_ERTS_MSEG static ERTS_INLINE void * -alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p) +alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; - res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, &allctr->mseg_opt); + res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_alloc); return res; } @@ -521,26 +771,30 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) void *res; res = erts_mseg_realloc_opt(allctr->alloc_no, seg, old_size, new_size_p, - &allctr->mseg_opt); + ERTS_MSEG_FLG_NONE, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_realloc); return res; } static ERTS_INLINE void -alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size) +alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { - erts_mseg_dealloc_opt(allctr->alloc_no, seg, size, &allctr->mseg_opt); + erts_mseg_dealloc_opt(allctr->alloc_no, seg, size, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_dealloc); } #endif static ERTS_INLINE void * -alcu_sys_alloc(Allctr_t *allctr, Uint size) +alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) { void *res; - - res = erts_sys_alloc(0, NULL, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size); + else +#endif + res = erts_sys_alloc(0, NULL, size); INC_CC(allctr->calls.sys_alloc); if (erts_mtrace_enabled) erts_mtrace_crr_alloc(res, allctr->alloc_no, ERTS_ALC_A_SYSTEM, size); @@ -548,11 +802,16 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size) } static ERTS_INLINE void * -alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) +alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) { void *res; - res = erts_sys_realloc(0, NULL, ptr, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_realloc(ERTS_SACRR_UNIT_SZ, ptr, size, old_size); + else +#endif + res = erts_sys_realloc(0, NULL, ptr, size); INC_CC(allctr->calls.sys_realloc); if (erts_mtrace_enabled) erts_mtrace_crr_realloc(res, @@ -564,9 +823,14 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) } static ERTS_INLINE void -alcu_sys_free(Allctr_t *allctr, void *ptr) +alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) { - erts_sys_free(0, NULL, ptr); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + erts_sys_aligned_free(ERTS_SACRR_UNIT_SZ, ptr); + else +#endif + erts_sys_free(0, NULL, ptr); INC_CC(allctr->calls.sys_free); if (erts_mtrace_enabled) erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr); @@ -661,10 +925,35 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } } -static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz); -static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); -static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); +#ifdef ERTS_SMP + +static ERTS_INLINE void +clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + if (crr) { + erts_aint_t max_size; + erts_aint_t new_val; + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + +#ifdef ERTS_ALC_CPOOL_DEBUG + { + erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; + + ERTS_ALC_CPOOL_ASSERT(old_val + == erts_smp_atomic_xchg_relb(&crr->allctr, + new_val)); + } +#else + erts_smp_atomic_set_relb(&crr->allctr, new_val); +#endif + } +} + +#endif #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -686,13 +975,154 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) #endif -erts_aint32_t -erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +static void *mbc_alloc(Allctr_t *allctr, Uint size); + +#ifdef ERTS_SMP +typedef struct { + ErtsAllctrDDBlock_t ddblock__; /* must be first */ + ErtsAlcType_t fix_type; +} ErtsAllctrFixDDBlock_t; +#endif + +static ERTS_INLINE void +dealloc_fix_block(Allctr_t *allctr, + ErtsAlcType_t type, + void *ptr, + int dec_cc_on_redirect) +{ +#ifdef ERTS_SMP + /* May be redirected... */ + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; +#endif + dealloc_block(allctr, ptr, dec_cc_on_redirect); +} + +static ERTS_INLINE void +sched_fix_shrink(Allctr_t *allctr, int on) +{ + if (on && !allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + else if (!on && allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 0; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 0); + } +} + +static ERTS_INLINE void +fix_cpool_check_shrink(Allctr_t *allctr, + ErtsAlcType_t type, + ErtsAlcFixList_t *fix, + Carrier_t **busy_pcrr_pp) +{ + if (fix->u.cpool.shrink_list > 0) { + if (fix->list_size == 0) + fix->u.cpool.shrink_list = 0; + else { + void *p; +#ifdef ERTS_SMP + if (busy_pcrr_pp) { + clear_busy_pool_carrier(allctr, *busy_pcrr_pp); + *busy_pcrr_pp = NULL; + } +#endif + fix->u.cpool.shrink_list--; + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + + fix->u.cpool.allocated--; + dealloc_fix_block(allctr, type, p, 0); + } + } +} + +static ERTS_INLINE void * +fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + void *res; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + res = fix->list; + if (res) { + fix->list = *((void **) res); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + fix->u.cpool.used++; + fix_cpool_check_shrink(allctr, type, fix, NULL); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); + if (res) { + fix->u.cpool.used++; + fix->u.cpool.allocated++; + } + return res; +} + +static ERTS_INLINE void +fix_cpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p, + Carrier_t **busy_pcrr_pp) +{ + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + fix->u.cpool.used--; + + if ((!busy_pcrr_pp || !*busy_pcrr_pp) + && !fix->u.cpool.shrink_list + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); + fix->u.cpool.allocated--; + fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp); + } +} + +static ERTS_INLINE erts_aint32_t +fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) { int all_empty = 1; erts_aint32_t res = 0; int ix, o; - ErtsAlcFixList_t *fix = allctr->fix; int flush = flgs == 0; #ifdef USE_THREADS @@ -701,56 +1131,204 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) #endif for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ErtsAlcType_t type; ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { - fix[ix].limit = fix[ix].max_used; - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); - - } - if (flush) { - fix[ix].limit = 0; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); + if (flush) + fix->u.cpool.shrink_list = fix->list_size; + else if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.cpool.shrink_list = fix->u.cpool.min_list_size; + fix->u.cpool.min_list_size = fix->list_size; } + type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE); for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { - Block_t *blk; void *ptr; - if (!flush && fix[ix].limit >= fix[ix].allocated) + if (fix->u.cpool.shrink_list == 0) break; - if (fix[ix].list_size == 0) + if (fix->list_size == 0) { + fix->u.cpool.shrink_list = 0; break; - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; + } + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + fix->u.cpool.shrink_list--; + fix->u.cpool.allocated--; + dealloc_fix_block(allctr, type, ptr, 0); + } + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + if (fix->list_size != 0) { + if (fix->u.cpool.shrink_list > 0) + res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; + all_empty = 0; + } + } + + if (all_empty) + sched_fix_shrink(allctr, 0); + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif + + return res; +} + +static ERTS_INLINE void * +fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + ErtsAlcFixList_t *fix; + void *res; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - blk = UMEM2BLK(ptr); + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used++; + res = fix->list; + if (res) { + fix->list_size--; + fix->list = *((void **) res); + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + Block_t *blk; + void *p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + blk = UMEM2BLK(p); if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, ptr); + mbc_free(allctr, p, NULL); + fix->u.nocpool.allocated--; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + if (fix->u.nocpool.max_used < fix->u.nocpool.used) + fix->u.nocpool.max_used = fix->u.nocpool.used; + fix->u.nocpool.allocated++; + + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); + + if (!res) { + fix->u.nocpool.allocated--; + fix->u.nocpool.used--; + } + return res; +} + +static ERTS_INLINE void +fix_nocpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p) +{ + Block_t *blk; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used--; + if (fix->u.nocpool.allocated < fix->u.nocpool.limit + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return; + } + fix->u.nocpool.allocated--; + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + fix->u.nocpool.allocated--; + } + + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); +} + +static ERTS_INLINE erts_aint32_t +fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + int all_empty = 1; + erts_aint32_t res = 0; + int ix, o; + int flush = flgs == 0; + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.nocpool.limit = fix->u.nocpool.max_used; + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); - fix[ix].allocated--; } - if (fix[ix].list_size != 0) { - if (fix[ix].limit < fix[ix].allocated) + if (flush) { + fix->u.nocpool.limit = 0; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); + } + for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { + void *ptr; + + if (!flush && fix->u.nocpool.limit >= fix->u.nocpool.allocated) + break; + if (fix->list_size == 0) + break; + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + dealloc_block(allctr, ptr, 0); + fix->u.nocpool.allocated--; + } + if (fix->list_size != 0) { + if (fix->u.nocpool.limit < fix->u.nocpool.allocated) res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; all_empty = 0; } ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } - if (all_empty && allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 0; - erts_set_aux_work_timeout(allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 0); - } + if (all_empty) + sched_fix_shrink(allctr, 0); #ifdef USE_THREADS if (allctr->thread_safe) @@ -760,18 +1338,21 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) return res; } -#ifdef ERTS_SMP +erts_aint32_t +erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc_shrink(allctr, flgs); + else + return fix_nocpool_alloc_shrink(allctr, flgs); +} -#define ERTS_ALCU_DD_FIX_TYPE_OFFS \ - ((sizeof(ErtsAllctrDDBlock_t)-1)/sizeof(UWord) + 1) +static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); -#define ERTS_AU_PREF_ALLOC_IX_MASK \ - ((((UWord) 1) << ERTS_AU_PREF_ALLOC_BITS) - 1) -#define ERTS_AU_PREF_ALLOC_SIZE_MASK \ - ((((UWord) 1) << (sizeof(UWord)*8 - ERTS_AU_PREF_ALLOC_BITS)) - 1) +#ifdef ERTS_SMP -static ERTS_INLINE int -get_pref_allctr(void *extra, Allctr_t **allctr) +static ERTS_INLINE Allctr_t* +get_pref_allctr(void *extra) { ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; int pref_ix; @@ -781,34 +1362,99 @@ get_pref_allctr(void *extra, Allctr_t **allctr) ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(0 <= pref_ix && pref_ix < tspec->size); - *allctr = tspec->allctr[pref_ix]; - return pref_ix; + return tspec->allctr[pref_ix]; } -static ERTS_INLINE void * -get_used_allctr(void *extra, void *p, Allctr_t **allctr, UWord *sizep) -{ - ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra; - void *ptr = (void *) (((char *) p) - sizeof(UWord)); - UWord ainfo = *((UWord *) ptr); - int aix = (int) (ainfo & ERTS_AU_PREF_ALLOC_IX_MASK); - *allctr = tspec->allctr[aix]; - if (sizep) - *sizep = ((ainfo >> ERTS_AU_PREF_ALLOC_BITS) - & ERTS_AU_PREF_ALLOC_SIZE_MASK); - return ptr; -} +#define ERTS_ALC_TS_PREF_LOCK_IF_USED (1) +#define ERTS_ALC_TS_PREF_LOCK_NO (0) -static ERTS_INLINE void * -put_used_allctr(void *p, int ix, UWord size) +/* SMP note: + * get_used_allctr() must be safe WITHOUT locking the allocator while + * concurrent threads may be updating adjacent blocks. + * We rely on getting a consistent result (without atomic op) when reading + * the block header word even if a concurrent thread is updating + * the "PREV_FREE" flag bit. + */ +static ERTS_INLINE Allctr_t* +get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, + Carrier_t **busy_pcrr_pp) { - UWord ainfo = (size >= ERTS_AU_PREF_ALLOC_SIZE_MASK - ? ERTS_AU_PREF_ALLOC_SIZE_MASK - : size); - ainfo <<= ERTS_AU_PREF_ALLOC_BITS; - ainfo |= (UWord) ix; - *((UWord *) p) = ainfo; - return (void *) (((char *) p) + sizeof(UWord)); + Block_t* blk = UMEM2BLK(p); + Carrier_t *crr; + erts_aint_t iallctr; + Allctr_t *used_allctr; + + *busy_pcrr_pp = NULL; + + if (IS_SBC_BLK(blk)) { + crr = BLK_TO_SBC(blk); + if (sizep) + *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ; + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); + } + else { + crr = ABLK_TO_MBC(blk); + + if (sizep) + *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ; + if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr)) + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); + else { + int locked_pref_allctr = 0; + iallctr = erts_smp_atomic_read_ddrb(&crr->allctr); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && pref_allctr->thread_safe) { + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + if (pref_allctr == used_allctr) { + erts_mtx_lock(&pref_allctr->mutex); + locked_pref_allctr = 1; + } + } + + while ((iallctr & ((~FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) { + erts_aint_t act; + + ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + if (act == iallctr) { + *busy_pcrr_pp = crr; + break; + } + iallctr = act; + } + + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) { + if (locked_pref_allctr && used_allctr != pref_allctr) { + /* Was taken out of pool; now owned by someone else */ + erts_mtx_unlock(&pref_allctr->mutex); + } + } + + ERTS_ALC_CPOOL_ASSERT( + (((iallctr & ~FLG_MASK) == (erts_aint_t) pref_allctr) + ? (((iallctr & FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) + || ((iallctr & FLG_MASK) == 0)) + : 1)); + + return used_allctr; + } + } + + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && used_allctr == pref_allctr + && pref_allctr->thread_safe) { + erts_mtx_lock(&pref_allctr->mutex); + } + + return used_allctr; } static void @@ -903,7 +1549,7 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast) } static ERTS_INLINE int -ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) +ddq_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) { int last_elem; int um_refc_ix = 0; @@ -1004,6 +1650,59 @@ store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq) } } +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); + +static void +handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) +{ + ErtsAlcType_t type; + + type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_nocpool_free(allctr, type, ptr); + else { + Block_t *blk = UMEM2BLK(ptr); + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + + if (IS_SBC_BLK(blk)) { + busy_pcrr_p = NULL; + goto doit; + } + + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + doit: + fix_cpool_free(allctr, type, ptr, &busy_pcrr_p); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + DEC_CC(allctr->calls.this_free); + + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +} + +static void +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, int allctr_locked, @@ -1035,7 +1734,6 @@ handle_delayed_dealloc(Allctr_t *allctr, while (1) { Block_t *blk; void *ptr; - int ix; if (use_limit && ++ops > ops_limit) { if (ddq->head.first != ddq->head.unref_end) { @@ -1064,52 +1762,36 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; - INC_CC(allctr->calls.this_free); + blk = UMEM2BLK(ptr); + if (IS_FREE_LAST_MBC_BLK(blk)) { + /* + * A multiblock carrier that previously has been migrated away + * from us and now is back to be deallocated. For more info + * see schedule_dealloc_carrier(). + * + * Note that we cannot use FBLK_TO_MBC(blk) since it + * data has been overwritten by the queue. + */ + Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); + ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + != (erts_smp_atomic_read_nob(&crr->allctr) + & ~FLG_MASK)); - if (fix) { - ErtsAlcType_t type; - - type = (ErtsAlcType_t) ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS]; - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) ptr) = fix[ix].list; - fix[ix].list = ptr; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - continue; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(ptr); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; - fix[ix].allocated--; - } + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + + schedule_dealloc_carrier(allctr, crr); } + else { - blk = UMEM2BLK(ptr); + INC_CC(allctr->calls.this_free); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + if (fix) + handle_delayed_fix_dealloc(allctr, ptr); + else + dealloc_block(allctr, ptr, 1); + } } if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { @@ -1119,6 +1801,12 @@ handle_delayed_dealloc(Allctr_t *allctr, store_earliest_thr_prgr(thr_prgr_p, ddq); } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + check_pending_dealloc_carrier(allctr, + need_thr_progress, + thr_prgr_p, + need_more_work); + if (allctr->thread_safe && !allctr_locked) erts_mtx_unlock(&allctr->mutex); return res; @@ -1131,15 +1819,61 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, int cinit) { if (allctr->fix) - ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS] = (UWord) type; + ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type; - if (ddq_enqueue(type, &allctr->dd.q, ptr, cinit)) + if (ddq_enqueue(&allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(allctr->ix); } #endif #ifdef ERTS_SMP +static void +set_new_allctr_abandon_limit(Allctr_t *allctr); +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr); + + +static ERTS_INLINE void +check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) +{ + Carrier_t *crr; + + if (busy_pcrr_pp && *busy_pcrr_pp) + return; + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return; + + allctr->cpool.check_limit_count--; + if (--allctr->cpool.check_limit_count <= 0) + set_new_allctr_abandon_limit(allctr); + + if (!erts_thr_progress_is_managed_thread()) + return; + + if (allctr->cpool.disable_abandon) + return; + + if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) + return; + + + crr = FBLK_TO_MBC(fblk); + + if (allctr->main_carrier == crr) + return; + + if (crr->cpool.blocks_size > crr->cpool.abandon_limit) + return; + + if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + abandon_carrier(allctr, crr); +} + void erts_alcu_check_delayed_dealloc(Allctr_t *allctr, int limit, @@ -1157,80 +1891,88 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, } #endif -#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ - handle_delayed_dealloc((Allctr), (Locked), 1, \ +#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ + handle_delayed_dealloc((Allctr), (Locked), 1, \ ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) +static void +dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect) +{ + Block_t *blk = UMEM2BLK(ptr); + + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); +#ifndef ERTS_SMP + else + mbc_free(allctr, ptr, NULL); +#else + else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbc_free(allctr, ptr, NULL); + else { + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + mbc_free(allctr, ptr, &busy_pcrr_p); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +#endif +} + /* Multi block carrier alloc/realloc/free ... */ /* NOTE! mbc_alloc() may in case of memory shortage place the requested * block in a sbc. */ static ERTS_INLINE void * -mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) +mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) { Block_t *blk; Uint get_blk_sz; - Uint sbmbct; ASSERT(size); ASSERT(size < allctr->sbc_threshold); *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size); - sbmbct = allctr->sbmbc_threshold; - if (sbmbct) { - if (get_blk_sz < sbmbct) { - *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC; - if (get_blk_sz + allctr->min_block_size > sbmbct) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz += allctr->min_block_size; - } - } - } - -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); - -#ifdef ERTS_SMP - if (!blk && allctr->dd.use) { - if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1)) - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, - *alcu_flgsp); - } -#endif + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0); if (!blk) { - if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) - blk = create_sbmbc(allctr, get_blk_sz); - else { -#if HALFWORD_HEAP - blk = create_carrier(allctr, get_blk_sz, CFLG_MBC|CFLG_FORCE_MSEG); -#else - blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); - if (!blk) { - /* Emergency! We couldn't create the carrier as we wanted. - Try to place it in a sys_alloced sbc. */ - blk = create_carrier(allctr, - size, - (CFLG_SBC - | CFLG_FORCE_SIZE - | CFLG_FORCE_SYS_ALLOC)); - } -#endif + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY + if (!blk) { + /* Emergency! We couldn't create the carrier as we wanted. + Try to place it in a sys_alloced sbc. */ + blk = create_carrier(allctr, + size, + (CFLG_SBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_SYS_ALLOC)); } +#endif } #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { - (*allctr->link_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->unlink_free_block)(allctr, blk); } #endif @@ -1242,9 +1984,9 @@ mbc_alloc_finalize(Allctr_t *allctr, Block_t *blk, Uint org_blk_sz, UWord flags, + Carrier_t *crr, Uint want_blk_sz, - int valid_blk_info, - Uint32 alcu_flgs) + int valid_blk_info) { Uint blk_sz; Uint nxt_blk_sz; @@ -1262,25 +2004,21 @@ mbc_alloc_finalize(Allctr_t *allctr, /* Shrink block... */ blk_sz = want_blk_sz; nxt_blk_sz = org_blk_sz - blk_sz; - SET_BLK_HDR(blk, - blk_sz, - SBH_THIS_ALLOCED|SBH_NOT_LAST_BLK|prev_free_flg); + SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr); - nxt_blk = NXT_BLK(blk); - SET_BLK_HDR(nxt_blk, - nxt_blk_sz, - (SBH_THIS_FREE - | SBH_PREV_ALLOCED - | (flags & LAST_BLK_HDR_FLG))); + nxt_blk = BLK_AFTER(blk, blk_sz); + SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, + SBH_THIS_FREE|(flags & LAST_BLK_HDR_FLG), + crr); if (!(flags & LAST_BLK_HDR_FLG)) { SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); if (!valid_blk_info) { - Block_t *nxt_nxt_blk = NXT_BLK(nxt_blk); - SET_PREV_BLK_FREE(nxt_nxt_blk); + Block_t *nxt_nxt_blk = BLK_AFTER(nxt_blk, nxt_blk_sz); + SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } } - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_NOT_LAST_BLK(blk)); ASSERT(IS_FREE_BLK(nxt_blk)); @@ -1291,40 +2029,41 @@ mbc_alloc_finalize(Allctr_t *allctr, || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk))); ASSERT((flags & LAST_BLK_HDR_FLG) || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk))); - ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk)); + ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(nxt_blk_sz >= allctr->min_block_size); + ASSERT(ABLK_TO_MBC(blk) == crr); + ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } else { + ASSERT(org_blk_sz <= MBC_ABLK_SZ_MASK); blk_sz = org_blk_sz; if (flags & LAST_BLK_HDR_FLG) { if (valid_blk_info) SET_BLK_ALLOCED(blk); else - SET_BLK_HDR(blk, - blk_sz, - SBH_THIS_ALLOCED|SBH_LAST_BLK|prev_free_flg); + SET_MBC_ABLK_HDR(blk, blk_sz, SBH_LAST_BLK|prev_free_flg, crr); } else { if (valid_blk_info) SET_BLK_ALLOCED(blk); else - SET_BLK_HDR(blk, - blk_sz, - SBH_THIS_ALLOCED|SBH_NOT_LAST_BLK|prev_free_flg); - nxt_blk = NXT_BLK(blk); + SET_MBC_ABLK_HDR(blk, blk_sz, prev_free_flg, crr); + nxt_blk = BLK_AFTER(blk, blk_sz); SET_PREV_BLK_ALLOCED(nxt_blk); } ASSERT((flags & LAST_BLK_HDR_FLG) ? IS_LAST_BLK(blk) : IS_NOT_LAST_BLK(blk)); + ASSERT(ABLK_TO_MBC(blk) == crr); } - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_ALLOC_OP(allctr); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); ASSERT(blk_sz >= want_blk_sz); @@ -1341,57 +2080,57 @@ mbc_alloc(Allctr_t *allctr, Uint size) { Block_t *blk; Uint blk_sz; - Uint32 alcu_flgs = 0; - blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs); + blk = mbc_alloc_block(allctr, size, &blk_sz); if (!blk) return NULL; if (IS_MBC_BLK(blk)) mbc_alloc_finalize(allctr, blk, - BLK_SZ(blk), + MBC_FBLK_SZ(blk), GET_BLK_HDR_FLGS(blk), + FBLK_TO_MBC(blk), blk_sz, - 1, - alcu_flgs); + 1); return BLK2UMEM(blk); } static void -mbc_free(Allctr_t *allctr, void *p) +mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) { Uint is_first_blk; Uint is_last_blk; - Uint32 alcu_flgs = 0; Uint blk_sz; Block_t *blk; Block_t *nxt_blk; - + Carrier_t *crr; ASSERT(p); blk = UMEM2BLK(p); - blk_sz = BLK_SZ(blk); - if (blk_sz < allctr->sbmbc_threshold) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; + blk_sz = MBC_ABLK_SZ(blk); ASSERT(IS_MBC_BLK(blk)); ASSERT(blk_sz >= allctr->min_block_size); HARD_CHECK_BLK_CARRIER(allctr, blk); - STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); + + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs); - is_first_blk = IS_FIRST_BLK(blk); + is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); - if (!is_first_blk && IS_PREV_BLK_FREE(blk)) { + if (IS_PREV_BLK_FREE(blk)) { + ASSERT(!is_first_blk); /* Coalesce with previous block... */ blk = PREV_BLK(blk); - (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, blk); - blk_sz += BLK_SZ(blk); - is_first_blk = IS_FIRST_BLK(blk); - SET_BLK_SZ(blk, blk_sz); + blk_sz += MBC_FBLK_SZ(blk); + is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk); + SET_MBC_FBLK_SZ(blk, blk_sz); } else { SET_BLK_FREE(blk); @@ -1400,12 +2139,12 @@ mbc_free(Allctr_t *allctr, void *p) if (is_last_blk) SET_LAST_BLK(blk); else { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, blk_sz); if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); - blk_sz += BLK_SZ(nxt_blk); - SET_BLK_SZ(blk, blk_sz); + (*allctr->unlink_free_block)(allctr, nxt_blk); + blk_sz += MBC_FBLK_SZ(nxt_blk); + SET_MBC_FBLK_SZ(blk, blk_sz); is_last_blk = IS_LAST_BLK(nxt_blk); if (is_last_blk) @@ -1416,39 +2155,40 @@ mbc_free(Allctr_t *allctr, void *p) } } else { - SET_PREV_BLK_FREE(nxt_blk); + SET_PREV_BLK_FREE(allctr, nxt_blk); SET_NOT_LAST_BLK(blk); SET_BLK_SZ_FTR(blk, blk_sz); } } - ASSERT(is_last_blk ? IS_LAST_BLK(blk) : IS_NOT_LAST_BLK(blk)); - ASSERT(is_first_blk ? IS_FIRST_BLK(blk) : IS_NOT_FIRST_BLK(blk)); ASSERT(IS_FREE_BLK(blk)); + ASSERT(!is_last_blk == !IS_LAST_BLK(blk)); + ASSERT(!is_first_blk == !IS_MBC_FIRST_FBLK(allctr, blk)); ASSERT(is_first_blk || IS_PREV_BLK_ALLOCED(blk)); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(blk))); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(is_last_blk || blk == PREV_BLK(NXT_BLK(blk))); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(IS_MBC_BLK(blk)); if (is_first_blk && is_last_blk - && allctr->main_carrier != FBLK2MBC(allctr, blk)) { - if (alcu_flgs & ERTS_ALCU_FLG_SBMBC) - destroy_sbmbc(allctr, blk); - else - destroy_carrier(allctr, blk); + && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { - (*allctr->link_free_block)(allctr, blk, alcu_flgs); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, blk, busy_pcrr_pp); +#endif } } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { void *new_p; Uint old_blk_sz; @@ -1462,17 +2202,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) Uint is_last_blk; #endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */ -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - ASSERT(p); ASSERT(size); ASSERT(size < allctr->sbc_threshold); blk = (Block_t *) UMEM2BLK(p); - old_blk_sz = BLK_SZ(blk); + old_blk_sz = MBC_ABLK_SZ(blk); ASSERT(old_blk_sz >= allctr->min_block_size); @@ -1480,13 +2215,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ + +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) + goto realloc_move; /* Don't want to use carrier in pool */ +#endif + get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); - if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC) - && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz = blk_sz + allctr->min_block_size; - } ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(IS_MBC_BLK(blk)); @@ -1497,6 +2232,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) return p; else if (blk_sz < old_blk_sz) { /* Shrink block... */ + Carrier_t* crr; Block_t *nxt_nxt_blk; Uint diff_sz_val = old_blk_sz - blk_sz; Uint old_blk_sz_val = old_blk_sz; @@ -1516,23 +2252,24 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) return NULL; cand_blk_sz = old_blk_sz; - if (!IS_PREV_BLK_FREE(blk) || IS_FIRST_BLK(blk)) + if (!IS_PREV_BLK_FREE(blk)) { cand_blk = blk; + } else { + ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk)); cand_blk = PREV_BLK(blk); cand_blk_sz += PREV_BLK_SZ(blk); } if (!is_last_blk) { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) - cand_blk_sz += BLK_SZ(nxt_blk); + cand_blk_sz += MBC_FBLK_SZ(nxt_blk); } new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); if (new_blk || cand_blk != blk) goto move_into_new_blk; } @@ -1541,52 +2278,50 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk_sz = old_blk_sz - blk_sz; - if ((is_last_blk || IS_ALLOCED_BLK(NXT_BLK(blk))) + if ((is_last_blk || IS_ALLOCED_BLK(BLK_AFTER(blk,old_blk_sz))) && (nxt_blk_sz < allctr->min_block_size)) return p; HARD_CHECK_BLK_CARRIER(allctr, blk); - SET_BLK_SZ(blk, blk_sz); + nxt_nxt_blk = BLK_AFTER(blk, old_blk_sz); + + SET_MBC_ABLK_SZ(blk, blk_sz); SET_NOT_LAST_BLK(blk); - nxt_blk = NXT_BLK(blk); - SET_BLK_HDR(nxt_blk, - nxt_blk_sz, - SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK); + nxt_blk = BLK_AFTER(blk, blk_sz); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); - ASSERT(BLK_SZ(blk) >= allctr->min_block_size); + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); - if (is_last_blk) - SET_LAST_BLK(nxt_blk); - else { - nxt_nxt_blk = NXT_BLK(nxt_blk); + ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); + + if (!is_last_blk) { if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ - nxt_blk_sz += BLK_SZ(nxt_nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); - SET_BLK_SZ(nxt_blk, nxt_blk_sz); + nxt_blk_sz += MBC_FBLK_SZ(nxt_nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_nxt_blk); - is_last_blk = IS_LAST_BLK(nxt_nxt_blk); - if (is_last_blk) - SET_LAST_BLK(nxt_blk); - else - SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); + is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk); } else { - SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); - SET_PREV_BLK_FREE(nxt_nxt_blk); + SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } + SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, + SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), + crr); + + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_ALLOCED_BLK(blk)); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); ASSERT(blk_sz >= size + ABLK_HDR_SZ); @@ -1594,37 +2329,43 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) ASSERT(IS_FREE_BLK(nxt_blk)); ASSERT(IS_PREV_BLK_ALLOCED(nxt_blk)); - ASSERT(nxt_blk_sz == BLK_SZ(nxt_blk)); + ASSERT(nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(nxt_blk_sz >= allctr->min_block_size); ASSERT(IS_MBC_BLK(nxt_blk)); ASSERT(is_last_blk ? IS_LAST_BLK(nxt_blk) : IS_NOT_LAST_BLK(nxt_blk)); ASSERT(is_last_blk || nxt_blk == PREV_BLK(NXT_BLK(nxt_blk))); ASSERT(is_last_blk || IS_PREV_BLK_FREE(NXT_BLK(nxt_blk))); - + ASSERT(FBLK_TO_MBC(nxt_blk) == crr); + HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, nxt_blk, NULL); +#endif + return p; } /* Need larger block... */ if (!is_last_blk) { - nxt_blk = NXT_BLK(blk); - nxt_blk_sz = BLK_SZ(nxt_blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); + nxt_blk_sz = MBC_BLK_SZ(nxt_blk); if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { + Carrier_t* crr = ABLK_TO_MBC(blk); /* Grow into next block... */ HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); nxt_blk_sz -= blk_sz - old_blk_sz; is_last_blk = IS_LAST_BLK(nxt_blk); if (nxt_blk_sz < allctr->min_block_size) { blk_sz += nxt_blk_sz; - SET_BLK_SZ(blk, blk_sz); + SET_MBC_ABLK_SZ(blk, blk_sz); if (is_last_blk) { SET_LAST_BLK(blk); @@ -1633,45 +2374,44 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #endif } else { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, blk_sz); SET_PREV_BLK_ALLOCED(nxt_blk); #ifdef DEBUG is_last_blk = IS_LAST_BLK(nxt_blk); - nxt_blk_sz = BLK_SZ(nxt_blk); + nxt_blk_sz = MBC_BLK_SZ(nxt_blk); #endif } } else { - SET_BLK_SZ(blk, blk_sz); + SET_MBC_ABLK_SZ(blk, blk_sz); - nxt_blk = NXT_BLK(blk); - SET_BLK_HDR(nxt_blk, - nxt_blk_sz, - SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK); + nxt_blk = BLK_AFTER(blk, blk_sz); + SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE, crr); if (is_last_blk) SET_LAST_BLK(nxt_blk); else SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_FREE_BLK(nxt_blk)); + ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); - + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); - ASSERT(blk_sz == BLK_SZ(blk)); + ASSERT(blk_sz == MBC_BLK_SZ(blk)); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); ASSERT(blk_sz >= size + ABLK_HDR_SZ); ASSERT(IS_MBC_BLK(blk)); ASSERT(!nxt_blk || IS_PREV_BLK_ALLOCED(nxt_blk)); - ASSERT(!nxt_blk || nxt_blk_sz == BLK_SZ(nxt_blk)); + ASSERT(!nxt_blk || nxt_blk_sz == MBC_BLK_SZ(nxt_blk)); ASSERT(!nxt_blk || nxt_blk_sz % sizeof(Unit_t) == 0); ASSERT(!nxt_blk || nxt_blk_sz >= allctr->min_block_size); ASSERT(!nxt_blk || IS_MBC_BLK(nxt_blk)); @@ -1696,30 +2436,34 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) /* Need to grow in another block */ - if (!IS_PREV_BLK_FREE(blk) || IS_FIRST_BLK(blk)) { + if (!IS_PREV_BLK_FREE(blk)) { cand_blk = NULL; cand_blk_sz = 0; } else { + ASSERT(!IS_MBC_FIRST_ABLK(allctr, blk)); cand_blk = PREV_BLK(blk); cand_blk_sz = old_blk_sz + PREV_BLK_SZ(blk); if (!is_last_blk) { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) - cand_blk_sz += BLK_SZ(nxt_blk); + cand_blk_sz += MBC_FBLK_SZ(nxt_blk); } } if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ +#ifdef ERTS_SMP + realloc_move: +#endif #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); if (!new_p) return NULL; sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, busy_pcrr_pp); return new_p; @@ -1732,8 +2476,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); move_into_new_blk: /* * new_blk, and cand_blk have to be correctly set @@ -1743,17 +2486,18 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (new_blk) { mbc_alloc_finalize(allctr, new_blk, - BLK_SZ(new_blk), + MBC_FBLK_SZ(new_blk), GET_BLK_HDR_FLGS(new_blk), + FBLK_TO_MBC(new_blk), blk_sz, - 1, - alcu_flgs); + 1); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, NULL); return new_p; } else { + Carrier_t* crr; Uint new_blk_sz; UWord new_blk_flgs; Uint prev_blk_sz; @@ -1769,16 +2513,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */ + (*allctr->unlink_free_block)(allctr, new_blk); /* prev */ if (is_last_blk) new_blk_flgs |= LAST_BLK_HDR_FLG; else { - nxt_blk = NXT_BLK(blk); + nxt_blk = BLK_AFTER(blk, old_blk_sz); if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); - new_blk_sz += BLK_SZ(nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + new_blk_sz += MBC_FBLK_SZ(nxt_blk); + (*allctr->unlink_free_block)(allctr, nxt_blk); } } @@ -1790,6 +2534,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_p = BLK2UMEM(new_blk); blk_cpy_sz = MIN(blk_sz, old_blk_sz); + crr = FBLK_TO_MBC(new_blk); if (prev_blk_sz >= blk_cpy_sz) sys_memcpy(new_p, p, blk_cpy_sz - ABLK_HDR_SZ); @@ -1800,11 +2545,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk, new_blk_sz, new_blk_flgs, + crr, blk_sz, - 0, - alcu_flgs); + 0); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); return new_p; } @@ -1812,138 +2558,689 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #endif /* !MBC_REALLOC_ALWAYS_MOVES */ } -#ifdef DEBUG +#ifdef ERTS_SMP -#if HAVE_ERTS_MSEG -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % mseg_unit_size == 0) -#else -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) -#endif +#define ERTS_ALC_MAX_DEALLOC_CARRIER 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10 +#define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 +#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5 +#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100 +#define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 -#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) \ -do { \ - ASSERT(IS_FIRST_BLK((B))); \ - ASSERT(IS_LAST_BLK((B))); \ - ASSERT((CSZ) == CARRIER_SZ((C))); \ - ASSERT((BSZ) == BLK_SZ((B))); \ - ASSERT((BSZ) % sizeof(Unit_t) == 0); \ - if ((SBC)) { \ - ASSERT(IS_SBC_BLK((B))); \ - ASSERT(IS_SB_CARRIER((C))); \ - } \ - else { \ - ASSERT(IS_MBC_BLK((B))); \ - ASSERT(IS_MB_CARRIER((C))); \ - } \ - if ((MSEGED)) { \ - ASSERT(IS_MSEG_CARRIER((C))); \ - ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ)); \ - } \ - else { \ - ASSERT(IS_SYS_ALLOC_CARRIER((C))); \ - ASSERT((CSZ) % sizeof(Unit_t) == 0); \ - } \ -} while (0) +#define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) +#define ERTS_ALC_CPOOL_PTR_DEL_MRK (((erts_aint_t) 1) << 1) -#else -#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) +#define ERTS_ALC_CPOOL_PTR_MRKS \ + (ERTS_ALC_CPOOL_PTR_MOD_MRK | ERTS_ALC_CPOOL_PTR_DEL_MRK) + +/* + * When setting multiple mod markers we always + * set mod markers in pointer order and always + * on next pointers before prev pointers. + */ + +typedef union { + ErtsAlcCPoolData_t sentinel; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAlcCPoolData_t))]; +} ErtsAlcCrrPool_t; + +#if ERTS_ALC_A_INVALID != 0 +# error "Carrier pool implementation assumes ERTS_ALC_A_INVALID == 0" +#endif +#if ERTS_ALC_A_MIN <= ERTS_ALC_A_INVALID +# error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID" #endif -static Block_t * -create_sbmbc(Allctr_t *allctr, Uint umem_sz) +/* + * The pool is only allowed to be manipulated by managed + * threads except in the alloc_SUITE:cpool case. In this + * test case carrier_pool[ERTS_ALC_A_INVALID] will be + * used. + */ + +static ErtsAlcCrrPool_t carrier_pool[ERTS_ALC_A_MAX+1] erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#define ERTS_ALC_CPOOL_MAX_BACKOFF (1 << 8) + +static int +backoff(int n) { - Block_t *blk; - Uint blk_sz; - Uint crr_sz = allctr->sbmbc_size; - Carrier_t *crr; + int i; -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz); + for (i = 0; i < n; i++) + ERTS_SPIN_BODY; + + if (n >= ERTS_ALC_CPOOL_MAX_BACKOFF) + return ERTS_ALC_CPOOL_MAX_BACKOFF; else -#endif - crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz); + return n << 1; +} - INC_CC(allctr->calls.sbmbc_alloc); - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC); +static int +cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *cpdp = sentinel; + Carrier_t *tmp_crr; + + while (1) { + cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~FLG_MASK); + if (cpdp == sentinel) + return 0; + tmp_crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + if (tmp_crr == crr) + return 1; + } +} - blk = MBC2FBLK(allctr, crr); +static int +cpool_is_empty(Allctr_t *allctr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + return ((erts_atomic_read_rb(&sentinel->next) == (erts_aint_t) sentinel) + && (erts_atomic_read_rb(&sentinel->prev) == (erts_aint_t) sentinel)); +} -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif +static ERTS_INLINE ErtsAlcCPoolData_t * +cpool_aint2cpd(erts_aint_t aint) +{ + return (ErtsAlcCPoolData_t *) (aint & ~ERTS_ALC_CPOOL_PTR_MRKS); +} - blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); +static ERTS_INLINE erts_aint_t +cpool_read(erts_atomic_t *aptr) +{ + return erts_atomic_read_acqb(aptr); +} - SET_MBC_BLK_FTR(((UWord *) blk)[-1]); - SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); +static ERTS_INLINE void +cpool_init(erts_atomic_t *aptr, erts_aint_t val) +{ + erts_atomic_set_nob(aptr, val); +} -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - *((Carrier_t **) NXT_BLK(blk)) = crr; +static ERTS_INLINE void +cpool_set_mod_marked(erts_atomic_t *aptr, erts_aint_t new, erts_aint_t old) +{ +#ifdef ERTS_ALC_CPOOL_DEBUG + erts_aint_t act = erts_atomic_xchg_relb(aptr, new); + ERTS_ALC_CPOOL_ASSERT(act == (old | ERTS_ALC_CPOOL_PTR_MOD_MRK)); +#else + erts_atomic_set_relb(aptr, new); #endif +} - link_carrier(&allctr->sbmbc_list, crr); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz += sizeof(UWord); -#endif +static ERTS_INLINE erts_aint_t +cpool_try_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) +{ + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + return erts_atomic_cmpxchg_nob(aptr, exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, exp); +} - STAT_SBMBC_ALLOC(allctr, crr_sz); - CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif - if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); +static ERTS_INLINE erts_aint_t +cpool_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) +{ + int b; + erts_aint_t act; + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + while (1) { + act = erts_atomic_cmpxchg_nob(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; + b = 1; + do { + if ((act & ~ERTS_ALC_CPOOL_PTR_MOD_MRK) != exp) + return act; + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } while (act != exp); + } +} - DEBUG_SAVE_ALIGNMENT(crr); - return blk; +static ERTS_INLINE erts_aint_t +cpool_mod_mark(erts_atomic_t *aptr) +{ + int b; + erts_aint_t act, exp; + act = cpool_read(aptr); + while (1) { + b = 1; + while (act & ERTS_ALC_CPOOL_PTR_MOD_MRK) { + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } + exp = act; + act = erts_atomic_cmpxchg_acqb(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; + } } static void -destroy_sbmbc(Allctr_t *allctr, Block_t *blk) +cpool_insert(Allctr_t *allctr, Carrier_t *crr) { - Uint crr_sz; - Carrier_t *crr; + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; - ASSERT(IS_FIRST_BLK(blk)); + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (erts_aint_t) allctr); - ASSERT(IS_MBC_BLK(blk)); + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + (erts_aint_t) crr->cpool.blocks_size); + erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + (erts_aint_t) crr->cpool.blocks); + erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + (erts_aint_t) CARRIER_SZ(crr)); + erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); - crr = FBLK2MBC(allctr, blk); - crr_sz = CARRIER_SZ(crr); + erts_smp_atomic_set_nob(&crr->allctr, + ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); -#ifdef DEBUG - if (!allctr->stopped) { - ASSERT(IS_LAST_BLK(blk)); + /* + * We search in 'next' direction and begin by passing + * one element before trying to insert. This in order to + * avoid contention with threads fetching elements. + */ -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); - HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); + val = cpool_read(&sentinel->next); + + /* Find a predecessor to be, and set mod marker on its next ptr */ + + while (1) { + cpd1p = cpool_aint2cpd(val); + if (cpd1p == sentinel) { + val = cpool_mod_mark(&cpd1p->next); + break; + } + val = cpool_read(&cpd1p->next); + if (!(val & ERTS_ALC_CPOOL_PTR_MRKS)) { + erts_aint_t tmp = cpool_try_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) { + val = tmp; + break; + } + val = tmp; + } + } + + /* Set mod marker on prev ptr of the to be successor */ + + cpd2p = cpool_aint2cpd(val); + + cpool_init(&crr->cpool.next, (erts_aint_t) cpd2p); + cpool_init(&crr->cpool.prev, (erts_aint_t) cpd1p); + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Write pointers to this element in successor and predecessor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd2p); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd1p); +} + +static void +cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; +#ifdef ERTS_ALC_CPOOL_DEBUG + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; #endif + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool); + + /* Set mod marker on next ptr of our predecessor */ + + val = (erts_aint_t) &crr->cpool; + while (1) { + erts_aint_t tmp; + cpd1p = cpool_aint2cpd(cpool_read(&crr->cpool.prev)); + tmp = cpool_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) + break; } + + /* Set mod marker on our next ptr */ + + val = cpool_mod_mark(&crr->cpool.next); + + /* Set mod marker on the prev ptr of our successor */ + + cpd2p = cpool_aint2cpd(val); + + val = (erts_aint_t) &crr->cpool; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Set mod marker on our prev ptr */ + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&crr->cpool.prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Write pointers past this element in predecessor and successor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) cpd2p, + (erts_aint_t) &crr->cpool); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) cpd1p, + (erts_aint_t) &crr->cpool); + + /* Repleace mod markers with delete markers on this element */ + cpool_set_mod_marked(&crr->cpool.next, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + cpool_set_mod_marked(&crr->cpool.prev, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + + crr->cpool.thr_prgr = erts_thr_progress_later(NULL); + + erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size, + -((erts_aint_t) crr->cpool.blocks_size)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks, + -((erts_aint_t) crr->cpool.blocks)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size, + -((erts_aint_t) CARRIER_SZ(crr))); + erts_atomic_dec_wb(&prev_allctr->cpool.stat.no_carriers); + +} + +static Carrier_t * +cpool_fetch(Allctr_t *allctr, UWord size) +{ + int i; + Carrier_t *crr; + ErtsAlcCPoolData_t *cpdp; + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + + i = 0; + + /* First; check our own pending dealloc carrier list... */ + crr = allctr->cpool.dc_list.last; + while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + unlink_carrier(&allctr->cpool.dc_list, crr); +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ((erts_aint_t) allctr)) + == (((erts_aint_t) allctr) & ~FLG_MASK)); +#else + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); #endif + return crr; + } + crr = crr->prev; + i++; + } - STAT_SBMBC_FREE(allctr, crr_sz); + /* ... then the pool ... */ - unlink_carrier(&allctr->sbmbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); + /* + * We search in 'prev' direction and begin by passing + * one element before trying to fetch. This in order to + * avoid contention with threads inserting elements. + */ - INC_CC(allctr->calls.sbmbc_free); + cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev)); + if (cpdp == sentinel) + return NULL; -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - erts_free(ERTS_ALC_T_SBMBC_LOW, crr); + while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + erts_aint_t exp; + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) { + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + return NULL; + i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */ + } + crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY)) + == ERTS_CRR_ALCTR_FLG_IN_POOL) + && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + erts_aint_t act; + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~FLG_MASK)), crr); + return crr; + } + } + i++; + } + return NULL; +} + +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + Carrier_t *crr = allctr->cpool.dc_list.first; + + if (crr) { + ErtsThrPrgrVal current = erts_thr_progress_current(); + int i = 0; + + do { + Carrier_t *dcrr; + + if (!erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + break; + + dcrr = crr; + crr = crr->next; + dealloc_carrier(allctr, dcrr, 1); + i++; + } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); + + allctr->cpool.dc_list.first = crr; + if (!crr) + allctr->cpool.dc_list.last = NULL; + else { + crr->prev = NULL; + + if (need_more_work) { + ERTS_ALC_CPOOL_ASSERT(need_thr_progress && thr_prgr_p); + if (erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + *need_more_work = 1; + else { + *need_thr_progress = 1; + if (*thr_prgr_p == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(crr->cpool.thr_prgr, + *thr_prgr_p) < 0) { + *thr_prgr_p = crr->cpool.thr_prgr; + } + } + } + } + } +} + +static void +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + Allctr_t *orig_allctr; + int check_pending_dealloc; + erts_aint_t max_size; + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + dealloc_carrier(allctr, crr, 1); + return; + } + + orig_allctr = crr->cpool.orig_allctr; + + if (allctr != orig_allctr) { + Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); + int cinit = orig_allctr->dd.ix - allctr->dd.ix; + + /* + * We send the carrier to its origin for deallocation. + * This in order: + * - not to complicate things for the thread specific + * instances of mseg_alloc, and + * - to ensure that we always only reuse empty carriers + * originating from our own thread specific mseg_alloc + * instance which is beneficial on NUMA systems. + * + * The receiver will recognize that this is a carrier to + * deallocate (and not a block which is the common case) + * since the block is an mbc block that is free and last + * in the carrier. + */ + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + == (erts_smp_atomic_read_nob(&crr->allctr) + & ~FLG_MASK)); + + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); + return; + } + + if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID + || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { + dealloc_carrier(allctr, crr, 1); + return; + } + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + crr->next = NULL; + crr->prev = allctr->cpool.dc_list.last; + if (allctr->cpool.dc_list.last) { + check_pending_dealloc = 1; + allctr->cpool.dc_list.last->next = crr; + } + else { + check_pending_dealloc = 0; + allctr->cpool.dc_list.first = crr; + } + allctr->cpool.dc_list.last = crr; + if (check_pending_dealloc) + check_pending_dealloc_carrier(allctr, NULL, NULL, NULL); + erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix); +} + +static ERTS_INLINE void +cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) +{ + erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); + erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); + crr->cpool.orig_allctr = allctr; + crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID; + erts_atomic_init_nob(&crr->cpool.max_size, 0); + crr->cpool.blocks = 0; + crr->cpool.blocks_size = 0; + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + crr->cpool.abandon_limit = 0; + else { + UWord csz = CARRIER_SZ(crr); + UWord limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; + else + limit = (csz/100)*allctr->cpool.util_limit; + crr->cpool.abandon_limit = limit; + } +} + +static void +set_new_allctr_abandon_limit(Allctr_t *allctr) +{ + UWord limit; + UWord csz; + + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + + csz = allctr->mbcs.curr.norm.mseg.size; + csz += allctr->mbcs.curr.norm.sys_alloc.size; + + limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; else + limit = (csz/100)*allctr->cpool.util_limit; + + allctr->cpool.abandon_limit = limit; +} + +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + erts_aint_t max_size; + + STAT_MBC_CPOOL_INSERT(allctr, crr); + + unlink_carrier(&allctr->mbc_list, crr); + + allctr->remove_mbc(allctr, crr); + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + cpool_insert(allctr, crr); + + set_new_allctr_abandon_limit(allctr); +} + +static void +cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) +{ + int i; + UWord noc = 0, csz = 0, nob = 0, bsz = 0; + + /* + * We try to get consistent values, but after + * ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS failed + * tries we give up and present what we got... + */ + for (i = 0; i <= ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS; i++) { + UWord tnoc, tcsz, tnob, tbsz; + + tnoc = (UWord) (nocp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_carriers) + : 0); + tcsz = (UWord) (cszp + ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size) + : 0); + tnob = (UWord) (nobp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks) + : 0); + tbsz = (UWord) (bszp + ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size) + : 0); + if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz) + break; + noc = tnoc; + csz = tcsz; + nob = tnob; + bsz = tbsz; + ERTS_THR_READ_MEMORY_BARRIER; + } + + if (nocp) + *nocp = noc; + if (cszp) + *cszp = csz; + if (nobp) + *nobp = nob; + if (bszp) + *bszp = bsz; +} + + +#endif /* ERTS_SMP */ + +#ifdef DEBUG + +#if ERTS_SA_MB_CARRIERS +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % ERTS_SACRR_UNIT_SZ == 0) +#else +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) #endif - erts_free(ERTS_ALC_T_SBMBC, crr); + +static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, + UWord CSZ, Block_t* B, UWord BSZ) +{ + ASSERT(IS_LAST_BLK((B))); + ASSERT((CSZ) == CARRIER_SZ((C))); + ASSERT((BSZ) % sizeof(Unit_t) == 0); + if ((SBC)) { + ASSERT((BSZ) == SBC_BLK_SZ((B))); + ASSERT((char*)B == (char*)C + SBC_HEADER_SIZE); + ASSERT(IS_SBC_BLK((B))); + ASSERT(IS_SB_CARRIER((C))); + } + else { + ASSERT(IS_FREE_BLK(B)); + ASSERT((BSZ) == MBC_FBLK_SZ((B))); + ASSERT(IS_MBC_FIRST_FBLK(A, (B))); + ASSERT(IS_MBC_BLK((B))); + ASSERT(IS_MB_CARRIER((C))); + ASSERT(FBLK_TO_MBC(B) == (C)); + } + if ((MSEGED)) { + ASSERT(IS_MSEG_CARRIER((C))); + ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ)); + } + else { + ASSERT(IS_SYS_ALLOC_CARRIER((C))); + ASSERT((CSZ) % sizeof(Unit_t) == 0); + } } +#else +#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) +#endif + static Block_t * create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) { @@ -1952,16 +3249,45 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) Uint blk_sz, bcrr_sz, crr_sz; #if HAVE_ERTS_MSEG int have_tried_sys_alloc = 0, have_tried_mseg = 0; + Uint mseg_flags; #endif #ifdef DEBUG int is_mseg = 0; #endif +#if HALFWORD_HEAP + flags |= CFLG_FORCE_MSEG; +#elif ERTS_SUPER_ALIGNED_MSEG_ONLY + if (flags & CFLG_MBC) { + flags |= CFLG_FORCE_MSEG; + } +#endif + ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC)) || (flags & CFLG_MBC && !(flags & CFLG_SBC))); + ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC)); + blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); +#ifdef ERTS_SMP + allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; + + if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC + && ERTS_ALC_IS_CPOOL_ENABLED(allctr) + && erts_thr_progress_is_managed_thread()) { + crr = cpool_fetch(allctr, blk_sz); + if (crr) { + STAT_MBC_CPOOL_FETCH(allctr, crr); + link_carrier(&allctr->mbc_list, crr); + (*allctr->add_mbc)(allctr, crr); + blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); + ASSERT(blk); + return blk; + } + } +#endif + #if HAVE_ERTS_MSEG if (flags & CFLG_FORCE_SYS_ALLOC) @@ -1974,29 +3300,27 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY else { if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) goto try_sys_alloc; } +#endif try_mseg: if (flags & CFLG_SBC) { - crr_sz = blk_sz + allctr->sbc_header_size; + crr_sz = blk_sz + SBC_HEADER_SIZE; + mseg_flags = ERTS_MSEG_FLG_NONE; } else { crr_sz = (*allctr->get_next_mbc_size)(allctr); - if (crr_sz < allctr->mbc_header_size + blk_sz) - crr_sz = allctr->mbc_header_size + blk_sz; -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz += sizeof(UWord); -#endif + if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) + crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; + mseg_flags = ERTS_MSEG_FLG_2POW; } - crr_sz = MSEG_UNIT_CEILING(crr_sz); - ASSERT(crr_sz % mseg_unit_size == 0); - crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz); + crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags); if (!crr) { have_tried_mseg = 1; if (!(have_tried_sys_alloc || flags & CFLG_FORCE_MSEG)) @@ -2008,44 +3332,43 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) is_mseg = 1; #endif if (flags & CFLG_SBC) { - SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_SBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_SBC, allctr); STAT_MSEG_SBC_ALLOC(allctr, crr_sz, blk_sz); goto sbc_final_touch; } else { - SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_MBC); +#ifndef ARCH_64 + ASSERT(crr_sz <= MBC_SZ_MAX_LIMIT); +#endif + SET_CARRIER_HDR(crr, crr_sz, SCH_MSEG|SCH_MBC, allctr); STAT_MSEG_MBC_ALLOC(allctr, crr_sz); goto mbc_final_touch; } try_sys_alloc: + #endif /* #if HAVE_ERTS_MSEG */ if (flags & CFLG_SBC) { - bcrr_sz = blk_sz + allctr->sbc_header_size; + bcrr_sz = blk_sz + SBC_HEADER_SIZE; } else { - bcrr_sz = allctr->mbc_header_size + blk_sz; + bcrr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; if (!(flags & CFLG_MAIN_CARRIER) && bcrr_sz < allctr->smallest_mbc_size) bcrr_sz = allctr->smallest_mbc_size; -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - bcrr_sz += sizeof(UWord); -#endif - } crr_sz = (flags & CFLG_FORCE_SIZE ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -2057,7 +3380,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) } } if (flags & CFLG_SBC) { - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_SBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_SBC, allctr); STAT_SYS_ALLOC_SBC_ALLOC(allctr, crr_sz, blk_sz); #if HAVE_ERTS_MSEG @@ -2066,8 +3389,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk = SBC2BLK(allctr, crr); - SET_SBC_BLK_FTR(((UWord *) blk)[-1]); - SET_BLK_HDR(blk, blk_sz, SBH_THIS_ALLOCED|SBH_PREV_FREE|SBH_LAST_BLK); + SET_SBC_BLK_HDR(blk, blk_sz); link_carrier(&allctr->sbc_list, crr); @@ -2075,47 +3397,33 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) } else { - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC); + SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr); STAT_SYS_ALLOC_MBC_ALLOC(allctr, crr_sz); #if HAVE_ERTS_MSEG mbc_final_touch: #endif - blk = MBC2FBLK(allctr, crr); - -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif - - blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size); + blk = MBC_TO_FIRST_BLK(allctr, crr); - SET_MBC_BLK_FTR(((UWord *) blk)[-1]); - SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK); + blk_sz = UNIT_FLOOR(crr_sz - MBC_HEADER_SIZE(allctr)); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - *((Carrier_t **) NXT_BLK(blk)) = crr; -#endif + SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); if (flags & CFLG_MAIN_CARRIER) { ASSERT(!allctr->main_carrier); allctr->main_carrier = crr; } +#ifdef ERTS_SMP + cpool_init_carrier_data(allctr, crr); +#endif + link_carrier(&allctr->mbc_list, crr); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz += sizeof(UWord); -#endif CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - if (allctr->mbc_header_size % sizeof(Unit_t) == 0) - crr_sz -= sizeof(UWord); -#endif if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, 0); + (*allctr->creating_mbc)(allctr, crr); } @@ -2142,8 +3450,8 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) HARD_CHECK_BLK_CARRIER(allctr, old_blk); - old_blk_sz = BLK_SZ(old_blk); - old_crr = BLK2SBC(allctr, old_blk); + old_blk_sz = SBC_BLK_SZ(old_blk); + old_crr = BLK_TO_SBC(old_blk); old_crr_sz = CARRIER_SZ(old_crr); ASSERT(IS_SB_CARRIER(old_crr)); ASSERT(IS_SBC_BLK(old_blk)); @@ -2157,8 +3465,8 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) if (!(flags & CFLG_FORCE_SYS_ALLOC)) { - new_crr_sz = new_blk_sz + allctr->sbc_header_size; - new_crr_sz = MSEG_UNIT_CEILING(new_crr_sz); + new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; + new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, old_crr, old_crr_sz, @@ -2166,7 +3474,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) if (new_crr) { SET_CARRIER_SZ(new_crr, new_crr_sz); new_blk = SBC2BLK(allctr, new_crr); - SET_BLK_SZ(new_blk, new_blk_sz); + SET_SBC_BLK_SZ(new_blk, new_blk_sz); STAT_MSEG_SBC_ALLOC(allctr, new_crr_sz, new_blk_sz); relink_carrier(&allctr->sbc_list, new_crr); CHECK_1BLK_CARRIER(allctr, 1, 1, new_crr, new_crr_sz, @@ -2174,6 +3482,11 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } +#if HALFWORD_HEAP + /* Old carrier unchanged; restore stat */ + STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); + return NULL; +#endif create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc() failed */ } @@ -2184,7 +3497,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_mseg_dealloc(allctr, old_crr, old_crr_sz); + alcu_mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE); } else { /* Old carrier unchanged; restore stat */ @@ -2196,19 +3509,21 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) else { if (!(flags & CFLG_FORCE_MSEG)) { #endif /* #if HAVE_ERTS_MSEG */ - new_bcrr_sz = new_blk_sz + allctr->sbc_header_size; + new_bcrr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = (flags & CFLG_FORCE_SIZE ? UNIT_CEILING(new_bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(new_bcrr_sz)); new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) { sys_realloc_success: SET_CARRIER_SZ(new_crr, new_crr_sz); new_blk = SBC2BLK(allctr, new_crr); - SET_BLK_SZ(new_blk, new_blk_sz); + SET_SBC_BLK_SZ(new_blk, new_blk_sz); STAT_SYS_ALLOC_SBC_FREE(allctr, old_crr_sz, old_blk_sz); STAT_SYS_ALLOC_SBC_ALLOC(allctr, new_crr_sz, new_blk_sz); relink_carrier(&allctr->sbc_list, new_crr); @@ -2218,11 +3533,13 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) return new_blk; } else if (new_crr_sz > UNIT_CEILING(new_bcrr_sz)) { - new_crr_sz = new_blk_sz + allctr->sbc_header_size; + new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) goto sys_realloc_success; } @@ -2241,7 +3558,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_sys_free(allctr, old_crr); + alcu_sys_free(allctr, old_crr, 0); } else { /* Old carrier unchanged; restore... */ @@ -2254,19 +3571,28 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) } static void -destroy_carrier(Allctr_t *allctr, Block_t *blk) +dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned) { - Uint crr_sz; - Carrier_t *crr; #if HAVE_ERTS_MSEG - Uint is_mseg = 0; + if (IS_MSEG_CARRIER(crr)) + alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), + (superaligned + ? ERTS_MSEG_FLG_2POW + : ERTS_MSEG_FLG_NONE)); + else #endif + alcu_sys_free(allctr, crr, superaligned); +} - ASSERT(IS_FIRST_BLK(blk)); +static void +destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) +{ + Uint crr_sz; + Carrier_t *crr; if (IS_SBC_BLK(blk)) { - Uint blk_sz = BLK_SZ(blk); - crr = BLK2SBC(allctr, blk); + Uint blk_sz = SBC_BLK_SZ(blk); + crr = BLK_TO_SBC(blk); crr_sz = CARRIER_SZ(crr); ASSERT(IS_LAST_BLK(blk)); @@ -2275,8 +3601,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - is_mseg++; - ASSERT(crr_sz % mseg_unit_size == 0); + ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } else @@ -2285,9 +3610,11 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) unlink_carrier(&allctr->sbc_list, crr); + dealloc_carrier(allctr, crr, 0); } else { - crr = FBLK2MBC(allctr, blk); + ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); + crr = FIRST_BLK_TO_MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); #ifdef DEBUG @@ -2302,29 +3629,35 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } #endif -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(crr)) { - is_mseg++; - ASSERT(crr_sz % mseg_unit_size == 0); - STAT_MSEG_MBC_FREE(allctr, crr_sz); + if (allctr->destroying_mbc) + (*allctr->destroying_mbc)(allctr, crr); + +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) { + ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); + *busy_pcrr_pp = NULL; + cpool_delete(allctr, allctr, crr); } else #endif - STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); - - unlink_carrier(&allctr->mbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, 0); - } - - + { + unlink_carrier(&allctr->mbc_list, crr); #if HAVE_ERTS_MSEG - if (is_mseg) { - alcu_mseg_dealloc(allctr, crr, crr_sz); - } - else + if (IS_MSEG_CARRIER(crr)) { + ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); + STAT_MSEG_MBC_FREE(allctr, crr_sz); + } + else +#endif + STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + } + +#ifdef ERTS_SMP + schedule_dealloc_carrier(allctr, crr); +#else + dealloc_carrier(allctr, crr, 1); #endif - alcu_sys_free(allctr, crr); + } } @@ -2358,19 +3691,19 @@ static struct { Eterm lmbcs; Eterm smbcs; Eterm mbcgs; - Eterm sbmbcs; - Eterm sbmbct; + Eterm acul; #if HAVE_ERTS_MSEG Eterm mmc; #endif Eterm ycs; - /* Eterm sbmbcs; */ - Eterm fix_types; Eterm mbcs; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -2395,8 +3728,6 @@ static struct { Eterm mseg_dealloc; Eterm mseg_realloc; #endif - Eterm sbmbc_alloc; - Eterm sbmbc_free; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -2415,12 +3746,6 @@ static erts_mtx_t init_atoms_mtx; static void init_atoms(Allctr_t *allctr) { - -#ifdef USE_THREADS - if (allctr && allctr->thread_safe) - erts_mtx_unlock(&allctr->mutex); -#endif - erts_mtx_lock(&init_atoms_mtx); if (!atoms_initialized) { @@ -2458,19 +3783,19 @@ init_atoms(Allctr_t *allctr) AM_INIT(lmbcs); AM_INIT(smbcs); AM_INIT(mbcgs); - AM_INIT(sbmbcs); - AM_INIT(sbmbct); + AM_INIT(acul); #if HAVE_ERTS_MSEG AM_INIT(mmc); #endif AM_INIT(ycs); - /*AM_INIT(sbmbcs);*/ - AM_INIT(fix_types); AM_INIT(mbcs); +#ifdef ERTS_SMP + AM_INIT(mbcs_pool); +#endif AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -2495,8 +3820,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(mseg_dealloc); AM_INIT(mseg_realloc); #endif - AM_INIT(sbmbc_free); - AM_INIT(sbmbc_alloc); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -2511,18 +3834,13 @@ init_atoms(Allctr_t *allctr) fix_type_atoms[ix] = am_atom_put(name, len); } } - - if (allctr) { + if (allctr && !allctr->atoms_initialized) { make_name_atoms(allctr); (*allctr->init_atoms)(); -#ifdef USE_THREADS - if (allctr->thread_safe) - erts_mtx_lock(&allctr->mutex); -#endif allctr->atoms_initialized = 1; } @@ -2549,19 +3867,22 @@ ensure_atoms_initialized(Allctr_t *allctr) * that would fit a small when size check is done may need to be built * as a big when the actual build is performed. Caller is required to * HRelease after build. + * + * Note, bld_unstable_uint() should have been called bld_unstable_uword() + * but we do not want to rename it... */ static ERTS_INLINE Eterm -bld_unstable_uint(Uint **hpp, Uint *szp, Uint ui) +bld_unstable_uint(Uint **hpp, Uint *szp, UWord ui) { Eterm res = THE_NON_VALUE; if (szp) - *szp += BIG_UINT_HEAP_SIZE; + *szp += BIG_UWORD_HEAP_SIZE(~((UWord) 0)); if (hpp) { if (IS_USMALL(0, ui)) res = make_small(ui); else { - res = uint_to_big(ui, *hpp); - *hpp += BIG_UINT_HEAP_SIZE; + res = uword_to_big(ui, *hpp); + *hpp += BIG_UWORD_HEAP_SIZE(ui); } } return res; @@ -2587,8 +3908,24 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp, bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp); } +static ERTS_INLINE void +add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp, + Eterm *lp, Eterm fix) +{ + if (allctr->fix) { + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, lp, am.fix_types, fix); + else if (internal) + add_3tup(hpp, szp, lp, + am.fix_types, + erts_bld_uword(hpp, szp, ~((UWord) 0)), + fix); + } +} + static Eterm sz_info_fix(Allctr_t *allctr, + int internal, int *print_to_p, void *print_to_arg, Uint **hpp, @@ -2596,36 +3933,67 @@ sz_info_fix(Allctr_t *allctr, { Eterm res; int ix; - ErtsAlcFixList_t *fix = allctr->fix; - ASSERT(fix); + ASSERT(allctr->fix); res = NIL; - for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { - ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE; - Uint alloced = (fix[ix].type_size * fix[ix].allocated); - Uint used = fix[ix].type_size*fix[ix].used; - - if (print_to_p) { - int to = *print_to_p; - void *arg = print_to_arg; - erts_print(to, - arg, - "fix type: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), - alloced, - used); - } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + + if (internal) { + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.cpool.allocated; + UWord used = fix->type_size * fix->u.cpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type internal: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } - if (hpp || szp) { - add_3tup(hpp, szp, &res, - fix_type_atoms[ix], - bld_unstable_uint(hpp, szp, alloced), - bld_unstable_uint(hpp, szp, used)); + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } } } + else { + + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.nocpool.allocated; + UWord used = fix->type_size*fix->u.nocpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } + } return res; } @@ -2639,9 +4007,7 @@ sz_info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_size = (cs == &allctr->sbmbcs - ? cs->curr.small_block.size - : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size); + UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2655,7 +4021,7 @@ sz_info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, @@ -2679,6 +4045,62 @@ sz_info_carriers(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + +static Eterm +info_cpool(Allctr_t *allctr, + int sz_only, + char *prefix, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res = THE_NON_VALUE; + UWord noc, csz, nob, bsz; + + noc = csz = nob = bsz = ~0; + if (print_to_p || hpp) { + if (sz_only) + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + else + cpool_read_stat(allctr, &noc, &csz, &nob, &bsz); + } + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + if (!sz_only) + erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); + erts_print(to, arg, "%sblocks size: %bpu\n", prefix, bsz); + if (!sz_only) + erts_print(to, arg, "%scarriers: %bpu\n", prefix, noc); + erts_print(to, arg, "%scarriers size: %bpu\n", prefix, csz); + } + + if (hpp || szp) { + res = NIL; + add_2tup(hpp, szp, &res, + am.carriers_size, + bld_unstable_uint(hpp, szp, csz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.carriers, + bld_unstable_uint(hpp, szp, noc)); + add_2tup(hpp, szp, &res, + am.blocks_size, + bld_unstable_uint(hpp, szp, bsz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.blocks, + bld_unstable_uint(hpp, szp, nob)); + } + + return res; +} + +#endif /* ERTS_SMP */ + static Eterm info_carriers(Allctr_t *allctr, CarriersStats_t *cs, @@ -2689,17 +4111,10 @@ info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_no, curr_size; - int small_block = cs == &allctr->sbmbcs; - - if (small_block) { - curr_no = cs->curr.small_block.no; - curr_size = cs->curr.small_block.size; - } - else { - curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; - curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; - } + UWord curr_no, curr_size; + + curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2720,75 +4135,67 @@ info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers: %beu %bpu %bpu\n", + "%scarriers: %bpu %bpu %bpu\n", prefix, curr_no, cs->max.no, cs->max_ever.no); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers: %bpu\n", - prefix, - cs->curr.norm.mseg.no); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.no); - } erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%smseg carriers: %bpu\n", + prefix, + cs->curr.norm.mseg.no); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.no); + erts_print(to, + arg, + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, cs->max_ever.size); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers size: %bpu\n", - prefix, - cs->curr.norm.mseg.size); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers size: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.size); - } + erts_print(to, + arg, + "%smseg carriers size: %bpu\n", + prefix, + cs->curr.norm.mseg.size); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers size: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.size); } if (hpp || szp) { res = NIL; - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); #endif - } add_4tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, curr_size), bld_unstable_uint(hpp, szp, cs->max.size), bld_unstable_uint(hpp, szp, cs->max_ever.size)); - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); #endif - } add_4tup(hpp, szp, &res, am.carriers, bld_unstable_uint(hpp, szp, curr_no), @@ -2815,10 +4222,10 @@ make_name_atoms(Allctr_t *allctr) char alloc[] = "alloc"; char realloc[] = "realloc"; char free[] = "free"; - char buf[MAX_ATOM_LENGTH]; + char buf[MAX_ATOM_CHARACTERS]; size_t prefix_len = strlen(allctr->name_prefix); - if (prefix_len > MAX_ATOM_LENGTH + sizeof(realloc) - 1) + if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix); memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); @@ -2847,16 +4254,10 @@ info_calls(Allctr_t *allctr, if (print_to_p) { #define PRINT_CC_4(TO, TOA, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no); \ - else \ - erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no) + erts_print(TO, TOA, "%s calls: %b64u\n", NAME, CC) #define PRINT_CC_5(TO, TOA, PRFX, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no); \ - else \ - erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no) + erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC) char *prefix = allctr->name_prefix; int to = *print_to_p; @@ -2866,9 +4267,6 @@ info_calls(Allctr_t *allctr, PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free); PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc); - PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc); - PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free); - #if HAVE_ERTS_MSEG PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc); PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc); @@ -2895,50 +4293,42 @@ info_calls(Allctr_t *allctr, add_3tup(hpp, szp, &res, am.sys_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_realloc))); add_3tup(hpp, szp, &res, am.sys_free, - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_free))); add_3tup(hpp, szp, &res, am.sys_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_alloc))); #if HAVE_ERTS_MSEG add_3tup(hpp, szp, &res, am.mseg_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_realloc))); add_3tup(hpp, szp, &res, am.mseg_dealloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_dealloc))); add_3tup(hpp, szp, &res, am.mseg_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_alloc))); #endif add_3tup(hpp, szp, &res, - am.sbmbc_free, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no)); - add_3tup(hpp, szp, &res, - am.sbmbc_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no)); - add_3tup(hpp, szp, &res, allctr->name.realloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_realloc))); add_3tup(hpp, szp, &res, allctr->name.free, - bld_unstable_uint(hpp, szp, allctr->calls.this_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_free))); add_3tup(hpp, szp, &res, allctr->name.alloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_alloc))); } return res; @@ -2952,6 +4342,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; + int acul; if (!allctr) { if (print_to_p) @@ -2963,6 +4354,12 @@ info_options(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + acul = allctr->cpool.util_limit; +#else + acul = 0; +#endif + if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ if (allctr->t) @@ -2992,8 +4389,7 @@ info_options(Allctr_t *allctr, "option lmbcs: %beu\n" "option smbcs: %beu\n" "option mbcgs: %beu\n" - "option sbmbcs: %beu\n" - "option sbmbct: %beu\n", + "option acul: %d\n", topt, allctr->ramv ? "true" : "false", #if HALFWORD_HEAP @@ -3014,8 +4410,7 @@ info_options(Allctr_t *allctr, allctr->largest_mbc_size, allctr->smallest_mbc_size, allctr->mbc_growth_stages, - allctr->sbmbc_size, - allctr->sbmbc_threshold); + acul); } res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg, @@ -3023,11 +4418,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { add_2tup(hpp, szp, &res, - am.sbmbct, - bld_uint(hpp, szp, allctr->sbmbc_threshold)); - add_2tup(hpp, szp, &res, - am.sbmbcs, - bld_uint(hpp, szp, allctr->sbmbc_size)); + am.acul, + bld_uint(hpp, szp, (UWord) acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -3150,17 +4542,21 @@ erts_alcu_info_options(Allctr_t *allctr, { Eterm res; + if (hpp || szp) + ensure_atoms_initialized(allctr); #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif - if (hpp || szp) - ensure_atoms_initialized(allctr); res = info_options(allctr, print_to_p, print_to_arg, hpp, szp); #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; } @@ -3169,13 +4565,17 @@ erts_alcu_info_options(Allctr_t *allctr, Eterm erts_alcu_sz_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE; + Eterm res, mbcs, sbcs, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3187,67 +4587,81 @@ erts_alcu_sz_info(Allctr_t *allctr, return am_false; } + if (hpp || szp) + ensure_atoms_initialized(allctr); + #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - if (hpp || szp) - ensure_atoms_initialized(allctr); - /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 1, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; } + Eterm erts_alcu_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE; + Eterm res, sett, mbcs, sbcs, calls, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3259,22 +4673,23 @@ erts_alcu_info(Allctr_t *allctr, return am_false; } + if (hpp || szp) + ensure_atoms_initialized(allctr); + #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - if (hpp || szp) - ensure_atoms_initialized(allctr); - /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -3288,11 +4703,16 @@ erts_alcu_info(Allctr_t *allctr, sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 0, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp); @@ -3302,10 +4722,12 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.calls, calls); add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -3314,15 +4736,16 @@ erts_alcu_info(Allctr_t *allctr, } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; @@ -3340,22 +4763,37 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * size->carriers = allctr->mbcs.curr.norm.mseg.size; size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; - size->carriers += allctr->sbmbcs.curr.small_block.size; size->carriers += allctr->sbcs.curr.norm.mseg.size; size->carriers += allctr->sbcs.curr.norm.sys_alloc.size; size->blocks = allctr->mbcs.blocks.curr.size; - size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + UWord csz, bsz; + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + size->blocks += bsz; + size->carriers += csz; + } +#endif + if (fi) { int ix; for (ix = 0; ix < fisz; ix++) { if (allctr->fix) { - fi[ix].allocated += (allctr->fix[ix].type_size - * allctr->fix[ix].allocated); - fi[ix].used += (allctr->fix[ix].type_size - * allctr->fix[ix].used); + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.used); + } + else { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.used); + } } } } @@ -3373,7 +4811,6 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; void *res; - ErtsAlcFixList_t *fix; ASSERT(initialized); @@ -3391,61 +4828,21 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); - fix = allctr->fix; - if (fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used++; - res = fix[ix].list; - if (res) { - fix[ix].list_size--; - fix[ix].list = *((void **) res); - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - void *p = fix[ix].list; - Block_t *blk; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - fix[ix].allocated--; - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return res; - } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - if (fix[ix].max_used < fix[ix].used) - fix[ix].max_used = fix[ix].used; - fix[ix].allocated++; + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc(allctr, type, size); + else + return fix_nocpool_alloc(allctr, type, size); } if (size >= allctr->sbc_threshold) { Block_t *blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif -#if HALFWORD_HEAP - blk = create_carrier(allctr, size, - CFLG_SBC | CFLG_FORCE_MSEG); -#else blk = create_carrier(allctr, size, CFLG_SBC); -#endif res = blk ? BLK2UMEM(blk) : NULL; } else res = mbc_alloc(allctr, size); - if (!res && fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - fix[ix].allocated--; - fix[ix].used--; - } return res; } @@ -3506,24 +4903,33 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) void * erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) { - int pref_ix; Allctr_t *pref_allctr; void *res; - pref_ix = get_pref_allctr(extra, &pref_allctr); + pref_allctr = get_pref_allctr(extra); if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); +#endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); + res = do_erts_alcu_alloc(type, pref_allctr, size); + +#ifdef ERTS_SMP + if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + res = do_erts_alcu_alloc(type, pref_allctr, size); + } +#endif + if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); - if (res) - res = put_used_allctr(res, pref_ix, size); - DEBUG_CHECK_ALIGNMENT(res); @@ -3537,9 +4943,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) /* ------------------------------------------------------------------------- */ static ERTS_INLINE void -do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) +do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, + Carrier_t **busy_pcrr_pp) { - int ix; Allctr_t *allctr = (Allctr_t *) extra; ASSERT(initialized); @@ -3551,57 +4957,28 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { - ErtsAlcFixList_t *fix = allctr->fix; - Block_t *blk; INC_CC(allctr->calls.this_free); - if (fix) { - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) p) = fix[ix].list; - fix[ix].list = p; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - p = fix[ix].list; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - fix[ix].allocated--; - } + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_cpool_free(allctr, type, p, busy_pcrr_pp); + else + fix_nocpool_free(allctr, type, p); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); } - - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } } void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) { - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } #ifdef USE_THREADS @@ -3611,7 +4988,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) { Allctr_t *allctr = (Allctr_t *) extra; erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); erts_mtx_unlock(&allctr->mutex); } @@ -3633,7 +5010,7 @@ erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p) if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, allctr, p); + do_erts_alcu_free(type, allctr, p, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -3643,24 +5020,24 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) { if (p) { + Carrier_t *busy_pcrr_p; Allctr_t *pref_allctr, *used_allctr; - void *ptr; - get_pref_allctr(extra, &pref_allctr); - ptr = get_used_allctr(extra, p, &used_allctr, NULL); + pref_allctr = get_pref_allctr(extra); + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, + p, NULL, &busy_pcrr_p); if (pref_allctr != used_allctr) enqueue_dealloc_other_instance(type, used_allctr, - ptr, + p, (used_allctr->dd.ix - pref_allctr->dd.ix)); else { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); - do_erts_alcu_free(type, used_allctr, ptr); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); } } } @@ -3676,7 +5053,8 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - Uint32 alcu_flgs) + Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -3701,7 +5079,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, #if ALLOC_ZERO_EQ_NULL if (!size) { ASSERT(p); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, busy_pcrr_pp); INC_CC(allctr->calls.this_realloc); DEC_CC(allctr->calls.this_free); return NULL; @@ -3712,45 +5090,22 @@ do_erts_alcu_realloc(ErtsAlcType_t type, blk = UMEM2BLK(p); - if (allctr->sbmbc_threshold > 0) { - Uint old_sz, new_sz, lim; - lim = allctr->sbmbc_threshold; - old_sz = BLK_SZ(blk); - new_sz = UMEMSZ2BLKSZ(allctr, size); - if ((old_sz < lim && lim <= new_sz) - || (new_sz < lim && lim <= old_sz)) { - /* *Need* to move it... */ - - INC_CC(allctr->calls.this_realloc); - res = do_erts_alcu_alloc(type, extra, size); - DEC_CC(allctr->calls.this_alloc); - - sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ)); - - do_erts_alcu_free(type, extra, p); - DEC_CC(allctr->calls.this_free); - return res; - } - if (old_sz < lim) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; - } - if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, alcu_flgs); + res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp); else { - Uint used_sz = allctr->sbc_header_size + ABLK_HDR_SZ + size; + Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size; Uint crr_sz; Uint diff_sz_val; Uint crr_sz_val; #if HAVE_ERTS_MSEG - if (IS_SYS_ALLOC_CARRIER(BLK2SBC(allctr, blk))) + if (IS_SYS_ALLOC_CARRIER(BLK_TO_SBC(blk))) #endif crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz); #if HAVE_ERTS_MSEG else - crr_sz = MSEG_UNIT_CEILING(used_sz); + crr_sz = ERTS_SACRR_UNIT_CEILING(used_sz); #endif diff_sz_val = crr_sz - used_sz; if (diff_sz_val < (~((Uint) 0) / 100)) @@ -3775,17 +5130,13 @@ do_erts_alcu_realloc(ErtsAlcType_t type, if (res) { sys_memcpy((void*) res, (void*) p, - MIN(BLK_SZ(blk) - ABLK_HDR_SZ, size)); - destroy_carrier(allctr, blk); + MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size)); + destroy_carrier(allctr, blk, NULL); } } } else { Block_t *new_blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif if(IS_SBC_BLK(blk)) { do_carrier_resize: #if HALFWORD_HEAP @@ -3798,17 +5149,13 @@ do_erts_alcu_realloc(ErtsAlcType_t type, else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; else { -#if HALFWORD_HEAP - new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG); -#else new_blk = create_carrier(allctr, size, CFLG_SBC); -#endif if (new_blk) { res = BLK2UMEM(new_blk); sys_memcpy((void *) res, (void *) p, - MIN(BLK_SZ(blk) - ABLK_HDR_SZ, size)); - mbc_free(allctr, p); + MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size)); + mbc_free(allctr, p, busy_pcrr_pp); } else res = NULL; @@ -3822,7 +5169,7 @@ void * erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size) { void *res; - res = do_erts_alcu_realloc(type, extra, p, size, 0); + res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL); DEBUG_CHECK_ALIGNMENT(res); return res; } @@ -3843,7 +5190,7 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3858,7 +5205,7 @@ erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size) Allctr_t *allctr = (Allctr_t *) extra; void *res; erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, extra, ptr, size, 0); + res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL); erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3882,7 +5229,7 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -3909,7 +5256,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, allctr, ptr, size, 0); + res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -3952,7 +5299,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, if (cpy_size > size) cpy_size = size; sys_memcpy(res, ptr, cpy_size); - do_erts_alcu_free(type, allctr, ptr); + do_erts_alcu_free(type, allctr, ptr, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } @@ -3966,76 +5313,82 @@ static ERTS_INLINE void * realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, int force_move) { - int pref_ix; - void *ptr, *res; + void *res; Allctr_t *pref_allctr, *used_allctr; UWord old_user_size; + Carrier_t *busy_pcrr_p; +#ifdef ERTS_SMP + int retried; +#endif if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); - pref_ix = get_pref_allctr(extra, &pref_allctr); - ptr = get_used_allctr(extra, p, &used_allctr, &old_user_size); + pref_allctr = get_pref_allctr(extra); + + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); + +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); + retried = 0; +restart: +#endif + + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO, + p, &old_user_size, &busy_pcrr_p); ASSERT(used_allctr && pref_allctr); if (!force_move && used_allctr == pref_allctr) { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); res = do_erts_alcu_realloc(type, used_allctr, - ptr, - size + sizeof(UWord), - 0); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); - if (res) - res = put_used_allctr(res, pref_ix, size); - } - else { + p, + size, + 0, + &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); +#ifdef ERTS_SMP + if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + retried = 1; + goto restart; + } +#endif if (pref_allctr->thread_safe) - erts_mtx_lock(&pref_allctr->mutex); - res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord)); - if (pref_allctr->thread_safe && (!force_move - || used_allctr != pref_allctr)) erts_mtx_unlock(&pref_allctr->mutex); - if (res) { - Block_t *blk; - size_t cpy_size; - - res = put_used_allctr(res, pref_ix, size); + } + else { + res = do_erts_alcu_alloc(type, pref_allctr, size); + if (!res) + goto unlock_ts_return; + else { DEBUG_CHECK_ALIGNMENT(res); - blk = UMEM2BLK(ptr); - if (old_user_size != ERTS_AU_PREF_ALLOC_SIZE_MASK) - cpy_size = old_user_size; - else { - if (used_allctr->thread_safe && (!force_move - || used_allctr != pref_allctr)) - erts_mtx_lock(&used_allctr->mutex); - ERTS_SMP_LC_ASSERT(!used_allctr->thread_safe || - erts_lc_mtx_is_locked(&used_allctr->mutex)); - cpy_size = BLK_SZ(blk); - if (used_allctr->thread_safe && (!force_move - || used_allctr != pref_allctr)) - erts_mtx_unlock(&used_allctr->mutex); - cpy_size -= ABLK_HDR_SZ + sizeof(UWord); - } - if (cpy_size > size) - cpy_size = size; - sys_memcpy(res, p, cpy_size); + if (used_allctr != pref_allctr) { + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + + sys_memcpy(res, p, MIN(size, old_user_size)); - if (!force_move || used_allctr != pref_allctr) enqueue_dealloc_other_instance(type, used_allctr, - ptr, + p, (used_allctr->dd.ix - pref_allctr->dd.ix)); + } else { - do_erts_alcu_free(type, used_allctr, ptr); + + sys_memcpy(res, p, MIN(size, old_user_size)); + + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); ASSERT(pref_allctr == used_allctr); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + + unlock_ts_return: if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); } @@ -4111,7 +5464,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->ramv = init->ramv; allctr->main_carrier_size = init->mmbcs; - allctr->sbc_threshold = init->sbct; + #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th = init->asbcst; allctr->mseg_opt.rel_shrink_th = init->rsbcst; @@ -4120,49 +5473,64 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mbc_move_threshold = init->rmbcmt; #if HAVE_ERTS_MSEG allctr->max_mseg_sbcs = init->mmsbc; +# if ERTS_SUPER_ALIGNED_MSEG_ONLY + allctr->max_mseg_mbcs = ~(Uint)0; +# else allctr->max_mseg_mbcs = init->mmmbc; +# endif #endif allctr->largest_mbc_size = MAX(init->lmbcs, init->smbcs); +#ifndef ARCH_64 + if (allctr->largest_mbc_size > MBC_SZ_MAX_LIMIT) { + allctr->largest_mbc_size = MBC_SZ_MAX_LIMIT; + } +#endif allctr->smallest_mbc_size = init->smbcs; allctr->mbc_growth_stages = MAX(1, init->mbcgs); if (allctr->min_block_size < ABLK_HDR_SZ) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size - + sizeof(UWord)); + + sizeof(FreeBlkFtr_t)); #if ERTS_SMP if (init->tpref) { - Uint sz = sizeof(Block_t); - sz += ERTS_ALCU_DD_FIX_TYPE_OFFS*sizeof(UWord); - if (init->fix) - sz += sizeof(UWord); + Uint sz = ABLK_HDR_SZ; + sz += (init->fix ? + sizeof(ErtsAllctrFixDDBlock_t) : sizeof(ErtsAllctrDDBlock_t)); sz = UNIT_CEILING(sz); if (sz > allctr->min_block_size) allctr->min_block_size = sz; } -#endif - - - allctr->sbmbc_threshold = init->sbmbct; - - if (!erts_have_sbmbc_alloc - || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)) - allctr->sbmbc_threshold = 0; + allctr->cpool.dc_list.first = NULL; + allctr->cpool.dc_list.last = NULL; + allctr->cpool.abandon_limit = 0; + allctr->cpool.disable_abandon = 0; + erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0); + erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + allctr->cpool.util_limit = init->acul; +#endif - if (!allctr->sbmbc_threshold) - allctr->sbmbc_size = 0; - else { - Uint min_size; - allctr->sbmbc_size = init->sbmbcs; - min_size = allctr->sbmbc_threshold; - min_size += allctr->min_block_size; - min_size += allctr->mbc_header_size; - if (allctr->sbmbc_size < min_size) - allctr->sbmbc_size = min_size; + allctr->sbc_threshold = init->sbct; +#ifndef ARCH_64 + if (allctr->sbc_threshold > 0) { + Uint max_mbc_block_sz = UNIT_CEILING(allctr->sbc_threshold - 1 + ABLK_HDR_SZ); + if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK + || max_mbc_block_sz < allctr->sbc_threshold) { /* wrap around */ + /* + * By limiting sbc_threshold to (hard limit - min_block_size) + * we avoid having to split off free "residue blocks" + * smaller than min_block_size. + */ + max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); + allctr->sbc_threshold = max_mbc_block_sz - ABLK_HDR_SZ + 1; + } } - +#endif #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) @@ -4175,16 +5543,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", + "alcu_allocator", make_small(allctr->alloc_no), ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", + "alcu_allocator", make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ @@ -4208,58 +5572,31 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_SMP allctr->dd.use = 0; if (init->tpref) { - allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size - + FBLK_FTR_SZ - + ABLK_HDR_SZ - + sizeof(UWord)) - - ABLK_HDR_SZ - - sizeof(UWord)); - allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t) - + FBLK_FTR_SZ - + ABLK_HDR_SZ - + sizeof(UWord)) - - ABLK_HDR_SZ - - sizeof(UWord)); - allctr->dd.use = 1; init_dd_queue(&allctr->dd.q); allctr->dd.ix = init->ix; } - else #endif - { - allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size - + FBLK_FTR_SZ - + ABLK_HDR_SZ) - - ABLK_HDR_SZ); - allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t) - + FBLK_FTR_SZ - + ABLK_HDR_SZ) - - ABLK_HDR_SZ); - } + allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + + ABLK_HDR_SZ) + - ABLK_HDR_SZ); if (allctr->main_carrier_size) { Block_t *blk; -#if HALFWORD_HEAP - blk = create_carrier(allctr, - allctr->main_carrier_size, - CFLG_MBC - | CFLG_FORCE_SIZE - | CFLG_FORCE_MSEG - | CFLG_MAIN_CARRIER); -#else blk = create_carrier(allctr, allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE + | CFLG_NO_CPOOL +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY | CFLG_FORCE_SYS_ALLOC - | CFLG_MAIN_CARRIER); #endif + | CFLG_MAIN_CARRIER); if (!blk) goto error; - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -4270,13 +5607,24 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->fix = init->fix; allctr->fix_shrink_scheduled = 0; for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { - allctr->fix[i].max_used = 0; - allctr->fix[i].limit = 0; allctr->fix[i].type_size = init->fix_type_size[i]; allctr->fix[i].list_size = 0; allctr->fix[i].list = NULL; - allctr->fix[i].allocated = 0; - allctr->fix[i].used = 0; +#ifdef ERTS_SMP + ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t)); +#endif + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + allctr->fix[i].u.cpool.min_list_size = 0; + allctr->fix[i].u.cpool.shrink_list = 0; + allctr->fix[i].u.cpool.allocated = 0; + allctr->fix[i].u.cpool.used = 0; + } + else { + allctr->fix[i].u.nocpool.max_used = 0; + allctr->fix[i].u.nocpool.limit = 0; + allctr->fix[i].u.nocpool.allocated = 0; + allctr->fix[i].u.nocpool.used = 0; + } } } @@ -4301,11 +5649,9 @@ erts_alcu_stop(Allctr_t *allctr) allctr->stopped = 1; while (allctr->sbc_list.first) - destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first)); + destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first), NULL); while (allctr->mbc_list.first) - destroy_carrier(allctr, MBC2FBLK(allctr, allctr->mbc_list.first)); - while (allctr->sbmbc_list.first) - destroy_sbmbc(allctr, MBC2FBLK(allctr, allctr->sbmbc_list.first)); + destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL); #ifdef USE_THREADS if (allctr->thread_safe) @@ -4319,19 +5665,19 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { - +#ifdef ERTS_SMP + int i; + for (i = 0; i <= ERTS_ALC_A_MAX; i++) { + ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + } +#endif + ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG - mseg_unit_size = erts_mseg_unit_size(); - - if (mseg_unit_size % sizeof(Unit_t)) /* A little paranoid... */ - erl_exit(-1, - "Mseg unit size (%d) not evenly divideble by " - "internal unit size of alloc_util (%d)\n", - mseg_unit_size, - sizeof(Unit_t)); - + ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); max_mseg_carriers = init->mmc; - sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs); + sys_alloc_carrier_size = ERTS_SACRR_UNIT_CEILING(init->ycs); #else /* #if HAVE_ERTS_MSEG */ sys_alloc_carrier_size = ((init->ycs + 4095) / 4096) * 4096; #endif @@ -4354,44 +5700,70 @@ erts_alcu_init(AlcUInit_t *init) * to erts_alcu_test() * \* */ -unsigned long -erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_alcu_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x000: return (unsigned long) BLK_SZ((Block_t *) a1); - case 0x001: return (unsigned long) BLK_UMEM_SZ((Block_t *) a1); - case 0x002: return (unsigned long) IS_PREV_BLK_FREE((Block_t *) a1); - case 0x003: return (unsigned long) IS_FREE_BLK((Block_t *) a1); - case 0x004: return (unsigned long) IS_LAST_BLK((Block_t *) a1); - case 0x005: return (unsigned long) UMEM2BLK((void *) a1); - case 0x006: return (unsigned long) BLK2UMEM((Block_t *) a1); - case 0x007: return (unsigned long) IS_SB_CARRIER((Carrier_t *) a1); - case 0x008: return (unsigned long) IS_SBC_BLK((Block_t *) a1); - case 0x009: return (unsigned long) IS_MB_CARRIER((Carrier_t *) a1); - case 0x00a: return (unsigned long) IS_MSEG_CARRIER((Carrier_t *) a1); - case 0x00b: return (unsigned long) CARRIER_SZ((Carrier_t *) a1); - case 0x00c: return (unsigned long) SBC2BLK((Allctr_t *) a1, + case 0x000: return (UWord) BLK_SZ((Block_t *) a1); + case 0x001: return (UWord) BLK_UMEM_SZ((Block_t *) a1); + case 0x002: return (UWord) IS_PREV_BLK_FREE((Block_t *) a1); + case 0x003: return (UWord) IS_FREE_BLK((Block_t *) a1); + case 0x004: return (UWord) IS_LAST_BLK((Block_t *) a1); + case 0x005: return (UWord) UMEM2BLK((void *) a1); + case 0x006: return (UWord) BLK2UMEM((Block_t *) a1); + case 0x007: return (UWord) IS_SB_CARRIER((Carrier_t *) a1); + case 0x008: return (UWord) IS_SBC_BLK((Block_t *) a1); + case 0x009: return (UWord) IS_MB_CARRIER((Carrier_t *) a1); + case 0x00a: return (UWord) IS_MSEG_CARRIER((Carrier_t *) a1); + case 0x00b: return (UWord) CARRIER_SZ((Carrier_t *) a1); + case 0x00c: return (UWord) SBC2BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00d: return (unsigned long) BLK2SBC((Allctr_t *) a1, - (Block_t *) a2); - case 0x00e: return (unsigned long) MBC2FBLK((Allctr_t *) a1, + case 0x00d: return (UWord) BLK_TO_SBC((Block_t *) a2); + case 0x00e: return (UWord) MBC_TO_FIRST_BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00f: return (unsigned long) FBLK2MBC((Allctr_t *) a1, + case 0x00f: return (UWord) FIRST_BLK_TO_MBC((Allctr_t *) a1, (Block_t *) a2); - case 0x010: return (unsigned long) ((Allctr_t *) a1)->mbc_list.first; - case 0x011: return (unsigned long) ((Allctr_t *) a1)->mbc_list.last; - case 0x012: return (unsigned long) ((Allctr_t *) a1)->sbc_list.first; - case 0x013: return (unsigned long) ((Allctr_t *) a1)->sbc_list.last; - case 0x014: return (unsigned long) ((Carrier_t *) a1)->next; - case 0x015: return (unsigned long) ((Carrier_t *) a1)->prev; - case 0x016: return (unsigned long) ABLK_HDR_SZ; - case 0x017: return (unsigned long) ((Allctr_t *) a1)->min_block_size; - case 0x018: return (unsigned long) NXT_BLK((Block_t *) a1); - case 0x019: return (unsigned long) PREV_BLK((Block_t *) a1); - case 0x01a: return (unsigned long) IS_FIRST_BLK((Block_t *) a1); - case 0x01b: return (unsigned long) sizeof(Unit_t); - default: ASSERT(0); return ~((unsigned long) 0); + case 0x010: return (UWord) ((Allctr_t *) a1)->mbc_list.first; + case 0x011: return (UWord) ((Allctr_t *) a1)->mbc_list.last; + case 0x012: return (UWord) ((Allctr_t *) a1)->sbc_list.first; + case 0x013: return (UWord) ((Allctr_t *) a1)->sbc_list.last; + case 0x014: return (UWord) ((Carrier_t *) a1)->next; + case 0x015: return (UWord) ((Carrier_t *) a1)->prev; + case 0x016: return (UWord) ABLK_HDR_SZ; + case 0x017: return (UWord) ((Allctr_t *) a1)->min_block_size; + case 0x018: return (UWord) NXT_BLK((Block_t *) a1); + case 0x019: return (UWord) PREV_BLK((Block_t *) a1); + case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); + case 0x01b: return (UWord) sizeof(Unit_t); + case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; + case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; +#ifdef ERTS_SMP + case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t); + case 0x020: + SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1); + cpool_init_carrier_data((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x021: + cpool_insert((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x022: + cpool_delete((Allctr_t *) a1, (Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1); + case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2); +#else + case 0x01f: return (UWord) 0; + case 0x020: return (UWord) 0; + case 0x021: return (UWord) 0; + case 0x022: return (UWord) 0; + case 0x023: return (UWord) 0; + case 0x024: return (UWord) 0; +#endif + + default: ASSERT(0); return ~((UWord) 0); } + return 0; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -4399,6 +5771,20 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) \* */ void +erts_alcu_assert_failed(char* expr, char* file, int line, char *func) +{ + fflush(stdout); + fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n", + file, line, func, expr); + fflush(stderr); +#if defined(__WIN__) || defined(__WIN32__) + DebugBreak(); +#else + abort(); +#endif +} + +void erts_alcu_verify_unused(Allctr_t *allctr) { UWord no; @@ -4406,12 +5792,10 @@ erts_alcu_verify_unused(Allctr_t *allctr) no = allctr->sbcs.curr.norm.mseg.no; no += allctr->sbcs.curr.norm.sys_alloc.no; no += allctr->mbcs.blocks.curr.no; - no += allctr->sbmbcs.blocks.curr.no; if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - sz += allctr->sbmbcs.blocks.curr.size; erl_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" @@ -4432,6 +5816,13 @@ erts_alcu_verify_unused_ts(Allctr_t *allctr) #endif } +#ifdef DEBUG +int is_sbc_blk(Block_t* blk) +{ + return IS_SBC_BLK(blk); +} +#endif + #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void @@ -4441,34 +5832,37 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) CarrierList_t *cl; if (IS_SBC_BLK(iblk)) { - Carrier_t *sbc = BLK2SBC(allctr, iblk); + Carrier_t *sbc = BLK_TO_SBC(iblk); ASSERT(SBC2BLK(allctr, sbc) == iblk); - ASSERT(IS_ALLOCED_BLK(iblk)); - ASSERT(IS_FIRST_BLK(iblk)); - ASSERT(IS_LAST_BLK(iblk)); - ASSERT(CARRIER_SZ(sbc) - allctr->sbc_header_size >= BLK_SZ(iblk)); + ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % mseg_unit_size == 0); + ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); } #endif crr = sbc; cl = &allctr->sbc_list; } else { - Carrier_t *mbc = NULL; Block_t *prev_blk = NULL; Block_t *blk; char *carrier_end; Uint is_free_blk; Uint tot_blk_sz; Uint blk_sz; + int has_wrapped_around = 0; blk = iblk; tot_blk_sz = 0; + crr = BLK_TO_MBC(blk); + ASSERT(IS_MB_CARRIER(crr)); + /* Step around the carrier one whole lap starting at 'iblk' + */ while (1) { + ASSERT(IS_MBC_BLK(blk)); + ASSERT(BLK_TO_MBC(blk) == crr); if (prev_blk) { ASSERT(NXT_BLK(prev_blk) == blk); @@ -4481,18 +5875,16 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) } } - if (mbc) { + if (has_wrapped_around) { + ASSERT(((Block_t *) crr) < blk); if (blk == iblk) break; - ASSERT(((Block_t *) mbc) < blk && blk < iblk); + ASSERT(blk < iblk); } else ASSERT(blk >= iblk); - - ASSERT(IS_MBC_BLK(blk)); - - blk_sz = BLK_SZ(blk); + blk_sz = MBC_BLK_SZ(blk); ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(blk_sz >= allctr->min_block_size); @@ -4500,48 +5892,44 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) tot_blk_sz += blk_sz; is_free_blk = (int) IS_FREE_BLK(blk); - if(is_free_blk) { - if (IS_NOT_LAST_BLK(blk)) - ASSERT(*((UWord *) (((char *) blk)+blk_sz-sizeof(UWord))) - == blk_sz); - } + ASSERT(!is_free_blk + || IS_LAST_BLK(blk) + || PREV_BLK_SZ(((char *) blk)+blk_sz) == blk_sz); if (allctr->check_block) (*allctr->check_block)(allctr, blk, (int) is_free_blk); if (IS_LAST_BLK(blk)) { carrier_end = ((char *) NXT_BLK(blk)); - mbc = *((Carrier_t **) NXT_BLK(blk)); + has_wrapped_around = 1; prev_blk = NULL; - blk = MBC2FBLK(allctr, mbc); - ASSERT(IS_FIRST_BLK(blk)); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(IS_MBC_FIRST_BLK(allctr,blk)); } else { prev_blk = blk; blk = NXT_BLK(blk); } } - - ASSERT(IS_MB_CARRIER(mbc)); - ASSERT((((char *) mbc) - + allctr->mbc_header_size + + ASSERT((((char *) crr) + + MBC_HEADER_SIZE(allctr) + tot_blk_sz) == carrier_end); - ASSERT(((char *) mbc) + CARRIER_SZ(mbc) - sizeof(Unit_t) <= carrier_end - && carrier_end <= ((char *) mbc) + CARRIER_SZ(mbc)); + ASSERT(((char *) crr) + CARRIER_SZ(crr) - sizeof(Unit_t) <= carrier_end + && carrier_end <= ((char *) crr) + CARRIER_SZ(crr)); if (allctr->check_mbc) - (*allctr->check_mbc)(allctr, mbc); + (*allctr->check_mbc)(allctr, crr); #if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(mbc)) { - ASSERT(CARRIER_SZ(mbc) % mseg_unit_size == 0); + if (IS_MSEG_CARRIER(crr)) { + ASSERT(CARRIER_SZ(crr) % ERTS_SACRR_UNIT_SZ == 0); } #endif - crr = mbc; cl = &allctr->mbc_list; } -#if 0 /* FIXIT sbmbc */ +#ifdef DEBUG if (cl->first == crr) { ASSERT(!crr->prev); } @@ -4559,4 +5947,5 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) #endif } -#endif +#endif /* ERTS_ALLOC_UTIL_HARD_DEBUG */ + diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index cedf4ccf85..70ecf28172 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -55,8 +55,7 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - UWord sbmbct; - UWord sbmbcs; + int acul; void *fix; size_t *fix_type_size; @@ -76,7 +75,7 @@ typedef struct { #define ERTS_DEFAULT_ALCU_INIT { \ 1024*1024, /* (bytes) ycs: sys_alloc carrier size */\ - 1024 /* (amount) mmc: max mseg carriers */\ + ~((UWord) 0) /* (amount) mmc: max mseg carriers */ \ } #define ERTS_DEFAULT_ALLCTR_INIT { \ @@ -96,12 +95,11 @@ typedef struct { 50, /* (%) rmbcmt: rel mbc move threshold */\ 1024*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 256, /* (amount) mmsbc: max mseg sbcs */\ - 10, /* (amount) mmmbc: max mseg mbcs */\ + ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 10*1024*1024, /* (bytes) lmbcs: largest mbc size */\ 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ - 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + 0, /* (%) acul: abandon carrier utilization limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -130,12 +128,11 @@ typedef struct { 80, /* (%) rsbcmt: rel sbc move threshold */\ 128*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 256, /* (amount) mmsbc: max mseg sbcs */\ - 10, /* (amount) mmmbc: max mseg mbcs */\ + ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 1024*1024, /* (bytes) lmbcs: largest mbc size */\ 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ - 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + 0, /* (%) acul: abandon carrier utilization limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -165,8 +162,8 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); #endif Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *); Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *); -Eterm erts_alcu_sz_info(Allctr_t *, int, int *, void *, Uint **, Uint *); -Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); @@ -181,7 +178,6 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define ERL_ALLOC_UTIL_IMPL__ #define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) -#define ERTS_ALCU_FLG_SBMBC (((Uint32) 1) << 1) #ifdef USE_THREADS #define ERL_THREADS_EMU_INTERNAL__ @@ -216,40 +212,143 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define UNIT_FLOOR(X) ((X) & UNIT_MASK) #define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK) +#define FLG_MASK INV_UNIT_MASK +#define SBC_BLK_SZ_MASK UNIT_MASK +#define MBC_FBLK_SZ_MASK UNIT_MASK +#define CARRIER_SZ_MASK UNIT_MASK + +#if ERTS_HAVE_MSEG_SUPER_ALIGNED \ + || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) +# ifndef MSEG_ALIGN_BITS +# define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS +# else +# define ERTS_SUPER_ALIGN_BITS 18 +# endif +# ifdef ARCH_64 +# define MBC_ABLK_OFFSET_BITS 24 +# else +# define MBC_ABLK_OFFSET_BITS 9 + /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ +# endif +# define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS +# define ERTS_SACRR_UNIT_SZ (1 << ERTS_SACRR_UNIT_SHIFT) +# define ERTS_SACRR_UNIT_MASK ((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT) +# define ERTS_SACRR_UNIT_FLOOR(X) ((X) & ERTS_SACRR_UNIT_MASK) +# define ERTS_SACRR_UNIT_CEILING(X) ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK) +# define ERTS_SA_MB_CARRIERS 1 +#else +# define ERTS_SA_MB_CARRIERS 0 +# define MBC_ABLK_OFFSET_BITS 0 /* no carrier offset in block header */ +#endif +#if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +# define ERTS_SUPER_ALIGNED_MSEG_ONLY 1 +#else +# define ERTS_SUPER_ALIGNED_MSEG_ONLY 0 +#endif -#define SZ_MASK (~((UWord) 0) << 3) -#define FLG_MASK (~(SZ_MASK)) - +#if MBC_ABLK_OFFSET_BITS +# define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) +# define MBC_ABLK_OFFSET_MASK (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) +# define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~FLG_MASK) +#else +# define MBC_ABLK_SZ_MASK (~FLG_MASK) +#endif -#define BLK_SZ(B) \ - (*((Block_t *) (B)) & SZ_MASK) +#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) +#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) +#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) #define CARRIER_SZ(C) \ - ((C)->chdr & SZ_MASK) - -extern int erts_have_sbmbc_alloc; + ((C)->chdr & CARRIER_SZ_MASK) typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; +#ifdef ERTS_SMP + +typedef struct { + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; +} ErtsAlcCPoolData_t; + +#endif + typedef struct Carrier_t_ Carrier_t; struct Carrier_t_ { UWord chdr; Carrier_t *next; Carrier_t *prev; + erts_smp_atomic_t allctr; +#ifdef ERTS_SMP + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +#endif }; +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + typedef struct { Carrier_t *first; Carrier_t *last; } CarrierList_t; -typedef UWord Block_t; -typedef UWord FreeBlkFtr_t; - typedef struct { - UWord giga_no; - UWord no; -} CallCounter_t; + UWord bhdr; +#if !MBC_ABLK_OFFSET_BITS + Carrier_t *carrier; +#else + union { + Carrier_t *carrier; /* if free */ + char udata__[1]; /* if allocated */ + }u; +#endif +} Block_t; + +#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) +#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) +#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) + +#define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\ + (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) + +/* + * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for + * distinguishing empty mbc's from allocated blocks in + * handle_delayed_dealloc(). + */ +#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) + +#define IS_FREE_LAST_MBC_BLK(B) \ + (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + +#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) +#define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) +#define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \ + (B)->bhdr & THIS_FREE_BLK_HDR_FLG) + +#if MBC_ABLK_OFFSET_BITS +# define FBLK_TO_MBC(B) (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ + (B)->u.carrier) +# define ABLK_TO_MBC(B) \ + (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \ + (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ + (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT)))) +# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) +#else +# define FBLK_TO_MBC(B) ((B)->carrier) +# define ABLK_TO_MBC(B) ((B)->carrier) +# define BLK_TO_MBC(B) ((B)->carrier) +#endif +#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) + +typedef UWord FreeBlkFtr_t; /* Footer of a free block */ + +typedef Uint64 CallCounter_t; typedef struct { UWord no; @@ -262,7 +361,6 @@ typedef struct { StatValues_t mseg; StatValues_t sys_alloc; } norm; - StatValues_t small_block; } curr; StatValues_t max; StatValues_t max_ever; @@ -322,10 +420,20 @@ typedef struct { size_t type_size; SWord list_size; void *list; - SWord max_used; - SWord limit; - SWord allocated; - SWord used; + union { + struct { + SWord max_used; + SWord limit; + SWord allocated; + SWord used; + } nocpool; + struct { + int min_list_size; + int shrink_list; + UWord allocated; + UWord used; + } cpool; + } u; } ErtsAlcFixList_t; struct Allctr_t_ { @@ -373,8 +481,6 @@ struct Allctr_t_ { Uint largest_mbc_size; Uint smallest_mbc_size; Uint mbc_growth_stages; - Uint sbmbc_threshold; - Uint sbmbc_size; #if HAVE_ERTS_MSEG ErtsMsegOpt_t mseg_opt; @@ -382,30 +488,49 @@ struct Allctr_t_ { /* */ Uint mbc_header_size; - Uint sbc_header_size; Uint min_mbc_size; Uint min_mbc_first_free_size; Uint min_block_size; /* Carriers */ - CarrierList_t sbmbc_list; CarrierList_t mbc_list; CarrierList_t sbc_list; +#ifdef ERTS_SMP + struct { + CarrierList_t dc_list; + UWord abandon_limit; + int disable_abandon; + int check_limit_count; + int util_limit; + struct { + erts_atomic_t blocks_size; + erts_atomic_t no_blocks; + erts_atomic_t carriers_size; + erts_atomic_t no_carriers; + } stat; + } cpool; +#endif /* Main carrier (if there is one) */ Carrier_t * main_carrier; /* Callback functions (first 4 are mandatory) */ Block_t * (*get_free_block) (Allctr_t *, Uint, - Block_t *, Uint, Uint32); - void (*link_free_block) (Allctr_t *, Block_t *, Uint32); - void (*unlink_free_block) (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); + void (*link_free_block) (Allctr_t *, Block_t *); + void (*unlink_free_block) (Allctr_t *, Block_t *); Eterm (*info_options) (Allctr_t *, char *, int *, void *, Uint **, Uint *); Uint (*get_next_mbc_size) (Allctr_t *); - void (*creating_mbc) (Allctr_t *, Carrier_t *, Uint32); - void (*destroying_mbc) (Allctr_t *, Carrier_t *, Uint32); + void (*creating_mbc) (Allctr_t *, Carrier_t *); + void (*destroying_mbc) (Allctr_t *, Carrier_t *); + + /* The three callbacks below are needed to support carrier migration */ + void (*add_mbc) (Allctr_t *, Carrier_t *); + void (*remove_mbc) (Allctr_t *, Carrier_t *); + UWord (*largest_fblk_in_mbc) (Allctr_t *, Carrier_t *); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -437,8 +562,6 @@ struct Allctr_t_ { CallCounter_t this_alloc; CallCounter_t this_free; CallCounter_t this_realloc; - CallCounter_t sbmbc_alloc; - CallCounter_t sbmbc_free; CallCounter_t mseg_alloc; CallCounter_t mseg_dealloc; CallCounter_t mseg_realloc; @@ -449,8 +572,7 @@ struct Allctr_t_ { CarriersStats_t sbcs; CarriersStats_t mbcs; - CarriersStats_t sbmbcs; - + #ifdef DEBUG #ifdef USE_THREADS struct { @@ -467,8 +589,13 @@ void erts_alcu_stop(Allctr_t *); void erts_alcu_verify_unused(Allctr_t *); void erts_alcu_verify_unused_ts(Allctr_t *allctr); -unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long); +UWord erts_alcu_test(UWord, UWord, UWord); + +void erts_alcu_assert_failed(char* expr, char* file, int line, char *func); +#ifdef DEBUG +int is_sbc_blk(Block_t*); +#endif #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL) diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 5bdb752d3a..4e6c8b317e 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,11 +28,17 @@ * * This module is a callback-module for erl_alloc_util.c * - * Algorithm: The tree nodes are free-blocks ordered in address order. + * AOFF Algorithm: + * The tree nodes are ordered in address order. * Every node also keeps the size of the largest block in its - * sub-tree ('max_size'). By that we can start from root and keep + * sub-tree ('max_sz'). By that we can start from root and keep * left (for low addresses) while dismissing entire sub-trees with * too small blocks. + * Bestfit within carrier: + * The only difference for "bestfit within carrier" is the tree + * sorting order. Blocks within the same carrier are sorted + * wrt size instead of address. The 'max_sz' field is maintained + * in order to dismiss entire carriers with too small blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -62,6 +68,15 @@ # define LEFT_VISITED_FLG (((Uint) 1) << 2) # define RIGHT_VISITED_FLG (((Uint) 1) << 3) #endif +#ifdef DEBUG +# define IS_BF_FLG (((Uint) 1) << 4) +#endif + +#define IS_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags & TREE_NODE_FLG) +#define IS_LIST_ELEM(N) (!IS_TREE_NODE(((AOFF_RBTree_t *) (N)))) + +#define SET_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags |= TREE_NODE_FLG) +#define SET_LIST_ELEM(N) (((AOFF_RBTree_t *) (N))->flags &= ~TREE_NODE_FLG) #define IS_RED(N) (((AOFF_RBTree_t *) (N)) \ && ((AOFF_RBTree_t *) (N))->flags & RED_FLG) @@ -85,24 +100,59 @@ typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; struct AOFF_RBTree_t_ { Block_t hdr; - Uint flags; AOFF_RBTree_t *parent; AOFF_RBTree_t *left; AOFF_RBTree_t *right; - Uint max_sz; /* of all blocks in this sub-tree */ + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ }; +#define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) + +/* BF block nodes keeps list of all with equal size + */ +typedef struct { + AOFF_RBTree_t t; + AOFF_RBTree_t *next; +}AOFF_RBTreeList_t; + +#define LIST_NEXT(N) (((AOFF_RBTreeList_t*) (N))->next) +#define LIST_PREV(N) (((AOFF_RBTreeList_t*) (N))->t.parent) + +typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; + +struct AOFF_Carrier_t_ { + Carrier_t crr; + AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + AOFF_RBTree_t* root; /* Root of my block tree */ +}; +#define RBT_NODE_TO_MBC(PTR) ((AOFF_Carrier_t*)((char*)(PTR) - offsetof(AOFF_Carrier_t, rbt_node))) + +/* + To support carrier migration we keep two kinds of rb-trees: + 1. One tree of carriers for each allocator instance. + 2. One tree of free blocks for each carrier. + Both trees use the same node structure AOFF_RBTree_t and implementation. + Carrier nodes thus contain a phony Block_t header 'rbt_node.hdr'. + The size value of such a phony block is the size of the largest free block in + that carrier, i.e same as 'max_sz' of the root node of its block tree. +*/ #ifdef HARD_DEBUG -static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); +#else +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) #endif -/* Calculate 'max_size' of tree node x by only looking at the direct children - * of x and x itself. +/* Calculate 'max_sz' of tree node x by only looking at 'max_sz' of the + * direct children of x and the size x itself. */ static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) { - Uint sz = BLK_SZ(x); + Uint sz = AOFF_BLK_SZ(x); if (x->left && x->left->max_sz > sz) { sz = x->left->max_sz; } @@ -112,7 +162,7 @@ static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) return sz; } -/* Set new possibly lower 'max_size' of node and propagate change toward root +/* Set new possibly lower 'max_sz' of node and propagate change toward root */ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, AOFF_RBTree_t* stop_at) @@ -131,32 +181,53 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } +static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, + AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) +{ + ASSERT(lhs != rhs); + ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (flavor != AOFF_AOFF) { + SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); + if (diff || flavor == AOFF_BF) return diff; + } + return (char*)lhs - (char*)rhs; +} + +static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, + Block_t* cand_blk, AOFF_RBTree_t* rhs) +{ + if (flavor != AOFF_AOFF) { + if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { + SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); + if (diff || flavor == AOFF_BF) return diff; + } + } + return (char*)cand_blk - (char*)rhs; +} + /* Prototypes of callback functions */ -static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint, Uint32 flags); -static void aoff_link_free_block(Allctr_t *, Block_t*, Uint32 flags); -static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags); +static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint); +static void aoff_link_free_block(Allctr_t *, Block_t*); +static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del); +static void aoff_creating_mbc(Allctr_t*, Carrier_t*); +static void aoff_destroying_mbc(Allctr_t*, Carrier_t*); +static void aoff_add_mbc(Allctr_t*, Carrier_t*); +static void aoff_remove_mbc(Allctr_t*, Carrier_t*); +static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); + +/* Generic tree functions used by both carrier and block trees. */ +static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); +static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); +#ifdef HARD_DEBUG +static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); +#endif static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); static void init_atoms(void); - -#ifdef DEBUG - -/* Destroy all tree fields */ -#define DESTROY_TREE_NODE(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(AOFF_RBTree_t) - sizeof(Block_t))) - -#else - -#define DESTROY_TREE_NODE(N) - -#endif - - static int atoms_initialized = 0; void @@ -183,10 +254,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); - allctr->mbc_header_size = sizeof(Carrier_t); + alc->flavor = aoffinit->flavor; + allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = sizeof(AOFF_RBTree_t); + allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -195,12 +268,15 @@ erts_aoffalc_start(AOFFAllctr_t *alc, allctr->get_free_block = aoff_get_free_block; allctr->link_free_block = aoff_link_free_block; - allctr->unlink_free_block = aoff_unlink_free_block; + allctr->unlink_free_block = aoff_unlink_free_block; allctr->info_options = info_options; allctr->get_next_mbc_size = NULL; - allctr->creating_mbc = NULL; - allctr->destroying_mbc = NULL; + allctr->creating_mbc = aoff_creating_mbc; + allctr->destroying_mbc = aoff_destroying_mbc; + allctr->add_mbc = aoff_add_mbc; + allctr->remove_mbc = aoff_remove_mbc; + allctr->largest_fblk_in_mbc = aoff_largest_fblk_in_mbc; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -303,10 +379,7 @@ replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y) y->parent = x->parent; y->right = x->right; y->left = x->left; - - y->max_sz = x->max_sz; - lower_max_size(y, NULL); - DESTROY_TREE_NODE(x); + y->max_sz = x->max_sz; } static void @@ -403,21 +476,72 @@ tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) } static void -aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) +aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) +{ + AOFFAllctr_t* alc = (AOFFAllctr_t*)allctr; + AOFF_RBTree_t* del = (AOFF_RBTree_t*)blk; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); + + ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + + if (alc->flavor == AOFF_BF) { + ASSERT(del->flags & IS_BF_FLG); + if (IS_LIST_ELEM(del)) { + /* Remove from list */ + ASSERT(LIST_PREV(del)); + ASSERT(LIST_PREV(del)->flags & IS_BF_FLG); + LIST_NEXT(LIST_PREV(del)) = LIST_NEXT(del); + if (LIST_NEXT(del)) { + ASSERT(LIST_NEXT(del)->flags & IS_BF_FLG); + LIST_PREV(LIST_NEXT(del)) = LIST_PREV(del); + } + return; + } + else if (LIST_NEXT(del)) { + /* Replace tree node by next element in list... */ + + ASSERT(AOFF_BLK_SZ(LIST_NEXT(del)) == AOFF_BLK_SZ(del)); + ASSERT(IS_LIST_ELEM(LIST_NEXT(del))); + + replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); + + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + return; + } + } + + rbt_delete(&crr->root, (AOFF_RBTree_t*)del); + + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + + /* Update the carrier tree with a potentially new (lower) max_sz + */ + if (crr->root) { + if (crr->rbt_node.hdr.bhdr == crr->root->max_sz) { + return; + } + ASSERT(crr->rbt_node.hdr.bhdr > crr->root->max_sz); + crr->rbt_node.hdr.bhdr = crr->root->max_sz; + } + else { + crr->rbt_node.hdr.bhdr = 0; + } + lower_max_size(&crr->rbt_node, NULL); +} + + +static void +rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &alc->sbmbc_root : &alc->mbc_root); Uint spliced_is_black; - AOFF_RBTree_t *x, *y, *z = (AOFF_RBTree_t *) del; + AOFF_RBTree_t *x, *y, *z = del; AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ - null_x.parent = NULL; + HARD_CHECK_IS_MEMBER(*root, del); -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif + null_x.parent = NULL; /* Remove node from tree... */ @@ -461,7 +585,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) } if (y != z) { /* We spliced out the successor of z; replace z by the successor */ + ASSERT(z != &null_x); replace(root, z, y); + lower_max_size(y, NULL); } if (spliced_is_black) { @@ -572,28 +698,48 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) RBT_ASSERT(!null_x.right); } } - - DESTROY_TREE_NODE(del); - -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif } static void -aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +aoff_link_free_block(Allctr_t *allctr, Block_t *block) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; - AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &alc->sbmbc_root : &alc->mbc_root); - Uint blk_sz = BLK_SZ(blk); + AOFF_RBTree_t *crr_node; + AOFF_Carrier_t *blk_crr = (AOFF_Carrier_t*) FBLK_TO_MBC(block); + Uint blk_sz = AOFF_BLK_SZ(blk); + + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); + ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + + rbt_insert(alc->flavor, &blk_crr->root, blk); + + /* Update the carrier tree with a potentially new (larger) max_sz + */ + crr_node = &blk_crr->rbt_node; + if (blk_sz > crr_node->hdr.bhdr) { + ASSERT(blk_sz == blk_crr->root->max_sz); + crr_node->hdr.bhdr = blk_sz; + while (blk_sz > crr_node->max_sz) { + crr_node->max_sz = blk_sz; + crr_node = crr_node->parent; + if (!crr_node) break; + } + } + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); +} -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif +static void +rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) +{ + Uint blk_sz = AOFF_BLK_SZ(blk); - blk->flags = 0; +#ifdef DEBUG + blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; +#else + blk->flags = 0; +#endif blk->left = NULL; blk->right = NULL; blk->max_sz = blk_sz; @@ -606,10 +752,12 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) else { AOFF_RBTree_t *x = *root; while (1) { + SWord diff; if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - if (blk < x) { + diff = cmp_blocks(flavor, blk, x); + if (diff < 0) { if (!x->left) { blk->parent = x; x->left = blk; @@ -617,7 +765,7 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } x = x->left; } - else { + else if (diff > 0) { if (!x->right) { blk->parent = x; x->right = blk; @@ -625,7 +773,18 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } x = x->right; } - + else { + ASSERT(flavor == AOFF_BF); + ASSERT(blk->flags & IS_BF_FLG); + ASSERT(x->flags & IS_BF_FLG); + SET_LIST_ELEM(blk); + LIST_NEXT(blk) = LIST_NEXT(x); + LIST_PREV(blk) = x; + if (LIST_NEXT(x)) + LIST_PREV(LIST_NEXT(x)) = blk; + LIST_NEXT(x) = blk; + return; + } } /* Insert block into size tree */ @@ -635,38 +794,63 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } + if (flavor == AOFF_BF) { + SET_TREE_NODE(blk); + LIST_NEXT(blk) = NULL; + } +} -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif +static AOFF_RBTree_t* +rbt_search(AOFF_RBTree_t* root, Uint size) +{ + AOFF_RBTree_t* x = root; + + ASSERT(x); + for (;;) { + if (x->left && x->left->max_sz >= size) { + x = x->left; + } + else if (AOFF_BLK_SZ(x) >= size) { + return x; + } + else { + x = x->right; + if (!x) { + return NULL; + } + } + } } static Block_t * aoff_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, Uint32 flags) + Block_t *cand_blk, Uint cand_size) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC) - ? alc->sbmbc_root : alc->mbc_root); + AOFF_RBTree_t *crr_node = alc->mbc_root; + AOFF_Carrier_t* crr; AOFF_RBTree_t *blk = NULL; #ifdef HARD_DEBUG - AOFF_RBTree_t* dbg_blk = check_tree(x, size); + AOFF_RBTree_t* dbg_blk; #endif - + ASSERT(!cand_blk || cand_size >= size); - while (x) { - if (x->left && x->left->max_sz >= size) { - x = x->left; - } - else if (BLK_SZ(x) >= size) { - blk = x; - break; - } - else { - x = x->right; - } + /* Get first-fit carrier + */ + if (!crr_node || !(blk=rbt_search(crr_node, size))) { + return NULL; } + crr = RBT_NODE_TO_MBC(blk); + + /* Get block within carrier tree + */ +#ifdef HARD_DEBUG + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); +#endif + + blk = rbt_search(crr->root, size); + ASSERT(blk); #ifdef HARD_DEBUG ASSERT(blk == dbg_blk); @@ -675,15 +859,87 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cand_blk < &blk->hdr) { + if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } - aoff_unlink_free_block(allctr, (Block_t *) blk, flags); + aoff_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } +static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + HARD_CHECK_TREE(NULL, 0, *root, 0); + + /* Link carrier in address order tree + */ + crr->rbt_node.hdr.bhdr = 0; + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + + /* aoff_link_free_block will add free block later */ + crr->root = NULL; + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static void aoff_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t *root = alc->mbc_root; + + if (crr->rbt_node.parent || &crr->rbt_node == root) { + aoff_remove_mbc(allctr, carrier); + } + /*else already removed */ +} + +static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + HARD_CHECK_TREE(NULL, 0, *root, 0); + + /* Link carrier in address order tree + */ + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + HARD_CHECK_TREE(NULL, 0, *root, 0); + + rbt_delete(root, &crr->rbt_node); + crr->rbt_node.parent = NULL; + crr->rbt_node.left = NULL; + crr->rbt_node.right = NULL; + crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr; + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) +{ + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + ASSERT(crr->rbt_node.hdr.bhdr == (crr->root ? crr->root->max_sz : 0)); + return crr->rbt_node.hdr.bhdr; +} /* * info_options() @@ -692,6 +948,8 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, static struct { Eterm as; Eterm aoff; + Eterm aoffcaobf; + Eterm aoffcbf; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -720,6 +978,8 @@ init_atoms(void) #endif AM_INIT(as); AM_INIT(aoff); + AM_INIT(aoffcaobf); + AM_INIT(aoffcbf); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -749,14 +1009,17 @@ info_options(Allctr_t *allctr, Uint **hpp, Uint *szp) { + AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; + const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; + Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - "aoff"); + flavor_str[alc->flavor]); } if (hpp || szp) { @@ -766,7 +1029,7 @@ info_options(Allctr_t *allctr, __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, am.aoff); + add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); } return res; @@ -780,19 +1043,28 @@ info_options(Allctr_t *allctr, * to erts_aoffalc_test() * \* */ -unsigned long -erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x500: return (unsigned long) 0; /* IS_AOBF */ - case 0x501: return (unsigned long) ((AOFFAllctr_t *) a1)->mbc_root; - case 0x502: return (unsigned long) ((AOFF_RBTree_t *) a1)->parent; - case 0x503: return (unsigned long) ((AOFF_RBTree_t *) a1)->left; - case 0x504: return (unsigned long) ((AOFF_RBTree_t *) a1)->right; - case 0x506: return (unsigned long) IS_BLACK((AOFF_RBTree_t *) a1); - case 0x508: return (unsigned long) 1; /* IS_AOFF */ - case 0x509: return (unsigned long) ((AOFF_RBTree_t *) a1)->max_sz; - default: ASSERT(0); return ~((unsigned long) 0); + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; + case 0x501: { + AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; + Uint size = (Uint) a2; + node = node ? rbt_search(node, size) : NULL; + return (UWord) (node ? RBT_NODE_TO_MBC(node)->root : NULL); + } + case 0x502: return (UWord) ((AOFF_RBTree_t *) a1)->parent; + case 0x503: return (UWord) ((AOFF_RBTree_t *) a1)->left; + case 0x504: return (UWord) ((AOFF_RBTree_t *) a1)->right; + case 0x505: return (UWord) LIST_NEXT(a1); + case 0x506: return (UWord) IS_BLACK((AOFF_RBTree_t *) a1); + case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); + case 0x508: return (UWord) 0; /* IS_BF_ALGO */ + case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50b: return (UWord) LIST_PREV(a1); + default: ASSERT(0); return ~((UWord) 0); } } @@ -804,6 +1076,16 @@ erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) #ifdef HARD_DEBUG +static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) +{ + while (node != root) { + ASSERT(node->parent); + ASSERT(node->parent->left == node || node->parent->right == node); + node = node->parent; + } + return 1; +} + #define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) #define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) @@ -840,16 +1122,19 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; Sint curr_blacks; AOFF_RBTree_t *x; + Carrier_t* crr; + Uint depth, max_depth, node_cnt; #ifdef PRINT_TREE print_tree(root); #endif + ASSERT(within_crr || flavor == AOFF_AOFF); if (!root) return res; @@ -859,12 +1144,16 @@ check_tree(AOFF_RBTree_t* root, Uint size) ASSERT(!x->parent); curr_blacks = 1; blacks = -1; + depth = 1; + max_depth = 0; + node_cnt = 0; while (x) { if (!IS_LEFT_VISITED(x)) { SET_LEFT_VISITED(x); if (x->left) { x = x->left; + ++depth; if (IS_BLACK(x)) curr_blacks++; continue; @@ -880,6 +1169,7 @@ check_tree(AOFF_RBTree_t* root, Uint size) SET_RIGHT_VISITED(x); if (x->right) { x = x->right; + ++depth; if (IS_BLACK(x)) curr_blacks++; continue; @@ -891,6 +1181,30 @@ check_tree(AOFF_RBTree_t* root, Uint size) } } + ++node_cnt; + if (depth > max_depth) + max_depth = depth; + + if (within_crr) { + crr = FBLK_TO_MBC(&x->hdr); + ASSERT(crr == within_crr); + ASSERT((char*)x > (char*)crr); + ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); + + } + if (flavor == AOFF_BF) { + AOFF_RBTree_t* y = x; + AOFF_RBTree_t* nxt = LIST_NEXT(y); + ASSERT(IS_TREE_NODE(x)); + while (nxt) { + ASSERT(IS_LIST_ELEM(nxt)); + ASSERT(AOFF_BLK_SZ(nxt) == AOFF_BLK_SZ(x)); + ASSERT(FBLK_TO_MBC(&nxt->hdr) == within_crr); + ASSERT(LIST_PREV(nxt) == y); + y = nxt; + nxt = LIST_NEXT(nxt); + } + } if (IS_RED(x)) { ASSERT(IS_BLACK(x->right)); @@ -901,22 +1215,22 @@ check_tree(AOFF_RBTree_t* root, Uint size) if (x->left) { ASSERT(x->left->parent == x); - ASSERT(x->left < x); + ASSERT(cmp_blocks(flavor, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(x->right > x); + ASSERT(cmp_blocks(flavor, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } - ASSERT(x->max_sz >= BLK_SZ(x)); - ASSERT(x->max_sz == BLK_SZ(x) + ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); + ASSERT(x->max_sz == AOFF_BLK_SZ(x) || x->max_sz == (x->left ? x->left->max_sz : 0) || x->max_sz == (x->right ? x->right->max_sz : 0)); - if (size && BLK_SZ(x) >= size) { - if (!res || x < res) { + if (size && AOFF_BLK_SZ(x) >= size) { + if (!res || cmp_blocks(flavor, x, res) < 0) { res = x; } } @@ -926,10 +1240,11 @@ check_tree(AOFF_RBTree_t* root, Uint size) if (IS_BLACK(x)) curr_blacks--; x = x->parent; - + --depth; } - + ASSERT(depth == 0 || (!root && depth==1)); ASSERT(curr_blacks == 0); + ASSERT((1 << (max_depth/2)) <= node_cnt); UNSET_LEFT_VISITED(root); UNSET_RIGHT_VISITED(root); @@ -954,9 +1269,9 @@ print_tree_aux(AOFF_RBTree_t *x, int indent) for (i = 0; i < indent; i++) { putc(' ', stderr); } - fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n", + fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%u\r\n", IS_BLACK(x) ? "BLACK" : "RED", - BLK_SZ(x), (Uint)x, x->max_sz); + AOFF_BLK_SZ(x), (Uint)x, (unsigned)x->max_sz); print_tree_aux(x->left, indent + INDENT_STEP); } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 6fa626f723..25b344c6a8 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -27,8 +27,14 @@ typedef struct AOFFAllctr_t_ AOFFAllctr_t; +enum AOFF_Flavor { + AOFF_AOFF = 0, + AOFF_AOBF = 1, + AOFF_BF = 2 +}; + typedef struct { - int dummy; + enum AOFF_Flavor flavor; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -51,10 +57,10 @@ struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - struct AOFF_RBTree_t_* sbmbc_root; + enum AOFF_Flavor flavor; }; -unsigned long erts_aoffalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_aoffalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_AOFF_ALLOC_IMPL) && !defined(ERL_AOFF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index f308039baf..e6d72f569b 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2012. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -122,6 +122,8 @@ typedef struct { #endif } ErtsAsyncData; +#if defined(USE_THREADS) && defined(USE_VM_PROBES) + /* * Some compilers, e.g. GCC 4.2.1 and -O3, will optimize away DTrace * calls if they're the last thing in the function. :-( @@ -129,6 +131,7 @@ typedef struct { * https://github.com/memcached/memcached/commit/6298b3978687530bc9d219b6ac707a1b681b2a46 */ static unsigned gcc_optimizer_hack = 0; +#endif int erts_async_max_threads; /* Initialized by erl_init.c */ int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */ @@ -281,8 +284,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) len = -1; DTRACE2(aio_pool_add, port_str, len); } -#endif gcc_optimizer_hack++; +#endif } static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, @@ -379,10 +382,15 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, static ERTS_INLINE void call_async_ready(ErtsAsync *a) { +#if ERTS_USE_ASYNC_READY_Q Port *p = erts_id2port_sflgs(a->port, NULL, 0, ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); +#else + Port *p = erts_thr_id2port_sflgs(a->port, + ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); +#endif if (!p) { if (a->async_free) a->async_free(a->async_data); @@ -392,7 +400,11 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a) if (a->async_free) a->async_free(a->async_data); } +#if ERTS_USE_ASYNC_READY_Q erts_port_release(p); +#else + erts_thr_port_release(p); +#endif } if (a->pdl) driver_pdl_dec_refc(a->pdl); @@ -571,6 +583,20 @@ int erts_async_ready_clean(void *varq, void *val) #endif /* +** Generate a fair async key prom an ErlDrvPort +** The port data gives a fair distribution grom port pointer +** to unsigned integer - to be used in key for driver_async below. +*/ +unsigned int driver_async_port_key(ErlDrvPort port) +{ + ErlDrvTermData td = driver_mk_port(port); + if (td == (ErlDrvTermData) NIL) { + return 0; + } + return (unsigned int) (UWord) internal_port_data(td); +} + +/* ** Schedule async_invoke on a worker thread ** NOTE will be syncrounous when threads are unsupported ** return values: @@ -601,7 +627,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key, #endif prt = erts_drvport2port(ix); - if (!prt) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -612,7 +638,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key, a->sched_id = sched_id; #endif a->hndl = (DE_Handle*)prt->drv_ptr->handle; - a->port = prt->id; + a->port = prt->common.id; a->pdl = NULL; a->async_data = async_data; a->async_invoke = async_invoke; diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index c50fdeb4e8..41f449bb28 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -73,6 +73,8 @@ #define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG) #define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG) +#define BF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) + #undef ASSERT #define ASSERT ASSERT_EXPR @@ -87,21 +89,21 @@ static RBTree_t * check_tree(RBTree_t, int, Uint); #endif -static void tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags); +static void tree_delete(Allctr_t *allctr, Block_t *del); /* Prototypes of callback functions */ /* "address order best fit" specific callback functions */ static Block_t * aobf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void aobf_link_free_block (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); +static void aobf_link_free_block (Allctr_t *, Block_t *); #define aobf_unlink_free_block tree_delete /* "best fit" specific callback functions */ static Block_t * bf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void bf_link_free_block (Allctr_t *, Block_t *, Uint32); -static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); +static void bf_link_free_block (Allctr_t *, Block_t *); +static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *); static Eterm info_options (Allctr_t *, char *, int *, @@ -206,6 +208,9 @@ erts_bfalc_start(BFAllctr_t *bfallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = NULL; allctr->destroying_mbc = NULL; + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -406,13 +411,11 @@ tree_insert_fixup(RBTree_t **root, RBTree_t *blk) * callback function in the address order case. */ static void -tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) +tree_delete(Allctr_t *allctr, Block_t *del) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; Uint spliced_is_black; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x, *y, *z = (RBTree_t *) del; RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ @@ -585,14 +588,12 @@ tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) \* */ static void -aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +aobf_link_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *blk = (RBTree_t *) block; - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = BF_BLK_SZ(blk); @@ -610,7 +611,7 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) while (1) { Uint size; - size = BLK_SZ(x); + size = BF_BLK_SZ(x); if (blk_sz < size || (blk_sz == size && blk < x)) { if (!x->left) { @@ -646,21 +647,18 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) #if 0 /* tree_delete() is directly used instead */ static void -aobf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +aobf_unlink_free_block(Allctr_t *allctr, Block_t *block) { - tree_delete(allctr, block, flags); + tree_delete(allctr, block); } #endif static Block_t * aobf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, - Uint32 flags) + Block_t *cand_blk, Uint cand_size) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -668,7 +666,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, ASSERT(!cand_blk || cand_size >= size); while (x) { - blk_sz = BLK_SZ(x); + blk_sz = BF_BLK_SZ(x); if (blk_sz < size) { x = x->right; } @@ -686,14 +684,14 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, #endif if (cand_blk) { - blk_sz = BLK_SZ(blk); + blk_sz = BF_BLK_SZ(blk); if (cand_size < blk_sz) return NULL; /* cand_blk was better */ if (cand_size == blk_sz && ((void *) cand_blk) < ((void *) blk)) return NULL; /* cand_blk was better */ } - aobf_unlink_free_block(allctr, (Block_t *) blk, flags); + aobf_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } @@ -704,14 +702,12 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, \* */ static void -bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +bf_link_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *blk = (RBTree_t *) block; - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = BF_BLK_SZ(blk); SET_TREE_NODE(blk); @@ -730,7 +726,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) while (1) { Uint size; - size = BLK_SZ(x); + size = BF_BLK_SZ(x); if (blk_sz == size) { @@ -778,12 +774,10 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static ERTS_INLINE void -bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +bf_unlink_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = (RBTree_t *) block; if (IS_LIST_ELEM(x)) { @@ -796,7 +790,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) else if (LIST_NEXT(x)) { /* Replace tree node by next element in list... */ - ASSERT(BLK_SZ(LIST_NEXT(x)) == BLK_SZ(x)); + ASSERT(BF_BLK_SZ(LIST_NEXT(x)) == BF_BLK_SZ(x)); ASSERT(IS_TREE_NODE(x)); ASSERT(IS_LIST_ELEM(LIST_NEXT(x))); @@ -811,7 +805,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } else { /* Remove from tree */ - tree_delete(allctr, block, flags); + tree_delete(allctr, block); } DESTROY_LIST_ELEM(x); @@ -820,13 +814,10 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) static Block_t * bf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, - Uint32 flags) + Block_t *cand_blk, Uint cand_size) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -834,7 +825,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, ASSERT(!cand_blk || cand_size >= size); while (x) { - blk_sz = BLK_SZ(x); + blk_sz = BF_BLK_SZ(x); if (blk_sz < size) { x = x->right; } @@ -855,18 +846,18 @@ bf_get_free_block(Allctr_t *allctr, Uint size, #ifdef HARD_DEBUG { RBTree_t *ct_blk = check_tree(root, 0, size); - ASSERT(BLK_SZ(ct_blk) == BLK_SZ(blk)); + ASSERT(BF_BLK_SZ(ct_blk) == BF_BLK_SZ(blk)); } #endif - if (cand_blk && cand_size <= BLK_SZ(blk)) + if (cand_blk && cand_size <= BF_BLK_SZ(blk)) return NULL; /* cand_blk was better */ /* Use next block if it exist in order to avoid replacing the tree node */ blk = LIST_NEXT(blk) ? LIST_NEXT(blk) : blk; - bf_unlink_free_block(allctr, (Block_t *) blk, flags); + bf_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } @@ -971,20 +962,22 @@ info_options(Allctr_t *allctr, * to erts_bfalc_test() * \* */ -unsigned long -erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_bfalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x200: return (unsigned long) ((BFAllctr_t *) a1)->address_order; - case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->mbc_root; - case 0x202: return (unsigned long) ((RBTree_t *) a1)->parent; - case 0x203: return (unsigned long) ((RBTree_t *) a1)->left; - case 0x204: return (unsigned long) ((RBTree_t *) a1)->right; - case 0x205: return (unsigned long) ((RBTreeList_t *) a1)->next; - case 0x206: return (unsigned long) IS_BLACK((RBTree_t *) a1); - case 0x207: return (unsigned long) IS_TREE_NODE((RBTree_t *) a1); - case 0x208: return (unsigned long) 0; /* IS_AOFF */ - default: ASSERT(0); return ~((unsigned long) 0); + case 0x200: return (UWord) ((BFAllctr_t *) a1)->address_order; /* IS_AOBF */ + case 0x201: return (UWord) ((BFAllctr_t *) a1)->mbc_root; + case 0x202: return (UWord) ((RBTree_t *) a1)->parent; + case 0x203: return (UWord) ((RBTree_t *) a1)->left; + case 0x204: return (UWord) ((RBTree_t *) a1)->right; + case 0x205: return (UWord) LIST_NEXT(a1); + case 0x206: return (UWord) IS_BLACK((RBTree_t *) a1); + case 0x207: return (UWord) IS_TREE_NODE((RBTree_t *) a1); + case 0x208: return (UWord) 1; /* IS_BF_ALGO */ + case 0x20a: return (UWord) !((BFAllctr_t *) a1)->address_order; /* IS_BF */ + case 0x20b: return (UWord) LIST_PREV(a1); + default: ASSERT(0); return ~((UWord) 0); } } @@ -1093,36 +1086,36 @@ check_tree(RBTree_t *root, int ao, Uint size) if (x->left) { ASSERT(x->left->parent == x); if (ao) { - ASSERT(BLK_SZ(x->left) < BLK_SZ(x) - || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x)); + ASSERT(BF_BLK_SZ(x->left) < BF_BLK_SZ(x) + || (BF_BLK_SZ(x->left) == BF_BLK_SZ(x) && x->left < x)); } else { ASSERT(IS_TREE_NODE(x->left)); - ASSERT(BLK_SZ(x->left) < BLK_SZ(x)); + ASSERT(BF_BLK_SZ(x->left) < BF_BLK_SZ(x)); } } if (x->right) { ASSERT(x->right->parent == x); if (ao) { - ASSERT(BLK_SZ(x->right) > BLK_SZ(x) - || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x)); + ASSERT(BF_BLK_SZ(x->right) > BF_BLK_SZ(x) + || (BF_BLK_SZ(x->right) == BF_BLK_SZ(x) && x->right > x)); } else { ASSERT(IS_TREE_NODE(x->right)); - ASSERT(BLK_SZ(x->right) > BLK_SZ(x)); + ASSERT(BF_BLK_SZ(x->right) > BF_BLK_SZ(x)); } } - if (size && BLK_SZ(x) >= size) { + if (size && BF_BLK_SZ(x) >= size) { if (ao) { if (!res - || BLK_SZ(x) < BLK_SZ(res) - || (BLK_SZ(x) == BLK_SZ(res) && x < res)) + || BF_BLK_SZ(x) < BF_BLK_SZ(res) + || (BF_BLK_SZ(x) == BF_BLK_SZ(res) && x < res)) res = x; } else { - if (!res || BLK_SZ(x) < BLK_SZ(res)) + if (!res || BF_BLK_SZ(x) < BF_BLK_SZ(res)) res = x; } } @@ -1168,7 +1161,7 @@ print_tree_aux(RBTree_t *x, int indent) } fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n", IS_BLACK(x) ? "BLACK" : "RED", - BLK_SZ(x), + BF_BLK_SZ(x), (Uint) x); print_tree_aux(x->left, indent + INDENT_STEP); } diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h index 0c29662852..870439e886 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.h +++ b/erts/emulator/beam/erl_bestfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -55,11 +55,10 @@ struct BFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ RBTree_t * mbc_root; - RBTree_t * sbmbc_root; int address_order; }; -unsigned long erts_bfalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_bfalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_BF_ALLOC_IMPL) && !defined(ERL_BF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index cc4f2be8eb..0db19a1ee6 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2011. All Rights Reserved. + * Copyright Ericsson AB 2010-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -72,52 +72,29 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); void erts_init_bif_binary(void) { - sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export)); - binary_match_trap_export.address = &binary_match_trap_export.code[3]; - binary_match_trap_export.code[0] = am_erlang; - binary_match_trap_export.code[1] = am_binary_match_trap; - binary_match_trap_export.code[2] = 3; - binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap; - - sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export)); - binary_matches_trap_export.address = &binary_matches_trap_export.code[3]; - binary_matches_trap_export.code[0] = am_erlang; - binary_matches_trap_export.code[1] = am_binary_matches_trap; - binary_matches_trap_export.code[2] = 3; - binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap; - - sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export)); - binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3]; - binary_longest_prefix_trap_export.code[0] = am_erlang; - binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap; - binary_longest_prefix_trap_export.code[2] = 3; - binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap; - - sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export)); - binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3]; - binary_longest_suffix_trap_export.code[0] = am_erlang; - binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap; - binary_longest_suffix_trap_export.code[2] = 3; - binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap; - - sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export)); - binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3]; - binary_bin_to_list_trap_export.code[0] = am_erlang; - binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap; - binary_bin_to_list_trap_export.code[2] = 3; - binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap; - sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export)); - binary_copy_trap_export.address = &binary_copy_trap_export.code[3]; - binary_copy_trap_export.code[0] = am_erlang; - binary_copy_trap_export.code[1] = am_binary_copy_trap; - binary_copy_trap_export.code[2] = 2; - binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif; - binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap; + erts_init_trap_export(&binary_match_trap_export, + am_erlang, am_binary_match_trap, 3, + &binary_match_trap); + + erts_init_trap_export(&binary_matches_trap_export, + am_erlang, am_binary_matches_trap, 3, + &binary_matches_trap); + + erts_init_trap_export(&binary_longest_prefix_trap_export, + am_erlang, am_binary_longest_prefix_trap, 3, + &binary_longest_prefix_trap); + + erts_init_trap_export(&binary_longest_suffix_trap_export, + am_erlang, am_binary_longest_suffix_trap, 3, + &binary_longest_suffix_trap); + + erts_init_trap_export(&binary_bin_to_list_trap_export, + am_erlang, am_binary_bin_to_list_trap, 3, + &binary_bin_to_list_trap); + + erts_init_trap_export(&binary_copy_trap_export, + am_erlang, am_binary_copy_trap, 2, + &binary_copy_trap); max_loop_limit = 0; return; diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 06b7ffdf32..4302fe8f79 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2010. All Rights Reserved. + * Copyright Ericsson AB 2008-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -42,16 +42,9 @@ static Export chksum_md5_2_exp; void erts_init_bif_chksum(void) { /* Non visual BIF to trap to. */ - memset(&chksum_md5_2_exp, 0, sizeof(Export)); - chksum_md5_2_exp.address = - &chksum_md5_2_exp.code[3]; - chksum_md5_2_exp.code[0] = am_erlang; - chksum_md5_2_exp.code[1] = am_atom_put("md5_trap",8); - chksum_md5_2_exp.code[2] = 2; - chksum_md5_2_exp.code[3] = - (BeamInstr) em_apply_bif; - chksum_md5_2_exp.code[4] = - (BeamInstr) &md5_2; + erts_init_trap_export(&chksum_md5_2_exp, + am_erlang, am_atom_put("md5_trap",8), 2, + &md5_2); } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index c338ee1c4b..1c3e955f47 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2012. All Rights Reserved. + * Copyright Ericsson AB 2006-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -104,16 +104,49 @@ static void dereference_all_processes(DE_Handle *dh); static void restore_process_references(DE_Handle *dh); static void ddll_no_more_references(void *vdh); -#define lock_drv_list() erts_smp_mtx_lock(&erts_driver_list_lock) -#define unlock_drv_list() erts_smp_mtx_unlock(&erts_driver_list_lock) +#define lock_drv_list() erts_smp_rwmtx_rwlock(&erts_driver_list_lock) +#define unlock_drv_list() erts_smp_rwmtx_rwunlock(&erts_driver_list_lock) #define assert_drv_list_locked() \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&erts_driver_list_lock)) + ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \ + || erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) +#define assert_drv_list_rwlocked() \ + ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock)) +#define assert_drv_list_rlocked() \ + ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) #define assert_drv_list_not_locked() \ - ERTS_SMP_LC_ASSERT(!erts_smp_lc_mtx_is_locked(&erts_driver_list_lock)) + ERTS_SMP_LC_ASSERT(!erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \ + && !erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) #define FREE_PORT_FLAGS (ERTS_PORT_SFLGS_DEAD & (~ERTS_PORT_SFLG_INITIALIZING)) +static void +kill_ports_driver_unloaded(DE_Handle *dh) +{ + int ix, max = erts_ptab_max(&erts_port); + + for (ix = 0; ix < max; ix++) { + erts_aint32_t state; + Port* prt = erts_pix2port(ix); + if (!prt) + continue; + + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + + state = erts_atomic32_read_nob(&prt->state); + if (state & FREE_PORT_FLAGS) + continue; + + erts_smp_port_lock(prt); + + state = erts_atomic32_read_nob(&prt->state); + if (!(state & ERTS_PORT_SFLGS_DEAD) && prt->drv_ptr->handle == dh) + driver_failure_atom(ERTS_Port2ErlDrvPort(prt), "driver_unloaded"); + + erts_port_release(prt); + } +} + /* * try_load(Path, Name, OptionList) -> {ok,Status} | * {ok, PendingStatus, Ref} | @@ -149,7 +182,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) Eterm name_term = BIF_ARG_2; Eterm options = BIF_ARG_3; char *path = NULL; - Uint path_len; + ErlDrvSizeT path_len; char *name = NULL; DE_Handle *dh; erts_driver_t *drv; @@ -228,7 +261,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) goto error; } path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1); - if (io_list_to_buf(path_term, path, path_len) != 0) { + if (erts_iolist_to_buf(path_term, path, path_len) != 0) { goto error; } while (path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/')) { @@ -356,42 +389,16 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) ok_term = mkatom("loaded"); } } - assert_drv_list_locked(); + assert_drv_list_rwlocked(); if (kill_ports) { - int j; - /* Avoid closing the driver by referencing it */ + /* Avoid closing the driver by referencing it */ erts_ddll_reference_driver(dh); ASSERT(dh->status == ERL_DE_RELOAD); dh->status = ERL_DE_FORCE_RELOAD; #if DDLL_SMP unlock_drv_list(); #endif - for (j = 0; j < erts_max_ports; j++) { - Port* prt = &erts_port[j]; -#ifdef DDLL_SMP - erts_smp_port_state_lock(prt); -#endif - if (!(prt->status & FREE_PORT_FLAGS) && - prt->drv_ptr->handle == dh) { -#if DDLL_SMP - erts_smp_atomic_inc_nob(&prt->refc); - /* Extremely rare spinlock */ - while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { - erts_smp_port_state_unlock(prt); - erts_smp_port_state_lock(prt); - } - erts_smp_port_state_unlock(prt); - erts_smp_mtx_lock(prt->lock); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) { - driver_failure_atom(j, "driver_unloaded"); - } -#else - driver_failure_atom(j, "driver_unloaded"); -#endif - erts_port_release(prt); - } - else erts_smp_port_state_unlock(prt); - } + kill_ports_driver_unloaded(dh); /* Dereference, eventually causing driver destruction */ #if DDLL_SMP lock_drv_list(); @@ -581,47 +588,21 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2) dh->reload_full_path = dh->reload_driver_name = NULL; dh->reload_flags = 0; } - if (dh->port_count > 0) { + if (erts_smp_atomic32_read_nob(&dh->port_count) > 0) { ++kill_ports; } dh->status = ERL_DE_UNLOAD; ok_term = am_pending_driver; done: - assert_drv_list_locked(); + assert_drv_list_rwlocked(); if (kill_ports > 1) { - int j; /* Avoid closing the driver by referencing it */ erts_ddll_reference_driver(dh); dh->status = ERL_DE_FORCE_UNLOAD; #if DDLL_SMP unlock_drv_list(); #endif - for (j = 0; j < erts_max_ports; j++) { - Port* prt = &erts_port[j]; -#if DDLL_SMP - erts_smp_port_state_lock(prt); -#endif - if (!(prt->status & FREE_PORT_FLAGS) - && prt->drv_ptr->handle == dh) { -#if DDLL_SMP - erts_smp_atomic_inc_nob(&prt->refc); - /* Extremely rare spinlock */ - while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { - erts_smp_port_state_unlock(prt); - erts_smp_port_state_lock(prt); - } - erts_smp_port_state_unlock(prt); - erts_smp_mtx_lock(prt->lock); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) { - driver_failure_atom(j, "driver_unloaded"); - } -#else - driver_failure_atom(j, "driver_unloaded"); -#endif - erts_port_release(prt); - } - else erts_smp_port_state_unlock(prt); - } + kill_ports_driver_unloaded(dh); #if DDLL_SMP lock_drv_list(); #endif @@ -791,7 +772,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) } else if (drv->handle->status == ERL_DE_PERMANENT) { res = am_permanent; } else { - res = make_small(drv->handle->port_count); + res = make_small(erts_smp_atomic32_read_nob(&drv->handle->port_count)); } goto done; case am_linked_in_driver: @@ -1049,40 +1030,16 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) } dh->status = ERL_DE_UNLOAD; } - if (!left && drv->handle->port_count > 0) { + if (!left + && erts_smp_atomic32_read_nob(&drv->handle->port_count) > 0) { if (kill_ports) { - int j; DE_Handle *dh = drv->handle; erts_ddll_reference_driver(dh); dh->status = ERL_DE_FORCE_UNLOAD; #if DDLL_SMP unlock_drv_list(); #endif - for (j = 0; j < erts_max_ports; j++) { - Port* prt = &erts_port[j]; -#if DDLL_SMP - erts_smp_port_state_lock(prt); -#endif - if (!(prt->status & FREE_PORT_FLAGS) && - prt->drv_ptr->handle == dh) { -#if DDLL_SMP - erts_smp_atomic_inc_nob(&prt->refc); - while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { - erts_smp_port_state_unlock(prt); - erts_smp_port_state_lock(prt); - } - erts_smp_port_state_unlock(prt); - erts_smp_mtx_lock(prt->lock); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) { - driver_failure_atom(j, "driver_unloaded"); - } -#else - driver_failure_atom(j, "driver_unloaded"); -#endif - erts_port_release(prt); - } - else erts_smp_port_state_unlock(prt); - } + kill_ports_driver_unloaded(dh); #if DDLL_SMP lock_drv_list(); /* Needed for future list operations */ #endif @@ -1104,7 +1061,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) void erts_ddll_lock_driver(DE_Handle *dh, char *name) { DE_ProcEntry *p,*q; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); notify_all(dh, name, ERL_DE_PROC_AWAIT_LOAD, am_UP, am_permanent); notify_all(dh, name, @@ -1127,19 +1084,22 @@ void erts_ddll_lock_driver(DE_Handle *dh, char *name) void erts_ddll_increment_port_count(DE_Handle *dh) { assert_drv_list_locked(); - dh->port_count++; + erts_smp_atomic32_inc_nob(&dh->port_count); } void erts_ddll_decrement_port_count(DE_Handle *dh) { assert_drv_list_locked(); - ASSERT(dh->port_count > 0); - dh->port_count--; +#if DEBUG + ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0); +#else + erts_smp_atomic32_dec_nob(&dh->port_count); +#endif } static void first_ddll_reference(DE_Handle *dh) { - assert_drv_list_locked(); + assert_drv_list_rwlocked(); erts_refc_init(&(dh->refc),1); } @@ -1167,7 +1127,7 @@ void erts_ddll_dereference_driver(DE_Handle *dh) static void dereference_all_processes(DE_Handle *dh) { DE_ProcEntry *p; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); for(p = dh->procs;p != NULL; p = p->next) { if (p->awaiting_status == ERL_DE_PROC_LOADED) { ASSERT(!(p->flags & ERL_DE_FL_DEREFERENCED)); @@ -1180,7 +1140,7 @@ static void dereference_all_processes(DE_Handle *dh) static void restore_process_references(DE_Handle *dh) { DE_ProcEntry *p; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); ASSERT(erts_refc_read(&(dh->refc),0) == 0); for(p = dh->procs;p != NULL; p = p->next) { if (p->awaiting_status == ERL_DE_PROC_LOADED) { @@ -1408,7 +1368,7 @@ static int is_last_user(DE_Handle *dh, Process *proc) { DE_ProcEntry *p = dh->procs; int found = 0; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); while (p != NULL) { if (p->proc == proc && p->awaiting_status == ERL_DE_PROC_LOADED) { @@ -1429,7 +1389,7 @@ static DE_ProcEntry *find_proc_entry(DE_Handle *dh, Process *proc, Uint status) { DE_ProcEntry *p = dh->procs; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); while (p != NULL) { if (p->proc == proc && p->awaiting_status == status) { @@ -1456,7 +1416,7 @@ static int num_procs(DE_Handle *dh, Uint status) { DE_ProcEntry *p = dh->procs; int i = 0; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); while (p != NULL) { if (p->awaiting_status == status) { @@ -1471,7 +1431,7 @@ static int num_entries(DE_Handle *dh, Process *proc, Uint status) { DE_ProcEntry *p = dh->procs; int i = 0; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); while (p != NULL) { if (p->awaiting_status == status && p->proc == proc) { ++i; @@ -1484,7 +1444,7 @@ static int num_entries(DE_Handle *dh, Process *proc, Uint status) { static void add_proc_loaded(DE_Handle *dh, Process *proc) { DE_ProcEntry *p; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry)); p->proc = proc; p->flags = 0; @@ -1496,7 +1456,7 @@ static void add_proc_loaded(DE_Handle *dh, Process *proc) static void add_proc_loaded_deref(DE_Handle *dh, Process *proc) { DE_ProcEntry *p; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry)); p->proc = proc; p->awaiting_status = ERL_DE_PROC_LOADED; @@ -1516,7 +1476,7 @@ static void add_proc_waiting(DE_Handle *dh, Process *proc, Uint status, Eterm ref) { DE_ProcEntry *p; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry)); p->proc = proc; p->flags = 0; @@ -1530,7 +1490,7 @@ static Eterm add_monitor(Process *p, DE_Handle *dh, Uint status) { Eterm r; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); r = erts_make_ref(p); add_proc_waiting(dh, p, status, r); return r; @@ -1541,7 +1501,7 @@ static void set_driver_reloading(DE_Handle *dh, Process *proc, char *path, char { DE_ProcEntry *p; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry)); p->proc = proc; p->awaiting_status = ERL_DE_OK; @@ -1562,7 +1522,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) int res; ErlDrvEntry *dp; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); if ((res = erts_sys_ddll_open(path, &(dh->handle))) != ERL_DE_NO_ERROR) { return res; @@ -1600,7 +1560,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) goto error; } erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); - dh->port_count = 0; + erts_smp_atomic32_init_nob(&dh->port_count, 0); dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); dh->flags = 0; @@ -1626,7 +1586,7 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name) { erts_driver_t *q, *p = driver_list; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); while (p != NULL) { if (p->handle == dh) { @@ -1666,11 +1626,11 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name) int res; DE_Handle *dh = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sizeof(DE_Handle)); - assert_drv_list_locked(); + assert_drv_list_rwlocked(); dh->handle = NULL; dh->procs = NULL; - dh->port_count = 0; + erts_smp_atomic32_init_nob(&dh->port_count, 0); erts_refc_init(&(dh->refc), (erts_aint_t) 0); dh->status = -1; dh->reload_full_path = NULL; @@ -1704,7 +1664,7 @@ static int reload_driver_entry(DE_Handle *dh) int loadres; Uint flags = dh->reload_flags; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); dh->reload_full_path = NULL; dh->reload_driver_name = NULL; @@ -1742,7 +1702,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, ErtsProcLocks rp_locks = 0; ERTS_SMP_CHK_NO_PROC_LOCKS; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); if (errcode != 0) { int need = load_error_need(errcode); Eterm e; @@ -1775,7 +1735,7 @@ static void notify_all(DE_Handle *dh, char *name, Uint awaiting, Eterm type, Ete { DE_ProcEntry **p; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); p = &(dh->procs); while (*p != NULL) { @@ -1875,13 +1835,16 @@ static Eterm build_load_error_hp(Eterm *hp, int code) static Eterm mkatom(char *str) { - return am_atom_put(str, sys_strlen(str)); + return erts_atom_put((byte *) str, + sys_strlen(str), + ERTS_ATOM_ENC_LATIN1, + 1); } static char *pick_list_or_atom(Eterm name_term) { char *name = NULL; - Uint name_len; + ErlDrvSizeT name_len; if (is_atom(name_term)) { Atom *ap = atom_tab(atom_val(name_term)); if (ap->len == 0) { @@ -1897,7 +1860,7 @@ static char *pick_list_or_atom(Eterm name_term) goto error; } name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1); - if (io_list_to_buf(name_term, name, name_len) != 0) { + if (erts_iolist_to_buf(name_term, name, name_len) != 0) { goto error; } name[name_len] = '\0'; @@ -1918,10 +1881,10 @@ static int build_proc_info(DE_Handle *dh, ProcEntryInfo **out_pei, Uint filter) int i; DE_ProcEntry *pe; - assert_drv_list_locked(); + assert_drv_list_rwlocked(); for (pe = dh->procs; pe != NULL; pe = pe->next) { - Eterm id = pe->proc->id; + Eterm id = pe->proc->common.id; Uint stat = pe->awaiting_status; if (stat == ERL_DE_PROC_AWAIT_UNLOAD_ONLY) { stat = ERL_DE_PROC_AWAIT_UNLOAD; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 45dc5fb11c..673dfc658c 100755 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -30,6 +30,7 @@ #include "bif.h" #include "big.h" #include "erl_version.h" +#include "erl_compile_flags.h" #include "erl_db_util.h" #include "erl_message.h" #include "erl_binary.h" @@ -40,6 +41,8 @@ #include "erl_cpu_topology.h" #include "erl_async.h" #include "erl_thr_progress.h" +#define ERTS_PTAB_WANT_DEBUG_FUNCS__ +#include "erl_ptab.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -57,6 +60,7 @@ static Export* alloc_info_trap = NULL; static Export* alloc_sizes_trap = NULL; static Export *gather_sched_wall_time_res_trap; +static Export *gather_gc_info_res_trap; #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -67,7 +71,11 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE " [no-c-stack-objects]" #endif #ifndef OTP_RELEASE +#ifdef ERLANG_GIT_VERSION + " [source-" ERLANG_GIT_VERSION "]" +#else " [source]" +#endif #endif #ifdef ARCH_64 #if HALFWORD_HEAP @@ -108,6 +116,9 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #ifdef VALGRIND " [valgrind-compiled]" #endif +#ifdef ERTS_FRMPTR + " [frame-pointer]" +#endif #ifdef USE_DTRACE " [dtrace]" #endif @@ -128,8 +139,6 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE static Eterm os_type_tuple; static Eterm os_version_tuple; -static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item); - static Eterm current_function(Process* p, Process* rp, Eterm** hpp, int full_info); static Eterm current_stacktrace(Process* p, Process* rp, Eterm** hpp); @@ -482,27 +491,6 @@ collect_one_suspend_monitor(ErtsSuspendMonitor *smon, void *vsmicp) } } - -static void one_link_size(ErtsLink *lnk, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_LINK_SIZE*sizeof(Uint); - if(!IS_CONST(lnk->pid)) - *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); - if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { - erts_doforall_links(ERTS_LINK_ROOT(lnk),&one_link_size,vpu); - } -} -static void one_mon_size(ErtsMonitor *mon, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_MONITOR_SIZE*sizeof(Uint); - if(!IS_CONST(mon->pid)) - *pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint); - if(!IS_CONST(mon->ref)) - *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); -} - /* * process_info/[1,2] */ @@ -873,8 +861,7 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1) && external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) BIF_RET(am_undefined); - if (is_not_internal_pid(BIF_ARG_1) - || internal_pid_index(BIF_ARG_1) >= erts_max_processes) { + if (is_not_internal_pid(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } @@ -909,8 +896,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) && external_pid_dist_entry(pid) == erts_this_dist_entry) BIF_RET(am_undefined); - if (is_not_internal_pid(pid) - || internal_pid_index(BIF_ARG_1) >= erts_max_processes) { + if (is_not_internal_pid(pid)) { BIF_ERROR(BIF_P, BADARG); } @@ -1002,9 +988,9 @@ process_info_aux(Process *BIF_P, switch (item) { case am_registered_name: - if (rp->reg != NULL) { + if (rp->common.u.alive.reg) { hp = HAlloc(BIF_P, 3); - res = rp->reg->name; + res = rp->common.u.alive.reg->name; } else { if (always_wrap) { hp = HAlloc(BIF_P, 3); @@ -1050,7 +1036,7 @@ process_info_aux(Process *BIF_P, ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); n = rp->msg.len; - if (n == 0 || rp->trace_flags & F_SENSITIVE) { + if (n == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { hp = HAlloc(BIF_P, 3); } else { int remove_bad_messages = 0; @@ -1209,7 +1195,7 @@ process_info_aux(Process *BIF_P, INIT_MONITOR_INFOS(mic); - erts_doforall_links(rp->nlinks,&collect_one_link,&mic); + erts_doforall_links(ERTS_P_LINKS(rp),&collect_one_link,&mic); hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; @@ -1227,7 +1213,7 @@ process_info_aux(Process *BIF_P, int i; INIT_MONITOR_INFOS(mic); - erts_doforall_monitors(rp->monitors,&collect_one_origin_monitor,&mic); + erts_doforall_monitors(ERTS_P_MONITORS(rp),&collect_one_origin_monitor,&mic); hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { @@ -1264,7 +1250,7 @@ process_info_aux(Process *BIF_P, Eterm item; INIT_MONITOR_INFOS(mic); - erts_doforall_monitors(rp->monitors,&collect_one_target_monitor,&mic); + erts_doforall_monitors(ERTS_P_MONITORS(rp),&collect_one_target_monitor,&mic); hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; @@ -1330,7 +1316,7 @@ process_info_aux(Process *BIF_P, } case am_dictionary: - if (rp->trace_flags & F_SENSITIVE) { + if (ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { res = NIL; } else { res = erts_dictionary_copy(BIF_P, rp->dictionary); @@ -1338,13 +1324,15 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3); break; - case am_trap_exit: + case am_trap_exit: { + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); hp = HAlloc(BIF_P, 3); - if (rp->flags & F_TRAPEXIT) + if (state & ERTS_PSFLG_TRAP_EXIT) res = am_true; else res = am_false; break; + } case am_error_handler: hp = HAlloc(BIF_P, 3); @@ -1416,41 +1404,8 @@ process_info_aux(Process *BIF_P, } case am_memory: { /* Memory consumed in bytes */ - ErlMessage *mp; - Uint size = 0; Uint hsz = 3; - struct saved_calls *scb; - size += sizeof(Process); - - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - - erts_doforall_links(rp->nlinks, &one_link_size, &size); - erts_doforall_monitors(rp->monitors, &one_mon_size, &size); - size += (rp->heap_sz + rp->mbuf_sz) * sizeof(Eterm); - if (rp->old_hend && rp->old_heap) - size += (rp->old_hend - rp->old_heap) * sizeof(Eterm); - - size += rp->msg.len * sizeof(ErlMessage); - - for (mp = rp->msg.first; mp; mp = mp->next) - if (mp->data.attached) - size += erts_msg_attached_data_size(mp)*sizeof(Eterm); - - if (rp->arg_reg != rp->def_arg_reg) { - size += rp->arity * sizeof(rp->arg_reg[0]); - } - - if (rp->psd) - size += sizeof(ErtsPSD); - - scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp); - if (scb) { - size += (sizeof(struct saved_calls) - + (scb->len-1) * sizeof(scb->ct[0])); - } - - size += erts_dicts_mem_size(rp); - + Uint size = erts_process_memory(rp); (void) erts_bld_uint(NULL, &hsz, size); hp = HAlloc(BIF_P, hsz); res = erts_bld_uint(&hp, NULL, size); @@ -1498,7 +1453,7 @@ process_info_aux(Process *BIF_P, case am_trace: hp = HAlloc(BIF_P, 3); - res = make_small(rp->trace_flags & TRACEE_FLAGS); + res = make_small(ERTS_TRACE_FLAGS(rp) & TRACEE_FLAGS); break; case am_binary: { @@ -1603,7 +1558,7 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) } } - if (BIF_P->id == rp->id) { + if (BIF_P == rp) { FunctionInfo fi2; /* @@ -1717,13 +1672,22 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ sel = *tp++; - if (sel == am_allocator_sizes) { + if (sel == am_memory_internal) { + switch (arity) { + case 3: + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 1)) + return am_true; + default: + goto badarg; + } + } + else if (sel == am_allocator_sizes) { switch (arity) { case 2: ERTS_BIF_PREP_TRAP1(ret, alloc_sizes_trap, BIF_P, *tp); return ret; case 3: - if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1)) + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 0)) return am_true; default: goto badarg; @@ -1782,7 +1746,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ ERTS_BIF_PREP_TRAP1(ret, alloc_info_trap, BIF_P, *tp); return ret; case 3: - if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0)) + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0, 0)) return am_true; default: goto badarg; @@ -1835,17 +1799,17 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ # define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML # endif #endif - Uint buf_size = 8*1024; /* Try with 8KB first */ + ErlDrvSizeT buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); - int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); - if (r < 0) { + ErlDrvSizeT r = erts_iolist_to_buf(*tp, (char*) buf, buf_size - 1); + if (ERTS_IOLIST_TO_BUF_FAILED(r)) { erts_free(ERTS_ALC_T_TMP, (void *) buf); if (erts_iolist_size(*tp, &buf_size)) { goto badarg; } buf_size++; buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); - r = io_list_to_buf(*tp, (char*) buf, buf_size - 1); + r = erts_iolist_to_buf(*tp, (char*) buf, buf_size - 1); ASSERT(r == buf_size - 1); } buf[buf_size - 1 - r] = '\0'; @@ -2092,6 +2056,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #elif defined(ERTS_ENABLE_LOCK_COUNT) ERTS_DECL_AM(lcnt); BIF_RET(AM_lcnt); +#elif defined(ERTS_FRMPTR) + ERTS_DECL_AM(frmptr); + BIF_RET(AM_frmptr); #else BIF_RET(am_opt); #endif @@ -2157,9 +2124,13 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE2(hp, am_min_bin_vheap_size,make_small(BIN_VH_MIN_SIZE)); BIF_RET(res); } else if (BIF_ARG_1 == am_process_count) { - BIF_RET(make_small(erts_process_count())); + BIF_RET(make_small(erts_ptab_count(&erts_proc))); } else if (BIF_ARG_1 == am_process_limit) { - BIF_RET(make_small(erts_max_processes)); + BIF_RET(make_small(erts_ptab_max(&erts_proc))); + } else if (BIF_ARG_1 == am_port_count) { + BIF_RET(make_small(erts_ptab_count(&erts_port))); + } else if (BIF_ARG_1 == am_port_limit) { + BIF_RET(make_small(erts_ptab_max(&erts_port))); } else if (BIF_ARG_1 == am_info || BIF_ARG_1 == am_procs || BIF_ARG_1 == am_loaded @@ -2294,8 +2265,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) for (i = num_instructions-1; i >= 0; i--) { res = erts_bld_cons(hpp, hszp, erts_bld_tuple(hpp, hszp, 2, - am_atom_put(opc[i].name, - strlen(opc[i].name)), + erts_atom_put(opc[i].name, + strlen(opc[i].name), + ERTS_ATOM_ENC_LATIN1, + 1), erts_bld_uint(hpp, hszp, opc[i].count)), res); @@ -2532,6 +2505,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { res = make_small(erts_no_run_queues); BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("port_parallelism", BIF_ARG_1)) { + res = erts_port_parallelism ? am_true : am_false; + BIF_RET(res); } else if (ERTS_IS_ATOM_STR("c_compiler_used", BIF_ARG_1)) { Eterm *hp = NULL; Uint sz = 0; @@ -2602,74 +2578,14 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); - } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) { -#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ - || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ - || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) - int i; - char **str; -#endif -#ifdef ETHR_NATIVE_ATOMIC32_IMPL - erts_printf("32-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC32_IMPL); - str = ethr_native_atomic32_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic32_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_ATOMIC64_IMPL - erts_printf("64-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC64_IMPL); - str = ethr_native_atomic64_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic64_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL - if (ethr_have_native_dw_atomic()) { - erts_printf("Double word native atomics: %s\n", - ETHR_NATIVE_DW_ATOMIC_IMPL); - str = ethr_native_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_dw_atomic_%s()\n", str[i]); - str = ethr_native_su_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]); - } -#endif -#ifdef ETHR_NATIVE_SPINLOCK_IMPL - erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL); -#endif -#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL - erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL); -#endif -#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - ? "yes" : "no")); -#endif -#ifdef ETHR_X86_OUT_OF_ORDER - erts_printf("x86" -#ifdef ARCH_64 - "_64" -#endif - " out of order\n"); -#endif -#ifdef ETHR_SPARC_TSO - erts_printf("Sparc TSO\n"); -#endif -#ifdef ETHR_SPARC_PSO - erts_printf("Sparc PSO\n"); -#endif -#ifdef ETHR_SPARC_RMO - erts_printf("Sparc RMO\n"); -#endif -#if defined(ETHR_PPC_HAVE_LWSYNC) - erts_printf("Have lwsync instruction: yes\n"); -#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) - erts_printf("Have lwsync instruction: no\n"); -#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) - erts_printf("Have lwsync instruction: %s (runtime test)\n", - ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"); -#endif - BIF_RET(am_true); + } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) { + BIF_RET(erts_get_ethread_info(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("emu_args", BIF_ARG_1)) { + BIF_RET(erts_get_emu_args(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("beam_jump_table", BIF_ARG_1)) { + BIF_RET(erts_beam_jump_table() ? am_true : am_false); } else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) { #if defined(USE_DTRACE) @@ -2695,70 +2611,35 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_true); } #endif + else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) { + Uint sz; + Eterm res = NIL, tup, text; + Eterm *hp = HAlloc(BIF_P, 3*(2 + 3) + /* three 2-tuples and three cons */ + 2*(strlen(erts_build_flags_CONFIG_H) + + strlen(erts_build_flags_CFLAGS) + + strlen(erts_build_flags_LDFLAGS))); - BIF_ERROR(BIF_P, BADARG); -} - -BIF_RETTYPE -port_info_1(BIF_ALIST_1) -{ - Process* p = BIF_P; - Eterm pid = BIF_ARG_1; - static Eterm keys[] = { - am_name, - am_links, - am_id, - am_connected, - am_input, - am_output, - am_os_pid - }; - Eterm items[ASIZE(keys)]; - Eterm result = NIL; - Eterm reg_name; - Eterm* hp; - Uint need; - int i; - - /* - * Collect all information about the port. - */ + sz = strlen(erts_build_flags_CONFIG_H); + text = buf_to_intlist(&hp, erts_build_flags_CONFIG_H, sz, NIL); + tup = TUPLE2(hp, am_config_h, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; - for (i = 0; i < ASIZE(keys); i++) { - Eterm item; + sz = strlen(erts_build_flags_CFLAGS); + text = buf_to_intlist(&hp, erts_build_flags_CFLAGS, sz, NIL); + tup = TUPLE2(hp, am_cflags, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; - item = port_info(p, pid, keys[i]); - if (is_non_value(item)) { - return THE_NON_VALUE; - } - if (item == am_undefined) { - return am_undefined; - } - items[i] = item; - } - reg_name = port_info(p, pid, am_registered_name); + sz = strlen(erts_build_flags_LDFLAGS); + text = buf_to_intlist(&hp, erts_build_flags_LDFLAGS, sz, NIL); + tup = TUPLE2(hp, am_ldflags, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; - /* - * Build the resulting list. - */ - - need = 2*ASIZE(keys); - if (is_tuple(reg_name)) { - need += 2; - } - hp = HAlloc(p, need); - for (i = ASIZE(keys) - 1; i >= 0; i--) { - result = CONS(hp, items[i], result); - hp += 2; - } - if (is_tuple(reg_name)) { - result = CONS(hp, reg_name, result); + BIF_RET(res); } - return result; + BIF_ERROR(BIF_P, BADARG); } - /**********************************************************************/ /* Return information on ports */ /* Info: @@ -2771,38 +2652,20 @@ port_info_1(BIF_ALIST_1) ** os_pid The child's process ID */ -BIF_RETTYPE port_info_2(BIF_ALIST_2) -{ - return port_info(BIF_P, BIF_ARG_1, BIF_ARG_2); -} - -static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item) +Eterm +erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm item) { - BIF_RETTYPE ret; - Port *prt; - Eterm res; - Eterm* hp; - int count; - - if (is_internal_port(portid)) - prt = erts_id2port(portid, p, ERTS_PROC_LOCK_MAIN); - else if (is_atom(portid)) - erts_whereis_name(p, ERTS_PROC_LOCK_MAIN, - portid, NULL, 0, 0, &prt); - else if (is_external_port(portid) - && external_port_dist_entry(portid) == erts_this_dist_entry) - BIF_RET(am_undefined); - else { - BIF_ERROR(p, BADARG); - } + Eterm res = THE_NON_VALUE; - if (!prt) { - BIF_RET(am_undefined); - } + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (item == am_id) { - hp = HAlloc(p, 3); - res = make_small(internal_port_number(portid)); + if (hpp) + res = make_small(internal_port_index(prt->common.id)); + if (szp) { + res = am_true; + goto done; + } } else if (item == am_links) { MonitorInfoCollection mic; @@ -2811,17 +2674,26 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item) INIT_MONITOR_INFOS(mic); - erts_doforall_links(prt->nlinks, &collect_one_link, &mic); + erts_doforall_links(ERTS_P_LINKS(prt), &collect_one_link, &mic); - hp = HAlloc(p, 3 + mic.sz); - res = NIL; - for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity); - res = CONS(hp, item, res); - hp += 2; + if (szp) + *szp += mic.sz; + + if (hpp) { + res = NIL; + for (i = 0; i < mic.mi_i; i++) { + item = STORE_NC(hpp, ohp, mic.mi[i].entity); + res = CONS(*hpp, item, res); + *hpp += 2; + } } + DESTROY_MONITOR_INFOS(mic); + if (szp) { + res = am_true; + goto done; + } } else if (item == am_monitors) { MonitorInfoCollection mic; @@ -2830,82 +2702,97 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item) INIT_MONITOR_INFOS(mic); - erts_doforall_monitors(prt->monitors, &collect_one_origin_monitor, &mic); + erts_doforall_monitors(ERTS_P_MONITORS(prt), &collect_one_origin_monitor, &mic); - hp = HAlloc(p, 3 + mic.sz); - res = NIL; - for (i = 0; i < mic.mi_i; i++) { - Eterm t; - item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity); - t = TUPLE2(hp, am_process, item); - hp += 3; - res = CONS(hp, t, res); - hp += 2; + if (szp) + *szp += mic.sz; + + if (hpp) { + res = NIL; + for (i = 0; i < mic.mi_i; i++) { + Eterm t; + item = STORE_NC(hpp, ohp, mic.mi[i].entity); + t = TUPLE2(*hpp, am_process, item); + *hpp += 3; + res = CONS(*hpp, t, res); + *hpp += 2; + } } + DESTROY_MONITOR_INFOS(mic); + if (szp) { + res = am_true; + goto done; + } } else if (item == am_name) { - count = sys_strlen(prt->name); + int count = sys_strlen(prt->name); - hp = HAlloc(p, 3 + 2*count); - res = buf_to_intlist(&hp, prt->name, count, NIL); + if (hpp) + res = buf_to_intlist(hpp, prt->name, count, NIL); + + if (szp) { + *szp += 2*count; + res = am_true; + goto done; + } } else if (item == am_connected) { - hp = HAlloc(p, 3); - res = prt->connected; /* internal pid */ + if (hpp) + res = ERTS_PORT_GET_CONNECTED(prt); /* internal pid */ + if (szp) { + res = am_true; + goto done; + } } else if (item == am_input) { - Uint hsz = 3; - Uint n = prt->bytes_in; - (void) erts_bld_uint(NULL, &hsz, n); - hp = HAlloc(p, hsz); - res = erts_bld_uint(&hp, NULL, n); + res = erts_bld_uint(hpp, szp, prt->bytes_in); + if (szp) { + res = am_true; + goto done; + } } else if (item == am_output) { - Uint hsz = 3; - Uint n = prt->bytes_out; - (void) erts_bld_uint(NULL, &hsz, n); - hp = HAlloc(p, hsz); - res = erts_bld_uint(&hp, NULL, n); + res = erts_bld_uint(hpp, szp, prt->bytes_out); + if (szp) { + res = am_true; + goto done; + } } else if (item == am_os_pid) { - if (prt->os_pid >= 0) { - Uint hsz = 3; - UWord n = prt->os_pid; - (void) erts_bld_uword(NULL, &hsz, n); - hp = HAlloc(p, hsz); - res = erts_bld_uword(&hp, NULL, n); - } else { - hp = HAlloc(p, 3); - res = am_undefined; - } + res = (prt->os_pid < 0 + ? am_undefined + : erts_bld_uword(hpp, szp, (UWord) prt->os_pid)); + if (szp) { + res = am_true; + goto done; + } } else if (item == am_registered_name) { - RegProc *reg; - reg = prt->reg; - if (reg == NULL) { - ERTS_BIF_PREP_RET(ret, NIL); - goto done; - } else { - hp = HAlloc(p, 3); + RegProc *reg = prt->common.u.alive.reg; + if (reg) { res = reg->name; + if (szp) { + res = am_true; + goto done; + } + } + else { + if (szp) + return am_undefined; + return NIL; } } else if (item == am_memory) { /* All memory consumed in bytes (the Port struct should not be included though). */ - Uint hsz = 3; Uint size = 0; - ErlHeapFragment* bp; - - hp = HAlloc(p, 3); - erts_doforall_links(prt->nlinks, &one_link_size, &size); + erts_doforall_links(ERTS_P_LINKS(prt), &erts_one_link_size, &size); - for (bp = prt->bp; bp; bp = bp->next) - size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm); + size += erts_port_data_size(prt); if (prt->linebuf) size += sizeof(LineBuf) + prt->linebuf->ovsiz; @@ -2916,51 +2803,72 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item) /* All memory allocated by the driver should be included, but it is hard to retrieve... */ - (void) erts_bld_uint(NULL, &hsz, size); - hp = HAlloc(p, hsz); - res = erts_bld_uint(&hp, NULL, size); + res = erts_bld_uint(hpp, szp, size); + if (szp) { + res = am_true; + goto done; + } } else if (item == am_queue_size) { Uint ioq_size = erts_port_ioq_size(prt); - Uint hsz = 3; - (void) erts_bld_uint(NULL, &hsz, ioq_size); - hp = HAlloc(p, hsz); - res = erts_bld_uint(&hp, NULL, ioq_size); + res = erts_bld_uint(hpp, szp, ioq_size); + if (szp) { + res = am_true; + goto done; + } } else if (ERTS_IS_ATOM_STR("locking", item)) { - hp = HAlloc(p, 3); + if (hpp) { #ifndef ERTS_SMP - res = am_false; + res = am_false; #else - if (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { - DECL_AM(port_level); - ASSERT(prt->drv_ptr->flags - & ERL_DRV_FLAG_USE_PORT_LOCKING); - res = AM_port_level; + if (erts_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { + DECL_AM(port_level); + ASSERT(prt->drv_ptr->flags + & ERL_DRV_FLAG_USE_PORT_LOCKING); + res = AM_port_level; + } + else { + DECL_AM(driver_level); + ASSERT(!(prt->drv_ptr->flags + & ERL_DRV_FLAG_USE_PORT_LOCKING)); + res = AM_driver_level; + } +#endif } - else { - DECL_AM(driver_level); - ASSERT(!(prt->drv_ptr->flags - & ERL_DRV_FLAG_USE_PORT_LOCKING)); - res = AM_driver_level; + if (szp) { + res = am_true; + goto done; } -#endif + } + else if (item == am_parallelism) { + if (szp) { + res = am_true; + goto done; + } + res = ((ERTS_PTS_FLG_PARALLELISM & + erts_smp_atomic32_read_nob(&prt->sched.flags)) + ? am_true + : am_false); } else { - ERTS_BIF_PREP_ERROR(ret, p, BADARG); - goto done; + if (szp) + return am_false; + return THE_NON_VALUE; } - ERTS_BIF_PREP_RET(ret, TUPLE2(hp, item, res)); - - done: - - erts_smp_port_unlock(prt); +done: + if (szp) + *szp += 3; + if (hpp) { + res = TUPLE2(*hpp, item, res); + *hpp += 3; + } - return ret; + return res; } - BIF_RETTYPE fun_info_2(BIF_ALIST_2) { @@ -3092,21 +3000,16 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) if(is_internal_pid(BIF_ARG_1)) { Process *rp; - if (BIF_ARG_1 == BIF_P->id) + if (BIF_ARG_1 == BIF_P->common.id) BIF_RET(am_true); - if(internal_pid_index(BIF_ARG_1) >= erts_max_processes) - BIF_ERROR(BIF_P, BADARG); - - rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_STATUS); + rp = erts_proc_lookup_raw(BIF_ARG_1); if (!rp) { BIF_RET(am_false); } else { - int have_pending_exit = ERTS_PROC_PENDING_EXIT(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - if (have_pending_exit) + if (erts_smp_atomic32_read_acqb(&rp->state) + & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false); else BIF_RET(am_true); @@ -3176,18 +3079,10 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, cs, SMALL_ZERO); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection) { - Uint hsz = 4; - ErtsGCInfo gc_info; - Eterm gcs; - Eterm recl; - erts_gc_info(&gc_info); - (void) erts_bld_uint(NULL, &hsz, gc_info.garbage_collections); - (void) erts_bld_uint(NULL, &hsz, gc_info.reclaimed); - hp = HAlloc(BIF_P, hsz); - gcs = erts_bld_uint(&hp, NULL, gc_info.garbage_collections); - recl = erts_bld_uint(&hp, NULL, gc_info.reclaimed); - res = TUPLE3(hp, gcs, recl, SMALL_ZERO); - BIF_RET(res); + res = erts_gc_info_request(BIF_P); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_gc_info_res_trap, BIF_P, res); } else if (BIF_ARG_1 == am_reductions) { Uint reds; Uint diff; @@ -3315,12 +3210,11 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("next_pid", BIF_ARG_1) || ERTS_IS_ATOM_STR("next_port", BIF_ARG_1)) { /* Used by node_container_SUITE (emulator) */ - Eterm res; + Sint res; if (ERTS_IS_ATOM_STR("next_pid", BIF_ARG_1)) - res = erts_test_next_pid(0, 0); - else { - res = erts_test_next_port(0, 0); - } + res = erts_ptab_test_next_id(&erts_proc, 0, 0); + else + res = erts_ptab_test_next_id(&erts_port, 0, 0); if (res < 0) BIF_RET(am_false); BIF_RET(erts_make_integer(res, BIF_P)); @@ -3356,11 +3250,11 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("processes", BIF_ARG_1)) { /* Used by process_SUITE (emulator) */ - BIF_RET(erts_debug_processes(BIF_P)); + BIF_RET(erts_debug_ptab_list(BIF_P, &erts_proc)); } else if (ERTS_IS_ATOM_STR("processes_bif_info", BIF_ARG_1)) { /* Used by process_SUITE (emulator) */ - BIF_RET(erts_debug_processes_bif_info(BIF_P)); + BIF_RET(erts_debug_ptab_list_bif_info(BIF_P, &erts_proc)); } else if (ERTS_IS_ATOM_STR("max_atom_out_cache_index", BIF_ARG_1)) { /* Used by distribution_SUITE (emulator) */ @@ -3421,17 +3315,20 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P); BIF_RET(am_undefined); } - res = make_link_list(BIF_P, p->nlinks, NIL); + res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); BIF_RET(res); } else if(is_internal_port(tp[2])) { Eterm res; - Port *p = erts_id2port(tp[2], BIF_P, ERTS_PROC_LOCK_MAIN); + Port *p = erts_id2port_sflgs(tp[2], + BIF_P, + ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_LOOKUP); if(!p) BIF_RET(am_undefined); - res = make_link_list(BIF_P, p->nlinks, NIL); - erts_smp_port_unlock(p); + res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL); + erts_port_release(p); BIF_RET(res); } else if(is_node_name_atom(tp[2])) { @@ -3463,7 +3360,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P); BIF_RET(am_undefined); } - res = make_monitor_list(BIF_P, p->monitors); + res = make_monitor_list(BIF_P, ERTS_P_MONITORS(p)); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); BIF_RET(res); } else if(is_node_name_atom(tp[2])) { @@ -3606,7 +3503,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) erts_aint_t prev_on = erts_smp_atomic_xchg_nob(&available_internal_state, on); if (on) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Process %T ", BIF_P->id); + erts_dsprintf(dsbufp, "Process %T ", BIF_P->common.id); if (erts_is_alive) erts_dsprintf(dsbufp, "on node %T ", erts_this_node->sysname); erts_dsprintf(dsbufp, @@ -3671,13 +3568,12 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) Uint next; if (term_to_Uint(BIF_ARG_2, &next) != 0) { - Eterm res; + Sint res; if (ERTS_IS_ATOM_STR("next_pid", BIF_ARG_1)) - res = erts_test_next_pid(1, next); - else { - res = erts_test_next_port(1, next); - } + res = erts_ptab_test_next_id(&erts_proc, 1, next); + else + res = erts_ptab_test_next_id(&erts_port, 1, next); if (res < 0) BIF_RET(am_false); BIF_RET(erts_make_integer(res, BIF_P)); @@ -3710,9 +3606,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) && is_internal_pid(tp[2])) { int xres; ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process *rp = erts_pid2proc_opt(BIF_P, ERTS_PROC_LOCK_MAIN, - tp[2], rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); + Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, + tp[2], rp_locks); if (!rp) { DECL_AM(dead); BIF_RET(AM_dead); @@ -3738,7 +3633,6 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) rp_locks &= ~ERTS_PROC_LOCK_MAIN; #endif erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); if (xres > 1) { DECL_AM(message); BIF_RET(AM_message); @@ -3903,7 +3797,7 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s timer_ns = stats->timer.ns; timer_n = stats->timer_n; - af = am_atom_put(stats->file, strlen(stats->file)); + af = erts_atom_put(stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); uil = erts_bld_uint( hpp, szp, line); tloc = erts_bld_tuple(hpp, szp, 2, af, uil); @@ -3940,18 +3834,18 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock ASSERT(ltype); - type = am_atom_put(ltype, strlen(ltype)); - name = am_atom_put(lock->name, strlen(lock->name)); + type = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); + name = erts_atom_put(lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); if (lock->flag & ERTS_LCNT_LT_ALLOC) { /* use allocator types names as id's for allocator locks */ ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id)); - id = am_atom_put(ltype, strlen(ltype)); + id = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) { /* use registered names as id's for process locks if available */ - proc = erts_pid2proc_unlocked(lock->id); - if (proc && proc->reg) { - id = proc->reg->name; + proc = erts_proc_lookup(lock->id); + if (proc && proc->common.u.alive.reg) { + id = proc->common.u.alive.reg->name; } else { /* otherwise use process id */ id = lock->id; @@ -3986,12 +3880,12 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *da dtns = erts_bld_uint( hpp, szp, data->duration.ns); tdt = erts_bld_tuple(hpp, szp, 2, dts, dtns); - adur = am_atom_put(str_duration, strlen(str_duration)); + adur = erts_atom_put(str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt); /* lock tuple */ - aloc = am_atom_put(str_locks, strlen(str_locks)); + aloc = erts_atom_put(str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) { lloc = lcnt_build_lock_term(hpp, szp, lock, lloc); @@ -4127,14 +4021,14 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) static void os_info_init(void) { - Eterm type = am_atom_put(os_type, strlen(os_type)); + Eterm type = erts_atom_put((byte *) os_type, strlen(os_type), ERTS_ATOM_ENC_LATIN1, 1); Eterm flav; int major, minor, build; char* buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */ Eterm* hp; os_flavor(buf, 1024); - flav = am_atom_put(buf, strlen(buf)); + flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1); erts_free(ERTS_ALC_T_TMP, (void *) buf); hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm)); os_type_tuple = TUPLE2(hp, type, flav); @@ -4156,6 +4050,8 @@ erts_bif_info_init(void) alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); gather_sched_wall_time_res_trap = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); + gather_gc_info_res_trap + = erts_export_put(am_erlang, am_gather_gc_info_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index 13f8b1f63c..adac0052d6 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -261,11 +261,6 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); } - } else if (is_tuple(arg1)) { - Eterm* tp = tuple_val(arg1); - if (tp[0] == make_arityval(2) && is_atom(tp[1]) && is_atom(tp[2])) { - BIF_RET(am_true); - } } BIF_RET(am_false); } diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 831e05493a..1062d4379b 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -58,7 +58,7 @@ BIF_RETTYPE os_getpid_0(BIF_ALIST_0) char pid_string[21]; /* enough for a 64 bit number */ int n; Eterm* hp; - sys_get_pid(pid_string); /* In sys.c */ + sys_get_pid(pid_string, sizeof(pid_string)); /* In sys.c */ n = sys_strlen(pid_string); hp = HAlloc(BIF_P, n*2); BIF_RET(buf_to_intlist(&hp, pid_string, n, NIL)); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index f9009166c0..109c54fd7f 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2012. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -42,577 +42,544 @@ #include "erl_bits.h" #include "dtrace-wrapper.h" -static int open_port(Process* p, Eterm name, Eterm settings, int *err_nump); +static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); static byte* convert_environment(Process* p, Eterm env); static char **convert_args(Eterm); static void free_args(char **); char *erts_default_arg0 = "default"; -static BIF_RETTYPE -port_call(Process* p, Eterm arg1, Eterm arg2, Eterm arg3); - BIF_RETTYPE open_port_2(BIF_ALIST_2) { - int port_num; - Eterm port_val; + Port *port; + Eterm port_id; char *str; - int err_num; + int err_type, err_num; - if ((port_num = open_port(BIF_P, BIF_ARG_1, BIF_ARG_2, &err_num)) < 0) { - if (port_num == -3) { + port = open_port(BIF_P, BIF_ARG_1, BIF_ARG_2, &err_type, &err_num); + if (!port) { + if (err_type == -3) { ASSERT(err_num == BADARG || err_num == SYSTEM_LIMIT); BIF_ERROR(BIF_P, err_num); - } else if (port_num == -2) { + } else if (err_type == -2) { str = erl_errno_id(err_num); } else { str = "einval"; } - BIF_P->fvalue = am_atom_put(str, strlen(str)); + BIF_P->fvalue = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1); BIF_ERROR(BIF_P, EXC_ERROR); } erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); - port_val = erts_port[port_num].id; - erts_add_link(&(erts_port[port_num].nlinks), LINK_PID, BIF_P->id); - erts_add_link(&(BIF_P->nlinks), LINK_PID, port_val); + port_id = port->common.id; + erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id); + erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port_id); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - erts_port_release(&erts_port[port_num]); + erts_port_release(port); - BIF_RET(port_val); + BIF_RET(port_id); } -/**************************************************************************** - - PORT BIFS: - - port_command/2 -- replace Port ! {..., {command, Data}} - port_command(Port, Data) -> true - when port(Port), io-list(Data) - - port_control/3 -- new port_control(Port, Ctl, Data) -> Reply - port_control(Port, Ctl, Data) -> Reply - where integer(Ctl), io-list(Data), io-list(Reply) - - port_close/1 -- replace Port ! {..., close} - port_close(Port) -> true - when port(Port) - - port_connect/2 -- replace Port ! {..., {connect, Pid}} - port_connect(Port, Pid) - when port(Port), pid(Pid) - - ***************************************************************************/ - -static Port* -id_or_name2port(Process *c_p, Eterm id) +static ERTS_INLINE Port * +lookup_port(Process *c_p, Eterm id_or_name) { - Port *port; - if (is_not_atom(id)) - port = erts_id2port(id, c_p, ERTS_PROC_LOCK_MAIN); + /* TODO: Implement nicer lookup in register... */ + Eterm id; + if (is_atom(id_or_name)) + id = erts_whereis_name_to_id(c_p, id_or_name); else - erts_whereis_name(c_p, ERTS_PROC_LOCK_MAIN, id, NULL, 0, 0, &port); - return port; + id = id_or_name; + return erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP); } -#define ERTS_PORT_COMMAND_FLAG_FORCE (((Uint32) 1) << 0) -#define ERTS_PORT_COMMAND_FLAG_NOSUSPEND (((Uint32) 1) << 1) +/* + * erts_internal:port_command/3 is used by the + * erlang:port_command/2 and erlang:port_command/3 + * BIFs. + */ -static BIF_RETTYPE -do_port_command(Process *BIF_P, Eterm arg1, Eterm arg2, Eterm arg3, - Uint32 flags) +BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3) { BIF_RETTYPE res; - Port *p; - - /* Trace sched out before lock check wait */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_out); - } - - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_inactive); - } - - p = id_or_name2port(BIF_P, arg1); - if (!p) { - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); + Port *prt; + int flags = 0; + Eterm ref; + + if (is_not_nil(BIF_ARG_3)) { + Eterm l = BIF_ARG_3; + while (is_list(l)) { + Eterm* cons = list_val(l); + Eterm car = CAR(cons); + if (car == am_force) + flags |= ERTS_PORT_SIG_FLG_FORCE; + else if (car == am_nosuspend) + flags |= ERTS_PORT_SIG_FLG_NOSUSPEND; + else + BIF_RET(am_badarg); + l = CDR(cons); } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); - } - BIF_ERROR(BIF_P, BADARG); + if (!is_nil(l)) + BIF_RET(am_badarg); } - - /* Trace port in, id_or_name2port causes wait */ - if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(p, am_in, am_command); - } - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { - profile_runnable_port(p, am_active); + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_RET(am_badarg); + + if (flags & ERTS_PORT_SIG_FLG_FORCE) { + if (!(prt->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY)) + BIF_RET(am_notsup); } - ERTS_BIF_PREP_RET(res, am_true); +#ifdef DEBUG + ref = NIL; +#endif - if ((flags & ERTS_PORT_COMMAND_FLAG_FORCE) - && !(p->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY)) { - ERTS_BIF_PREP_ERROR(res, BIF_P, EXC_NOTSUP); - } - else if (!(flags & ERTS_PORT_COMMAND_FLAG_FORCE) - && p->status & ERTS_PORT_SFLG_PORT_BUSY) { - if (flags & ERTS_PORT_COMMAND_FLAG_NOSUSPEND) { + switch (erts_port_output(BIF_P, flags, prt, prt->common.id, BIF_ARG_2, &ref)) { + case ERTS_PORT_OP_CALLER_EXIT: + case ERTS_PORT_OP_BADARG: + case ERTS_PORT_OP_DROPPED: + ERTS_BIF_PREP_RET(res, am_badarg); + break; + case ERTS_PORT_OP_BUSY: + ASSERT(!(flags & ERTS_PORT_SIG_FLG_FORCE)); + if (flags & ERTS_PORT_SIG_FLG_NOSUSPEND) ERTS_BIF_PREP_RET(res, am_false); - } else { - erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, p); - if (erts_system_monitor_flags.busy_port) { - monitor_generic(BIF_P, am_busy_port, p->id); - } - ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_port_command_3], BIF_P, - arg1, arg2, arg3); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, prt); + ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_erts_internal_port_command_3], + BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } - } else { - int wres; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - ERTS_SMP_CHK_NO_PROC_LOCKS; - wres = erts_write_to_port(BIF_P->id, p, arg2); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - if (wres != 0) { - ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); - } - } - - if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(p, am_out, am_command); - } - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { - profile_runnable_port(p, am_inactive); + break; + case ERTS_PORT_OP_BUSY_SCHEDULED: + ASSERT(!(flags & ERTS_PORT_SIG_FLG_FORCE)); + /* Fall through... */ + case ERTS_PORT_OP_SCHEDULED: + ASSERT(is_internal_ref(ref)); + ERTS_BIF_PREP_RET(res, ref); + break; + case ERTS_PORT_OP_DONE: + ERTS_BIF_PREP_RET(res, am_true); + break; + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_output() result"); + break; } - erts_port_release(p); - /* Trace sched in after port release */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); - } - if (ERTS_PROC_IS_EXITING(BIF_P)) { KILL_CATCHES(BIF_P); /* Must exit */ ERTS_BIF_PREP_ERROR(res, BIF_P, EXC_ERROR); } + return res; } -BIF_RETTYPE port_command_2(BIF_ALIST_2) +BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3) { - return do_port_command(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL, 0); + Port* prt; + Eterm retval; + Uint uint_op; + unsigned int op; + erts_aint32_t state; + + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_RET(am_badarg); + + if (!term_to_Uint(BIF_ARG_2, &uint_op)) + BIF_RET(am_badarg); + + if (uint_op > (Uint) UINT_MAX) + BIF_RET(am_badarg); + + op = (unsigned int) uint_op; + + switch (erts_port_call(BIF_P, prt, op, BIF_ARG_3, &retval)) { + case ERTS_PORT_OP_CALLER_EXIT: + case ERTS_PORT_OP_DROPPED: + case ERTS_PORT_OP_BADARG: + retval = am_badarg; + break; + case ERTS_PORT_OP_SCHEDULED: + ASSERT(is_internal_ref(retval)); + break; + case ERTS_PORT_OP_DONE: + ASSERT(is_not_internal_ref(retval)); + break; + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_call() result"); + retval = am_internal_error; + break; + } + + state = erts_smp_atomic32_read_acqb(&BIF_P->state); + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { +#ifdef ERTS_SMP + if (state & ERTS_PSFLG_PENDING_EXIT) + erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); +#endif + ERTS_BIF_EXITED(BIF_P); + } + + BIF_RET(retval); } -BIF_RETTYPE port_command_3(BIF_ALIST_3) +BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3) { - Eterm l = BIF_ARG_3; - Uint32 flags = 0; - while (is_list(l)) { - Eterm* cons = list_val(l); - Eterm car = CAR(cons); - if (car == am_force) { - flags |= ERTS_PORT_COMMAND_FLAG_FORCE; - } else if (car == am_nosuspend) { - flags |= ERTS_PORT_COMMAND_FLAG_NOSUSPEND; - } else { - BIF_ERROR(BIF_P, BADARG); - } - l = CDR(cons); + Port* prt; + Eterm retval; + Uint uint_op; + unsigned int op; + erts_aint32_t state; + + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_RET(am_badarg); + + if (!term_to_Uint(BIF_ARG_2, &uint_op)) + BIF_RET(am_badarg); + + if (uint_op > (Uint) UINT_MAX) + BIF_RET(am_badarg); + + op = (unsigned int) uint_op; + + switch (erts_port_control(BIF_P, prt, op, BIF_ARG_3, &retval)) { + case ERTS_PORT_OP_CALLER_EXIT: + case ERTS_PORT_OP_BADARG: + case ERTS_PORT_OP_DROPPED: + retval = am_badarg; + break; + case ERTS_PORT_OP_SCHEDULED: + ASSERT(is_internal_ref(retval)); + break; + case ERTS_PORT_OP_DONE: + ASSERT(is_not_internal_ref(retval)); + break; + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_control() result"); + retval = am_internal_error; + break; } - if(!is_nil(l)) { - BIF_ERROR(BIF_P, BADARG); + + state = erts_smp_atomic32_read_acqb(&BIF_P->state); + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { +#ifdef ERTS_SMP + if (state & ERTS_PSFLG_PENDING_EXIT) + erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); +#endif + ERTS_BIF_EXITED(BIF_P); } - return do_port_command(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, flags); -} -BIF_RETTYPE port_call_2(BIF_ALIST_2) -{ - return port_call(BIF_P,BIF_ARG_1, make_small(0), BIF_ARG_2); + BIF_RET(retval); } -BIF_RETTYPE port_call_3(BIF_ALIST_3) +/* + * erts_internal:port_close/1 is used by the + * erlang:port_close/1 BIF. + */ +BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1) { - return port_call(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); -} + Eterm ref; + Port *prt; -static BIF_RETTYPE -port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3) -{ - Uint op; - Port *p; - Uint size; - byte *bytes; - byte *endp; - ErlDrvSizeT real_size; - erts_driver_t *drv; - byte port_input[256]; /* Default input buffer to encode in */ - byte port_result[256]; /* Buffer for result from port. */ - byte* port_resp; /* Pointer to result buffer. */ - char *prc; - ErlDrvSSizeT ret; - Eterm res; - Sint result_size; - Eterm *hp; - Eterm *hp_end; - unsigned ret_flags = 0U; - int fpe_was_unmasked; - - bytes = &port_input[0]; - port_resp = port_result; - /* trace of port scheduling with virtual process descheduling - * lock wait - */ - if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(c_p, am_out); - } +#ifdef DEBUG + ref = NIL; +#endif - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(c_p, am_inactive); + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_RET(am_badarg); + + + switch (erts_port_exit(BIF_P, 0, prt, prt->common.id, am_normal, &ref)) { + case ERTS_PORT_OP_CALLER_EXIT: + case ERTS_PORT_OP_BADARG: + case ERTS_PORT_OP_DROPPED: + BIF_RET(am_badarg); + case ERTS_PORT_OP_SCHEDULED: + ASSERT(is_internal_ref(ref)); + BIF_RET(ref); + case ERTS_PORT_OP_DONE: + BIF_RET(am_true); + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_exit() result"); + BIF_RET(am_internal_error); } +} - p = id_or_name2port(c_p, arg1); - if (!p) { - error: - if (port_resp != port_result && - !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) { - driver_free(port_resp); - } - if (bytes != &port_input[0]) - erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes); - /* Need to virtual schedule in the process if there - * was an error. - */ - if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(c_p, am_in); - } +/* + * erts_internal:port_connect/2 is used by the + * erlang:port_connect/2 BIF. + */ +BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2) +{ + Eterm ref; + Port* prt; - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(c_p, am_active); - } + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_RET(am_badarg); - if (p) - erts_port_release(p); -#ifdef ERTS_SMP - ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); -#else - ERTS_BIF_CHK_EXITED(c_p); +#ifdef DEBUG + ref = NIL; #endif - BIF_ERROR(c_p, BADARG); - } - if ((drv = p->drv_ptr) == NULL) { - goto error; - } - if (drv->call == NULL) { - goto error; + switch (erts_port_connect(BIF_P, 0, prt, prt->common.id, BIF_ARG_2, &ref)) { + case ERTS_PORT_OP_CALLER_EXIT: + case ERTS_PORT_OP_BADARG: + case ERTS_PORT_OP_DROPPED: + BIF_RET(am_badarg); + case ERTS_PORT_OP_SCHEDULED: + ASSERT(is_internal_ref(ref)); + BIF_RET(ref); + break; + case ERTS_PORT_OP_DONE: + BIF_RET(am_true); + break; + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_connect() result"); + BIF_RET(am_internal_error); } - if (!term_to_Uint(arg2, &op)) { - goto error; +} + +BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1) +{ + Eterm retval; + Port* prt; + + if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) { + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_RET(am_undefined); } - p->caller = c_p->id; - - /* Lock taken, virtual schedule of port */ - if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(p, am_in, am_call); + else if (is_external_port(BIF_ARG_1)) { + if (external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry) + BIF_RET(am_undefined); + else + BIF_RET(am_badarg); } - - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { - profile_runnable_port(p, am_active); + else { + BIF_RET(am_badarg); + } + + switch (erts_port_info(BIF_P, prt, THE_NON_VALUE, &retval)) { + case ERTS_PORT_OP_CALLER_EXIT: + case ERTS_PORT_OP_BADARG: + BIF_RET(am_badarg); + case ERTS_PORT_OP_DROPPED: + BIF_RET(am_undefined); + case ERTS_PORT_OP_SCHEDULED: + ASSERT(is_internal_ref(retval)); + BIF_RET(retval); + case ERTS_PORT_OP_DONE: + ASSERT(is_not_internal_ref(retval)); + BIF_RET(retval); + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_info() result"); + BIF_RET(am_internal_error); } - size = erts_encode_ext_size(arg3); - if (size > sizeof(port_input)) - bytes = erts_alloc(ERTS_ALC_T_PORT_CALL_BUF, size); +} - endp = bytes; - erts_encode_ext(arg3, &endp); - real_size = endp - bytes; - if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", - __FILE__, __LINE__, endp - (bytes + size)); - } - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(driver_call)) { - DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); +BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2) +{ + Eterm retval; + Port* prt; - dtrace_pid_str(p->connected, process_str); - dtrace_port_str(p, port_str); - DTRACE5(driver_call, process_str, port_str, p->name, op, real_size); - } -#endif - prc = (char *) port_resp; - fpe_was_unmasked = erts_block_fpe(); - ret = drv->call((ErlDrvData)p->drv_data, - (unsigned) op, - (char *) bytes, - (int) real_size, - &prc, - (int) sizeof(port_result), - &ret_flags); - erts_unblock_fpe(fpe_was_unmasked); - if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(p, am_out, am_call); - } - - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { - profile_runnable_port(p, am_inactive); - } - - port_resp = (byte *) prc; - p->caller = NIL; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); -#ifdef HARDDEBUG - { - ErlDrvSizeT z; - printf("real_size = %ld,%d, ret = %ld,%d\r\n", (unsigned long) real_size, - (int) real_size, (unsigned long)ret, (int) ret); - printf("["); - for(z = 0; z < real_size; ++z) { - printf("%d, ",(int) bytes[z]); - } - printf("]\r\n"); - printf("["); - for(z = 0; z < ret; ++z) { - printf("%d, ",(int) port_resp[z]); - } - printf("]\r\n"); + if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) { + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_RET(am_undefined); } -#endif - if (ret <= 0 || port_resp[0] != VERSION_MAGIC) { - /* Error or a binary without magic/ with wrong magic */ - goto error; - } - result_size = erts_decode_ext_size(port_resp, ret); - if (result_size < 0) { - goto error; - } - hp = HAlloc(c_p, result_size); - hp_end = hp + result_size; - endp = port_resp; - res = erts_decode_ext(&hp, &MSO(c_p), &endp); - if (res == THE_NON_VALUE) { - goto error; - } - HRelease(c_p, hp_end, hp); - if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) { - driver_free(port_resp); - } - if (bytes != &port_input[0]) - erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes); - if (p) - erts_port_release(p); -#ifdef ERTS_SMP - ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN); -#else - ERTS_BIF_CHK_EXITED(c_p); -#endif - if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(c_p, am_in); + else if (is_external_port(BIF_ARG_1)) { + if (external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry) + BIF_RET(am_undefined); + else + BIF_RET(am_badarg); } - - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(c_p, am_active); + else { + BIF_RET(am_badarg); + } + + switch (erts_port_info(BIF_P, prt, BIF_ARG_2, &retval)) { + case ERTS_PORT_OP_CALLER_EXIT: + case ERTS_PORT_OP_BADARG: + BIF_RET(am_badarg); + case ERTS_PORT_OP_DROPPED: + BIF_RET(am_undefined); + case ERTS_PORT_OP_SCHEDULED: + ASSERT(is_internal_ref(retval)); + BIF_RET(retval); + case ERTS_PORT_OP_DONE: + ASSERT(is_not_internal_ref(retval)); + BIF_RET(retval); + default: + ERTS_INTERNAL_ERROR("Unexpected erts_port_info() result"); + BIF_RET(am_internal_error); } - - return res; } - -BIF_RETTYPE port_control_3(BIF_ALIST_3) -{ - Port* p; - Uint op; - Eterm res = THE_NON_VALUE; - - /* Virtual schedule out calling process before lock wait */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_out); - } - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_inactive); - } +/* + * The erlang:port_set_data()/erlang:port_get_data() operations should + * be viewed as operations on a table (inet_db) with data values + * associated with port identifier keys. That is, these operations are + * *not* signals to/from ports. + */ - p = id_or_name2port(BIF_P, BIF_ARG_1); - if (!p) { - /* Schedule the process before exiting */ - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); - } - - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); - } - - BIF_ERROR(BIF_P, BADARG); - } +#if (TAG_PRIMARY_IMMED1 & 0x3) == 0 +# error "erlang:port_set_data()/erlang:port_get_data() needs to be rewritten!" +#endif - /* Trace the port for scheduling in */ - if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(p, am_in, am_control); - } - - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { - profile_runnable_port(p, am_active); - } +typedef struct { + ErtsThrPrgrLaterOp later_op; + Uint hsize; + Eterm data; + ErlOffHeap off_heap; + Eterm heap[1]; +} ErtsPortDataHeap; - if (term_to_Uint(BIF_ARG_2, &op)) - res = erts_port_control(BIF_P, p, op, BIF_ARG_3); - - /* Trace the port for scheduling out */ - if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(p, am_out, am_control); - } +static void +free_port_data_heap(void *vpdhp) +{ + erts_cleanup_offheap(&((ErtsPortDataHeap *) vpdhp)->off_heap); + erts_free(ERTS_ALC_T_PORT_DATA_HEAP, vpdhp); +} - if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) { - profile_runnable_port(p, am_inactive); +static ERTS_INLINE void +cleanup_old_port_data(erts_aint_t data) +{ + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) data)); } - - erts_port_release(p); + else { #ifdef ERTS_SMP - ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN); + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + size_t size; + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + size = sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm) - 1); + erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap, + (void *) pdhp, + &pdhp->later_op, + size); #else - ERTS_BIF_CHK_EXITED(BIF_P); + free_port_data_heap((void *) data); #endif - - if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) { - trace_virtual_sched(BIF_P, am_in); } - - if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) { - profile_runnable_proc(BIF_P, am_active); - } - - if (is_non_value(res)) { - BIF_ERROR(BIF_P, BADARG); - } - BIF_RET(res); } -BIF_RETTYPE port_close_1(BIF_ALIST_1) +void +erts_init_port_data(Port *prt) { - Port* p; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - p = id_or_name2port(NULL, BIF_ARG_1); - if (!p) { - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_ERROR(BIF_P, BADARG); - } - erts_do_exit_port(p, p->connected, am_normal); - /* if !ERTS_SMP: since we terminate port with reason normal - we SHOULD never get an exit signal ourselves - */ - erts_port_release(p); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_RET(am_true); + erts_smp_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined); } -BIF_RETTYPE port_connect_2(BIF_ALIST_2) +void +erts_cleanup_port_data(Port *prt) { - Port* prt; - Process* rp; - Eterm pid = BIF_ARG_2; + ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP); + cleanup_old_port_data(erts_smp_atomic_read_nob(&prt->data)); + erts_smp_atomic_set_nob(&prt->data, (erts_aint_t) THE_NON_VALUE); +} - if (is_not_internal_pid(pid)) { - error: - BIF_ERROR(BIF_P, BADARG); - } - prt = id_or_name2port(BIF_P, BIF_ARG_1); - if (!prt) { - goto error; - } +Uint +erts_port_data_size(Port *prt) +{ + erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); - rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - pid, ERTS_PROC_LOCK_LINK); - if (!rp) { - erts_smp_port_unlock(prt); - ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P); - goto error; + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) (UWord) data)); + return (Uint) 0; } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + return (Uint) sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm)-1); + } +} - erts_add_link(&(rp->nlinks), LINK_PID, prt->id); - erts_add_link(&(prt->nlinks), LINK_PID, pid); - - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - - prt->connected = pid; /* internal pid */ - erts_smp_port_unlock(prt); -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(port_connect)) { - DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); +ErlOffHeap * +erts_port_data_offheap(Port *prt) +{ + erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); - dtrace_pid_str(prt->connected, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->id); - dtrace_proc_str(rp, newprocess_str); - DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) (UWord) data)); + return NULL; + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + return &pdhp->off_heap; } -#endif - BIF_RET(am_true); } BIF_RETTYPE port_set_data_2(BIF_ALIST_2) { + /* + * This is not a signal. See comment above. + */ + erts_aint_t data; Port* prt; - Eterm portid = BIF_ARG_1; - Eterm data = BIF_ARG_2; - - prt = id_or_name2port(BIF_P, portid); - if (!prt) { - BIF_ERROR(BIF_P, BADARG); - } - if (prt->bp != NULL) { - free_message_buffer(prt->bp); - prt->bp = NULL; - } - if (IS_CONST(data)) { - prt->data = data; - } else { - Uint size; - ErlHeapFragment* bp; - Eterm* hp; - - size = size_object(data); - prt->bp = bp = new_message_buffer(size); - hp = bp->mem; - prt->data = copy_struct(data, size, &hp, &bp->off_heap); - } - erts_smp_port_unlock(prt); + + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_ERROR(BIF_P, BADARG); + + if (is_immed(BIF_ARG_2)) { + data = (erts_aint_t) BIF_ARG_2; + ASSERT((data & 0x3) != 0); + } + else { + ErtsPortDataHeap *pdhp; + Uint hsize; + Eterm *hp; + + hsize = size_object(BIF_ARG_2); + pdhp = erts_alloc(ERTS_ALC_T_PORT_DATA_HEAP, + sizeof(ErtsPortDataHeap) + hsize*(sizeof(Eterm)-1)); + hp = &pdhp->heap[0]; + pdhp->off_heap.first = NULL; + pdhp->off_heap.overhead = 0; + pdhp->data = copy_struct(BIF_ARG_2, hsize, &hp, &pdhp->off_heap); + data = (erts_aint_t) pdhp; + ASSERT((data & 0x3) == 0); + } + + data = erts_smp_atomic_xchg_wb(&prt->data, data); + + cleanup_old_port_data(data); + BIF_RET(am_true); } BIF_RETTYPE port_get_data_1(BIF_ALIST_1) { - BIF_RETTYPE res; + /* + * This is not a signal. See comment above. + */ + Eterm res; + erts_aint_t data; Port* prt; - Eterm portid = BIF_ARG_1; - prt = id_or_name2port(BIF_P, portid); - if (!prt) { - BIF_ERROR(BIF_P, BADARG); + prt = lookup_port(BIF_P, BIF_ARG_1); + if (!prt) + BIF_ERROR(BIF_P, BADARG); + + data = erts_smp_atomic_read_ddrb(&prt->data); + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); } - if (prt->bp == NULL) { /* MUST be CONST! */ - res = prt->data; - } else { - Eterm* hp = HAlloc(BIF_P, prt->bp->used_size); - res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P)); + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + Eterm *hp = HAlloc(BIF_P, pdhp->hsize); + res = copy_struct(pdhp->data, pdhp->hsize, &hp, &MSO(BIF_P)); } - erts_smp_port_unlock(prt); + BIF_RET(res); } @@ -625,11 +592,10 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) * either BADARG or SYSTEM_LIMIT). */ -static int -open_port(Process* p, Eterm name, Eterm settings, int *err_nump) +static Port * +open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) { -#define OPEN_PORT_ERROR(VAL) do { port_num = (VAL); goto do_return; } while (0) - int i, port_num; + int i; Eterm option; Uint arity; Eterm* tp; @@ -637,11 +603,11 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) erts_driver_t* driver; char* name_buf = NULL; SysDriverOpts opts; - int binary_io; - int soft_eof; Sint linebuf; Eterm edir = NIL; byte dir[MAXPATHLEN]; + erts_aint32_t sflgs = 0; + Port *port; /* These are the defaults */ opts.packet_bytes = 0; @@ -655,8 +621,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) opts.overlapped_io = 0; opts.spawn_type = ERTS_SPAWN_ANY; opts.argv = NULL; - binary_io = 0; - soft_eof = 0; + opts.parallelism = erts_port_parallelism; linebuf = 0; *err_nump = 0; @@ -734,6 +699,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) } } else if (option == am_cd) { edir = *tp; + } else if (option == am_parallelism) { + if (*tp == am_true) + opts.parallelism = 1; + else if (*tp == am_false) + opts.parallelism = 0; + else + goto badarg; } else { goto badarg; } @@ -748,13 +720,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) } else if (*nargs == am_nouse_stdio) { opts.use_stdio = 0; } else if (*nargs == am_binary) { - binary_io = 1; + sflgs |= ERTS_PORT_SFLG_BINARY_IO; } else if (*nargs == am_in) { opts.read_write |= DO_READ; } else if (*nargs == am_out) { opts.read_write |= DO_WRITE; } else if (*nargs == am_eof) { - soft_eof = 1; + sflgs |= ERTS_PORT_SFLG_SOFT_EOF; } else if (*nargs == am_hide) { opts.hide_window = 1; } else if (*nargs == am_exit_status) { @@ -902,9 +874,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) heap[2] = make_small(0); heap[3] = NIL; iolist = make_list(heap); - r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); + r = erts_iolist_to_buf(iolist, (char*) dir, MAXPATHLEN); UnUseTmpHeap(4,p); - if (r < 0) { + if (ERTS_IOLIST_TO_BUF_FAILED(r)) { goto badarg; } opts.wd = (char *) dir; @@ -926,44 +898,40 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - port_num = erts_open_driver(driver, p->id, name_buf, &opts, err_nump); + port = erts_open_driver(driver, p->common.id, name_buf, &opts, err_typep, err_nump); #ifdef USE_VM_PROBES - if (port_num >= 0 && DTRACE_ENABLED(port_open)) { + if (port && DTRACE_ENABLED(port_open)) { DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(p, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", erts_port[port_num].id); + erts_snprintf(port_str, sizeof(port_str), "%T", port->common.id); DTRACE3(port_open, process_str, name_buf, port_str); } #endif erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - if (port_num < 0) { - DEBUGF(("open_driver returned %d(%d)\n", port_num, *err_nump)); + if (!port) { + DEBUGF(("open_driver returned (%d:%d)\n", + err_typep ? *err_typep : 4711, + err_nump ? *err_nump : 4711)); if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { trace_virtual_sched(p, am_in); } - OPEN_PORT_ERROR(port_num); + goto do_return; } if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { trace_virtual_sched(p, am_in); } - if (binary_io) { - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_BINARY_IO); - } - if (soft_eof) { - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_SOFT_EOF); - } - if (linebuf && erts_port[port_num].linebuf == NULL){ - erts_port[port_num].linebuf = allocate_linebuf(linebuf); - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_LINEBUF_IO); + if (linebuf && port->linebuf == NULL){ + port->linebuf = allocate_linebuf(linebuf); + sflgs |= ERTS_PORT_SFLG_LINEBUF_IO; } + + if (sflgs) + erts_atomic32_read_bor_relb(&port->state, sflgs); do_return: if (name_buf) @@ -974,13 +942,15 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) if (opts.wd && opts.wd != ((char *)dir)) { erts_free(ERTS_ALC_T_TMP, (void *) opts.wd); } - return port_num; + return port; badarg: - *err_nump = BADARG; - OPEN_PORT_ERROR(-3); + if (err_typep) + *err_typep = -3; + if (err_nump) + *err_nump = BADARG; + port = NULL; goto do_return; -#undef OPEN_PORT_ERROR } /* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */ diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 6b843d2e08..3d34c2a77f 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2011. All Rights Reserved. + * Copyright Ericsson AB 2008-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -71,14 +71,9 @@ void erts_init_bif_re(void) erts_pcre_stack_free = &erts_erts_pcre_stack_free; default_table = NULL; /* ISO8859-1 default, forced into pcre */ max_loop_limit = CONTEXT_REDS * LOOP_FACTOR; - - sys_memset((void *) &re_exec_trap_export, 0, sizeof(Export)); - re_exec_trap_export.address = &re_exec_trap_export.code[3]; - re_exec_trap_export.code[0] = am_erlang; - re_exec_trap_export.code[1] = am_re_run_trap; - re_exec_trap_export.code[2] = 3; - re_exec_trap_export.code[3] = (BeamInstr) em_apply_bif; - re_exec_trap_export.code[4] = (BeamInstr) &re_exec_trap; + + erts_init_trap_export(&re_exec_trap_export, am_erlang, am_re_run_trap, 3, + &re_exec_trap); grun_trap_exportp = erts_export_put(am_re,am_grun,3); urun_trap_exportp = erts_export_put(am_re,am_urun,3); @@ -189,6 +184,7 @@ static Eterm make_signed_integer(int x, Process *p) #define CAPSPEC_VALUES 0 #define CAPSPEC_TYPE 1 #define CAPSPEC_SIZE 2 +#define CAPSPEC_INIT {0,0} static int /* 0 == ok, < 0 == error */ parse_options(Eterm listp, /* in */ @@ -418,7 +414,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con static BIF_RETTYPE re_compile(Process* p, Eterm arg1, Eterm arg2) { - Uint slen; + ErlDrvSizeT slen; char *expr; pcre *result; int errcode = 0; @@ -449,7 +445,7 @@ re_compile(Process* p, Eterm arg1, Eterm arg2) BIF_ERROR(p,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (io_list_to_buf(arg1, expr, slen) != 0) { + if (erts_iolist_to_buf(arg1, expr, slen) != 0) { erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); BIF_ERROR(p,BADARG); } @@ -802,7 +798,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) memcpy(tmpb,ap->name,ap->len); tmpb[ap->len] = '\0'; } else { - Uint slen; + ErlDrvSizeT slen; if (erts_iolist_size(val, &slen)) { goto error; } @@ -814,7 +810,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) (tmpbsiz = slen + 1)); } } - if (io_list_to_buf(val, tmpb, slen) != 0) { + if (erts_iolist_to_buf(val, tmpb, slen) != 0) { goto error; } tmpb[slen] = '\0'; @@ -858,7 +854,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) const pcre *code_tmp; RestartContext restart; byte *temp_alloc = NULL; - Uint slength; + ErlDrvSizeT slength; int startoffset = 0; int options = 0, comp_options = 0; int ovsize; @@ -869,7 +865,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) size_t code_size; Uint loop_limit_tmp; unsigned long loop_count; - Eterm capture[CAPSPEC_SIZE]; + Eterm capture[CAPSPEC_SIZE] = CAPSPEC_INIT; int is_list_cap; if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture) @@ -882,7 +878,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 4)) { if (is_binary(arg2) || is_list(arg2) || is_nil(arg2)) { /* Compile from textual RE */ - Uint slen; + ErlDrvSizeT slen; char *expr; pcre *result; int errcode = 0; @@ -901,7 +897,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (io_list_to_buf(arg2, expr, slen) != 0) { + if (erts_iolist_to_buf(arg2, expr, slen) != 0) { erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); BIF_ERROR(p,BADARG); } @@ -1044,7 +1040,7 @@ handle_iolist: } restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength); - if (io_list_to_buf(arg1, restart.subject, slength) != 0) { + if (erts_iolist_to_buf(arg1, restart.subject, slength) != 0) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); erts_free(ERTS_ALC_T_RE_SUBJECT, restart.subject); diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index d806be0704..d67695e533 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -265,10 +265,10 @@ link_proc(Process *p, ErtsBifTimer* btm) { btm->receiver.proc.ess = p; btm->receiver.proc.prev = NULL; - btm->receiver.proc.next = p->bif_timers; - if (p->bif_timers) - p->bif_timers->receiver.proc.prev = btm; - p->bif_timers = btm; + btm->receiver.proc.next = p->u.bif_timers; + if (p->u.bif_timers) + p->u.bif_timers->receiver.proc.prev = btm; + p->u.bif_timers = btm; } static ERTS_INLINE void @@ -277,7 +277,7 @@ unlink_proc(ErtsBifTimer* btm) if (btm->receiver.proc.prev) btm->receiver.proc.prev->receiver.proc.next = btm->receiver.proc.next; else - btm->receiver.proc.ess->bif_timers = btm->receiver.proc.next; + btm->receiver.proc.ess->u.bif_timers = btm->receiver.proc.next; if (btm->receiver.proc.next) btm->receiver.proc.next->receiver.proc.prev = btm->receiver.proc.prev; } @@ -324,10 +324,9 @@ bif_timer_timeout(ErtsBifTimer* btm) ASSERT(!erts_get_current_process()); if (btm->flags & BTM_FLG_BYNAME) - rp = erts_whereis_process(NULL,0,btm->receiver.name,0,ERTS_P2P_FLG_SMP_INC_REFC); + rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0); else { rp = btm->receiver.proc.ess; - erts_smp_proc_inc_refc(rp); unlink_proc(btm); } @@ -379,7 +378,6 @@ bif_timer_timeout(ErtsBifTimer* btm) #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); } } @@ -615,7 +613,7 @@ erts_print_bif_timer_info(int to, void *to_arg) for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) { Eterm receiver = (btm->flags & BTM_FLG_BYNAME ? btm->receiver.name - : btm->receiver.proc.ess->id); + : btm->receiver.proc.ess->common.id); erts_print(to, to_arg, "=timer:%T\n", receiver); erts_print(to, to_arg, "Message: %T\n", btm->message); erts_print(to, to_arg, "Time left: %u ms\n", @@ -639,7 +637,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) erts_smp_proc_lock(p, plocks); } - btm = p->bif_timers; + btm = p->u.bif_timers; while (btm) { ErtsBifTimer *tmp_btm; ASSERT(!(btm->flags & BTM_FLG_CANCELED)); @@ -649,7 +647,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) erts_cancel_timer(&tmp_btm->tm); } - p->bif_timers = NULL; + p->u.bif_timers = NULL; erts_smp_btm_rwunlock(); } @@ -698,7 +696,7 @@ erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *), for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) { (*func)((btm->flags & BTM_FLG_BYNAME ? btm->receiver.name - : btm->receiver.proc.ess->id), + : btm->receiver.proc.ess->common.id), btm->message, btm->bp, arg); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 3d1afdd11c..06fbbea123 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -42,14 +42,33 @@ #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) const struct trace_pattern_flags erts_trace_pattern_flags_off = {0, 0, 0, 0, 0}; + +/* + * The following variables are protected by code write permission. + */ static int erts_default_trace_pattern_is_on; static Binary *erts_default_match_spec; static Binary *erts_default_meta_match_spec; static struct trace_pattern_flags erts_default_trace_pattern_flags; static Eterm erts_default_meta_tracer_pid; +static struct { /* Protected by code write permission */ + int current; + int install; + int local; + BpFunctions f; /* Local functions */ + BpFunctions e; /* Export entries */ +#ifdef ERTS_SMP + Process* stager; + ErtsThrPrgrLaterOp lop; +#endif +} finish_bp; + static Eterm trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist); +#ifdef ERTS_SMP +static void smp_bp_finisher(void* arg); +#endif static BIF_RETTYPE system_monitor(Process *p, Eterm monitor_pid, Eterm list); @@ -60,12 +79,11 @@ static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_on_load(Process* p, Eterm key); -static int setup_func_trace(Export* ep, void* match_prog); -static int reset_func_trace(Export* ep); -static void reset_bif_trace(int bif_index); -static void setup_bif_trace(int bif_index); -static void set_trace_bif(int bif_index, void* match_prog); -static void clear_trace_bif(int bif_index); +static void reset_bif_trace(void); +static void setup_bif_trace(void); +static void install_exp_breakpoints(BpFunctions* f); +static void uninstall_exp_breakpoints(BpFunctions* f); +static void clean_export_entries(BpFunctions* f); void erts_bif_trace_init(void) @@ -98,7 +116,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; - int matches = 0; + int matches = -1; int specified = 0; enum erts_break_op on; Binary* match_prog_set; @@ -106,10 +124,12 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) struct trace_pattern_flags flags = erts_trace_pattern_flags_off; int is_global; Process *meta_tracer_proc = p; - Eterm meta_tracer_pid = p->id; + Eterm meta_tracer_pid = p->common.id; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + if (!erts_try_seize_code_write_permission(p)) { + ERTS_BIF_YIELD3(bif_export[BIF_trace_pattern_3], p, MFA, Pattern, flaglist); + } + finish_bp.current = -1; UseTmpHeap(3,p); /* @@ -151,14 +171,12 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) } } else if (is_internal_port(meta_tracer_pid)) { Port *meta_tracer_port; - meta_tracer_proc = NULL; - if (internal_port_index(meta_tracer_pid) >= erts_max_ports) - goto error; - meta_tracer_port = - &erts_port[internal_port_index(meta_tracer_pid)]; - if (INVALID_TRACER_PORT(meta_tracer_port, meta_tracer_pid)) { + meta_tracer_proc = NULL; + meta_tracer_port = (erts_port_lookup( + meta_tracer_pid, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)); + if (!meta_tracer_port) goto error; - } } else { goto error; } @@ -234,14 +252,13 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) MatchSetRef(erts_default_meta_match_spec); erts_default_meta_tracer_pid = meta_tracer_pid; if (meta_tracer_proc) { - meta_tracer_proc->trace_flags |= F_TRACER; + ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER; } } else if (! flags.breakpoint) { MatchSetUnref(erts_default_meta_match_spec); erts_default_meta_match_spec = NULL; erts_default_meta_tracer_pid = NIL; } - MatchSetUnref(match_prog_set); if (erts_default_trace_pattern_flags.breakpoint && flags.breakpoint) { /* Breakpoint trace -> breakpoint trace */ @@ -297,8 +314,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) erts_default_trace_pattern_is_on = !!flags.breakpoint; } } - - goto done; + matches = 0; } else if (is_tuple(MFA)) { Eterm *tp = tuple_val(MFA); if (tp[0] != make_arityval(3)) { @@ -322,36 +338,64 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) if (is_small(mfa[2])) { mfa[2] = signed_val(mfa[2]); } - } else { - goto error; - } - if (meta_tracer_proc) { - meta_tracer_proc->trace_flags |= F_TRACER; - } + if (meta_tracer_proc) { + ERTS_TRACE_FLAGS(meta_tracer_proc) |= F_TRACER; + } + matches = erts_set_trace_pattern(p, mfa, specified, + match_prog_set, match_prog_set, + on, flags, meta_tracer_pid, 0); + } - matches = erts_set_trace_pattern(mfa, specified, - match_prog_set, match_prog_set, - on, flags, meta_tracer_pid); + error: MatchSetUnref(match_prog_set); - - done: UnUseTmpHeap(3,p); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - return make_small(matches); +#ifdef ERTS_SMP + if (finish_bp.current >= 0) { + ASSERT(matches >= 0); + ASSERT(finish_bp.stager == NULL); + finish_bp.stager = p; + erts_schedule_thr_prgr_later_op(smp_bp_finisher, NULL, &finish_bp.lop); + erts_smp_proc_inc_refc(p); + erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(p, make_small(matches)); + } +#endif - error: + erts_release_code_write_permission(); - MatchSetUnref(match_prog_set); + if (matches >= 0) { + return make_small(matches); + } + else { + BIF_ERROR(p, BADARG); + } +} - UnUseTmpHeap(3,p); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - BIF_ERROR(p, BADARG); +#ifdef ERTS_SMP +static void smp_bp_finisher(void* null) +{ + if (erts_finish_breakpointing()) { /* Not done */ + /* Arrange for being called again */ + erts_schedule_thr_prgr_later_op(smp_bp_finisher, NULL, &finish_bp.lop); + } + else { /* Done */ + Process* p = finish_bp.stager; +#ifdef DEBUG + finish_bp.stager = NULL; +#endif + erts_release_code_write_permission(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(p)) { + erts_resume(p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_dec_refc(p); + } } +#endif /* ERTS_SMP */ void erts_get_default_trace_pattern(int *trace_pattern_is_on, @@ -360,6 +404,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, struct trace_pattern_flags *trace_pattern_flags, Eterm *meta_tracer_pid) { + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() || + erts_smp_thr_progress_is_blocking()); if (trace_pattern_is_on) *trace_pattern_is_on = erts_default_trace_pattern_is_on; if (match_spec) @@ -372,7 +418,12 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, *meta_tracer_pid = erts_default_meta_tracer_pid; } - +int erts_is_default_trace_enabled(void) +{ + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() || + erts_smp_thr_progress_is_blocking()); + return erts_default_trace_pattern_is_on; +} Uint erts_trace_flag2bit(Eterm flag) @@ -466,27 +517,31 @@ Eterm trace_3(BIF_ALIST_3) BIF_ERROR(p, BADARG); } + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD3(bif_export[BIF_trace_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + } + if (is_nil(tracer) || is_internal_pid(tracer)) { Process *tracer_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, - is_nil(tracer) ? p->id : tracer, + is_nil(tracer) ? p->common.id : tracer, ERTS_PROC_LOCKS_ALL); if (!tracer_proc) goto error; - tracer_proc->trace_flags |= F_TRACER; + ERTS_TRACE_FLAGS(tracer_proc) |= F_TRACER; erts_smp_proc_unlock(tracer_proc, (tracer_proc == p ? ERTS_PROC_LOCKS_ALL_MINOR : ERTS_PROC_LOCKS_ALL)); } else if (is_internal_port(tracer)) { - Port *tracer_port = erts_id2port(tracer, p, ERTS_PROC_LOCK_MAIN); - if (!erts_is_valid_tracer_port(tracer)) { - if (tracer_port) - erts_smp_port_unlock(tracer_port); + Port *tracer_port = erts_id2port_sflgs(tracer, + p, + ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); + if (!tracer_port) goto error; - } - tracer_port->trace_flags |= F_TRACER; - erts_smp_port_unlock(tracer_port); + ERTS_TRACE_FLAGS(tracer_port) |= F_TRACER; + erts_port_release(tracer_port); } else goto error; @@ -497,7 +552,7 @@ Eterm trace_3(BIF_ALIST_3) case am_true: on = 1; if (is_nil(tracer)) - tracer = p->id; + tracer = p->common.id; break; default: goto error; @@ -519,26 +574,29 @@ Eterm trace_3(BIF_ALIST_3) if (pid_spec == tracer) goto error; - tracee_port = erts_id2port(pid_spec, p, ERTS_PROC_LOCK_MAIN); + tracee_port = erts_id2port_sflgs(pid_spec, + p, + ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_LOOKUP); if (!tracee_port) goto error; if (tracer != NIL && port_already_traced(p, tracee_port, tracer)) { - erts_smp_port_unlock(tracee_port); + erts_port_release(tracee_port); goto already_traced; } if (on) - tracee_port->trace_flags |= mask; + ERTS_TRACE_FLAGS(tracee_port) |= mask; else - tracee_port->trace_flags &= ~mask; + ERTS_TRACE_FLAGS(tracee_port) &= ~mask; - if (!tracee_port->trace_flags) - tracee_port->tracer_proc = NIL; + if (!ERTS_TRACE_FLAGS(tracee_port)) + ERTS_TRACER_PROC(tracee_port) = NIL; else if (tracer != NIL) - tracee_port->tracer_proc = tracer; + ERTS_TRACER_PROC(tracee_port) = tracer; - erts_smp_port_unlock(tracee_port); + erts_port_release(tracee_port); matches = 1; } else if (is_pid(pid_spec)) { @@ -570,14 +628,14 @@ Eterm trace_3(BIF_ALIST_3) } if (on) - tracee_p->trace_flags |= mask; + ERTS_TRACE_FLAGS(tracee_p) |= mask; else - tracee_p->trace_flags &= ~mask; + ERTS_TRACE_FLAGS(tracee_p) &= ~mask; - if ((tracee_p->trace_flags & TRACEE_FLAGS) == 0) - tracee_p->tracer_proc = NIL; + if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS) == 0) + ERTS_TRACER_PROC(tracee_p) = NIL; else if (tracer != NIL) - tracee_p->tracer_proc = tracer; + ERTS_TRACER_PROC(tracee_p) = tracer; erts_smp_proc_unlock(tracee_p, (tracee_p == p @@ -651,48 +709,56 @@ Eterm trace_3(BIF_ALIST_3) ok = 1; if (procs || mods) { + int max = erts_ptab_max(&erts_proc); /* tracing of processes */ - for (i = 0; i < erts_max_processes; i++) { - Process* tracee_p = process_tab[i]; - + for (i = 0; i < max; i++) { + Process* tracee_p = erts_pix2proc(i); if (! tracee_p) continue; if (tracer != NIL) { - if (tracee_p->id == tracer) + if (tracee_p->common.id == tracer) continue; if (already_traced(NULL, tracee_p, tracer)) continue; } if (on) { - tracee_p->trace_flags |= mask; + ERTS_TRACE_FLAGS(tracee_p) |= mask; } else { - tracee_p->trace_flags &= ~mask; + ERTS_TRACE_FLAGS(tracee_p) &= ~mask; } - if(!(tracee_p->trace_flags & TRACEE_FLAGS)) { - tracee_p->tracer_proc = NIL; + if(!(ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS)) { + ERTS_TRACER_PROC(tracee_p) = NIL; } else if (tracer != NIL) { - tracee_p->tracer_proc = tracer; + ERTS_TRACER_PROC(tracee_p) = tracer; } matches++; } } if (ports || mods) { + int max = erts_ptab_max(&erts_port); /* tracing of ports */ - for (i = 0; i < erts_max_ports; i++) { - Port *tracee_port = &erts_port[i]; - if (tracee_port->status & ERTS_PORT_SFLGS_DEAD) continue; + for (i = 0; i < max; i++) { + erts_aint32_t state; + Port *tracee_port = erts_pix2port(i); + if (!tracee_port) + continue; + state = erts_atomic32_read_nob(&tracee_port->state); + if (state & ERTS_PORT_SFLGS_DEAD) + continue; if (tracer != NIL) { - if (tracee_port->id == tracer) continue; - if (port_already_traced(NULL, tracee_port, tracer)) continue; + if (tracee_port->common.id == tracer) + continue; + if (port_already_traced(NULL, tracee_port, tracer)) + continue; } - if (on) tracee_port->trace_flags |= mask; - else tracee_port->trace_flags &= ~mask; + if (on) ERTS_TRACE_FLAGS(tracee_port) |= mask; + else ERTS_TRACE_FLAGS(tracee_port) &= ~mask; - if (!(tracee_port->trace_flags & TRACEE_FLAGS)) { - tracee_port->tracer_proc = NIL; + if (!(ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS)) { + ERTS_TRACER_PROC(tracee_port) = NIL; } else if (tracer != NIL) { - tracee_port->tracer_proc = tracer; + ERTS_TRACER_PROC(tracee_port) = tracer; } /* matches are not counted for ports since it would violate compatibility */ /* This could be a reason to modify this function or make a new one. */ @@ -730,6 +796,7 @@ Eterm trace_3(BIF_ALIST_3) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif + erts_release_code_write_permission(); BIF_RET(make_small(matches)); @@ -745,6 +812,7 @@ Eterm trace_3(BIF_ALIST_3) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); } #endif + erts_release_code_write_permission(); BIF_ERROR(p, BADARG); } @@ -759,21 +827,20 @@ static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer) * * main lock is held on c_p * * all locks are held on port tracee_p */ - if ((tracee_port->trace_flags & TRACEE_FLAGS) - && tracee_port->tracer_proc != tracer) { + if ((ERTS_TRACE_FLAGS(tracee_port) & TRACEE_FLAGS) + && ERTS_TRACER_PROC(tracee_port) != tracer) { /* This tracee is already being traced, and not by the * tracer to be */ - if (is_internal_port(tracee_port->tracer_proc)) { - if (!erts_is_valid_tracer_port(tracee_port->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(tracee_port))) { + if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_port))) { /* Current trace port now invalid * - discard it and approve the new. */ goto remove_tracer; } else return 1; } - else if(is_internal_pid(tracee_port->tracer_proc)) { - Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - tracee_port->tracer_proc, 0); + else if(is_internal_pid(ERTS_TRACER_PROC(tracee_port))) { + Process *tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_port)); if (!tracer_p) { /* Current trace process now invalid * - discard it and approve the new. */ @@ -783,8 +850,8 @@ static int port_already_traced(Process *c_p, Port *tracee_port, Eterm tracer) } else { remove_tracer: - tracee_port->trace_flags &= ~TRACEE_FLAGS; - tracee_port->tracer_proc = NIL; + ERTS_TRACE_FLAGS(tracee_port) &= ~TRACEE_FLAGS; + ERTS_TRACER_PROC(tracee_port) = NIL; } } return 0; @@ -800,21 +867,22 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer) * * main lock is held on c_p * * all locks multiple are held on tracee_p */ - if ((tracee_p->trace_flags & TRACEE_FLAGS) - && tracee_p->tracer_proc != tracer) { + if ((ERTS_TRACE_FLAGS(tracee_p) & TRACEE_FLAGS) + && ERTS_TRACER_PROC(tracee_p) != tracer) { /* This tracee is already being traced, and not by the * tracer to be */ - if (is_internal_port(tracee_p->tracer_proc)) { - if (!erts_is_valid_tracer_port(tracee_p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(tracee_p))) { + if (!erts_is_valid_tracer_port(ERTS_TRACER_PROC(tracee_p))) { /* Current trace port now invalid * - discard it and approve the new. */ goto remove_tracer; } else return 1; } - else if(is_internal_pid(tracee_p->tracer_proc)) { - Process *tracer_p = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - tracee_p->tracer_proc, 0); + else if(is_internal_pid(ERTS_TRACER_PROC(tracee_p))) { + Process *tracer_p; + + tracer_p = erts_proc_lookup(ERTS_TRACER_PROC(tracee_p)); if (!tracer_p) { /* Current trace process now invalid * - discard it and approve the new. */ @@ -824,8 +892,8 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer) } else { remove_tracer: - tracee_p->trace_flags &= ~TRACEE_FLAGS; - tracee_p->tracer_proc = NIL; + ERTS_TRACE_FLAGS(tracee_p) &= ~TRACEE_FLAGS; + ERTS_TRACER_PROC(tracee_p) = NIL; } } return 0; @@ -841,6 +909,11 @@ Eterm trace_info_2(BIF_ALIST_2) Eterm What = BIF_ARG_1; Eterm Key = BIF_ARG_2; Eterm res; + + if (!erts_try_seize_code_write_permission(p)) { + ERTS_BIF_YIELD2(bif_export[BIF_trace_info_2], p, What, Key); + } + if (What == am_on_load) { res = trace_info_on_load(p, Key); } else if (is_atom(What) || is_pid(What)) { @@ -848,8 +921,10 @@ Eterm trace_info_2(BIF_ALIST_2) } else if (is_tuple(What)) { res = trace_info_func(p, What, Key); } else { + erts_release_code_write_permission(); BIF_ERROR(p, BADARG); } + erts_release_code_write_permission(); BIF_RET(res); } @@ -862,8 +937,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) if (pid_spec == am_new) { erts_get_default_tracing(&trace_flags, &tracer); - } else if (is_internal_pid(pid_spec) - && internal_pid_index(pid_spec) < erts_max_processes) { + } else if (is_internal_pid(pid_spec)) { Process *tracee; tracee = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid_spec, ERTS_PROC_LOCKS_ALL); @@ -871,16 +945,16 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) if (!tracee) { return am_undefined; } else { - tracer = tracee->tracer_proc; - trace_flags = tracee->trace_flags; + tracer = ERTS_TRACER_PROC(tracee); + trace_flags = ERTS_TRACE_FLAGS(tracee); } if (is_internal_pid(tracer)) { - if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, tracer, 0)) { + if (!erts_proc_lookup(tracer)) { reset_tracer: - tracee->trace_flags &= ~TRACEE_FLAGS; - trace_flags = tracee->trace_flags; - tracer = tracee->tracer_proc = NIL; + ERTS_TRACE_FLAGS(tracee) &= ~TRACEE_FLAGS; + trace_flags = ERTS_TRACE_FLAGS(tracee); + tracer = ERTS_TRACER_PROC(tracee) = NIL; } } else if (is_internal_port(tracer)) { @@ -977,64 +1051,54 @@ static int function_is_traced(Process *p, Binary **ms, /* out */ Binary **ms_meta, /* out */ Eterm *tracer_pid_meta, /* out */ - Sint *count, /* out */ + Uint *count, /* out */ Eterm *call_time) /* out */ { Export e; Export* ep; - int i; - BeamInstr *code; + BeamInstr* pc; /* First look for an export entry */ e.code[0] = mfa[0]; e.code[1] = mfa[1]; e.code[2] = mfa[2]; if ((ep = export_get(&e)) != NULL) { - if (ep->address == ep->code+3 && - ep->code[3] != (BeamInstr) em_call_error_handler) { - if (ep->code[3] == (BeamInstr) em_call_traced_function) { - *ms = ep->match_prog_set; + pc = ep->code+3; + if (ep->addressv[erts_active_code_ix()] == pc && + *pc != (BeamInstr) em_call_error_handler) { + + int r = 0; + + ASSERT(*pc == (BeamInstr) em_apply_bif || + *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); + + if (erts_is_trace_break(pc, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; } - if (ep->code[3] == (BeamInstr) em_apply_bif) { - for (i = 0; i < BIF_SIZE; ++i) { - if (bif_export[i] == ep) { - int r = 0; - - if (erts_bif_trace_flags[i] & BIF_TRACE_AS_GLOBAL) { - *ms = ep->match_prog_set; - return FUNC_TRACE_GLOBAL_TRACE; - } else { - if (erts_bif_trace_flags[i] & BIF_TRACE_AS_LOCAL) { - r |= FUNC_TRACE_LOCAL_TRACE; - *ms = ep->match_prog_set; - } - if (erts_is_mtrace_break(ep->code+3, ms_meta, - tracer_pid_meta)) { - r |= FUNC_TRACE_META_TRACE; - } - if (erts_is_time_break(p, ep->code+3, call_time)) { - r |= FUNC_TRACE_TIME_TRACE; - } - } - return r ? r : FUNC_TRACE_UNTRACED; - } - } - erl_exit(1,"Impossible ghost bif encountered in trace_info."); + + if (erts_is_trace_break(pc, ms, 1)) { + r |= FUNC_TRACE_LOCAL_TRACE; + } + if (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta)) { + r |= FUNC_TRACE_META_TRACE; } + if (erts_is_time_break(p, pc, call_time)) { + r |= FUNC_TRACE_TIME_TRACE; + } + return r ? r : FUNC_TRACE_UNTRACED; } } /* OK, now look for breakpoint tracing */ - if ((code = erts_find_local_func(mfa)) != NULL) { + if ((pc = erts_find_local_func(mfa)) != NULL) { int r = - (erts_is_trace_break(code, ms, NULL) + (erts_is_trace_break(pc, ms, 1) ? FUNC_TRACE_LOCAL_TRACE : 0) - | (erts_is_mtrace_break(code, ms_meta, tracer_pid_meta) + | (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) - | (erts_is_count_break(code, count) + | (erts_is_count_break(pc, count) ? FUNC_TRACE_COUNT_TRACE : 0) - | (erts_is_time_break(p, code, call_time) + | (erts_is_time_break(p, pc, call_time) ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; @@ -1049,7 +1113,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) Eterm* hp; DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ Binary *ms = NULL, *ms_meta = NULL; - Sint count = 0; + Uint count = 0; Eterm traced = am_false; Eterm match_spec = am_false; Eterm retval = am_false; @@ -1137,9 +1201,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) break; case am_call_count: if (r & FUNC_TRACE_COUNT_TRACE) { - retval = count < 0 ? - erts_make_integer(-count-1, p) : - erts_make_integer(count, p); + retval = erts_make_integer(count, p); } break; case am_call_time: @@ -1162,9 +1224,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) match_spec_meta = NIL; } if (r & FUNC_TRACE_COUNT_TRACE) { - c = count < 0 ? - erts_make_integer(-count-1, p) : - erts_make_integer(count, p); + c = erts_make_integer(count, p); } if (r & FUNC_TRACE_TIME_TRACE) { ct = call_time; @@ -1328,38 +1388,53 @@ trace_info_on_load(Process* p, Eterm key) #undef FUNC_TRACE_LOCAL_TRACE int -erts_set_trace_pattern(Eterm* mfa, int specified, +erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags flags, - Eterm meta_tracer_pid) + Eterm meta_tracer_pid, int is_blocking) { + const ErtsCodeIndex code_ix = erts_active_code_ix(); int matches = 0; int i; + int n; + BpFunction* fp; /* * First work on normal functions (not real BIFs). */ - - for (i = 0; i < export_list_size(); i++) { - Export* ep = export_list(i); - int j; - - if (ExportIsBuiltIn(ep)) { - continue; - } - - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j == specified) { - if (on) { - if (! flags.breakpoint) - matches += setup_func_trace(ep, match_prog_set); - else - reset_func_trace(ep); - } else if (! flags.breakpoint) { - matches += reset_func_trace(ep); + erts_bp_match_export(&finish_bp.e, mfa, specified); + fp = finish_bp.e.matching; + n = finish_bp.e.matched; + + for (i = 0; i < n; i++) { + BeamInstr* pc = fp[i].pc; + Export* ep = (Export *)(((char *)(pc-3)) - offsetof(Export, code)); + + if (on && !flags.breakpoint) { + /* Turn on global call tracing */ + if (ep->addressv[code_ix] != pc) { + fp[i].mod->curr.num_traced_exports++; +#ifdef DEBUG + pc[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); +#endif + pc[0] = (BeamInstr) BeamOp(op_jump_f); + pc[1] = (BeamInstr) ep->addressv[code_ix]; + } + erts_set_call_trace_bif(pc, match_prog_set, 0); + if (ep->addressv[code_ix] != pc) { + pc[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + } + } else if (!on && flags.breakpoint) { + /* Turn off breakpoint tracing -- nothing to do here. */ + } else { + /* + * Turn off global tracing, either explicitly or implicitly + * before turning on breakpoint tracing. + */ + erts_clear_call_trace_bif(pc, 0); + if (pc[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + pc[0] = (BeamInstr) BeamOp(op_jump_f); } } } @@ -1384,26 +1459,15 @@ erts_set_trace_pattern(Eterm* mfa, int specified, /* Empty loop body */ } if (j == specified) { + BeamInstr* pc = (BeamInstr *)bif_export[i]->code + 3; + if (! flags.breakpoint) { /* Export entry call trace */ if (on) { - if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) { - ASSERT(ExportIsBuiltIn(bif_export[i])); - erts_clear_mtrace_bif - ((BeamInstr *)bif_export[i]->code + 3); - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META; - } - set_trace_bif(i, match_prog_set); - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_LOCAL; - erts_bif_trace_flags[i] |= BIF_TRACE_AS_GLOBAL; - setup_bif_trace(i); + erts_clear_call_trace_bif(pc, 1); + erts_clear_mtrace_bif(pc); + erts_set_call_trace_bif(pc, match_prog_set, 0); } else { /* off */ - if (erts_bif_trace_flags[i] & BIF_TRACE_AS_GLOBAL) { - clear_trace_bif(i); - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL; - } - if (! erts_bif_trace_flags[i]) { - reset_bif_trace(i); - } + erts_clear_call_trace_bif(pc, 0); } matches++; } else { /* Breakpoint call trace */ @@ -1411,52 +1475,33 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (on) { if (flags.local) { - set_trace_bif(i, match_prog_set); - erts_bif_trace_flags[i] |= BIF_TRACE_AS_LOCAL; - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL; + erts_clear_call_trace_bif(pc, 0); + erts_set_call_trace_bif(pc, match_prog_set, 1); m = 1; } if (flags.meta) { - erts_set_mtrace_bif - ((BeamInstr *)bif_export[i]->code + 3, - meta_match_prog_set, meta_tracer_pid); - erts_bif_trace_flags[i] |= BIF_TRACE_AS_META; - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL; + erts_set_mtrace_bif(pc, meta_match_prog_set, + meta_tracer_pid); m = 1; } if (flags.call_time) { - erts_set_time_trace_bif(bif_export[i]->code + 3, on); + erts_set_time_trace_bif(pc, on); /* I don't want to remove any other tracers */ - erts_bif_trace_flags[i] |= BIF_TRACE_AS_CALL_TIME; m = 1; } - if (erts_bif_trace_flags[i]) { - setup_bif_trace(i); - } } else { /* off */ if (flags.local) { - if (erts_bif_trace_flags[i] & BIF_TRACE_AS_LOCAL) { - clear_trace_bif(i); - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_LOCAL; - } + erts_clear_call_trace_bif(pc, 1); m = 1; } if (flags.meta) { - if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) { - erts_clear_mtrace_bif - ((BeamInstr *)bif_export[i]->code + 3); - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META; - } + erts_clear_mtrace_bif(pc); m = 1; } if (flags.call_time) { - erts_clear_time_trace_bif(bif_export[i]->code + 3); - erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_CALL_TIME; + erts_clear_time_trace_bif(pc); m = 1; } - if (! erts_bif_trace_flags[i]) { - reset_bif_trace(i); - } } matches += m; } @@ -1466,176 +1511,241 @@ erts_set_trace_pattern(Eterm* mfa, int specified, /* ** So, now for breakpoint tracing */ + erts_bp_match_functions(&finish_bp.f, mfa, specified); if (on) { if (! flags.breakpoint) { - erts_clear_trace_break(mfa, specified); - erts_clear_mtrace_break(mfa, specified); - erts_clear_count_break(mfa, specified); - erts_clear_time_break(mfa, specified); + erts_clear_all_breaks(&finish_bp.f); } else { - int m = 0; if (flags.local) { - m = erts_set_trace_break(mfa, specified, match_prog_set, - am_true); + erts_set_trace_break(&finish_bp.f, match_prog_set); } if (flags.meta) { - m = erts_set_mtrace_break(mfa, specified, meta_match_prog_set, - meta_tracer_pid); + erts_set_mtrace_break(&finish_bp.f, meta_match_prog_set, + meta_tracer_pid); } if (flags.call_count) { - m = erts_set_count_break(mfa, specified, on); + erts_set_count_break(&finish_bp.f, on); } if (flags.call_time) { - m = erts_set_time_break(mfa, specified, on); + erts_set_time_break(&finish_bp.f, on); } - /* All assignments to 'm' above should give the same value, - * so just use the last */ - matches += m; } } else { - int m = 0; if (flags.local) { - m = erts_clear_trace_break(mfa, specified); + erts_clear_trace_break(&finish_bp.f); } if (flags.meta) { - m = erts_clear_mtrace_break(mfa, specified); + erts_clear_mtrace_break(&finish_bp.f); } if (flags.call_count) { - m = erts_clear_count_break(mfa, specified); + erts_clear_count_break(&finish_bp.f); } if (flags.call_time) { - m = erts_clear_time_break(mfa, specified); + erts_clear_time_break(&finish_bp.f); + } + } + + finish_bp.current = 0; + finish_bp.install = on; + finish_bp.local = flags.breakpoint; + +#ifdef ERTS_SMP + if (is_blocking) { + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); +#endif + while (erts_finish_breakpointing()) { + /* Empty loop body */ } - /* All assignments to 'm' above should give the same value, - * so just use the last */ - matches += m; +#ifdef ERTS_SMP + finish_bp.current = -1; } +#endif + if (flags.breakpoint) { + matches += finish_bp.f.matched; + } else { + matches += finish_bp.e.matched; + } return matches; } -/* - * Setup function tracing for the given exported function. - * - * Return Value: 1 if entry refers to a BIF or loaded function, - * 0 if the entry refers to a function not loaded. - */ - -static int -setup_func_trace(Export* ep, void* match_prog) +int +erts_finish_breakpointing(void) { - if (ep->address == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_call_error_handler) { - return 0; - } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { - MatchSetUnref(ep->match_prog_set); - ep->match_prog_set = match_prog; - MatchSetRef(ep->match_prog_set); - return 1; - } else { - /* - * We ignore apply/3 and anything else. - */ - return 0; - } - } - + ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + /* - * Currently no trace support for native code. + * Memory barriers will be issued for all processes *before* + * each of the stages below. (Unless the other schedulers + * are blocked, in which case memory barriers will be issued + * when they are awaken.) */ - if (erts_is_native_break(ep->address)) { + switch (finish_bp.current++) { + case 0: + /* + * At this point, in all functions that are to be breakpointed, + * a pointer to a GenericBp struct has already been added, + * + * Insert the new breakpoints (if any) into the + * code. Different schedulers may see breakpoint instruction + * at different times, but it does not matter since the newly + * added breakpoints are disabled. + */ + if (finish_bp.install) { + if (finish_bp.local) { + erts_install_breakpoints(&finish_bp.f); + } else { + install_exp_breakpoints(&finish_bp.e); + } + } + setup_bif_trace(); + return 1; + case 1: + /* + * Switch index for the breakpoint data, activating the staged + * data. (Depending on the changes in the breakpoint data, + * that could either activate breakpoints or disable + * breakpoints.) + */ + erts_commit_staged_bp(); + return 1; + case 2: + /* + * Remove breakpoints instructions for disabled breakpoints + * (if any). + */ + if (finish_bp.install) { + if (finish_bp.local) { + uninstall_exp_breakpoints(&finish_bp.e); + } else { + erts_uninstall_breakpoints(&finish_bp.f); + } + } else { + if (finish_bp.local) { + erts_uninstall_breakpoints(&finish_bp.f); + } else { + uninstall_exp_breakpoints(&finish_bp.e); + } + } + reset_bif_trace(); + return 1; + case 3: + /* + * Now all breakpoints have either been inserted or removed. + * For all updated breakpoints, copy the active breakpoint + * data to the staged breakpoint data to make them equal + * (simplifying for the next time breakpoints are to be + * updated). If any breakpoints have been totally disabled, + * deallocate the GenericBp structs for them. + */ + erts_consolidate_bif_bp_data(); + clean_export_entries(&finish_bp.e); + erts_consolidate_bp_data(&finish_bp.e, 0); + erts_consolidate_bp_data(&finish_bp.f, 1); + erts_bp_free_matched_functions(&finish_bp.e); + erts_bp_free_matched_functions(&finish_bp.f); return 0; + default: + ASSERT(0); } - - ep->code[3] = (BeamInstr) em_call_traced_function; - ep->code[4] = (BeamInstr) ep->address; - ep->address = ep->code+3; - ep->match_prog_set = match_prog; - MatchSetRef(ep->match_prog_set); - return 1; + return 0; } -static void setup_bif_trace(int bif_index) { - Export *ep = bif_export[bif_index]; - - ASSERT(ExportIsBuiltIn(ep)); - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[bif_index].traced; -} +static void +install_exp_breakpoints(BpFunctions* f) +{ + const ErtsCodeIndex code_ix = erts_active_code_ix(); + BpFunction* fp = f->matching; + Uint ne = f->matched; + Uint i; + Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); -static void set_trace_bif(int bif_index, void* match_prog) { - Export *ep = bif_export[bif_index]; - -#ifdef HARDDEBUG - erts_fprintf(stderr, "set_trace_bif: %T:%T/%bpu\n", - ep->code[0], ep->code[1], ep->code[2]); -#endif - ASSERT(ExportIsBuiltIn(ep)); - MatchSetUnref(ep->match_prog_set); - ep->match_prog_set = match_prog; - MatchSetRef(ep->match_prog_set); -} + for (i = 0; i < ne; i++) { + BeamInstr* pc = fp[i].pc; + Export* ep = (Export *) (((char *)pc)-offset); -/* - * Reset function tracing for the given exported function. - * - * Return Value: 1 if entry refers to a BIF or loaded function, - * 0 if the entry refers to a function not loaded. - */ + ep->addressv[code_ix] = pc; + } +} -static int -reset_func_trace(Export* ep) +static void +uninstall_exp_breakpoints(BpFunctions* f) { - if (ep->address == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_call_error_handler) { - return 0; - } else if (ep->code[3] == (BeamInstr) em_call_traced_function) { - ep->address = (Uint *) ep->code[4]; - MatchSetUnref(ep->match_prog_set); - ep->match_prog_set = NULL; - return 1; - } else { - /* - * We ignore apply/3 and anything else. - */ - return 0; + const ErtsCodeIndex code_ix = erts_active_code_ix(); + BpFunction* fp = f->matching; + Uint ne = f->matched; + Uint i; + Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); + + for (i = 0; i < ne; i++) { + BeamInstr* pc = fp[i].pc; + Export* ep = (Export *) (((char *)pc)-offset); + + if (ep->addressv[code_ix] != pc) { + continue; } + ASSERT(*pc == (BeamInstr) BeamOp(op_jump_f)); + ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; } - - /* - * Currently no trace support for native code. - */ - if (erts_is_native_break(ep->address)) { - return 0; - } - - /* - * Nothing to do, but the export entry matches. - */ +} - return 1; +static void +clean_export_entries(BpFunctions* f) +{ + const ErtsCodeIndex code_ix = erts_active_code_ix(); + BpFunction* fp = f->matching; + Uint ne = f->matched; + Uint i; + Uint offset = offsetof(Export, code) + 3*sizeof(BeamInstr); + + for (i = 0; i < ne; i++) { + BeamInstr* pc = fp[i].pc; + Export* ep = (Export *) (((char *)pc)-offset); + + if (ep->addressv[code_ix] == pc) { + continue; + } + if (*pc == (BeamInstr) BeamOp(op_jump_f)) { + ep->code[3] = (BeamInstr) 0; + ep->code[4] = (BeamInstr) 0; + } + } } -static void reset_bif_trace(int bif_index) { - Export *ep = bif_export[bif_index]; - - ASSERT(ExportIsBuiltIn(ep)); - ASSERT(ep->code[4]); - ASSERT(! ep->match_prog_set); - ASSERT(! erts_is_mtrace_break((BeamInstr *)ep->code+3, NULL, NULL)); - ep->code[4] = (BeamInstr) bif_table[bif_index].f; +static void +setup_bif_trace(void) +{ + int i; + + for (i = 0; i < BIF_SIZE; ++i) { + Export *ep = bif_export[i]; + GenericBp* g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + if (g) { + if (ExportIsBuiltIn(ep)) { + ASSERT(ep->code[4]); + ep->code[4] = (BeamInstr) bif_table[i].traced; + } + } + } } -static void clear_trace_bif(int bif_index) { - Export *ep = bif_export[bif_index]; - -#ifdef HARDDEBUG - erts_fprintf(stderr, "clear_trace_bif: %T:%T/%bpu\n", - ep->code[0], ep->code[1], ep->code[2]); -#endif - ASSERT(ExportIsBuiltIn(ep)); - MatchSetUnref(ep->match_prog_set); - ep->match_prog_set = NULL; +static void +reset_bif_trace(void) +{ + int i; + ErtsBpIndex active = erts_active_bp_ix(); + + for (i = 0; i < BIF_SIZE; ++i) { + Export *ep = bif_export[i]; + BeamInstr* pc = ep->code+3; + GenericBp* g = (GenericBp *) pc[-4]; + if (g && g->data[active].flags == 0) { + if (ExportIsBuiltIn(ep)) { + ASSERT(ep->code[4]); + ep->code[4] = (BeamInstr) bif_table[i].f; + } + } + } } /* @@ -1776,7 +1886,7 @@ new_seq_trace_token(Process* p) SEQ_TRACE_TOKEN(p) = TUPLE5(hp, make_small(0), /* Flags */ make_small(0), /* Label */ make_small(0), /* Serial */ - p->id, /* Internal pid */ /* From */ + p->common.id, /* Internal pid */ /* From */ make_small(p->seq_trace_lastcnt)); } } @@ -2161,13 +2271,15 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) /* Check if valid process, no locks are taken */ if (is_internal_pid(profiler)) { - if (internal_pid_index(profiler) >= erts_max_processes) goto error; - profiler_p = process_tab[internal_pid_index(profiler)]; - if (INVALID_PID(profiler_p, profiler)) goto error; + profiler_p = erts_proc_lookup(profiler); + if (!profiler_p) + goto error; } else if (is_internal_port(profiler)) { - if (internal_port_index(profiler) >= erts_max_ports) goto error; - profiler_port = &erts_port[internal_port_index(profiler)]; - if (INVALID_TRACER_PORT(profiler_port, profiler)) goto error; + profiler_port = (erts_port_lookup( + profiler, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)); + if (!profiler_port) + goto error; } else { goto error; } @@ -2231,8 +2343,7 @@ trace_delivered_1(BIF_ALIST_1) p = NULL; } else if (! (p = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_ARG_1, ERTS_PROC_LOCKS_ALL))) { - if (is_not_internal_pid(BIF_ARG_1) - || internal_pid_index(BIF_ARG_1) >= erts_max_processes) { + if (is_not_internal_pid(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } } @@ -2251,7 +2362,7 @@ trace_delivered_1(BIF_ALIST_1) msg = TUPLE3(hp, AM_trace_delivered, BIF_ARG_1, msg_ref); #ifdef ERTS_SMP - erts_send_sys_msg_proc(BIF_P->id, BIF_P->id, msg, bp); + erts_send_sys_msg_proc(BIF_P->common.id, BIF_P->common.id, msg, bp); if (p) erts_smp_proc_unlock(p, (BIF_P == p diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 3753b618e1..43eb691338 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -484,8 +484,16 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer ERTS_FP_ERROR_THOROUGH(p, f32, return THE_NON_VALUE); f.fd = f32; } else { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef ftmp; + ftmp.fd = f64; + f.fw[0] = ftmp.fw[1]; + f.fw[1] = ftmp.fw[0]; + ERTS_FP_ERROR_THOROUGH(p, f.fd, return THE_NON_VALUE); +#else ERTS_FP_ERROR_THOROUGH(p, f64, return THE_NON_VALUE); f.fd = f64; +#endif } mb->offset += num_bits; hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT); @@ -1014,8 +1022,13 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) #endif } else if (is_small(arg)) { u.f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + a = u.i32[1]; + b = u.i32[0]; +#else a = u.i32[0]; b = u.i32[1]; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &u.f64) < 0) { return 0; @@ -1118,21 +1131,42 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) byte *bptr; double f64; float f32; +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef fbuf, ftmp; +#endif if (num_bits == 64) { if (is_float(arg)) { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef *fdp = (FloatDef*)(float_val(arg) + 1); + ftmp = *fdp; +#else bptr = (byte *) (float_val(arg) + 1); +#endif } else if (is_small(arg)) { f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &f64) < 0) { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + fbuf.fw[0] = ftmp.fw[1]; + fbuf.fw[1] = ftmp.fw[0]; + bptr = fbuf.fb; +#endif } else if (num_bits == 32) { if (is_float(arg)) { FloatDef f; diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index fe3693d0ca..88c6c34881 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2011. All Rights Reserved. + * Copyright Ericsson AB 2010-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -34,7 +34,7 @@ #include "bif.h" #include "erl_cpu_topology.h" -#define ERTS_MAX_READER_GROUPS 8 +#define ERTS_MAX_READER_GROUPS 64 /* * Cpu topology hierarchy. @@ -486,7 +486,7 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) erts_thr_set_main_status(1, (int) esdp->no); /* Make sure we check if we should bind to a cpu or not... */ - esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + (void) ERTS_RUNQ_FLGS_SET(esdp->run_queue, ERTS_RUNQ_FLG_CHK_CPU_BIND); } #endif @@ -498,9 +498,6 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) erts_cpu_groups_map_t *cgm; erts_cpu_groups_callback_list_t *cgcl; erts_cpu_groups_callback_call_t *cgcc; -#ifdef ERTS_SMP - esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; -#endif erts_smp_runq_unlock(esdp->run_queue); erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); cpu_id = scheduler2cpu_map[esdp->no].bind_id; @@ -623,30 +620,38 @@ write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) int erts_init_scheduler_bind_type_string(char *how) { + ErtsCpuBindOrder order; + if (sys_strcmp(how, "u") == 0) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) - return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; - else if (!system_cpudata && !user_cpudata) - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; + order = ERTS_CPU_BIND_NONE; else if (sys_strcmp(how, "db") == 0) - cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + order = ERTS_CPU_BIND_DEFAULT_BIND; else if (sys_strcmp(how, "s") == 0) - cpu_bind_order = ERTS_CPU_BIND_SPREAD; + order = ERTS_CPU_BIND_SPREAD; else if (sys_strcmp(how, "ps") == 0) - cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + order = ERTS_CPU_BIND_PROCESSOR_SPREAD; else if (sys_strcmp(how, "ts") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + order = ERTS_CPU_BIND_THREAD_SPREAD; else if (sys_strcmp(how, "tnnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; else if (sys_strcmp(how, "nnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; else if (sys_strcmp(how, "nnts") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; else if (sys_strcmp(how, "ns") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + order = ERTS_CPU_BIND_NO_SPREAD; else - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_BAD_TYPE; + + if (order != ERTS_CPU_BIND_NONE) { + if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) + return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; + else if (!system_cpudata && !user_cpudata) + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; + } + + cpu_bind_order = order; + return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; } diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index c5a9520b61..b502258dae 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -40,7 +40,7 @@ void erts_init_cpu_topology(void); #define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS 0 #define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED 1 #define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY 2 -#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE 3 +#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_BAD_TYPE 3 int erts_init_scheduler_bind_type_string(char *how); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 7409564167..98c2988323 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -224,8 +224,9 @@ Export ets_select_continue_exp; static Export ets_delete_continue_exp; static void -free_dbtable(DbTable* tb) +free_dbtable(void *vtb) { + DbTable *tb = (DbTable *) vtb; #ifdef HARDDEBUG if (erts_smp_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) { erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", @@ -250,20 +251,8 @@ free_dbtable(DbTable* tb) #endif ASSERT(is_immed(tb->common.heir_data)); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); - ERTS_SMP_MEMORY_BARRIER; } -#ifdef ERTS_SMP -static void -chk_free_dbtable(void *vtb) -{ - DbTable * tb = (DbTable *) vtb; - ERTS_THR_MEMORY_BARRIER; - if (erts_refc_dectest(&tb->common.ref, 0) == 0) - free_dbtable(tb); -} -#endif - static void schedule_free_dbtable(DbTable* tb) { /* @@ -274,15 +263,11 @@ static void schedule_free_dbtable(DbTable* tb) * need to unlock the table lock after this * function has returned). */ -#ifdef ERTS_SMP - int scheds = erts_get_max_no_executing_schedulers(); - ASSERT(scheds >= 1); ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); - erts_refc_init(&tb->common.ref, scheds); - erts_schedule_multi_misc_aux_work(0, scheds, chk_free_dbtable, tb); -#else - free_dbtable(tb); -#endif + erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, + (void *) tb, + &tb->release.data, + sizeof(DbTable)); } static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, @@ -441,7 +426,8 @@ DbTable* db_get_table_aux(Process *p, if (tb) { db_lock(tb, kind); if (tb->common.id != id - || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) { + || ((tb->common.status & what) == 0 + && p->common.id != tb->common.owner)) { db_unlock(tb, kind); tb = NULL; } @@ -541,10 +527,6 @@ static int remove_named_tab(DbTable *tb, int have_lock) &rwlock); #ifdef ERTS_SMP if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { - /* - * We keep our increased refc over this op in order to - * prevent the table from disapearing. - */ db_unlock(tb, LCK_WRITE); erts_smp_rwmtx_rwlock(rwlock); db_lock(tb, LCK_WRITE); @@ -635,7 +617,7 @@ BIF_RETTYPE ets_safe_fixtable_2(BIF_ALIST_2) #ifdef HARDDEBUG erts_fprintf(stderr, "ets:safe_fixtable(%T,%T); Process: %T, initial: %T:%T/%bpu\n", - BIF_ARG_1, BIF_ARG_2, BIF_P->id, + BIF_ARG_1, BIF_ARG_2, BIF_P->common.id, BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif kind = (BIF_ARG_2 == am_true) ? LCK_READ : LCK_WRITE_REC; @@ -1214,7 +1196,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) #ifdef HARDDEBUG erts_fprintf(stderr, "ets:rename(%T,%T); Process: %T, initial: %T:%T/%bpu\n", - BIF_ARG_1, BIF_ARG_2, BIF_P->id, + BIF_ARG_1, BIF_ARG_2, BIF_P->common.id, BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif @@ -1457,7 +1439,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), "db_tab", "db_tab_fix"); tb->common.keypos = keypos; - tb->common.owner = BIF_P->id; + tb->common.owner = BIF_P->common.id; set_heir(BIF_P, tb, heir, heir_data); erts_smp_atomic_init_nob(&tb->common.nitems, 0); @@ -1479,7 +1461,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) "** Too many db tables **\n"); free_heir_data(tb); tb->common.meth->db_free_table(tb); - free_dbtable(tb); + free_dbtable((void *) tb); BIF_ERROR(BIF_P, SYSTEM_LIMIT); } @@ -1526,7 +1508,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) #ifdef HARDDEBUG erts_fprintf(stderr, "ets:new(%T,%T)=%T; Process: %T, initial: %T:%T/%bpu\n", - BIF_ARG_1, BIF_ARG_2, ret, BIF_P->id, + BIF_ARG_1, BIF_ARG_2, ret, BIF_P->common.id, BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n", erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); @@ -1538,7 +1520,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); if (db_put_hash(meta_pid_to_tab, - TUPLE2(meta_tuple, BIF_P->id, make_small(slot)), + TUPLE2(meta_tuple, + BIF_P->common.id, + make_small(slot)), 0) != DB_ERROR_NONE) { erl_exit(1,"Could not update ets metadata."); } @@ -1657,7 +1641,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) #ifdef HARDDEBUG erts_fprintf(stderr, "ets:delete(%T); Process: %T, initial: %T:%T/%bpu\n", - BIF_ARG_1, BIF_P->id, + BIF_ARG_1, BIF_P->common.id, BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif @@ -1674,7 +1658,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - if (tb->common.owner != BIF_P->id) { + if (tb->common.owner != BIF_P->common.id) { DeclareTmpHeap(meta_tuple,3,BIF_P); /* @@ -1689,10 +1673,12 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) make_small(tb->common.slot)); BIF_P->flags |= F_USING_DB; - tb->common.owner = BIF_P->id; + tb->common.owner = BIF_P->common.id; db_put_hash(meta_pid_to_tab, - TUPLE2(meta_tuple,BIF_P->id,make_small(tb->common.slot)), + TUPLE2(meta_tuple, + BIF_P->common.id, + make_small(tb->common.slot)), 0); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); UnUseTmpHeap(3,BIF_P); @@ -1768,7 +1754,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) } if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL - || tb->common.owner != BIF_P->id) { + || tb->common.owner != BIF_P->common.id) { goto badarg; } from_pid = tb->common.owner; @@ -1791,7 +1777,10 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) db_unlock(tb,LCK_WRITE); erts_send_message(BIF_P, to_proc, &to_locks, - TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, from_pid, BIF_ARG_3), + TUPLE4(buf, am_ETS_TRANSFER, + tb->common.id, + from_pid, + BIF_ARG_3), 0); erts_smp_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); @@ -1853,7 +1842,7 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2) if (tail != NIL || (tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL - || tb->common.owner != BIF_P->id) { + || tb->common.owner != BIF_P->common.id) { goto badarg; } @@ -2667,7 +2656,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) */ /* If/when we implement lockless private tables: - if ((tb->common.status & DB_PRIVATE) && owner != BIF_P->id) { + if ((tb->common.status & DB_PRIVATE) && owner != BIF_P->common.id) { db_unlock(tb, LCK_READ); rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, owner, ERTS_PROC_LOCK_MAIN); @@ -2814,7 +2803,6 @@ void init_db(void) { DbTable init_tb; int i; - extern BeamInstr* em_apply_bif; Eterm *hp; unsigned bits; size_t size; @@ -2848,7 +2836,7 @@ void init_db(void) else db_max_tabs = user_requested_db_max_tabs; - bits = erts_fit_in_bits(db_max_tabs-1); + bits = erts_fit_in_bits_int32(db_max_tabs-1); if (bits > SMALL_BITS) { erl_exit(1,"Max limit for ets tabled too high %u (max %u).", db_max_tabs, ((Uint)1)<<SMALL_BITS); @@ -2945,49 +2933,24 @@ void init_db(void) } /* Non visual BIF to trap to. */ - memset(&ets_select_delete_continue_exp, 0, sizeof(Export)); - ets_select_delete_continue_exp.address = - &ets_select_delete_continue_exp.code[3]; - ets_select_delete_continue_exp.code[0] = am_ets; - ets_select_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); - ets_select_delete_continue_exp.code[2] = 1; - ets_select_delete_continue_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_delete_continue_exp.code[4] = - (BeamInstr) &ets_select_delete_1; + erts_init_trap_export(&ets_select_delete_continue_exp, + am_ets, am_atom_put("delete_trap",11), 1, + &ets_select_delete_1); /* Non visual BIF to trap to. */ - memset(&ets_select_count_continue_exp, 0, sizeof(Export)); - ets_select_count_continue_exp.address = - &ets_select_count_continue_exp.code[3]; - ets_select_count_continue_exp.code[0] = am_ets; - ets_select_count_continue_exp.code[1] = am_atom_put("count_trap",11); - ets_select_count_continue_exp.code[2] = 1; - ets_select_count_continue_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_count_continue_exp.code[4] = - (BeamInstr) &ets_select_count_1; + erts_init_trap_export(&ets_select_count_continue_exp, + am_ets, am_atom_put("count_trap",11), 1, + &ets_select_count_1); /* Non visual BIF to trap to. */ - memset(&ets_select_continue_exp, 0, sizeof(Export)); - ets_select_continue_exp.address = - &ets_select_continue_exp.code[3]; - ets_select_continue_exp.code[0] = am_ets; - ets_select_continue_exp.code[1] = am_atom_put("select_trap",11); - ets_select_continue_exp.code[2] = 1; - ets_select_continue_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_continue_exp.code[4] = - (BeamInstr) &ets_select_trap_1; + erts_init_trap_export(&ets_select_continue_exp, + am_ets, am_atom_put("select_trap",11), 1, + &ets_select_trap_1); /* Non visual BIF to trap to. */ - memset(&ets_delete_continue_exp, 0, sizeof(Export)); - ets_delete_continue_exp.address = &ets_delete_continue_exp.code[3]; - ets_delete_continue_exp.code[0] = am_ets; - ets_delete_continue_exp.code[1] = am_atom_put("delete_trap",11); - ets_delete_continue_exp.code[2] = 1; - ets_delete_continue_exp.code[3] = (BeamInstr) em_apply_bif; - ets_delete_continue_exp.code[4] = (BeamInstr) &ets_delete_trap; + erts_init_trap_export(&ets_delete_continue_exp, + am_ets, am_atom_put("delete_trap",11), 1, + &ets_delete_trap); hp = ms_delete_all_buff; ms_delete_all = CONS(hp, am_true, NIL); @@ -3085,9 +3048,9 @@ static int give_away_to_heir(Process* p, DbTable* tb) Eterm to_pid; UWord heir_data; - ASSERT(tb->common.owner == p->id); + ASSERT(tb->common.owner == p->common.id); ASSERT(is_internal_pid(tb->common.heir)); - ASSERT(tb->common.heir != p->id); + ASSERT(tb->common.heir != p->common.id); retry: to_pid = tb->common.heir; to_proc = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN, @@ -3100,7 +3063,7 @@ retry: db_lock(tb,LCK_WRITE); ASSERT(tb != NULL); - if (tb->common.owner != p->id) { + if (tb->common.owner != p->common.id) { if (to_proc != NULL ) { erts_smp_proc_unlock(to_proc, to_locks); } @@ -3111,7 +3074,7 @@ retry: if (to_proc != NULL ) { erts_smp_proc_unlock(to_proc, to_locks); } - if (to_pid == p->id || to_pid == am_none) { + if (to_pid == p->common.id || to_pid == am_none) { return 0; /* no real heir, table still mine */ } goto retry; @@ -3120,7 +3083,8 @@ retry: if (to_proc == NULL) { return 0; /* heir not alive, table still mine */ } - if (erts_cmp_timeval(&to_proc->started, &tb->common.heir_started) != 0) { + if (to_proc->common.u.alive.started_interval + != tb->common.heir_started_interval) { erts_smp_proc_unlock(to_proc, to_locks); return 0; /* heir dead and pid reused, table still mine */ } @@ -3145,7 +3109,11 @@ retry: heir_data = tpv[1]; } erts_send_message(p, to_proc, &to_locks, - TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, p->id, heir_data), + TUPLE4(buf, + am_ETS_TRANSFER, + tb->common.id, + p->common.id, + heir_data), 0); erts_smp_proc_unlock(to_proc, to_locks); return !0; @@ -3154,7 +3122,7 @@ retry: /* * erts_db_process_exiting() is called when a process terminates. * It returns 0 when completely done, and !0 when it wants to - * yield. c_p->u.exit_data can hold a pointer to a state while + * yield. c_p->u.terminate can hold a pointer to a state while * yielding. */ #define ERTS_DB_INTERNAL_ERROR(LSTR) \ @@ -3164,8 +3132,8 @@ retry: int erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) { - ErtsDbProcCleanupState *state = (ErtsDbProcCleanupState *) c_p->u.exit_data; - Eterm pid = c_p->id; + ErtsDbProcCleanupState *state = (ErtsDbProcCleanupState *) c_p->u.terminate; + Eterm pid = c_p->common.id; ErtsDbProcCleanupState default_state; int ret; @@ -3346,7 +3314,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) if (state != &default_state) erts_free(ERTS_ALC_T_DB_PROC_CLEANUP, state); - c_p->u.exit_data = NULL; + c_p->u.terminate = NULL; return 0; default: @@ -3367,13 +3335,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) break; } - ASSERT(c_p->u.exit_data == (void *) state + ASSERT(c_p->u.terminate == (void *) state || state == &default_state); if (state == &default_state) { - c_p->u.exit_data = erts_alloc(ERTS_ALC_T_DB_PROC_CLEANUP, + c_p->u.terminate = erts_alloc(ERTS_ALC_T_DB_PROC_CLEANUP, sizeof(ErtsDbProcCleanupState)); - sys_memcpy(c_p->u.exit_data, + sys_memcpy(c_p->u.terminate, (void*) state, sizeof(ErtsDbProcCleanupState)); } @@ -3399,7 +3367,7 @@ static void fix_table_locked(Process* p, DbTable* tb) } else { for (; fix != NULL; fix = fix->next) { - if (fix->pid == p->id) { + if (fix->pid == p->common.id) { ++(fix->counter); #ifdef ERTS_SMP erts_smp_mtx_unlock(&tb->common.fixlock); @@ -3411,7 +3379,7 @@ static void fix_table_locked(Process* p, DbTable* tb) fix = (DbFixation *) erts_db_alloc(ERTS_ALC_T_DB_FIXATION, tb, sizeof(DbFixation)); ERTS_ETS_MISC_MEM_ADD(sizeof(DbFixation)); - fix->pid = p->id; + fix->pid = p->common.id; fix->counter = 1; fix->next = tb->common.fixations; tb->common.fixations = fix; @@ -3422,7 +3390,9 @@ static void fix_table_locked(Process* p, DbTable* tb) UseTmpHeap(3,p); db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); if (db_put_hash(meta_pid_to_fixed_tab, - TUPLE2(meta_tuple, p->id, make_small(tb->common.slot)), + TUPLE2(meta_tuple, + p->common.id, + make_small(tb->common.slot)), 0) != DB_ERROR_NONE) { UnUseTmpHeap(3,p); erl_exit(1,"Could not insert ets metadata in safe_fixtable."); @@ -3442,7 +3412,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, erts_smp_mtx_lock(&tb->common.fixlock); #endif for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) { - if ((*pp)->pid == p->id) { + if ((*pp)->pid == p->common.id) { DbFixation* fix = *pp; erts_refc_dec(&tb->common.ref,0); --(fix->counter); @@ -3456,7 +3426,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, #endif db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_fixed_tab, - p->id, make_small(tb->common.slot)); + p->common.id, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, (void *) fix, sizeof(DbFixation)); @@ -3515,15 +3485,15 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) if (heir == am_none) { return; } - if (heir == me->id) { - tb->common.heir_started = me->started; + if (heir == me->common.id) { + erts_ensure_later_proc_interval(me->common.u.alive.started_interval); + tb->common.heir_started_interval = me->common.u.alive.started_interval; } else { - Process* heir_proc= erts_pid2proc_opt(me, ERTS_PROC_LOCK_MAIN, heir, - 0, ERTS_P2P_FLG_SMP_INC_REFC); + Process* heir_proc= erts_proc_lookup(heir); if (heir_proc != NULL) { - tb->common.heir_started = heir_proc->started; - erts_smp_proc_dec_refc(heir_proc); + erts_ensure_later_proc_interval(heir_proc->common.u.alive.started_interval); + tb->common.heir_started_interval = heir_proc->common.u.alive.started_interval; } else { tb->common.heir = am_none; } @@ -3861,7 +3831,7 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) while (index >= atom_table_size()) { char tmp[20]; erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); - am_atom_put(tmp,strlen(tmp)); + erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); } list = CONS(hp, make_atom(index), list); hp += 2; diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 2e5deaf338..6b62e10eb7 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -27,6 +27,10 @@ #define __DB_H__ #include "sys.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "bif.h" #include "erl_db_util.h" /* Flags */ @@ -36,6 +40,11 @@ Uint erts_get_ets_misc_mem_size(void); +typedef struct { + DbTableCommon common; + ErtsThrPrgrLaterOp data; +} DbTableRelease; + /* * So, the structure for a database table, NB this is only * interesting in db.c. @@ -44,6 +53,7 @@ union db_table { DbTableCommon common; /* Any type of db table */ DbTableHash hash; /* Linear hash array specific data */ DbTableTree tree; /* AVL tree specific data */ + DbTableRelease release; /*TT*/ }; diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index cddd8dfadd..d17bd9f693 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,7 +33,7 @@ typedef struct hash_db_term { DbTerm dbterm; /* The actual term */ } HashDbTerm; -#define DB_HASH_LOCK_CNT 16 +#define DB_HASH_LOCK_CNT 64 typedef struct db_table_hash_fine_locks { union { erts_smp_rwmtx_t lck; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 312050b931..25029ba90f 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -452,16 +452,8 @@ DbTableMethod db_tree = void db_initialize_tree(void) { - memset(&ets_select_reverse_exp, 0, sizeof(Export)); - ets_select_reverse_exp.address = - &ets_select_reverse_exp.code[3]; - ets_select_reverse_exp.code[0] = am_ets; - ets_select_reverse_exp.code[1] = am_reverse; - ets_select_reverse_exp.code[2] = 3; - ets_select_reverse_exp.code[3] = - (BeamInstr) em_apply_bif; - ets_select_reverse_exp.code[4] = - (BeamInstr) &ets_select_reverse; + erts_init_trap_export(&ets_select_reverse_exp, am_ets, am_reverse, 3, + &ets_select_reverse); return; }; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 42907e2e84..ef3749a2c4 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2012. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -138,21 +138,23 @@ set_tracee_flags(Process *tracee_p, Eterm tracer, Uint d_flags, Uint e_flags) { Uint flags; if (tracer == NIL) { - flags = tracee_p->trace_flags & ~TRACEE_FLAGS; + flags = ERTS_TRACE_FLAGS(tracee_p) & ~TRACEE_FLAGS; } else { - flags = ((tracee_p->trace_flags & ~d_flags) | e_flags); + flags = ((ERTS_TRACE_FLAGS(tracee_p) & ~d_flags) | e_flags); if (! flags) tracer = NIL; } - ret = tracee_p->tracer_proc != tracer || tracee_p->trace_flags != flags - ? am_true : am_false; - tracee_p->tracer_proc = tracer; - tracee_p->trace_flags = flags; + ret = ((ERTS_TRACER_PROC(tracee_p) != tracer + || ERTS_TRACE_FLAGS(tracee_p) != flags) + ? am_true + : am_false); + ERTS_TRACER_PROC(tracee_p) = tracer; + ERTS_TRACE_FLAGS(tracee_p) = flags; return ret; } /* ** Assuming all locks on tracee_p on entry ** -** Changes tracee_p->trace_flags and tracee_p->tracer_proc +** Changes ERTS_TRACE_FLAGS(tracee_p) and ERTS_TRACER_PROC(tracee_p) ** according to input disable/enable flags and tracer. ** ** Returns am_true|am_false on success, am_true if value changed, @@ -173,17 +175,20 @@ set_match_trace(Process *tracee_p, Eterm fail_term, Eterm tracer, tracer, ERTS_PROC_LOCKS_ALL))) { if (tracee_p != tracer_p) { ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - tracer_p->trace_flags |= tracee_p->trace_flags ? F_TRACER : 0; + ERTS_TRACE_FLAGS(tracer_p) |= (ERTS_TRACE_FLAGS(tracee_p) + ? F_TRACER + : 0); erts_smp_proc_unlock(tracer_p, ERTS_PROC_LOCKS_ALL); } } else if (is_internal_port(tracer)) { Port *tracer_port = - erts_id2port(tracer, tracee_p, ERTS_PROC_LOCKS_ALL); + erts_id2port_sflgs(tracer, + tracee_p, + ERTS_PROC_LOCKS_ALL, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); if (tracer_port) { - if (! INVALID_TRACER_PORT(tracer_port, tracer)) { - ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); - } - erts_smp_port_unlock(tracer_port); + ret = set_tracee_flags(tracee_p, tracer, d_flags, e_flags); + erts_port_release(tracer_port); } } else { ASSERT(is_nil(tracer)); @@ -2174,7 +2179,7 @@ restart: pc += n; break; case matchSelf: - *esp++ = c_p->id; + *esp++ = c_p->common.id; break; case matchWaste: --esp; @@ -2261,7 +2266,7 @@ restart: case matchEnableTrace: if ( (n = erts_trace_flag2bit(esp[-1]))) { BEGIN_ATOMIC_TRACE(c_p); - set_tracee_flags(c_p, c_p->tracer_proc, 0, n); + set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), 0, n); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; @@ -2274,7 +2279,7 @@ restart: BEGIN_ATOMIC_TRACE(c_p); if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { /* Always take over the tracer of the current process */ - set_tracee_flags(tmpp, c_p->tracer_proc, 0, n); + set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), 0, n); esp[-1] = am_true; } } @@ -2282,7 +2287,7 @@ restart: case matchDisableTrace: if ( (n = erts_trace_flag2bit(esp[-1]))) { BEGIN_ATOMIC_TRACE(c_p); - set_tracee_flags(c_p, c_p->tracer_proc, n, 0); + set_tracee_flags(c_p, ERTS_TRACER_PROC(c_p), n, 0); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; @@ -2295,7 +2300,7 @@ restart: BEGIN_ATOMIC_TRACE(c_p); if ( (tmpp = get_proc(c_p, 0, esp[0], 0))) { /* Always take over the tracer of the current process */ - set_tracee_flags(tmpp, c_p->tracer_proc, n, 0); + set_tracee_flags(tmpp, ERTS_TRACER_PROC(c_p), n, 0); esp[-1] = am_true; } } @@ -2314,14 +2319,16 @@ restart: break; case matchSilent: --esp; + if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT) + break; if (*esp == am_true) { erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - c_p->trace_flags |= F_TRACE_SILENT; + ERTS_TRACE_FLAGS(c_p) |= F_TRACE_SILENT; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } else if (*esp == am_false) { erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - c_p->trace_flags &= ~F_TRACE_SILENT; + ERTS_TRACE_FLAGS(c_p) &= ~F_TRACE_SILENT; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } break; @@ -2329,11 +2336,11 @@ restart: { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ - Eterm tracer = c_p->tracer_proc; + Eterm tracer = ERTS_TRACER_PROC(c_p); /* XXX Atomicity note: Not fully atomic. Default tracer * is sampled from current process but applied to * tracee and tracer later after releasing main - * locks on current process, so c_p->tracer_proc + * locks on current process, so ERTS_TRACER_PROC(c_p) * may actually have changed when tracee and tracer * gets updated. I do not think nobody will notice. * It is just the default value that is not fully atomic. @@ -2358,7 +2365,7 @@ restart: { /* disable enable */ Uint d_flags = 0, e_flags = 0; /* process trace flags */ - Eterm tracer = c_p->tracer_proc; + Eterm tracer = ERTS_TRACER_PROC(c_p); /* XXX Atomicity note. Not fully atomic. See above. * Above it could possibly be solved, but not here. */ @@ -2480,7 +2487,7 @@ Eterm db_format_dmc_err_info(Process *p, DMCErrInfo *ei) vnum = tmp->variable; } if (vnum >= 0) - sprintf(buff,tmp->error_string, vnum); + erts_snprintf(buff,sizeof(buff)+20,tmp->error_string, vnum); else strcpy(buff,tmp->error_string); sl = strlen(buff); @@ -3410,8 +3417,7 @@ static DMCRet dmc_one_term(DMCContext *context, } default: erl_exit(1, "db_match_compile: " - "Bad object on heap: 0x%08lx\n", - (unsigned long) c); + "Bad object on heap: 0x%bex\n", c); } return retOk; } @@ -4485,7 +4491,9 @@ static DMCRet dmc_fun(DMCContext *context, if (context->err_info != NULL) { /* Ugly, should define a better RETURN_TERM_ERROR interface... */ char buff[100]; - sprintf(buff, "Function %%T/%d does_not_exist.", (int)a - 1); + erts_snprintf(buff, sizeof(buff), + "Function %%T/%d does_not_exist.", + (int)a - 1); RETURN_TERM_ERROR(buff, p[1], context, *constant); } else { return retFail; @@ -4500,7 +4508,7 @@ static DMCRet dmc_fun(DMCContext *context, if (context->err_info != NULL) { /* Ugly, should define a better RETURN_TERM_ERROR interface... */ char buff[100]; - sprintf(buff, + erts_snprintf(buff, sizeof(buff), "Function %%T/%d cannot be called in this context.", (int)a - 1); RETURN_TERM_ERROR(buff, p[1], context, *constant); @@ -4764,9 +4772,10 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) for (j = 0; j < x && DMC_PEEK(heap,j) != n; ++j) ; ASSERT(j < x); - sprintf(buff+1,"%u", (unsigned) j); + erts_snprintf(buff+1, sizeof(buff) - 1, "%u", (unsigned) j); /* Yes, writing directly into terms, they ARE off heap */ - *p = am_atom_put(buff, strlen(buff)); + *p = erts_atom_put((byte *) buff, strlen(buff), + ERTS_ATOM_ENC_LATIN1, 1); } ++p; } @@ -4853,7 +4862,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) ret = copy_struct(b,sz,hp,off_heap); } else { erl_exit(1, "Trying to constant-copy non constant expression " - "0x%08x in (d)ets:match compilation.", (unsigned long) t); + "0x%bex in (d)ets:match compilation.", t); } } else { sz = size_object(t); @@ -4964,7 +4973,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) save_cp = p->cp; p->cp = NULL; res = erts_match_set_run(p, mps, arr, n, - ERTS_PAM_COPY_RESULT, &ret_flags); + ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, + &ret_flags); p->cp = save_cp; } else { n = 0; @@ -5002,7 +5012,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) static Eterm seq_trace_fake(Process *p, Eterm arg1) { Eterm result = erl_seq_trace_info(p, arg1); - if (is_tuple(result) && *tuple_val(result) == 2) { + if (!is_non_value(result) && is_tuple(result) && *tuple_val(result) == 2) { return (tuple_val(result))[2]; } return result; @@ -5387,7 +5397,7 @@ void db_match_dis(Binary *bp) erts_printf("Caller\n"); break; default: - erts_printf("??? (0x%08x)\n", *t); + erts_printf("??? (0x%bpx)\n", *t); ++t; break; } @@ -5399,13 +5409,13 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); - erts_printf("0x%08x", (unsigned long) tmp); + erts_printf("%p", tmp); } erts_printf("}\n"); erts_printf("num_bindings: %d\n", prog->num_bindings); erts_printf("heap_size: %beu\n", prog->heap_size); erts_printf("stack_offset: %beu\n", prog->stack_offset); - erts_printf("text: 0x%08x\n", (unsigned long) prog->text); + erts_printf("text: %p\n", prog->text); erts_printf("stack_size: %d (words)\n", prog->heap_size-prog->stack_offset); } diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 6a96e174e1..90b79e6044 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -209,7 +209,7 @@ typedef struct db_fixation { */ typedef struct db_table_common { - erts_refc_t ref; /* fixation counter and delete counter */ + erts_refc_t ref; /* fixation counter */ #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ @@ -219,7 +219,7 @@ typedef struct db_table_common { Eterm owner; /* Pid of the creator */ Eterm heir; /* Pid of the heir */ UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ - SysTimeval heir_started; /* To further identify the heir */ + Uint64 heir_started_interval; /* To further identify the heir */ Eterm the_name; /* an atom */ Eterm id; /* atom | integer */ DbTableMethod* meth; /* table methods */ @@ -320,10 +320,10 @@ ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) #define DB_INFO (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE) #define ONLY_WRITER(P,T) (((T)->common.status & (DB_PRIVATE|DB_PROTECTED)) \ - && (T)->common.owner == (P)->id) + && (T)->common.owner == (P)->common.id) #define ONLY_READER(P,T) (((T)->common.status & DB_PRIVATE) && \ -(T)->common.owner == (P)->id) +(T)->common.owner == (P)->common.id) /* Function prototypes */ BIF_RETTYPE db_get_trace_control_word(Process* p); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 22e873afc6..b90d00f236 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -252,16 +252,16 @@ void erts_check_stack(Process *p) if (p->stop > stack_start) erl_exit(1, "<%lu.%lu.%lu>: Stack underflow\n", - internal_pid_channel_no(p->id), - internal_pid_number(p->id), - internal_pid_serial(p->id)); + internal_pid_channel_no(p->common.id), + internal_pid_number(p->common.id), + internal_pid_serial(p->common.id)); if (p->stop < stack_end) erl_exit(1, "<%lu.%lu.%lu>: Stack overflow\n", - internal_pid_channel_no(p->id), - internal_pid_number(p->id), - internal_pid_serial(p->id)); + internal_pid_channel_no(p->common.id), + internal_pid_number(p->common.id), + internal_pid_serial(p->common.id)); for (elemp = p->stop; elemp < stack_start; elemp++) { int in_mbuf = 0; @@ -284,9 +284,9 @@ void erts_check_stack(Process *p) erl_exit(1, "<%lu.%lu.%lu>: Wild stack pointer\n", - internal_pid_channel_no(p->id), - internal_pid_number(p->id), - internal_pid_serial(p->id)); + internal_pid_channel_no(p->common.id), + internal_pid_number(p->common.id), + internal_pid_serial(p->common.id)); } } @@ -387,16 +387,16 @@ void verify_process(Process *p) #define VERIFY_AREA(name,ptr,sz) { \ int n = (sz); \ while (n--) if(!verify_eterm(p,*(ptr+n))) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->id); } + erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } #define VERIFY_ETERM(name,eterm) { \ if(!verify_eterm(p,eterm)) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->id); } + erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } ErlMessage* mp = p->msg.first; - VERBOSE(DEBUG_MEMORY,("Verify process: %T...\n",p->id)); + VERBOSE(DEBUG_MEMORY,("Verify process: %T...\n",p->common.id)); while (mp != NULL) { VERIFY_ETERM("message term",ERL_MESSAGE_TERM(mp)); @@ -516,7 +516,7 @@ static void print_process_memory(Process *p) ErlHeapFragment* bp = MBUF(p); erts_printf("==============================\n"); - erts_printf("|| Memory info for %T ||\n",p->id); + erts_printf("|| Memory info for %T ||\n",p->common.id); erts_printf("==============================\n"); erts_printf("-- %-*s ---%s-%s-%s-%s--\n", @@ -601,7 +601,7 @@ void print_memory_info(Process *p) { if (p != NULL) { erts_printf("======================================\n"); - erts_printf("|| Memory info for %-12T ||\n",p->id); + erts_printf("|| Memory info for %-12T ||\n",p->common.id); erts_printf("======================================\n"); erts_printf("+- local heap ----%s-%s-%s-%s-+\n", dashes,dashes,dashes,dashes); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 1ae9a211d7..b68fd46fcc 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2011. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -85,12 +85,9 @@ #include "erl_drv_nif.h" #include <stdlib.h> -#include <string.h> /* ssize_t on Mac OS X */ +#include <sys/types.h> /* ssize_t */ -#if defined(VXWORKS) -# include <ioLib.h> -typedef struct iovec SysIOVec; -#elif defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_) +#if defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_) #ifndef STATIC_ERLANG_DRIVER /* Windows dynamic drivers, everything is different... */ #define ERL_DRIVER_TYPES_ONLY @@ -136,7 +133,7 @@ typedef struct { #define ERL_DRV_EXTENDED_MARKER (0xfeeeeeed) #define ERL_DRV_EXTENDED_MAJOR_VERSION 2 -#define ERL_DRV_EXTENDED_MINOR_VERSION 0 +#define ERL_DRV_EXTENDED_MINOR_VERSION 2 /* * The emulator will refuse to load a driver with different major @@ -157,6 +154,7 @@ typedef struct { #define ERL_DRV_FLAG_USE_PORT_LOCKING (1 << 0) #define ERL_DRV_FLAG_SOFT_BUSY (1 << 1) +#define ERL_DRV_FLAG_NO_BUSY_MSGQ (1 << 2) /* * Integer types @@ -210,8 +208,8 @@ typedef struct erl_drv_binary { typedef struct _erl_drv_data* ErlDrvData; /* Data to be used by the driver itself. */ #ifndef ERL_SYS_DRV typedef struct _erl_drv_event* ErlDrvEvent; /* An event to be selected on. */ -typedef struct _erl_drv_port* ErlDrvPort; /* A port descriptor. */ #endif +typedef struct _erl_drv_port* ErlDrvPort; /* A port descriptor. */ typedef struct _erl_drv_port* ErlDrvThreadData; /* Thread data. */ #if !defined(__WIN32__) && !defined(_WIN32) && !defined(_WIN32_) && !defined(USE_SELECT) @@ -370,11 +368,7 @@ typedef struct erl_drv_entry { /* For windows dynamic drivers */ #ifndef ERL_DRIVER_TYPES_ONLY -#if defined(VXWORKS) -# define DRIVER_INIT(DRIVER_NAME) \ - ErlDrvEntry* DRIVER_NAME ## _init(void); \ - ErlDrvEntry* DRIVER_NAME ## _init(void) -#elif defined(__WIN32__) +#if defined(__WIN32__) # define DRIVER_INIT(DRIVER_NAME) \ __declspec(dllexport) ErlDrvEntry* driver_init(void); \ __declspec(dllexport) ErlDrvEntry* driver_init(void) @@ -384,9 +378,18 @@ typedef struct erl_drv_entry { ErlDrvEntry* driver_init(void) #endif +#define ERL_DRV_BUSY_MSGQ_DISABLED (~((ErlDrvSizeT) 0)) +#define ERL_DRV_BUSY_MSGQ_READ_ONLY ((ErlDrvSizeT) 0) +#define ERL_DRV_BUSY_MSGQ_LIM_MAX (ERL_DRV_BUSY_MSGQ_DISABLED - 1) +#define ERL_DRV_BUSY_MSGQ_LIM_MIN ((ErlDrvSizeT) 1) + /* * These are the functions available for driver writers. */ +EXTERN void erl_drv_busy_msgq_limits(ErlDrvPort port, + ErlDrvSizeT *low, + ErlDrvSizeT *high); + EXTERN int driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on); EXTERN int driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data); @@ -405,6 +408,11 @@ EXTERN int driver_cancel_timer(ErlDrvPort port); EXTERN int driver_read_timer(ErlDrvPort port, unsigned long *time_left); /* + * Inform runtime system about lengthy work. + */ +EXTERN int erl_drv_consume_timeslice(ErlDrvPort port, int percent); + +/* * Get plain-text error message from within a driver */ EXTERN char* erl_errno_id(int error); @@ -538,6 +546,11 @@ EXTERN int erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2); EXTERN void erl_drv_thread_exit(void *resp); EXTERN int erl_drv_thread_join(ErlDrvTid, void **respp); +EXTERN char* erl_drv_mutex_name(ErlDrvMutex *mtx); +EXTERN char* erl_drv_cond_name(ErlDrvCond *cnd); +EXTERN char* erl_drv_rwlock_name(ErlDrvRWLock *rwlck); +EXTERN char* erl_drv_thread_name(ErlDrvTid tid); + /* * Misc. */ @@ -601,13 +614,37 @@ EXTERN ErlDrvPort driver_create_port(ErlDrvPort creator_port, ErlDrvData drv_data); +/* + * driver_output_term() is deprecated, and scheduled for removal in + * OTP-R17. Use erl_drv_output_term() instead. For more information + * see the erl_driver(3) documentation. + */ +EXTERN int driver_output_term(ErlDrvPort ix, + ErlDrvTermData* data, + int len) ERL_DRV_DEPRECATED_FUNC; +/* + * driver_send_term() is deprecated, and scheduled for removal in + * OTP-R17. Use erl_drv_send_term() instead. For more information + * see the erl_driver(3) documentation. + */ +EXTERN int driver_send_term(ErlDrvPort ix, + ErlDrvTermData to, + ErlDrvTermData* data, + int len) ERL_DRV_DEPRECATED_FUNC; + /* output term data to the port owner */ -EXTERN int driver_output_term(ErlDrvPort ix, ErlDrvTermData* data, int len); +EXTERN int erl_drv_output_term(ErlDrvTermData port, + ErlDrvTermData* data, + int len); /* output term data to a specific process */ -EXTERN int driver_send_term(ErlDrvPort ix, ErlDrvTermData to, - ErlDrvTermData* data, int len); +EXTERN int erl_drv_send_term(ErlDrvTermData port, + ErlDrvTermData to, + ErlDrvTermData* data, + int len); /* Async IO functions */ +EXTERN unsigned int driver_async_port_key(ErlDrvPort port); + EXTERN long driver_async(ErlDrvPort ix, unsigned int* key, void (*async_invoke)(void*), @@ -651,6 +688,3 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); /* also in global.h, but driver's can't include global.h */ void dtrace_drvport_str(ErlDrvPort port, char *port_buf); - - - diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index a49a155701..4f1bba8657 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -188,6 +188,17 @@ erl_drv_mutex_destroy(ErlDrvMutex *dmtx) #endif } + +char * +erl_drv_mutex_name(ErlDrvMutex *dmtx) +{ +#ifdef USE_THREADS + return dmtx ? dmtx->name : NULL; +#else + return NULL; +#endif +} + int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { @@ -258,6 +269,15 @@ erl_drv_cond_destroy(ErlDrvCond *dcnd) #endif } +char * +erl_drv_cond_name(ErlDrvCond *dcnd) +{ +#ifdef USE_THREADS + return dcnd ? dcnd->name : NULL; +#else + return NULL; +#endif +} void erl_drv_cond_signal(ErlDrvCond *dcnd) @@ -331,6 +351,16 @@ erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) #endif } +char * +erl_drv_rwlock_name(ErlDrvRWLock *drwlck) +{ +#ifdef USE_THREADS + return drwlck ? drwlck->name : NULL; +#else + return NULL; +#endif +} + int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { @@ -617,6 +647,18 @@ erl_drv_thread_create(char *name, #endif } +char * +erl_drv_thread_name(ErlDrvTid tid) +{ +#ifdef USE_THREADS + struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid; + return dtid ? dtid->name : NULL; +#else + return NULL; +#endif +} + + ErlDrvTid erl_drv_thread_self(void) { diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 5c66a0bf73..b673ef6b3c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -77,8 +77,6 @@ ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index); ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, byte* uniq, int index, int arity); -ErlFunEntry* erts_get_fun_entry2(Eterm mod, int old_uniq, int old_index, - byte* uniq, int index, int arity); void erts_erase_fun_entry(ErlFunEntry* fe); void erts_cleanup_funs(ErlFunThing* funp); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index e2689f58c3..8ba94d89e9 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -47,10 +47,6 @@ */ #define ALENGTH(a) (sizeof(a)/sizeof(a[0])) -static erts_smp_spinlock_t info_lck; -static Uint garbage_cols; /* no of garbage collections */ -static Uint reclaimed; /* no of words reclaimed in GCs */ - # define STACK_SZ_ON_HEAP(p) ((p)->hend - (p)->stop) # define OverRunCheck(P) \ if ((P)->stop < (P)->htop) { \ @@ -59,7 +55,7 @@ static Uint reclaimed; /* no of words reclaimed in GCs */ erts_fprintf(stderr, "htop=%p\n", (p)->htop); \ erts_fprintf(stderr, "heap=%p\n", (p)->heap); \ erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ - __FILE__,__LINE__,(P)->id); \ + __FILE__,__LINE__,(P)->common.id); \ } #ifdef DEBUG @@ -120,6 +116,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); +static void init_gc_info(ErtsGCInfo *gcip); + #ifdef HARDDEBUG static void disallow_heap_frag_ref_in_heap(Process* p); static void disallow_heap_frag_ref_in_old_heap(Process* p); @@ -129,7 +127,7 @@ static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int n #if defined(ARCH_64) && !HALFWORD_HEAP # define MAX_HEAP_SIZES 154 #else -# define MAX_HEAP_SIZES 55 +# define MAX_HEAP_SIZES 59 #endif static Sint heap_sizes[MAX_HEAP_SIZES]; /* Suitable heap sizes. */ @@ -137,13 +135,42 @@ static int num_heap_sizes; /* Number of heap sizes. */ Uint erts_test_long_gc_sleep; /* Only used for testing... */ +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsGCInfoReq; + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, + ErtsGCInfoReq, + 5, + ERTS_ALC_T_GC_INFO_REQ) +#else +static ERTS_INLINE ErtsGCInfoReq * +gcireq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_GC_INFO_REQ, + sizeof(ErtsGCInfoReq)); +} + +static ERTS_INLINE void +gcireq_free(ErtsGCInfoReq *ptr) +{ + erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr); +} +#endif + /* * Initialize GC global data. */ void erts_init_gc(void) { - int i = 0; + int i = 0, ix; + Sint max_heap_size = 0; ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); @@ -155,9 +182,6 @@ erts_init_gc(void) ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); - erts_smp_spinlock_init(&info_lck, "gc_info"); - garbage_cols = 0; - reclaimed = 0; erts_test_long_gc_sleep = 0; /* @@ -168,22 +192,46 @@ erts_init_gc(void) * we really don't want that growth when the heaps are that big. */ - heap_sizes[0] = 34; - heap_sizes[1] = 55; - for (i = 2; i < 23; i++) { - heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2]; + /* Growth stage 1 - Fibonacci + 1*/ + /* 12,38 will hit size 233, the old default */ + + heap_sizes[0] = 12; + heap_sizes[1] = 38; + + for(i = 2; i < 23; i++) { + /* one extra word for block header */ + heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2] + 1; } + + /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] (and halfword) + * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words] + */ + + max_heap_size = sizeof(Eterm) < 8 ? (Sint)((~(Uint)0)/(sizeof(Eterm))) : + (Sint)(((Uint64)1 << 53)/sizeof(Eterm)); + + /* Growth stage 2 - 20% growth */ /* At 1.3 mega words heap, we start to slow down. */ for (i = 23; i < ALENGTH(heap_sizes); i++) { - heap_sizes[i] = 5*(heap_sizes[i-1]/4); - if (heap_sizes[i] < 0) { + heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-1]/5; + if ((heap_sizes[i] < 0) || heap_sizes[i] > max_heap_size) { /* Size turned negative. Discard this last size. */ i--; break; } } num_heap_sizes = i; + + for (ix = 0; ix < erts_no_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); + init_gc_info(&esdp->gc_info); + } + +#if !HALFWORD_HEAP + init_gcireq_alloc(); +#endif + } /* @@ -216,7 +264,7 @@ erts_next_heap_size(Uint size, Uint offset) low = mid + 1; } } - erl_exit(1, "no next heap size found: %lu, offset %lu\n", (unsigned long)size, (unsigned long)offset); + erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset); } return 0; } @@ -272,17 +320,6 @@ erts_heap_sizes(Process* p) return res; } -void -erts_gc_info(ErtsGCInfo *gcip) -{ - if (gcip) { - erts_smp_spin_lock(&info_lck); - gcip->garbage_collections = garbage_cols; - gcip->reclaimed = reclaimed; - erts_smp_spin_unlock(&info_lck); - } -} - void erts_offset_heap(Eterm* hp, Uint sz, Sint offs, Eterm* low, Eterm* high) { @@ -363,6 +400,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) Uint reclaimed_now = 0; int done = 0; Uint ms1, s1, us1; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif @@ -370,11 +408,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) trace_gc(p, am_gc_start); } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->gcstatus = p->status; - p->status = P_GARBING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - + erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) { get_now(&ms1, &s1, &us1); } @@ -418,9 +452,8 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) ErtsGcQuickSanityCheck(p); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->status = p->gcstatus; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_end); } @@ -445,11 +478,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) monitor_large_heap(p); } - erts_smp_spin_lock(&info_lck); - garbage_cols++; - reclaimed += reclaimed_now; - erts_smp_spin_unlock(&info_lck); - + esdp->gc_info.garbage_cols++; + esdp->gc_info.reclaimed += reclaimed_now; + FLAGS(p) &= ~F_FORCE_GC; #ifdef CHECK_FOR_HOLES @@ -504,10 +535,7 @@ erts_garbage_collect_hibernate(Process* p) /* * Preliminaries. */ - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->gcstatus = p->status; - p->status = P_GARBING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); ErtsGcQuickSanityCheck(p); ASSERT(p->mbuf_sz == 0); ASSERT(p->mbuf == 0); @@ -618,9 +646,7 @@ erts_garbage_collect_hibernate(Process* p) ErtsGcQuickSanityCheck(p); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->status = p->gcstatus; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); } @@ -644,10 +670,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, /* * Set GC state. */ - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->gcstatus = p->status; - p->status = P_GARBING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); /* * We assume that the caller has already done a major collection @@ -789,9 +812,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, /* * Restore status. */ - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - p->status = p->gcstatus; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); } static int @@ -883,14 +904,12 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - if (wanted < MIN_HEAP_SIZE(p)) { - wanted = MIN_HEAP_SIZE(p); - } else { - wanted = next_heap_size(p, wanted, 0); - } + wanted = wanted < MIN_HEAP_SIZE(p) ? MIN_HEAP_SIZE(p) + : next_heap_size(p, wanted, 0); if (wanted < HEAP_SIZE(p)) { shrink_new_heap(p, wanted, objv, nobj); } + ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); return 1; /* We are done. */ } @@ -1449,11 +1468,10 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int I think this is better as fullsweep is used mainly on small memory systems, but I could be wrong... */ wanted = 2 * need_after; - if (wanted < p->min_heap_size) { - sz = p->min_heap_size; - } else { - sz = next_heap_size(p, wanted, 0); - } + + sz = wanted < p->min_heap_size ? p->min_heap_size + : next_heap_size(p, wanted, 0); + if (sz < HEAP_SIZE(p)) { shrink_new_heap(p, sz, objv, nobj); } @@ -1946,6 +1964,17 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) ++n; } + /* + * A trapping BIF can add to rootset by setting the extra_root + * in the process_structure. + */ + if (p->extra_root != NULL) { + roots[n].v = p->extra_root->objv; + roots[n].sz = p->extra_root->sz; + ++n; + } + + ASSERT((is_nil(p->seq_trace_token) || is_tuple(follow_moved(p->seq_trace_token)) || is_atom(p->seq_trace_token))); @@ -1961,9 +1990,9 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) n++; } #endif - ASSERT(is_nil(p->tracer_proc) || - is_internal_pid(p->tracer_proc) || - is_internal_port(p->tracer_proc)); + ASSERT(is_nil(ERTS_TRACER_PROC(p)) || + is_internal_pid(ERTS_TRACER_PROC(p)) || + is_internal_port(ERTS_TRACER_PROC(p))); ASSERT(is_pid(follow_moved(p->group_leader))); if (is_not_immed(p->group_leader)) { @@ -2523,6 +2552,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, p->dictionary->used, offs, area, area_size); } + if (p->extra_root != NULL) { + offset_heap_ptr(p->extra_root->objv, + p->extra_root->sz, + offs, area, area_size); + } + offset_heap_ptr(&p->fvalue, 1, offs, area, area_size); offset_heap_ptr(&p->ftrace, 1, offs, area, area_size); offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size); @@ -2546,6 +2581,110 @@ offset_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_one_rootset(p, offs, area, area_size, objv, nobj); } +static void +init_gc_info(ErtsGCInfo *gcip) +{ + gcip->reclaimed = 0; + gcip->garbage_cols = 0; +} + +static void +reply_gc_info(void *vgcirp) +{ + Uint64 reclaimed = 0, garbage_cols = 0; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsGCInfoReq *gcirp = (ErtsGCInfoReq *) vgcirp; + ErtsProcLocks rp_locks = (gcirp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN + : 0); + Process *rp = gcirp->proc; + Eterm ref_copy = NIL, msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); + + reclaimed = esdp->gc_info.reclaimed; + garbage_cols = esdp->gc_info.garbage_cols; + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, gcirp->ref); + else + *szp += REF_THING_SIZE; + + msg = erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, garbage_cols), + erts_bld_uint64(hpp, szp, reclaimed)); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + szp = NULL; + hpp = &hp; + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); + + if (gcirp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_smp_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0) + gcireq_free(vgcirp); +} + +Eterm +erts_gc_info_request(Process *c_p) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsGCInfoReq *gcirp; + Eterm *hp; + + gcirp = gcireq_alloc(); + ref = erts_make_ref(c_p); + hp = &gcirp->ref_heap[0]; + + gcirp->proc = c_p; + gcirp->ref = STORE_NC(&hp, NULL, ref); + gcirp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&gcirp->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_gc_info, + (void *) gcirp); +#endif + + reply_gc_info((void *) gcirp); + + return ref; +} + #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) static int diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index e7d4ac2b67..e9d8249ee1 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -163,10 +163,10 @@ BKT_MIN_SZ(GFAllctr_t *gfallctr, int ix) /* Prototypes of callback functions */ static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void link_free_block (Allctr_t *, Block_t *, Uint32); -static void unlink_free_block (Allctr_t *, Block_t *, Uint32); -static void update_last_aux_mbc (Allctr_t *, Carrier_t *, Uint32); + Block_t *, Uint); +static void link_free_block (Allctr_t *, Block_t *); +static void unlink_free_block (Allctr_t *, Block_t *); +static void update_last_aux_mbc (Allctr_t *, Carrier_t *); static Eterm info_options (Allctr_t *, char *, int *, void *, Uint **, Uint *); static void init_atoms (void); @@ -203,8 +203,6 @@ erts_gfalc_start(GFAllctr_t *gfallctr, sys_memcpy((void *) gfallctr, (void *) &zero.allctr, sizeof(GFAllctr_t)); - init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ - allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; @@ -223,7 +221,9 @@ erts_gfalc_start(GFAllctr_t *gfallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = update_last_aux_mbc; allctr->destroying_mbc = update_last_aux_mbc; - + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -363,7 +363,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) blk && i < max_blk_search; blk = blk->next, i++) { - blk_sz = BLK_SZ(blk); + blk_sz = MBC_FBLK_SZ(&blk->block_head); blk_on_lambc = (((char *) blk) < gfallctr->last_aux_mbc_end && gfallctr->last_aux_mbc_start <= ((char *) blk)); @@ -385,7 +385,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) static Block_t * get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, Uint32 flags) + Block_t *cand_blk, Uint cand_size) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; int unsafe_bi, min_bi; @@ -402,9 +402,9 @@ get_free_block(Allctr_t *allctr, Uint size, if (min_bi == unsafe_bi) { blk = search_bucket(allctr, min_bi, size); if (blk) { - if (cand_blk && cand_size <= BLK_SZ(blk)) + if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk, flags); + unlink_free_block(allctr, blk); return blk; } if (min_bi < NO_OF_BKTS - 1) { @@ -422,20 +422,20 @@ get_free_block(Allctr_t *allctr, Uint size, /* We are guaranteed to find a block that fits in this bucket */ blk = search_bucket(allctr, min_bi, size); ASSERT(blk); - if (cand_blk && cand_size <= BLK_SZ(blk)) + if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk, flags); + unlink_free_block(allctr, blk); return blk; } static void -link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +link_free_block(Allctr_t *allctr, Block_t *block) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; - Uint sz = BLK_SZ(blk); + Uint sz = MBC_FBLK_SZ(&blk->block_head); int i = BKT_IX(gfallctr, sz); ASSERT(sz >= MIN_BLK_SZ); @@ -452,11 +452,11 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +unlink_free_block(Allctr_t *allctr, Block_t *block) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; - Uint sz = BLK_SZ(blk); + Uint sz = MBC_FBLK_SZ(&blk->block_head); int i = BKT_IX(gfallctr, sz); if (!blk->prev) { @@ -473,7 +473,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc, Uint32 flags) +update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; @@ -590,16 +590,16 @@ info_options(Allctr_t *allctr, * to erts_gfalc_test() * \* */ -unsigned long -erts_gfalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_gfalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x100: return (unsigned long) BKT_IX((GFAllctr_t *) a1, (Uint) a2); - case 0x101: return (unsigned long) BKT_MIN_SZ((GFAllctr_t *) a1, (int) a2); - case 0x102: return (unsigned long) NO_OF_BKTS; - case 0x103: return (unsigned long) + case 0x100: return (UWord) BKT_IX((GFAllctr_t *) a1, (Uint) a2); + case 0x101: return (UWord) BKT_MIN_SZ((GFAllctr_t *) a1, (int) a2); + case 0x102: return (UWord) NO_OF_BKTS; + case 0x103: return (UWord) find_bucket(&((GFAllctr_t *) a1)->bucket_mask, (int) a2); - default: ASSERT(0); return ~((unsigned long) 0); + default: ASSERT(0); return ~((UWord) 0); } } @@ -618,7 +618,7 @@ check_block(Allctr_t *allctr, Block_t * blk, int free_block) GFFreeBlock_t *fblk; if(free_block) { - Uint blk_sz = BLK_SZ(blk); + Uint blk_sz = is_sbc_blk(blk) ? SBC_BLK_SZ(blk) : MBC_BLK_SZ(blk); bi = BKT_IX(gfallctr, blk_sz); ASSERT(gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi))); diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h index a554a6f466..385de0da23 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.h +++ b/erts/emulator/beam/erl_goodfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -82,7 +82,7 @@ struct GFAllctr_t_ { }; -unsigned long erts_gfalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_gfalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_GF_ALLOC_IMPL) && !defined(ERL_GF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 7f52b111ac..8c4fffa75b 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,6 +44,7 @@ #include "erl_thr_progress.h" #include "erl_thr_queue.h" #include "erl_async.h" +#include "erl_ptab.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -54,6 +55,68 @@ # include <sys/resource.h> #endif +#define ERTS_DEFAULT_NO_ASYNC_THREADS 10 + +/* + * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands + * only. Do not remove even though they aren't used elsewhere in the emulator! + */ +#ifdef ERTS_SMP +const int etp_smp_compiled = 1; +#else +const int etp_smp_compiled = 0; +#endif +#ifdef USE_THREADS +const int etp_thread_compiled = 1; +#else +const int etp_thread_compiled = 0; +#endif +const char etp_erts_version[] = ERLANG_VERSION; +const char etp_otp_release[] = ERLANG_OTP_RELEASE; +const char etp_compile_date[] = ERLANG_COMPILE_DATE; +const char etp_arch[] = ERLANG_ARCHITECTURE; +#ifdef ERTS_ENABLE_KERNEL_POLL +const int etp_kernel_poll_support = 1; +#else +const int etp_kernel_poll_support = 0; +#endif +#if defined(ARCH_64) +const int etp_arch_bits = 64; +#elif defined(ARCH_32) +const int etp_arch_bits = 32; +#else +# error "Not 64-bit, nor 32-bit arch" +#endif +#if HALFWORD_HEAP +const int etp_halfword = 1; +#else +const int etp_halfword = 0; +#endif +#ifdef HIPE +const int etp_hipe = 1; +#else +const int etp_hipe = 0; +#endif +#ifdef DEBUG +const int etp_debug_compiled = 1; +#else +const int etp_debug_compiled = 0; +#endif +#ifdef ERTS_ENABLE_LOCK_COUNT +const int etp_lock_count = 1; +#else +const int etp_lock_count = 0; +#endif +#ifdef ERTS_ENABLE_LOCK_CHECK +const int etp_lock_check = 1; +#else +const int etp_lock_check = 0; +#endif +#ifdef WORDS_BIGENDIAN +const int etp_big_endian = 1; +#else +const int etp_big_endian = 0; +#endif /* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will @@ -66,9 +129,12 @@ extern void ConNormalExit(void); extern void ConWaitForExit(void); #endif -static void erl_init(int ncpu); - -#define ERTS_MIN_COMPAT_REL 7 +static void erl_init(int ncpu, + int proc_tab_sz, + int legacy_proc_tab, + int port_tab_sz, + int port_tab_sz_ignore_files, + int legacy_port_tab); static erts_atomic_t exiting; @@ -151,8 +217,6 @@ ErtsModifiedTimings erts_modified_timings[] = { Export *erts_delay_trap = NULL; -int erts_use_r9_pids_ports; - int ignore_break; int replace_intr; @@ -216,23 +280,33 @@ void erts_short_init(void) { int ncpu = early_init(NULL, NULL); - erl_init(ncpu); + erl_init(ncpu, + ERTS_DEFAULT_MAX_PROCESSES, + 0, + ERTS_DEFAULT_MAX_PORTS, + 0, + 0); erts_initialized = 1; } static void -erl_init(int ncpu) +erl_init(int ncpu, + int proc_tab_sz, + int legacy_proc_tab, + int port_tab_sz, + int port_tab_sz_ignore_files, + int legacy_port_tab) { init_benchmarking(); erts_init_monitors(); - erts_init_gc(); erts_init_time(); erts_init_sys_common_misc(); - erts_init_process(ncpu); + erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); erts_init_scheduling(no_schedulers, no_schedulers_online); erts_init_cpu_topology(); /* Must be after init_scheduling */ + erts_init_gc(); /* Must be after init_scheduling */ erts_alloc_late_init(); H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); @@ -241,6 +315,7 @@ erl_init(int ncpu) erts_init_trace(); erts_init_binary(); erts_init_bits(); + erts_code_ix_init(); erts_init_fun_table(); init_atom_table(); init_export_table(); @@ -250,6 +325,7 @@ erl_init(int ncpu) erts_bif_info_init(); erts_ddll_init(); init_emulator(); + erts_ptab_init(); /* Must be after init_emulator() */ erts_bp_init(); init_db(); /* Must be after init_emulator */ erts_bif_timer_init(); @@ -257,13 +333,14 @@ erl_init(int ncpu) init_dist(); erl_drv_thr_init(); erts_init_async(); - init_io(); + erts_init_io(port_tab_sz, port_tab_sz_ignore_files, legacy_port_tab); init_load(); erts_init_bif(); erts_init_bif_chksum(); erts_init_bif_binary(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ + erts_init_external(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -288,8 +365,9 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** ErlSpawnOpts so; Eterm env; - start_mod = am_atom_put(modname, sys_strlen(modname)); - if (erts_find_function(start_mod, am_start, 2) == NULL) { + start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); + if (erts_find_function(start_mod, am_start, 2, + erts_active_code_ix()) == NULL) { erl_exit(5, "No function %s:start/2\n", modname); } @@ -384,11 +462,11 @@ load_preloaded(void) i = 0; while ((name = preload_p[i].name) != NULL) { length = preload_p[i].size; - module_name = am_atom_put(name, sys_strlen(name)); + module_name = erts_atom_put((byte *) name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1); if ((code = sys_preload_begin(&preload_p[i])) == 0) erl_exit(1, "Failed to find preloaded code for module %s\n", name); - res = erts_load_module(NULL, 0, NIL, &module_name, code, length); + res = erts_preload_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); if (res != NIL) erl_exit(1,"Failed loading preloaded module %s (%T)\n", @@ -400,6 +478,7 @@ load_preloaded(void) /* be helpful (or maybe downright rude:-) */ void erts_usage(void) { + int this_rel = this_rel_num(); erts_fprintf(stderr, "Usage: %s [flags] [ -- [init_args] ]\n", progname(program)); erts_fprintf(stderr, "The flags are:\n\n"); @@ -424,7 +503,7 @@ void erts_usage(void) erts_fprintf(stderr, "-d don't write a crash dump for internally detected errors\n"); erts_fprintf(stderr, " (halt(String) will still produce a crash dump)\n"); - + erts_fprintf(stderr, "-fn[u|a|l] Control how filenames are interpreted\n"); erts_fprintf(stderr, "-hms size set minimum heap size in words (default %d)\n", H_DEFAULT_SIZE); erts_fprintf(stderr, "-hmbs size set minimum binary virtual heap size in words (default %d)\n", @@ -433,21 +512,25 @@ void erts_usage(void) /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); - + erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n"); + erts_fprintf(stderr, " Note that this flag is deprecated!\n"); erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n"); - + erts_fprintf(stderr, "-pc <set> Control what characters are considered printable (default latin1)\n"); erts_fprintf(stderr, "-P number set maximum number of processes on this node,\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", - ERTS_MIN_PROCESSES, ERTS_MAX_PROCESSES); + ERTS_MIN_PROCESSES, ERTS_MAX_PROCESSES); + erts_fprintf(stderr, "-Q number set maximum number of ports on this node,\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", + ERTS_MIN_PORTS, ERTS_MAX_PORTS); erts_fprintf(stderr, "-R number set compatibility release number,\n"); erts_fprintf(stderr, " valid range [%d-%d]\n", - ERTS_MIN_COMPAT_REL, this_rel_num()); + this_rel-2, this_rel); erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); - erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); + erts_fprintf(stderr, "-stbt type u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); erts_fprintf(stderr, "-sbwt val set scheduler busy wait threshold, valid values are:\n"); erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n"); erts_fprintf(stderr, "-scl bool enable/disable compaction of scheduler load,\n"); @@ -455,17 +538,23 @@ void erts_usage(void) erts_fprintf(stderr, "-sct cput set cpu topology,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); erts_fprintf(stderr, "-sws val set scheduler wakeup strategy, valid values are:\n"); - erts_fprintf(stderr, " default|legacy|proposal.\n"); + erts_fprintf(stderr, " default|legacy.\n"); + erts_fprintf(stderr, "-swct val set scheduler wake cleanup threshold, valid values are:\n"); + erts_fprintf(stderr, " very_lazy|lazy|medium|eager|very_eager.\n"); erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, ERTS_SCHED_THREAD_MAX_STACK_SIZE); + erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); - erts_fprintf(stderr, " schedulers online (n2), valid range for both\n"); - erts_fprintf(stderr, " numbers are [1-%d]\n", + erts_fprintf(stderr, " schedulers online (n2), maximum for both\n"); + erts_fprintf(stderr, " numbers is %d\n", ERTS_MAX_NO_OF_SCHEDULERS); + erts_fprintf(stderr, "-SP p1:p2 specify schedulers (p1) and schedulers online (p2)\n"); + erts_fprintf(stderr, " as percentages of logical processors configured and logical\n"); + erts_fprintf(stderr, " processors available, respectively\n"); erts_fprintf(stderr, "-t size set the maximum number of atoms the " "emulator can handle\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", @@ -545,18 +634,22 @@ early_init(int *argc, char **argv) /* int ncpuavail; int schdlrs; int schdlrs_onln; + int schdlrs_percentage = 100; + int schdlrs_onln_percentage = 100; int max_main_threads; int max_reader_groups; int reader_groups; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; + erts_save_emu_args(*argc, argv); + erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; display_items = 200; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; - erts_async_max_threads = 0; + erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; H_MIN_SIZE = H_DEFAULT_SIZE; BIN_VH_MIN_SIZE = VH_DEFAULT_SIZE; @@ -583,8 +676,6 @@ early_init(int *argc, char **argv) /* erts_compat_rel = this_rel_num(); - erts_use_r9_pids_ports = 0; - erts_sys_pre_init(); erts_atomic_init_nob(&exiting, 0); #ifdef ERTS_SMP @@ -627,7 +718,7 @@ early_init(int *argc, char **argv) /* if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) erts_async_max_threads = atoi(envbuf); else - erts_async_max_threads = 0; + erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; if (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS) erts_async_max_threads = ERTS_MAX_NO_OF_ASYNC_THREADS; @@ -672,63 +763,132 @@ early_init(int *argc, char **argv) /* } break; } - case 'S' : { - int tot, onln; - char *arg = get_arg(argv[i]+2, argv[i+1], &i); - switch (sscanf(arg, "%d:%d", &tot, &onln)) { - case 0: - switch (sscanf(arg, ":%d", &onln)) { + case 'S' : + if (argv[i][2] == 'P') { + int ptot, ponln; + char *arg = get_arg(argv[i]+3, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &ptot, &ponln)) { + case 0: + switch (sscanf(arg, ":%d", &ponln)) { + case 1: + if (ponln < 0) + goto bad_SP; + ptot = 100; + goto chk_SP; + default: + goto bad_SP; + } case 1: - tot = no_schedulers; - goto chk_S; + if (ptot < 0) + goto bad_SP; + ponln = ptot < 100 ? ptot : 100; + goto chk_SP; + case 2: + if (ptot < 0 || ponln < 0) + goto bad_SP; + chk_SP: + schdlrs_percentage = ptot; + schdlrs_onln_percentage = ponln; + break; default: - goto bad_S; - } - case 1: - onln = tot < schdlrs_onln ? tot : schdlrs_onln; - case 2: - chk_S: - if (tot > 0) - schdlrs = tot; - else - schdlrs = no_schedulers + tot; - if (onln > 0) - schdlrs_onln = onln; - else - schdlrs_onln = no_schedulers_online + onln; - if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { - erts_fprintf(stderr, - "bad amount of schedulers %d\n", - tot); - erts_usage(); - } - if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) { + bad_SP: + erts_fprintf(stderr, + "bad schedulers percentage specifier %s\n", + arg); + erts_usage(); + break; + } + + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d scheduler percentages\n", + schdlrs_percentage, schdlrs_onln_percentage)); + } else { + int tot, onln; + char *arg = get_arg(argv[i]+2, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &tot, &onln)) { + case 0: + switch (sscanf(arg, ":%d", &onln)) { + case 1: + tot = no_schedulers; + goto chk_S; + default: + goto bad_S; + } + case 1: + onln = tot < schdlrs_onln ? tot : schdlrs_onln; + case 2: + chk_S: + if (tot > 0) + schdlrs = tot; + else + schdlrs = no_schedulers + tot; + if (onln > 0) + schdlrs_onln = onln; + else + schdlrs_onln = no_schedulers_online + onln; + if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { + erts_fprintf(stderr, + "bad amount of schedulers %d\n", + tot); + erts_usage(); + } + if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) { + erts_fprintf(stderr, + "bad amount of schedulers online %d " + "(total amount of schedulers %d)\n", + schdlrs_onln, schdlrs); + erts_usage(); + } + break; + default: + bad_S: erts_fprintf(stderr, - "bad amount of schedulers online %d " - "(total amount of schedulers %d)\n", - schdlrs_onln, schdlrs); + "bad amount of schedulers %s\n", + arg); erts_usage(); + break; } - break; - default: - bad_S: - erts_fprintf(stderr, - "bad amount of schedulers %s\n", - arg); - erts_usage(); - break; - } - VERBOSE(DEBUG_SYSTEM, - ("using %d:%d scheduler(s)\n", tot, onln)); - break; - } + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d scheduler(s)\n", tot, onln)); + } + break; default: break; } } i++; } + +#ifdef ERTS_SMP + /* apply any scheduler percentages */ + if (schdlrs_percentage != 100 || schdlrs_onln_percentage != 100) { + schdlrs = schdlrs * schdlrs_percentage / 100; + schdlrs_onln = schdlrs_onln * schdlrs_onln_percentage / 100; + if (schdlrs < 1) + schdlrs = 1; + if (ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { + erts_fprintf(stderr, + "bad schedulers percentage %d " + "(total amount of schedulers %d)\n", + schdlrs_percentage, schdlrs); + erts_usage(); + } + if (schdlrs_onln < 1) + schdlrs_onln = 1; + if (schdlrs < schdlrs_onln) { + erts_fprintf(stderr, + "bad schedulers online percentage %d " + "(total amount of schedulers %d, online %d)\n", + schdlrs_onln_percentage, schdlrs, schdlrs_onln); + erts_usage(); + } + } +#else + /* Silence gcc warnings */ + (void)schdlrs_percentage; + (void)schdlrs_onln_percentage; +#endif } #ifndef USE_THREADS @@ -839,11 +999,16 @@ erl_start(int argc, char **argv) { int i = 1; char* arg=NULL; - char* Parg = NULL; int have_break_handler = 1; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; int ncpu = early_init(&argc, argv); + int proc_tab_sz = ERTS_DEFAULT_MAX_PROCESSES; + int port_tab_sz = ERTS_DEFAULT_MAX_PORTS; + int port_tab_sz_ignore_files = 0; + int legacy_proc_tab = 0; + int legacy_port_tab = 0; + envbufsz = sizeof(envbuf); if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -858,6 +1023,12 @@ erl_start(int argc, char **argv) (erts_aint32_t) max_gen_gcs); } + envbufsz = sizeof(envbuf); + if (erts_sys_getenv_raw("ERL_MAX_PORTS", envbuf, &envbufsz) == 0) { + port_tab_sz = atoi(envbuf); + port_tab_sz_ignore_files = 1; + } + #if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) /* * The default stack size on MacOS X is too small for pcre. @@ -902,20 +1073,83 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("using display items %d\n",display_items)); break; + case 'p': + if (!strncmp(argv[i],"-pc",3)) { + int printable_chars = ERL_PRINTABLE_CHARACTERS_LATIN1; + arg = get_arg(argv[i]+3, argv[i+1], &i); + if (!strcmp(arg,"unicode")) { + printable_chars = ERL_PRINTABLE_CHARACTERS_UNICODE; + } else if (strcmp(arg,"latin1")) { + erts_fprintf(stderr, "bad range of printable " + "characters: %s\n", arg); + erts_usage(); + } + erts_set_printable_characters(printable_chars); + break; + } else { + erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); + erts_usage(); + } case 'f': if (!strncmp(argv[i],"-fn",3)) { + int warning_type = ERL_FILENAME_WARNING_WARNING; arg = get_arg(argv[i]+3, argv[i+1], &i); switch (*arg) { case 'u': - erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8); + switch (*(arg+1)) { + case 'w': + case 0: + break; + case 'i': + warning_type = ERL_FILENAME_WARNING_IGNORE; + break; + case 'e': + warning_type = ERL_FILENAME_WARNING_ERROR; + break; + default: + erts_fprintf(stderr, "bad type of warnings for " + "wrongly coded filename: %s\n", arg+1); + erts_usage(); + } + erts_set_user_requested_filename_encoding + ( + ERL_FILENAME_UTF8, + warning_type + ); break; case 'l': - erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1); + erts_set_user_requested_filename_encoding + ( + ERL_FILENAME_LATIN1, + warning_type + ); break; case 'a': - erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN); + switch (*(arg+1)) { + case 'w': + case 0: + break; + case 'i': + warning_type = ERL_FILENAME_WARNING_IGNORE; + break; + case 'e': + warning_type = ERL_FILENAME_WARNING_ERROR; + break; + default: + erts_fprintf(stderr, "bad type of warnings for " + "wrongly coded filename: %s\n", arg+1); + erts_usage(); + } + erts_set_user_requested_filename_encoding + ( + ERL_FILENAME_UNKNOWN, + warning_type + ); + break; default: - erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg); + erts_fprintf(stderr, "bad filename encoding %s, can be " + "(l,u or a, optionally followed by w, " + "i or e)\n", arg); erts_usage(); } break; @@ -1094,16 +1328,68 @@ erl_start(int argc, char **argv) arg); break; - case 'P': - /* set maximum number of processes */ - Parg = get_arg(argv[i]+2, argv[i+1], &i); - erts_max_processes = atoi(Parg); - /* Check of result is delayed until later. This is because +R - may be given after +P. */ + case 'n': + arg = get_arg(argv[i]+2, argv[i+1], &i); + switch (arg[0]) { + case 's': /* synchronous */ + erts_port_synchronous_ops = 1; + erts_port_schedule_all_ops = 0; + break; + case 'a': /* asynchronous */ + erts_port_synchronous_ops = 0; + erts_port_schedule_all_ops = 1; + break; + case 'd': /* Default - schedule on conflict (asynchronous) */ + erts_port_synchronous_ops = 0; + erts_port_schedule_all_ops = 0; + break; + default: + bad_n_option: + erts_fprintf(stderr, "bad -n option %s\n", arg); + erts_usage(); + } + if (arg[1] != '\0') + goto bad_n_option; + break; + + case 'P': /* set maximum number of processes */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (strcmp(arg, "legacy") == 0) + legacy_proc_tab = 1; + else { + errno = 0; + proc_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || proc_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < proc_tab_sz) { + erts_fprintf(stderr, "bad number of processes %s\n", arg); + erts_usage(); + } + } + break; + + case 'Q': /* set maximum number of ports */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (strcmp(arg, "legacy") == 0) + legacy_port_tab = 1; + else { + errno = 0; + port_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || port_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < port_tab_sz) { + erts_fprintf(stderr, "bad number of ports %s\n", arg); + erts_usage(); + } + port_tab_sz_ignore_files = 1; + } break; case 'S' : /* Was handled in early_init() just read past it */ - (void) get_arg(argv[i]+2, argv[i+1], &i); + if (argv[i][2] == 'P') + (void) get_arg(argv[i]+3, argv[i+1], &i); + else + (void) get_arg(argv[i]+2, argv[i+1], &i); break; case 's' : { @@ -1121,7 +1407,7 @@ erl_start(int argc, char **argv) case ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY: estr = "no cpu topology available"; break; - case ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE: + case ERTS_INIT_SCHED_BIND_TYPE_ERROR_BAD_TYPE: estr = "invalid type"; break; default: @@ -1201,9 +1487,42 @@ erl_start(int argc, char **argv) erts_usage(); } } + else if (has_prefix("pp", sub_param)) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if (sys_strcmp(arg, "true") == 0) + erts_port_parallelism = 1; + else if (sys_strcmp(arg, "false") == 0) + erts_port_parallelism = 0; + else { + erts_fprintf(stderr, + "bad port parallelism scheduling hint %s\n", + arg); + erts_usage(); + } + } else if (sys_strcmp("nsp", sub_param) == 0) erts_use_sender_punish = 0; - else if (sys_strcmp("wt", sub_param) == 0) { + else if (has_prefix("tbt", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + res = erts_init_scheduler_bind_type_string(arg); + if (res == ERTS_INIT_SCHED_BIND_TYPE_ERROR_BAD_TYPE) { + erts_fprintf(stderr, + "setting scheduler bind type '%s' failed: invalid type\n", + arg); + erts_usage(); + } + } + else if (has_prefix("wct", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (erts_sched_set_wake_cleanup_threshold(arg) != 0) { + erts_fprintf(stderr, "scheduler wake cleanup threshold: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("scheduler wake cleanup threshold: %s\n", arg)); + } + else if (has_prefix("wt", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (erts_sched_set_wakeup_other_thresold(arg) != 0) { erts_fprintf(stderr, "scheduler wakeup threshold: %s\n", @@ -1213,7 +1532,7 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("scheduler wakeup threshold: %s\n", arg)); } - else if (sys_strcmp("ws", sub_param) == 0) { + else if (has_prefix("ws", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (erts_sched_set_wakeup_other_type(arg) != 0) { erts_fprintf(stderr, "scheduler wakeup strategy: %s\n", @@ -1298,22 +1617,19 @@ erl_start(int argc, char **argv) case 'R': { /* set compatibility release */ + int this_rel; arg = get_arg(argv[i]+2, argv[i+1], &i); erts_compat_rel = atoi(arg); - if (erts_compat_rel < ERTS_MIN_COMPAT_REL - || erts_compat_rel > this_rel_num()) { + this_rel = this_rel_num(); + if (erts_compat_rel < this_rel - 2 || this_rel < erts_compat_rel) { erts_fprintf(stderr, "bad compatibility release number %s\n", arg); erts_usage(); } - ASSERT(ERTS_MIN_COMPAT_REL >= 7); switch (erts_compat_rel) { - case 7: - case 8: - case 9: - erts_use_r9_pids_ports = 1; + /* Currently no compat features... */ default: break; } @@ -1355,8 +1671,6 @@ erl_start(int argc, char **argv) } break; } - case 'n': /* XXX obsolete */ - break; case 'c': if (argv[i][2] == 0) { /* -c: documented option */ erts_disable_tolerant_timeofday = 1; @@ -1411,14 +1725,13 @@ erl_start(int argc, char **argv) i++; } - /* Delayed check of +P flag */ - if (erts_max_processes < ERTS_MIN_PROCESSES - || erts_max_processes > ERTS_MAX_PROCESSES - || (erts_use_r9_pids_ports - && erts_max_processes > ERTS_MAX_R9_PROCESSES)) { - erts_fprintf(stderr, "bad number of processes %s\n", Parg); - erts_usage(); - } +/* Output format on windows for sprintf defaults to three exponents. + * We use two-exponent to mimic normal sprintf behaviour. + */ + +#if defined(__WIN32__) && defined(_TWO_DIGIT_EXPONENT) + _set_output_format(_TWO_DIGIT_EXPONENT); +#endif /* Restart will not reinstall the break handler */ #ifdef __WIN32__ @@ -1440,9 +1753,16 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; - erl_init(ncpu); + erl_init(ncpu, + proc_tab_sz, + legacy_proc_tab, + port_tab_sz, + port_tab_sz_ignore_files, + legacy_port_tab); load_preloaded(); + erts_end_staging_code_ix(); + erts_commit_staging_code_ix(); erts_initialized = 1; diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index 963c8b3c58..df7c443387 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -271,6 +271,18 @@ stat_upd_realloc(ErtsAlcType_t n, Uint size, Uint old_size) * stat instrumentation callback functions */ +static void stat_pre_lock(void) +{ + erts_mtx_lock(&instr_mutex); +} + +static void stat_pre_unlock(void) +{ + erts_mtx_unlock(&instr_mutex); +} + +static ErtsAllocatorWrapper_t instr_wrapper; + static void * stat_alloc(ErtsAlcType_t n, void *extra, Uint size) { @@ -278,7 +290,9 @@ stat_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint ssize; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } ssize = size + STAT_BLOCK_HEADER_SIZE; res = (*real_af->alloc)(n, real_af->extra, ssize); @@ -293,7 +307,9 @@ stat_alloc(ErtsAlcType_t n, void *extra, Uint size) res = (void *) ((StatBlock_t *) res)->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -307,7 +323,9 @@ stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) void *sptr; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } if (ptr) { sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); @@ -329,7 +347,9 @@ stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) res = (void *) ((StatBlock_t *) res)->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -340,7 +360,9 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *sptr; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } if (ptr) { sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); @@ -352,7 +374,9 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) (*real_af->free)(n, real_af->extra, sptr); - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } } @@ -360,6 +384,18 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) * map stat instrumentation callback functions */ +static void map_stat_pre_lock(void) +{ + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); +} + +static void map_stat_pre_unlock(void) +{ + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); +} + static void * map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) { @@ -367,7 +403,9 @@ map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint msize; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } msize = size + MAP_STAT_BLOCK_HEADER_SIZE; res = (*real_af->alloc)(n, real_af->extra, msize); @@ -388,7 +426,9 @@ map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) res = (void *) mb->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -402,8 +442,10 @@ map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) void *mptr; void *res; - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); + } if (ptr) { mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE); @@ -449,9 +491,10 @@ map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) res = (void *) mb->mem; } - - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); + } return res; } @@ -462,8 +505,10 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *mptr; - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); + } if (ptr) { MapStatBlock_t *mb; @@ -486,8 +531,10 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) (*real_af->free)(n, real_af->extra, mptr); - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); + } } @@ -496,8 +543,10 @@ static void dump_memory_map_to_stream(FILE *fp) ErtsAlcType_t n; MapStatBlock_t *bp; int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) + if (lock) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&instr_mutex); + } /* Write header */ @@ -775,8 +824,8 @@ Eterm erts_instr_get_memory_map(Process *proc) ASSERT(hp + 3 == end_hp); if (mem_anchor) { - for (bp = mem_anchor; bp->next; bp = bp->next); - + for (bp = mem_anchor; bp->next; bp = bp->next) + ; ASSERT(org_mem_anchor); org_mem_anchor->prev = bp; bp->next = org_mem_anchor; @@ -1155,6 +1204,7 @@ erts_instr_get_type_info(Process *proc) Uint erts_instr_init(int stat, int map_stat) { + Uint extra_sz; int i; am_tot = NULL; @@ -1186,8 +1236,6 @@ erts_instr_init(int stat, int map_stat) sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1)); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; if (erts_allctrs_info[i].enabled) stats->ap[i] = &stats->a[i]; else @@ -1201,27 +1249,28 @@ erts_instr_init(int stat, int map_stat) erts_instr_memory_map = 1; erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; erts_allctrs[i].alloc = map_stat_alloc; erts_allctrs[i].realloc = map_stat_realloc; erts_allctrs[i].free = map_stat_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } - return MAP_STAT_BLOCK_HEADER_SIZE; + instr_wrapper.lock = map_stat_pre_lock; + instr_wrapper.unlock = map_stat_pre_unlock; + extra_sz = MAP_STAT_BLOCK_HEADER_SIZE; } else { erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; erts_allctrs[i].alloc = stat_alloc; erts_allctrs[i].realloc = stat_realloc; erts_allctrs[i].free = stat_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } - return STAT_BLOCK_HEADER_SIZE; + instr_wrapper.lock = stat_pre_lock; + instr_wrapper.unlock = stat_pre_unlock; + extra_sz = STAT_BLOCK_HEADER_SIZE; } - + erts_allctr_wrapper_prelock_init(&instr_wrapper); + return extra_sz; } diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 1f388c1796..2114d0c001 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2012. All Rights Reserved. + * Copyright Ericsson AB 2005-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -82,8 +82,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { #ifdef ERTS_SMP { "bif_timers", NULL }, { "reg_tab", NULL }, - { "migration_info_update", NULL }, { "proc_main", "pid" }, + { "old_code", "address" }, #ifdef HIPE { "hipe_mfait_lock", NULL }, #endif @@ -93,8 +93,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_msgq", "pid" }, { "dist_entry", "address" }, { "dist_entry_links", "address" }, + { "code_write_permission", NULL }, { "proc_status", "pid" }, - { "proc_tab", NULL }, { "ports_snapshot", NULL }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, @@ -114,9 +114,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { #if defined(ENABLE_CHILD_WAITER_THREAD) || defined(ERTS_SMP) { "child_status", NULL }, #endif -#ifdef __WIN32__ - { "sys_driver_data_lock", NULL }, -#endif { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, @@ -124,7 +121,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "removed_fd_pre_alloc_lock", "address" }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, + { "migration_info_update", NULL }, { "run_queue", "address" }, + { "process_table", NULL }, { "cpu_info", NULL }, { "pollset", "address" }, #ifdef __WIN32__ @@ -144,23 +143,20 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "ptimer_pre_alloc_lock", "address", }, { "btm_pre_alloc_lock", NULL, }, { "dist_entry_out_queue", "address" }, + { "port_sched_lock", "port_id" }, + { "port_table", NULL }, #endif { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, { "alcu_allocator", "index" }, - { "sbmbc_alloc", "index" }, { "mseg", NULL }, #if HALFWORD_HEAP { "pmmap", NULL }, #endif #ifdef ERTS_SMP { "port_task_pre_alloc_lock", "address" }, - { "port_taskq_pre_alloc_lock", "address" }, { "proclist_pre_alloc_lock", "address" }, - { "port_tasks_lock", NULL }, - { "get_free_port", NULL }, - { "port_state", "address" }, { "xports_list_pre_alloc_lock", "address" }, { "inet_buffer_stack_lock", NULL }, { "gc_info", NULL }, @@ -184,6 +180,11 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "efile_drv dtrace mutex", NULL }, #endif { "mtrace_buf", NULL }, +#ifdef __WIN32__ +#ifdef ERTS_SMP + { "sys_gethrtime", NULL }, +#endif +#endif { "erts_alloc_hard_debug", NULL } }; @@ -245,6 +246,7 @@ typedef struct { typedef struct erts_lc_locked_locks_t_ erts_lc_locked_locks_t; struct erts_lc_locked_locks_t_ { char *thread_name; + int emu_thread; erts_tid_t tid; erts_lc_locked_locks_t *next; erts_lc_locked_locks_t *prev; @@ -362,6 +364,7 @@ create_locked_locks(char *thread_name) if (!l_lcks->thread_name) lc_abort(); + l_lcks->emu_thread = 0; l_lcks->tid = erts_thr_self(); l_lcks->required.first = NULL; l_lcks->required.last = NULL; @@ -669,7 +672,7 @@ erts_lc_set_thread_name(char *thread_name) { erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); if (!l_lcks) - (void) create_locked_locks(thread_name); + l_lcks = create_locked_locks(thread_name); else { ASSERT(l_lcks->thread_name); free((void *) l_lcks->thread_name); @@ -677,6 +680,14 @@ erts_lc_set_thread_name(char *thread_name) if (!l_lcks->thread_name) lc_abort(); } + l_lcks->emu_thread = 1; +} + +int +erts_lc_is_emu_thr(void) +{ + erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + return l_lcks->emu_thread; } int diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index df7b3758e1..068340abe7 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -102,6 +102,7 @@ void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_require_lock(erts_lc_lock_t *lck); void erts_lc_unrequire_lock(erts_lc_lock_t *lck); +int erts_lc_is_emu_thr(void); #define ERTS_LC_ASSERT(A) \ ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A))) diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 919567ab27..325d77e911 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -296,36 +296,6 @@ erts_msg_distext2heap(Process *pp, return THE_NON_VALUE; } -static ERTS_INLINE void -notify_new_message(Process *receiver) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS - & erts_proc_lc_my_proc_locks(receiver)); - - switch (receiver->status) { - case P_GARBING: - switch (receiver->gcstatus) { - case P_SUSPENDED: - goto suspended; - case P_WAITING: - goto waiting; - default: - break; - } - break; - case P_SUSPENDED: - suspended: - receiver->rstatus = P_RUNABLE; - break; - case P_WAITING: - waiting: - erts_add_to_runq(receiver); - break; - default: - break; - } -} - void erts_queue_dist_message(Process *rcvr, ErtsProcLocks *rcvr_locks, @@ -339,7 +309,7 @@ erts_queue_dist_message(Process *rcvr, Sint tok_serial = 0; #endif #ifdef ERTS_SMP - ErtsProcLocks need_locks; + erts_aint_t state; #endif ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); @@ -347,20 +317,21 @@ erts_queue_dist_message(Process *rcvr, mp = message_alloc(); #ifdef ERTS_SMP - need_locks = ~(*rcvr_locks) & (ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - if (need_locks) { - *rcvr_locks |= need_locks; - if (erts_smp_proc_trylock(rcvr, need_locks) == EBUSY) { - if (need_locks == ERTS_PROC_LOCK_MSGQ) { + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { + if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + if (*rcvr_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_STATUS); - need_locks = (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS); + need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(rcvr, need_locks); } } - if (rcvr->is_exiting || ERTS_PROC_PENDING_EXIT(rcvr)) { + state = erts_smp_atomic32_read_acqb(&rcvr->state); + if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ if (is_not_nil(token)) { ErlHeapFragment *heap_frag; @@ -376,6 +347,8 @@ erts_queue_dist_message(Process *rcvr, /* Ahh... need to decode it in order to trace it... */ ErlHeapFragment *mbuf; Eterm msg; + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); message_free(mp); msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext); if (is_value(msg)) @@ -437,26 +410,33 @@ erts_queue_dist_message(Process *rcvr, mp->data.dist_ext = dist_ext; LINK_MESSAGE(rcvr, mp); - notify_new_message(rcvr); + if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) + erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); + + erts_proc_notify_new_message(rcvr); } } /* Add a message last in message queue */ -void -erts_queue_message(Process* receiver, - ErtsProcLocks *receiver_locks, - ErlHeapFragment* bp, - Eterm message, - Eterm seq_trace_token +static Sint +queue_message(Process *c_p, + Process* receiver, + ErtsProcLocks *receiver_locks, + erts_aint32_t *receiver_state, + ErlHeapFragment* bp, + Eterm message, + Eterm seq_trace_token #ifdef USE_VM_PROBES , Eterm dt_utag #endif -) + ) { + Sint res; ErlMessage* mp; -#ifdef ERTS_SMP - ErtsProcLocks need_locks; -#else + int locked_msgq = 0; + erts_aint_t state; + +#ifndef ERTS_SMP ASSERT(bp != NULL || receiver->mbuf == NULL); #endif @@ -464,31 +444,45 @@ erts_queue_message(Process* receiver, mp = message_alloc(); + if (receiver_state) + state = *receiver_state; + else + state = erts_smp_atomic32_read_acqb(&receiver->state); + #ifdef ERTS_SMP - need_locks = ~(*receiver_locks) & (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS); - if (need_locks) { - *receiver_locks |= need_locks; - if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) { - if (need_locks == ERTS_PROC_LOCK_MSGQ) { + + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + goto exiting; + + if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { + if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); - need_locks = (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS); + need_locks |= ERTS_PROC_LOCK_STATUS; } erts_smp_proc_lock(receiver, need_locks); } + locked_msgq = 1; + state = erts_smp_atomic32_read_nob(&receiver->state); + if (receiver_state) + *receiver_state = state; } - if (receiver->is_exiting || ERTS_PROC_PENDING_EXIT(receiver)) { - /* Drop message if receiver is exiting or has a pending - * exit ... - */ +#endif + + if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { +#ifdef ERTS_SMP + exiting: +#endif + /* Drop message if receiver is exiting or has a pending exit... */ + if (locked_msgq) + erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); if (bp) free_message_buffer(bp); message_free(mp); - return; + return 0; } -#endif ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; @@ -498,7 +492,10 @@ erts_queue_message(Process* receiver, mp->next = NULL; mp->data.heap_frag = bp; -#ifdef ERTS_SMP +#ifndef ERTS_SMP + res = receiver->msg.len; +#else + res = receiver->msg_inq.len; if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place @@ -508,15 +505,15 @@ erts_queue_message(Process* receiver, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ + res += receiver->msg.len; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } - else { + else +#endif + { LINK_MESSAGE(receiver, mp); } -#else - LINK_MESSAGE(receiver, mp); -#endif #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { @@ -536,15 +533,43 @@ erts_queue_message(Process* receiver, tok_label, tok_lastcnt, tok_serial); } #endif - notify_new_message(receiver); - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { + if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) trace_receive(receiver, message); - } + + if (locked_msgq) + erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + + erts_proc_notify_new_message(receiver); #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); #endif + return res; +} + +void +erts_queue_message(Process* receiver, + ErtsProcLocks *receiver_locks, + ErlHeapFragment* bp, + Eterm message, + Eterm seq_trace_token +#ifdef USE_VM_PROBES + , Eterm dt_utag +#endif + ) +{ + queue_message(NULL, + receiver, + receiver_locks, + NULL, + bp, + message, + seq_trace_token +#ifdef USE_VM_PROBES + , dt_utag +#endif + ); } void @@ -576,9 +601,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif #ifdef HARD_DEBUG - ProcBin *dbg_mso_start = off_heap->mso; - ErlFunThing *dbg_fun_start = off_heap->funs; - ExternalThing *dbg_external_start = off_heap->externals; + struct erl_off_heap_header* dbg_oh_start = off_heap->first; Eterm dbg_term, dbg_token; ErlHeapFragment *dbg_bp; Uint *dbg_hp, *dbg_thp_start; @@ -752,48 +775,16 @@ copy_done: int i, j; ErlHeapFragment* frag; { - ProcBin *mso = off_heap->mso; + struct erl_off_heap_header* dbg_oh = off_heap->first; i = j = 0; - while (mso != dbg_mso_start) { - mso = mso->next; + while (dbg_oh != dbg_oh_start) { + dbg_oh = dbg_oh->next; i++; } for (frag=bp; frag; frag=frag->next) { - mso = frag->off_heap.mso; - while (mso) { - mso = mso->next; - j++; - } - } - ASSERT(i == j); - } - { - ErlFunThing *fun = off_heap->funs; - i = j = 0; - while (fun != dbg_fun_start) { - fun = fun->next; - i++; - } - for (frag=bp; frag; frag=frag->next) { - fun = frag->off_heap.funs; - while (fun) { - fun = fun->next; - j++; - } - } - ASSERT(i == j); - } - { - ExternalThing *external = off_heap->externals; - i = j = 0; - while (external != dbg_external_start) { - external = external->next; - i++; - } - for (frag=bp; frag; frag=frag->next) { - external = frag->off_heap.externals; - while (external) { - external = external->next; + dbg_oh = frag->off_heap.first; + while (dbg_oh) { + dbg_oh = dbg_oh->next; j++; } } @@ -878,7 +869,7 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms * Send a local message when sender & receiver processes are known. */ -void +Sint erts_send_message(Process* sender, Process* receiver, ErtsProcLocks *receiver_locks, @@ -888,6 +879,7 @@ erts_send_message(Process* sender, Uint msize; ErlHeapFragment* bp = NULL; Eterm token = NIL; + Sint res = 0; #ifdef USE_VM_PROBES DTRACE_CHARBUF(sender_name, 64); DTRACE_CHARBUF(receiver_name, 64); @@ -902,8 +894,8 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send)) { - erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->id); - erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->id); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id); } #endif if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { @@ -925,7 +917,7 @@ erts_send_message(Process* sender, seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, - receiver->id, sender); + receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ #ifdef USE_VM_PROBES } @@ -956,7 +948,7 @@ erts_send_message(Process* sender, #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) Spreading tag (%T) with " - "message %T!\r\n",sender->id, utag, message); + "message %T!\r\n",sender->common.id, utag, message); #endif } #endif @@ -974,15 +966,17 @@ erts_send_message(Process* sender, msize, tok_label, tok_lastcnt, tok_serial); } #endif - erts_queue_message(receiver, - receiver_locks, - bp, - message, - token + res = queue_message(NULL, + receiver, + receiver_locks, + NULL, + bp, + message, + token #ifdef USE_VM_PROBES - , utag + , utag #endif - ); + ); BM_SWAP_TIMER(send,system); } else if (sender == receiver) { /* Drop message if receiver has a pending exit ... */ @@ -1026,31 +1020,45 @@ erts_send_message(Process* sender, ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); + res = receiver->msg.len; + if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { trace_receive(receiver, message); } } BM_SWAP_TIMER(send,system); - return; } else { #ifdef ERTS_SMP ErlOffHeap *ohp; Eterm *hp; + erts_aint32_t state; + BM_SWAP_TIMER(send,size); msize = size_object(message); BM_SWAP_TIMER(size,send); - hp = erts_alloc_message_heap(msize,&bp,&ohp,receiver,receiver_locks); + hp = erts_alloc_message_heap_state(msize, + &bp, + &ohp, + receiver, + receiver_locks, + &state); BM_SWAP_TIMER(send,copy); message = copy_struct(message, msize, &hp, ohp); BM_MESSAGE_COPIED(msz); BM_SWAP_TIMER(copy,send); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - erts_queue_message(receiver, receiver_locks, bp, message, token + res = queue_message(sender, + receiver, + receiver_locks, + &state, + bp, + message, + token #ifdef USE_VM_PROBES - , NIL + , NIL #endif - ); + ); BM_SWAP_TIMER(send,system); #else ErlMessage* mp = message_alloc(); @@ -1080,19 +1088,16 @@ erts_send_message(Process* sender, mp->next = NULL; mp->data.attached = NULL; LINK_MESSAGE(receiver, mp); + res = receiver->msg.len; + erts_proc_notify_new_message(receiver); - if (receiver->status == P_WAITING) { - erts_add_to_runq(receiver); - } else if (receiver->status == P_SUSPENDED) { - receiver->rstatus = P_RUNABLE; - } if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { trace_receive(receiver, message); } BM_SWAP_TIMER(send,system); #endif /* #ifndef ERTS_SMP */ - return; } + return res; } /* @@ -1131,7 +1136,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, save = TUPLE3(hp, am_EXIT, from_copy, mess); hp += 4; /* the trace token must in this case be updated by the caller */ - seq_trace_output(token, save, SEQ_TRACE_SEND, to->id, NULL); + seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL); temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap); erts_queue_message(to, to_locksp, bp, save, temptoken #ifdef USE_VM_PROBES diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 3e9a24ee81..771eba431f 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -90,7 +90,7 @@ typedef struct { ErlMessage* first; ErlMessage** last; /* point to the last next pointer */ ErlMessage** save; - int len; /* queue length */ + Sint len; /* queue length */ /* * The following two fields are used by the recv_mark/1 and @@ -105,7 +105,7 @@ typedef struct { typedef struct { ErlMessage* first; ErlMessage** last; /* point to the last next pointer */ - int len; /* queue length */ + Sint len; /* queue length */ } ErlMessageInQueue; #endif @@ -125,16 +125,16 @@ typedef struct { #ifdef ERTS_SMP /* Move in message queue to end of private message queue */ -#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P) \ -do { \ - if ((P)->msg_inq.first) { \ - *(P)->msg.last = (P)->msg_inq.first; \ - (P)->msg.last = (P)->msg_inq.last; \ - (P)->msg.len += (P)->msg_inq.len; \ - (P)->msg_inq.first = NULL; \ - (P)->msg_inq.last = &(P)->msg_inq.first; \ - (P)->msg_inq.len = 0; \ - } \ +#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P) \ +do { \ + if ((P)->msg_inq.first) { \ + *(P)->msg.last = (P)->msg_inq.first; \ + (P)->msg.last = (P)->msg_inq.last; \ + (P)->msg.len += (P)->msg_inq.len; \ + (P)->msg_inq.first = NULL; \ + (P)->msg_inq.last = &(P)->msg_inq.first; \ + (P)->msg_inq.len = 0; \ + } \ } while (0) /* Add message last in message queue */ @@ -234,7 +234,7 @@ void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm #endif ); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); -void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); +Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); @@ -245,6 +245,9 @@ void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); +void erts_cleanup_offheap(ErlOffHeap *offheap); + + ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 1a84950120..244a2b26db 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2011. All Rights Reserved. + * Copyright Ericsson AB 2004-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -971,7 +971,7 @@ Eterm erts_debug_dump_monitors_1(BIF_ALIST_1) } } else { erts_printf("Dumping pid monitors--------------------\n"); - erts_dump_monitors(rp->monitors,0); + erts_dump_monitors(ERTS_P_MONITORS(rp),0); erts_printf("Monitors dumped-------------------------\n"); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); BIF_RET(am_true); @@ -985,12 +985,15 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1) Process *rp; DistEntry *dep; if (is_internal_port(pid)) { - Port *rport = erts_id2port(pid, p, ERTS_PROC_LOCK_MAIN); + Port *rport = erts_id2port_sflgs(pid, + p, + ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_LOOKUP); if (rport) { erts_printf("Dumping port links----------------------\n"); - erts_dump_links(rport->nlinks,0); + erts_dump_links(ERTS_P_LINKS(rport), 0); erts_printf("Links dumped----------------------------\n"); - erts_smp_port_unlock(rport); + erts_port_release(rport); BIF_RET(am_true); } else { BIF_ERROR(p,BADARG); @@ -1014,10 +1017,30 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1) } else { erts_printf("Dumping pid links-----------------------\n"); - erts_dump_links(rp->nlinks,0); + erts_dump_links(ERTS_P_LINKS(rp), 0); erts_printf("Links dumped----------------------------\n"); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); BIF_RET(am_true); } } } + +void erts_one_link_size(ErtsLink *lnk, void *vpu) +{ + Uint *pu = vpu; + *pu += ERTS_LINK_SIZE*sizeof(Uint); + if(!IS_CONST(lnk->pid)) + *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); + if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { + erts_doforall_links(ERTS_LINK_ROOT(lnk),&erts_one_link_size,vpu); + } +} +void erts_one_mon_size(ErtsMonitor *mon, void *vpu) +{ + Uint *pu = vpu; + *pu += ERTS_MONITOR_SIZE*sizeof(Uint); + if(!IS_CONST(mon->pid)) + *pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint); + if(!IS_CONST(mon->ref)) + *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); +} diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index d3f6d410dd..fb11dbbd22 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. + * Copyright Ericsson AB 2004-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -137,8 +137,6 @@ typedef struct erts_suspend_monitor { #define ERTS_LINK_ROOT(Linkp) ((Linkp)->shared.root) #define ERTS_LINK_REFC(Linkp) ((Linkp)->shared.refc) -#define ERTS_LINK_ROOT_AS_UINT(Linkp) (*((Uint *) &((Linkp)->root))) - Uint erts_tot_link_lh_size(void); @@ -172,6 +170,8 @@ ErtsSuspendMonitor *erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid); void erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid); void erts_init_monitors(void); +void erts_one_link_size(ErtsLink *lnk, void *vpu); +void erts_one_mon_size(ErtsMonitor *mon, void *vpu); #define erts_doforall_monitors erts_sweep_monitors #define erts_doforall_links erts_sweep_links diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index 358c67bf20..c8bb126687 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -222,6 +222,8 @@ static byte *tracep; static byte *endp; static SysTimeval last_tv; +static ErtsAllocatorWrapper_t mtrace_wrapper; + #if ERTS_MTRACE_SEGMENT_ID >= ERTS_ALC_A_MIN || ERTS_MTRACE_SEGMENT_ID < 0 #error ERTS_MTRACE_SEGMENT_ID >= ERTS_ALC_A_MIN || ERTS_MTRACE_SEGMENT_ID < 0 #endif @@ -555,6 +557,8 @@ write_trace_header(char *nodename, char *pid, char *hostname) return 1; } +static void mtrace_pre_lock(void); +static void mtrace_pre_unlock(void); static void *mtrace_alloc(ErtsAlcType_t, void *, Uint); static void *mtrace_realloc(ErtsAlcType_t, void *, void *, Uint); static void mtrace_free(ErtsAlcType_t, void *, void *); @@ -611,7 +615,7 @@ void erts_mtrace_init(char *receiver, char *nodename) if (erts_sock_gethostname(hostname, MAXHOSTNAMELEN) != 0) hostname[0] = '\0'; hostname[MAXHOSTNAMELEN-1] = '\0'; - sys_get_pid(pid); + sys_get_pid(pid, sizeof(pid)); write_trace_header(nodename ? nodename : "", pid, hostname); erts_mtrace_update_heap_size(); } @@ -635,12 +639,16 @@ erts_mtrace_install_wrapper_functions(void) erts_allctrs[i].free = mtrace_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } + mtrace_wrapper.lock = mtrace_pre_lock; + mtrace_wrapper.unlock = mtrace_pre_unlock; + erts_allctr_wrapper_prelock_init(&mtrace_wrapper); } } void erts_mtrace_stop(void) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&mtrace_op_mutex); erts_mtx_lock(&mtrace_buf_mutex); if (erts_mtrace_enabled) { @@ -677,6 +685,7 @@ erts_mtrace_stop(void) void erts_mtrace_exit(Uint32 exit_value) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&mtrace_op_mutex); erts_mtx_lock(&mtrace_buf_mutex); if (erts_mtrace_enabled) { @@ -935,18 +944,33 @@ write_free_entry(byte tag, erts_mtx_unlock(&mtrace_buf_mutex); } +static void mtrace_pre_lock(void) +{ + erts_mtx_lock(&mtrace_op_mutex); +} + +static void mtrace_pre_unlock(void) +{ + erts_mtx_unlock(&mtrace_op_mutex); +} + + static void * mtrace_alloc(ErtsAlcType_t n, void *extra, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *res; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } res = (*real_af->alloc)(n, real_af->extra, size); write_alloc_entry(ERTS_MT_ALLOC_BDY_TAG, res, n, 0, size); - erts_mtx_unlock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&mtrace_op_mutex); + } return res; } @@ -957,12 +981,16 @@ mtrace_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *res; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } res = (*real_af->realloc)(n, real_af->extra, ptr, size); write_realloc_entry(ERTS_MT_REALLOC_BDY_TAG, res, n, 0, ptr, size); - erts_mtx_unlock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&mtrace_op_mutex); + } return res; @@ -973,10 +1001,14 @@ mtrace_free(ErtsAlcType_t n, void *extra, void *ptr) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } (*real_af->free)(n, real_af->extra, ptr); - write_free_entry(ERTS_MT_FREE_BDY_TAG, n, 0, ptr); + if (!erts_is_allctr_wrapper_prelocked()) { + write_free_entry(ERTS_MT_FREE_BDY_TAG, n, 0, ptr); + } erts_mtx_unlock(&mtrace_op_mutex); } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 4109c20fa7..48f8be8dd3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2012. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -263,7 +263,7 @@ ErlNifEnv* enif_alloc_env(void) HEAP_LIMIT(&msg_env->phony_proc) = phony_heap; HEAP_END(&msg_env->phony_proc) = phony_heap; MBUF(&msg_env->phony_proc) = NULL; - msg_env->phony_proc.id = ERTS_INVALID_PID; + msg_env->phony_proc.common.id = ERTS_INVALID_PID; #ifdef FORCE_HEAP_FRAGS msg_env->phony_proc.space_verified = 0; msg_env->phony_proc.space_verified_from = NULL; @@ -287,7 +287,7 @@ void enif_clear_env(ErlNifEnv* env) struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env; Process* p = &menv->phony_proc; ASSERT(p == menv->env.proc); - ASSERT(p->id == ERTS_INVALID_PID); + ASSERT(p->common.id == ERTS_INVALID_PID); ASSERT(MBUF(p) == menv->env.heap_frag); if (MBUF(p) != NULL) { erts_cleanup_offheap(&MSO(p)); @@ -310,15 +310,13 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Process* rp; Process* c_p; ErlHeapFragment* frags; -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) - ErtsProcLocks rp_had_locks; -#endif Eterm receiver = to_pid->pid; int flush_me = 0; + int scheduler = erts_get_scheduler_id() != 0; if (env != NULL) { c_p = env->proc; - if (receiver == c_p->id) { + if (receiver == c_p->common.id) { rp_locks = ERTS_PROC_LOCK_MAIN; flush_me = 1; } @@ -331,13 +329,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #endif } -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) - rp_had_locks = rp_locks; -#endif - rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, - receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + rp = (scheduler + ? erts_proc_lookup(receiver) + : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC)); if (rp == NULL) { - ASSERT(env == NULL || receiver != c_p->id); + ASSERT(env == NULL || receiver != c_p->common.id); return 0; } flush_env(msg_env); @@ -362,12 +359,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, , NIL #endif ); - if (rp_locks) { - ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | - ERTS_PROC_LOCK_STATUS))); - erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS)); - } - erts_smp_proc_dec_refc(rp); + if (c_p == rp) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (!scheduler) + erts_smp_proc_dec_refc(rp); if (flush_me) { cache_env(env); } @@ -393,7 +390,7 @@ static int is_offheap(const ErlOffHeap* oh) ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid) { - pid->pid = caller_env->proc->id; + pid->pid = caller_env->proc->common.id; return pid; } int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid) @@ -501,7 +498,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { struct enif_tmp_obj_t* tobj; ErtsAlcType_t allocator; - Uint sz; + ErlDrvSizeT sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); } @@ -527,7 +524,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) bin->size = sz; bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; - io_list_to_buf(term, (char*) bin->data, sz); + erts_iolist_to_buf(term, (char*) bin->data, sz); ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } @@ -739,16 +736,23 @@ int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len, { Atom* ap; ASSERT(encoding == ERL_NIF_LATIN1); - if (is_not_atom(atom)) { + if (is_not_atom(atom) || len==0) { return 0; } ap = atom_tab(atom_val(atom)); - if (ap->len+1 > len) { + + if (ap->latin1_chars < 0 || ap->latin1_chars >= len) { return 0; } - sys_memcpy(buf, ap->name, ap->len); - buf[ap->len] = '\0'; - return ap->len + 1; + if (ap->latin1_chars == ap->len) { + sys_memcpy(buf, ap->name, ap->len); + } + else { + int dlen = erts_utf8_to_latin1((byte*)buf, ap->name, ap->len); + ASSERT(dlen == ap->latin1_chars); (void)dlen; + } + buf[ap->latin1_chars] = '\0'; + return ap->latin1_chars + 1; } int enif_get_int(ErlNifEnv* env, Eterm term, int* ip) @@ -850,7 +854,10 @@ int enif_get_atom_length(ErlNifEnv* env, Eterm atom, unsigned* len, ASSERT(enc == ERL_NIF_LATIN1); if (is_not_atom(atom)) return 0; ap = atom_tab(atom_val(atom)); - *len = ap->len; + if (ap->latin1_chars < 0) { + return 0; + } + *len = ap->latin1_chars; return 1; } @@ -957,7 +964,7 @@ ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len) { - return am_atom_put(name, len); + return erts_atom_put((byte*)name, len, ERTS_ATOM_ENC_LATIN1, 1); } int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, @@ -970,7 +977,7 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding encoding) { ASSERT(encoding == ERL_NIF_LATIN1); - return erts_atom_get(name, len, atom); + return erts_atom_get(name, len, atom, ERTS_ATOM_ENC_LATIN1); } ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) @@ -1392,6 +1399,57 @@ size_t enif_sizeof_resource(void* obj) return ERTS_MAGIC_BIN_DATA_SIZE(bin) - offsetof(ErlNifResource,data); } + +void* enif_dlopen(const char* lib, + void (*err_handler)(void*,const char*), void* err_arg) +{ + ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; + void* handle; + void* init_func; + if (erts_sys_ddll_open2(lib, &handle, &errdesc) == ERL_DE_NO_ERROR) { + if (erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) == ERL_DE_NO_ERROR) { + erts_sys_ddll_call_nif_init(init_func); + } + } + else { + if (err_handler != NULL) { + (*err_handler)(err_arg, errdesc.str); + } + handle = NULL; + } + erts_sys_ddll_free_error(&errdesc); + return handle; +} + +void* enif_dlsym(void* handle, const char* symbol, + void (*err_handler)(void*,const char*), void* err_arg) +{ + ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; + void* ret; + if (erts_sys_ddll_sym2(handle, symbol, &ret, &errdesc) != ERL_DE_NO_ERROR) { + if (err_handler != NULL) { + (*err_handler)(err_arg, errdesc.str); + } + erts_sys_ddll_free_error(&errdesc); + return NULL; + } + return ret; +} + +int enif_consume_timeslice(ErlNifEnv* env, int percent) +{ + Sint reds; + + ASSERT(is_proc_bound(env) && percent >= 1 && percent <= 100); + if (percent < 1) percent = 1; + else if (percent > 100) percent = 100; + + reds = ((CONTEXT_REDS+99) / 100) * percent; + ASSERT(reds > 0 && reds <= CONTEXT_REDS); + BUMP_REDS(env->proc, reds); + return ERTS_BIF_REDS_LEFT(env->proc) == 0; +} + /*************************************************************************** ** load_nif/2 ** ***************************************************************************/ @@ -1524,6 +1582,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (len < 0) { BIF_ERROR(BIF_P, BADARG); } + lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1); if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) { @@ -1532,6 +1591,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } lib_name[len] = '\0'; + if (!erts_try_seize_code_write_permission(BIF_P)) { + erts_free(ERTS_ALC_T_TMP, lib_name); + ERTS_BIF_YIELD2(bif_export[BIF_load_nif_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + /* Block system (is this the right place to do it?) */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); @@ -1545,11 +1610,11 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(caller != NULL); mod_atom = caller[0]; ASSERT(is_atom(mod_atom)); - mod=erts_get_module(mod_atom); + mod=erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(mod != NULL); - if (!in_area(caller, mod->code, mod->code_length)) { - ASSERT(in_area(caller, mod->old_code, mod->old_code_length)); + if (!in_area(caller, mod->curr.code, mod->curr.code_length)) { + ASSERT(in_area(caller, mod->old.code, mod->old.code_length)); ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " "module '%T' not allowed", mod_atom); @@ -1584,7 +1649,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) "this vm variant (%s).", entry->vm_variant, ERL_NIF_VM_VARIANT); } - else if (!erts_is_atom_str((char*)entry->name, mod_atom)) { + else if (!erts_is_atom_str((char*)entry->name, mod_atom, 1)) { ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not" " match calling module '%T'", entry->name, mod_atom); } @@ -1594,8 +1659,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { BeamInstr** code_pp; ErlNifFunc* f = &entry->funcs[i]; - if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom) - || (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { + if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) + || (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -1624,18 +1689,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_refc_init(&lib->rt_dtor_cnt, 0); lib->mod = mod; env.mod_nif = lib; - if (mod->nif != NULL) { /* Reload */ + if (mod->curr.nif != NULL) { /* Reload */ int k; - lib->priv_data = mod->nif->priv_data; + lib->priv_data = mod->curr.nif->priv_data; - ASSERT(mod->nif->entry != NULL); + ASSERT(mod->curr.nif->entry != NULL); if (entry->reload == NULL) { ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); goto error; } /* Check that no NIF is removed */ - for (k=0; k < mod->nif->entry->num_of_funcs; k++) { - ErlNifFunc* old_func = &mod->nif->entry->funcs[k]; + for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) { + ErlNifFunc* old_func = &mod->curr.nif->entry->funcs[k]; for (i=0; i < entry->num_of_funcs; i++) { if (old_func->arity == entry->funcs[i].arity && sys_strcmp(old_func->name, entry->funcs[i].name) == 0) { @@ -1656,24 +1721,24 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful."); } else { - mod->nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(mod->nif); + mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */ + erts_unload_nif(mod->curr.nif); reload_warning = 1; } } else { lib->priv_data = NULL; - if (mod->old_nif != NULL) { /* Upgrade */ - void* prev_old_data = mod->old_nif->priv_data; + if (mod->old.nif != NULL) { /* Upgrade */ + void* prev_old_data = mod->old.nif->priv_data; if (entry->upgrade == NULL) { ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); goto error; } erts_pre_nif(&env, BIF_P, lib); - veto = entry->upgrade(&env, &lib->priv_data, &mod->old_nif->priv_data, BIF_ARG_2); + veto = entry->upgrade(&env, &lib->priv_data, &mod->old.nif->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { - mod->old_nif->priv_data = prev_old_data; + mod->old.nif->priv_data = prev_old_data; ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); } /*else if (mod->old_nif->priv_data != prev_old_data) { @@ -1693,20 +1758,21 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* ** Everything ok, patch the beam code with op_call_nif */ - mod->nif = lib; + mod->curr.nif = lib; for (i=0; i < entry->num_of_funcs; i++) { BeamInstr* code_ptr; - erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom); - code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity); + erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom, ERTS_ATOM_ENC_LATIN1); + code_ptr = *get_func_pp(mod->curr.code, f_atom, entry->funcs[i].arity); if (code_ptr[1] == 0) { code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - BpData** bps = (BpData**) code_ptr[1]; - BpData* bp = (BpData*) bps[erts_bp_sched2ix()]; - bp->orig_instr = (BeamInstr) BeamOp(op_call_nif); + GenericBp* g = (GenericBp *) code_ptr[1]; + ASSERT(code_ptr[5+0] == + (BeamInstr) BeamOp(op_i_generic_breakpoint)); + g->orig_instr = (BeamInstr) BeamOp(op_call_nif); } code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr; code_ptr[5+2] = (BeamInstr) lib; @@ -1726,6 +1792,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); if (reload_warning) { @@ -1793,7 +1860,7 @@ void erl_nif_init() #ifdef USE_VM_PROBES void dtrace_nifenv_str(ErlNifEnv *env, char *process_buf) { - dtrace_pid_str(env->proc->id, process_buf); + dtrace_pid_str(env->proc->common.id, process_buf); } #endif diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 50f99c90c4..8006741a63 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2012. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -32,10 +32,11 @@ ** 2.0: R14A ** 2.1: R14B02 "vm_variant" ** 2.2: R14B03 enif_is_exception -** 2.3: R15 enif_make_reverse_list +** 2.3: R15 enif_make_reverse_list, enif_is_number +** 2.4: R16 enif_consume_timeslice */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 3 +#define ERL_NIF_MINOR_VERSION 4 #include <stdlib.h> @@ -187,11 +188,7 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #else # define ERL_NIF_INIT_GLOB # define ERL_NIF_INIT_BODY -# if defined(VXWORKS) -# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _init(void) -# else -# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) -# endif +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) #endif diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 6396af09d0..2f841645e1 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2011. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -138,6 +138,9 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list)); ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg)); +ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg)); +ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent)); /* ** Add new entries here to keep compatibility on Windows!!! @@ -260,6 +263,9 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term)); # define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) # define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list) # define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number) +# define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen) +# define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym) +# define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice) /* ** Add new entries here diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 329a2204cc..0f93a3a9f0 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -20,7 +20,7 @@ #ifndef ERL_NODE_CONTAINER_UTILS_H__ #define ERL_NODE_CONTAINER_UTILS_H__ -#include "erl_term.h" +#include "erl_ptab.h" /* * Note regarding node containers: @@ -29,9 +29,6 @@ * the emulator) for the Erlang data types that contain a reference * to a node, i.e. pids, ports, and references. * - * Observe! The layouts of the node container data types have been - * changed in R9. - * * Node containers are divided into internal and external node containers. * An internal node container refer to the current incarnation of the * node which it reside on. An external node container refer to @@ -52,13 +49,6 @@ * reference is a boxed data type. An internal node container have an * implicit reference to the 'erts_this_node' element in the node table. * - * Due to the R9 changes in layouts of node containers there are room to - * store more data than previously. Today (R9) this extra space is unused, - * but it is planned to be used in the future. For example only 18 bits - * are used for data in a pid but there is room for 28 bits of data (on a - * 32-bit machine). Some preparations have been made in the emulator for - * usage of this extra space. - * * OBSERVE! Pids doesn't use fixed size 'serial' and 'number' fields any * more. Previously the 15 bit 'number' field of a pid was used as index * into the process table, and the 3 bit 'serial' field was used as a @@ -104,8 +94,6 @@ #define internal_dist_entry(x) (erts_this_node->dist_entry) #define external_dist_entry(x) (external_node((x))->dist_entry) -extern int erts_use_r9_pids_ports; - /* * For this node (and previous incarnations of this node), 0 is used as * channel no. For other nodes, the atom index of the atom corresponding @@ -128,8 +116,20 @@ extern int erts_use_r9_pids_ports; * Pids * \* */ -#define internal_pid_index(x) (internal_pid_data((x)) \ - & erts_process_tab_index_mask) +extern ErtsPTab erts_proc; + +#define make_internal_pid(D) erts_ptab_make_id(&erts_proc, \ + (D), \ + _TAG_IMMED1_PID) + +#define internal_pid_index(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ + erts_ptab_id2pix(&erts_proc, (PID))) + +#define internal_pid_data(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ + erts_ptab_id2data(&erts_proc, (PID))) + +#define internal_pid_number(x) _GET_PID_NUM(internal_pid_data((x))) +#define internal_pid_serial(x) _GET_PID_SER(internal_pid_data((x))) #define internal_pid_node_name(x) (internal_pid_node((x))->sysname) #define external_pid_node_name(x) (external_pid_node((x))->sysname) @@ -169,34 +169,37 @@ extern int erts_use_r9_pids_ports; || is_external_pid((x))) #define is_not_pid(x) (!is_pid(x)) -#define ERTS_MAX_R9_PROCESSES (1 << ERTS_R9_PROC_BITS) - /* * Maximum number of processes. We want the number to fit in a SMALL on * 32-bit CPU. */ -#define ERTS_MAX_PROCESSES ((SWORD_CONSTANT(1) << 27)-1) -#if (ERTS_MAX_PROCESSES > MAX_SMALL) -# error "The maximum number of processes must fit in a SMALL." -#endif - +#define ERTS_MAX_PROCESSES (ERTS_PTAB_MAX_SIZE-1) #define ERTS_MAX_PID_DATA ((1 << _PID_DATA_SIZE) - 1) #define ERTS_MAX_PID_NUMBER ((1 << _PID_NUM_SIZE) - 1) #define ERTS_MAX_PID_SERIAL ((1 << _PID_SER_SIZE) - 1) -#define ERTS_MAX_PID_R9_SERIAL ((1 << _PID_R9_SER_SIZE) - 1) -#define ERTS_R9_PROC_BITS (_PID_R9_SER_SIZE + _PID_NUM_SIZE) #define ERTS_PROC_BITS (_PID_SER_SIZE + _PID_NUM_SIZE) -#define ERTS_INVALID_PID make_internal_pid(ERTS_MAX_PID_DATA) +#define ERTS_INVALID_PID ERTS_PTAB_INVALID_ID(_TAG_IMMED1_PID) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Ports * \* */ -#define internal_port_index(x) (internal_port_data((x)) \ - & erts_port_tab_index_mask) +extern ErtsPTab erts_port; + +#define make_internal_port(D) erts_ptab_make_id(&erts_port, \ + (D), \ + _TAG_IMMED1_PORT) + +#define internal_port_index(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ + erts_ptab_id2pix(&erts_port, (PRT))) + +#define internal_port_data(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ + erts_ptab_id2data(&erts_port, (PRT))) + +#define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x))) #define internal_port_node_name(x) (internal_port_node((x))->sysname) #define external_port_node_name(x) (external_port_node((x))->sysname) @@ -235,18 +238,18 @@ extern int erts_use_r9_pids_ports; #define is_not_port(x) (!is_port(x)) /* Highest port-ID part in a term of type Port - Not necessarily the same as the variable erts_max_ports + Not necessarily the same as current maximum port table size which defines the maximum number of simultaneous Ports in the Erlang node. ERTS_MAX_PORTS is a hard upper limit. */ -#define ERTS_MAX_R9_PORTS (1 << ERTS_R9_PORTS_BITS) -#define ERTS_MAX_PORTS (1 << ERTS_PORTS_BITS) - +#define ERTS_MAX_PORTS (ERTS_PTAB_MAX_SIZE-1) #define ERTS_MAX_PORT_DATA ((1 << _PORT_DATA_SIZE) - 1) #define ERTS_MAX_PORT_NUMBER ((1 << _PORT_NUM_SIZE) - 1) -#define ERTS_R9_PORTS_BITS (_PORT_R9_NUM_SIZE) #define ERTS_PORTS_BITS (_PORT_NUM_SIZE) + +#define ERTS_INVALID_PORT ERTS_PTAB_INVALID_ID(_TAG_IMMED1_PORT) + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Refs * \* */ diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 5574cb0ac4..c6d136f951 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2012. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -116,8 +116,7 @@ dist_table_alloc(void *dep_tmpl) dep->qsize = 0; dep->out_queue.first = NULL; dep->out_queue.last = NULL; - dep->suspended.first = NULL; - dep->suspended.last = NULL; + dep->suspended = NULL; dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; @@ -769,8 +768,7 @@ void erts_init_node_tables(void) erts_this_dist_entry->qsize = 0; erts_this_dist_entry->out_queue.first = NULL; erts_this_dist_entry->out_queue.last = NULL; - erts_this_dist_entry->suspended.first = NULL; - erts_this_dist_entry->suspended.last = NULL; + erts_this_dist_entry->suspended = NULL; erts_this_dist_entry->finalized_out_queue.first = NULL; erts_this_dist_entry->finalized_out_queue.last = NULL; @@ -1268,7 +1266,7 @@ setup_reference_table(void) ErlHeapFragment *hfp; DistEntry *dep; HashInfo hi; - int i; + int i, max; DeclareTmpHeapNoproc(heap,3); inserted_bins = NULL; @@ -1297,22 +1295,24 @@ setup_reference_table(void) UnUseTmpHeapNoproc(3); + max = erts_ptab_max(&erts_proc); /* Insert all processes */ - for (i = 0; i < erts_max_processes; i++) - if (process_tab[i]) { + for (i = 0; i < max; i++) { + Process *proc = erts_pix2proc(i); + if (proc) { ErlMessage *msg; /* Insert Heap */ - insert_offheap(&(process_tab[i]->off_heap), + insert_offheap(&(proc->off_heap), HEAP_REF, - process_tab[i]->id); + proc->common.id); /* Insert message buffers */ - for(hfp = process_tab[i]->mbuf; hfp; hfp = hfp->next) + for(hfp = proc->mbuf; hfp; hfp = hfp->next) insert_offheap(&(hfp->off_heap), HEAP_REF, - process_tab[i]->id); + proc->common.id); /* Insert msg msg buffers */ - for (msg = process_tab[i]->msg.first; msg; msg = msg->next) { + for (msg = proc->msg.first; msg; msg = msg->next) { ErlHeapFragment *heap_frag = NULL; if (msg->data.attached) { if (is_value(ERL_MESSAGE_TERM(msg))) @@ -1320,7 +1320,7 @@ setup_reference_table(void) else { if (msg->data.dist_ext->dep) insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, process_tab[i]->id, 0); + HEAP_REF, proc->common.id, 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); } @@ -1328,10 +1328,10 @@ setup_reference_table(void) if (heap_frag) insert_offheap(&(heap_frag->off_heap), HEAP_REF, - process_tab[i]->id); + proc->common.id); } #ifdef ERTS_SMP - for (msg = process_tab[i]->msg_inq.first; msg; msg = msg->next) { + for (msg = proc->msg_inq.first; msg; msg = msg->next) { ErlHeapFragment *heap_frag = NULL; if (msg->data.attached) { if (is_value(ERL_MESSAGE_TERM(msg))) @@ -1339,7 +1339,7 @@ setup_reference_table(void) else { if (msg->data.dist_ext->dep) insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, process_tab[i]->id, 0); + HEAP_REF, proc->common.id, 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); } @@ -1347,42 +1347,57 @@ setup_reference_table(void) if (heap_frag) insert_offheap(&(heap_frag->off_heap), HEAP_REF, - process_tab[i]->id); + proc->common.id); } #endif /* Insert links */ - if(process_tab[i]->nlinks) - insert_links(process_tab[i]->nlinks, process_tab[i]->id); - if(process_tab[i]->monitors) - insert_monitors(process_tab[i]->monitors, process_tab[i]->id); + if (ERTS_P_LINKS(proc)) + insert_links(ERTS_P_LINKS(proc), proc->common.id); + if (ERTS_P_MONITORS(proc)) + insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); /* Insert controller */ { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(process_tab[i]); + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); if (dep) - insert_dist_entry(dep, CTRL_REF, process_tab[i]->id, 0); + insert_dist_entry(dep, CTRL_REF, proc->common.id, 0); } } + } #ifdef ERTS_SMP erts_foreach_sys_msg_in_q(insert_sys_msg); #endif /* Insert all ports */ - for (i = 0; i < erts_max_ports; i++) { - if (erts_port[i].status & ERTS_PORT_SFLGS_DEAD) + max = erts_ptab_max(&erts_port); + for (i = 0; i < max; i++) { + ErlOffHeap *ohp; + erts_aint32_t state; + Port *prt; + + prt = erts_pix2port(i); + if (!prt) + continue; + + state = erts_atomic32_read_nob(&prt->state); + if (state & ERTS_PORT_SFLGS_DEAD) continue; /* Insert links */ - if(erts_port[i].nlinks) - insert_links(erts_port[i].nlinks, erts_port[i].id); + if (ERTS_P_LINKS(prt)) + insert_links(ERTS_P_LINKS(prt), prt->common.id); + /* Insert monitors */ + if (ERTS_P_MONITORS(prt)) + insert_monitors(ERTS_P_MONITORS(prt), prt->common.id); /* Insert port data */ - for(hfp = erts_port[i].bp; hfp; hfp = hfp->next) - insert_offheap(&(hfp->off_heap), HEAP_REF, erts_port[i].id); + ohp = erts_port_data_offheap(prt); + if (ohp) + insert_offheap(ohp, HEAP_REF, prt->common.id); /* Insert controller */ - if (erts_port[i].dist_entry) - insert_dist_entry(erts_port[i].dist_entry, + if (prt->dist_entry) + insert_dist_entry(prt->dist_entry, CTRL_REF, - erts_port[i].id, + prt->common.id, 0); } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 4a015bdef9..af60071ea5 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -84,10 +84,6 @@ typedef struct { } ErtsDistOutputQueue; struct ErtsProcList_; -typedef struct { - struct ErtsProcList_ *first; - struct ErtsProcList_ *last; -} ErtsDistSuspended; /* * Lock order: @@ -100,7 +96,6 @@ typedef struct { */ struct erl_link; -struct port; typedef struct dist_entry_ { HashBucket hash_bucket; /* Hash bucket */ @@ -135,13 +130,13 @@ typedef struct dist_entry_ { Uint32 qflgs; Sint qsize; ErtsDistOutputQueue out_queue; - ErtsDistSuspended suspended; + struct ErtsProcList_ *suspended; ErtsDistOutputQueue finalized_out_queue; erts_smp_atomic_t dist_cmd_scheduled; ErtsPortTaskHandle dist_cmd; - Uint (*send)(struct port *prt, ErtsDistOutputBuf *obuf); + Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); struct cache* cache; /* The atom cache */ } DistEntry; diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h new file mode 100644 index 0000000000..ad3f104a68 --- /dev/null +++ b/erts/emulator/beam/erl_port.h @@ -0,0 +1,957 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_PORT_TYPE__ +#define ERL_PORT_TYPE__ +typedef struct _erl_drv_port Port; +typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData; +#endif + +#if !defined(ERL_PORT_H__) && !defined(ERL_PORT_GET_PORT_TYPE_ONLY__) +#define ERL_PORT_H__ + +#include "erl_port_task.h" +#include "erl_ptab.h" +#include "erl_thr_progress.h" +#include "erl_trace.h" + +#ifndef __WIN32__ +#define ERTS_DEFAULT_MAX_PORTS (1 << 16) +#else +/* + * Do not default to as many max ports on Windows + * as there are no os limits to stop system + * from running amok. If allowed to go too high + * windows rarely recovers from the errors and + * other OS processes can be effected. + */ +#define ERTS_DEFAULT_MAX_PORTS (1 << 13) +#endif /* __WIN32__ */ +#define ERTS_MIN_PORTS 1024 + +extern int erts_port_synchronous_ops; +extern int erts_port_schedule_all_ops; +extern int erts_port_parallelism; + +typedef struct erts_driver_t_ erts_driver_t; + +/* + * It would have been preferred to use NULL as value of + * ERTS_INVALID_ERL_DRV_PORT. That would, however, not be + * backward compatible. In pre-R16 systems, 0 was a valid + * port handle and -1 was used as invalid handle, so we + * are stuck with it. + */ +#define ERTS_INVALID_ERL_DRV_PORT ((struct _erl_drv_port *) ((SWord) -1)) +#ifdef DEBUG +/* Make sure we use this api, and do not cast directly */ +#define ERTS_ErlDrvPort2Port(PH) \ + ((PH) == ERTS_INVALID_ERL_DRV_PORT \ + ? ERTS_INVALID_ERL_DRV_PORT \ + : ((Port *) ((PH) - 4711))) +#define ERTS_Port2ErlDrvPort(PH) \ + ((PH) == ERTS_INVALID_ERL_DRV_PORT \ + ? ERTS_INVALID_ERL_DRV_PORT \ + : ((ErlDrvPort) ((PH) + 4711))) +#else +#define ERTS_ErlDrvPort2Port(PH) ((Port *) (PH)) +#define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH)) +#endif + +#define SMALL_IO_QUEUE 5 /* Number of fixed elements */ + +typedef struct { + ErlDrvSizeT size; /* total size in bytes */ + + SysIOVec* v_start; + SysIOVec* v_end; + SysIOVec* v_head; + SysIOVec* v_tail; + SysIOVec v_small[SMALL_IO_QUEUE]; + + ErlDrvBinary** b_start; + ErlDrvBinary** b_end; + ErlDrvBinary** b_head; + ErlDrvBinary** b_tail; + ErlDrvBinary* b_small[SMALL_IO_QUEUE]; +} ErlIOQueue; + +typedef struct line_buf { /* Buffer used in line oriented I/O */ + ErlDrvSizeT bufsiz; /* Size of character buffer */ + ErlDrvSizeT ovlen; /* Length of overflow data */ + ErlDrvSizeT ovsiz; /* Actual size of overflow buffer */ + char data[1]; /* Starting point of buffer data, + data[0] is a flag indicating an unprocess CR, + The rest is the overflow buffer. */ +} LineBuf; + +/* + * Items part of erlang:port_info/1 result. Note am_registered_name + * *need* to be first. + */ + +#define ERTS_PORT_INFO_1_ITEMS \ + { am_registered_name, /* Needs to be first */ \ + am_name, \ + am_links, \ + am_id, \ + am_connected, \ + am_input, \ + am_output, \ + am_os_pid } + +/* + * Port Specific Data. + * + * Only use PrtSD for very rarely used data. + */ + +#define ERTS_PRTSD_SCHED_ID 0 + +#define ERTS_PRTSD_SIZE 1 + +typedef struct { + void *data[ERTS_PRTSD_SIZE]; +} ErtsPrtSD; + +#ifdef ERTS_SMP +typedef struct ErtsXPortsList_ ErtsXPortsList; +#endif + +/* + * Port locking: + * + * Locking is done either driver specific or port specific. When + * driver specific locking is used, all instances of the driver, + * i.e. ports running the driver, share the same lock. When port + * specific locking is used each instance have its own lock. + * + * Most fields in the Port structure are protected by the lock + * referred to by the 'lock' field. This lock is shared between + * all ports running the same driver when driver specific locking + * is used. + * + * The 'sched' field is protected by the run queue lock that the + * port currently is assigned to. + * + */ + +struct _erl_drv_port { + ErtsPTabElementCommon common; /* *Need* to be first in struct */ + + ErtsPortTaskSched sched; + ErtsPortTaskHandle timeout_task; +#ifdef ERTS_SMP + erts_mtx_t *lock; + ErtsXPortsList *xports; + erts_smp_atomic_t run_queue; +#else + erts_atomic32_t refc; + int cleanup; +#endif + erts_atomic_t connected; /* A connected process */ + Eterm caller; /* Current caller. */ + erts_smp_atomic_t data; /* Data associated with port. */ + Uint bytes_in; /* Number of bytes read */ + Uint bytes_out; /* Number of bytes written */ + + ErlIOQueue ioq; /* driver accessible i/o queue */ + DistEntry *dist_entry; /* Dist entry used in DISTRIBUTION */ + char *name; /* String used in the open */ + erts_driver_t* drv_ptr; + UWord drv_data; + SWord os_pid; /* Child process ID */ + ErtsProcList *suspended; /* List of suspended processes. */ + LineBuf *linebuf; /* Buffer to hold data not ready for + process to get (line oriented I/O)*/ + erts_atomic32_t state; /* Status and type flags */ + int control_flags; /* Flags for port_control() */ + ErlDrvPDL port_data_lock; + + ErtsPrtSD *psd; /* Port specific data */ + int reds; /* Only used while executing driver callbacks */ +}; + + +void erts_init_port_data(Port *); +void erts_cleanup_port_data(Port *); +Uint erts_port_data_size(Port *); +ErlOffHeap *erts_port_data_offheap(Port *); + +#define ERTS_PORT_GET_CONNECTED(PRT) \ + ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) +#define ERTS_PORT_SET_CONNECTED(PRT, PID) \ + erts_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID)) +#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \ + erts_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID)) + + +struct erl_drv_port_data_lock { + erts_mtx_t mtx; + erts_atomic_t refc; + Port *prt; +}; + +ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE ErtsRunQueue * +erts_port_runq(Port *prt) +{ +#ifdef ERTS_SMP + ErtsRunQueue *rq1, *rq2; + rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); + if (!rq1) + return NULL; + while (1) { + erts_smp_runq_lock(rq1); + rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); + if (rq1 == rq2) + return rq1; + erts_smp_runq_unlock(rq1); + rq1 = rq2; + if (!rq1) + return NULL; + } +#else + return ERTS_RUNQ_IX(0); +#endif +} + +#endif + + +ERTS_GLB_INLINE void *erts_prtsd_get(Port *p, int ix); +ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void * +erts_prtsd_get(Port *prt, int ix) +{ + return prt->psd ? prt->psd->data[ix] : NULL; +} + +ERTS_GLB_INLINE void * +erts_prtsd_set(Port *prt, int ix, void *data) +{ + if (prt->psd) { + void *old = prt->psd->data[ix]; + prt->psd->data[ix] = data; + return old; + } + else { + prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD)); + prt->psd->data[ix] = data; + return NULL; + } +} + +#endif + +extern erts_smp_atomic_t erts_bytes_out; /* no bytes written out */ +extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ + + +/* port status flags */ + +#define ERTS_PORT_SFLG_CONNECTED ((Uint32) (1 << 0)) +/* Port have begun exiting */ +#define ERTS_PORT_SFLG_EXITING ((Uint32) (1 << 1)) +/* Distribution port */ +#define ERTS_PORT_SFLG_DISTRIBUTION ((Uint32) (1 << 2)) +#define ERTS_PORT_SFLG_BINARY_IO ((Uint32) (1 << 3)) +#define ERTS_PORT_SFLG_SOFT_EOF ((Uint32) (1 << 4)) +/* Flow control */ +/* Port is closing (no i/o accepted) */ +#define ERTS_PORT_SFLG_CLOSING ((Uint32) (1 << 5)) +/* Send a closed message when terminating */ +#define ERTS_PORT_SFLG_SEND_CLOSED ((Uint32) (1 << 6)) +/* Line orinted io on port */ +#define ERTS_PORT_SFLG_LINEBUF_IO ((Uint32) (1 << 7)) +/* Immortal port (only certain system ports) */ +#define ERTS_PORT_SFLG_FREE ((Uint32) (1 << 8)) +#define ERTS_PORT_SFLG_INITIALIZING ((Uint32) (1 << 9)) +/* Port uses port specific locking (opposed to driver specific locking) */ +#define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 10)) +#define ERTS_PORT_SFLG_INVALID ((Uint32) (1 << 11)) +/* Last port to terminate halts the emulator */ +#define ERTS_PORT_SFLG_HALT ((Uint32) (1 << 12)) +#ifdef DEBUG +/* Only debug: make sure all flags aren't cleared unintentionally */ +#define ERTS_PORT_SFLG_PORT_DEBUG ((Uint32) (1 << 31)) +#endif + +/* Combinations of port status flags */ +#define ERTS_PORT_SFLGS_DEAD \ + (ERTS_PORT_SFLG_FREE | ERTS_PORT_SFLG_INITIALIZING) +#define ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP \ + (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID) +#define ERTS_PORT_SFLGS_INVALID_LOOKUP \ + (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP \ + | ERTS_PORT_SFLG_EXITING \ + | ERTS_PORT_SFLG_CLOSING) +#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP \ + (ERTS_PORT_SFLGS_INVALID_LOOKUP \ + | ERTS_PORT_SFLG_DISTRIBUTION) + +/* + * Costs in reductions for some port operations. + */ +#define ERTS_PORT_REDS_EXECUTE (CONTEXT_REDS/4) +#define ERTS_PORT_REDS_FREE (CONTEXT_REDS/400) +#define ERTS_PORT_REDS_TIMEOUT (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_INPUT (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_OUTPUT (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_EVENT (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_CMD_OUTPUTV (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_CMD_OUTPUT (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_EXIT (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_CONNECT (CONTEXT_REDS/200) +#define ERTS_PORT_REDS_UNLINK (CONTEXT_REDS/200) +#define ERTS_PORT_REDS_LINK (CONTEXT_REDS/200) +#define ERTS_PORT_REDS_BADSIG (CONTEXT_REDS/200) +#define ERTS_PORT_REDS_CONTROL (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_CALL (CONTEXT_REDS/50) +#define ERTS_PORT_REDS_INFO (CONTEXT_REDS/100) +#define ERTS_PORT_REDS_TERMINATE (CONTEXT_REDS/50) + +void print_port_info(Port *, int, void *); +void erts_port_free(Port *); +#ifndef ERTS_SMP +void erts_port_cleanup(Port *); +#endif +void erts_fire_port_monitor(Port *prt, Eterm ref); +#ifdef ERTS_SMP +int erts_port_handle_xports(Port *); +#endif + +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int erts_lc_is_port_locked(Port *); +#endif + +ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt); +ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt); +ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc); + +ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt); +ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt); +ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt) +{ +#ifdef ERTS_SMP + erts_ptab_inc_refc(&prt->common); +#else + erts_atomic32_inc_nob(&prt->refc); +#endif +} + +ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt) +{ +#ifdef ERTS_SMP + int referred = erts_ptab_dec_test_refc(&prt->common); + if (!referred) + erts_port_free(prt); +#else + int refc = erts_atomic32_dec_read_nob(&prt->refc); + if (refc == 0) + erts_port_free(prt); +#endif +} + +ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc) +{ +#ifdef ERTS_SMP + int referred = erts_ptab_add_test_refc(&prt->common, add_refc); + if (!referred) + erts_port_free(prt); +#else + int refc = erts_atomic32_add_read_nob(&prt->refc, add_refc); + if (refc == 0) + erts_port_free(prt); +#endif +} + +ERTS_GLB_INLINE int +erts_smp_port_trylock(Port *prt) +{ +#ifdef ERTS_SMP + /* *Need* to be a managed thread */ + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + return erts_mtx_trylock(prt->lock); +#else + return 0; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_port_lock(Port *prt) +{ +#ifdef ERTS_SMP + /* *Need* to be a managed thread */ + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + erts_mtx_lock(prt->lock); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_port_unlock(Port *prt) +{ +#ifdef ERTS_SMP + /* *Need* to be a managed thread */ + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + erts_mtx_unlock(prt->lock); +#endif +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS) \ + (!(PP) \ + || (erts_atomic32_read_nob(&(PP)->state) & (FLGS)) \ + || (PP)->common.id != (ID)) + +/* port lookup */ + +#define INVALID_PORT(PP, ID) \ + ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_LOOKUP) + +/* Invalidate trace port if anything suspicious, for instance + * that the port is a distribution port or it is busy. + */ +#define INVALID_TRACER_PORT(PP, ID) \ + ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP) + +#define ERTS_PORT_SCHED_ID(P, ID) \ + ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID))) + +extern const Port erts_invalid_port; +#define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port) + +int erts_is_port_ioq_empty(Port *); +void erts_terminate_port(Port *); + +#ifdef ERTS_SMP +Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks); +#endif + +ERTS_GLB_INLINE Port *erts_pix2port(int); +ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm); +ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32); +ERTS_GLB_INLINE Port*erts_id2port(Eterm id); +ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32); +ERTS_GLB_INLINE void erts_port_release(Port *); +#ifdef ERTS_SMP +ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs); +ERTS_GLB_INLINE void erts_thr_port_release(Port *prt); +#endif +ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int); +ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *); +ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort); +ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm); +ERTS_GLB_INLINE int erts_is_port_alive(Eterm); +ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm); +ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *); + +#define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL) + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Port *erts_pix2port(int ix) +{ + Port *prt; + ASSERT(0 <= ix && ix < erts_ptab_max(&erts_port)); + prt = (Port *) erts_ptab_pix2intptr_nob(&erts_port, ix); + return prt == ERTS_PORT_LOCK_BUSY ? NULL : prt; +} + +ERTS_GLB_INLINE Port * +erts_port_lookup_raw(Eterm id) +{ + Port *prt; + + ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); + + if (is_not_internal_port(id)) + return NULL; + + prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port, + internal_port_index(id)); + return prt && prt->common.id == id ? prt : NULL; +} + +ERTS_GLB_INLINE Port * +erts_port_lookup(Eterm id, Uint32 invalid_sflgs) +{ + Port *prt = erts_port_lookup_raw(id); + return (!prt + ? NULL + : ((invalid_sflgs & erts_atomic32_read_nob(&prt->state)) + ? NULL + : prt)); +} + + +ERTS_GLB_INLINE Port* +erts_id2port(Eterm id) +{ + erts_aint32_t state; + Port *prt; + + /* Only allowed to be called from managed threads */ + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + + if (is_not_internal_port(id)) + return NULL; + + prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port, + internal_port_index(id)); + + if (!prt || prt->common.id != id) + return NULL; + + erts_smp_port_lock(prt); + state = erts_atomic32_read_nob(&prt->state); + if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) { + erts_smp_port_unlock(prt); + return NULL; + } + + return prt; +} + + +ERTS_GLB_INLINE Port* +erts_id2port_sflgs(Eterm id, + Process *c_p, ErtsProcLocks c_p_locks, + Uint32 invalid_sflgs) +{ +#ifdef ERTS_SMP + int no_proc_locks = !c_p || !c_p_locks; +#endif + erts_aint32_t state; + Port *prt; + + /* Only allowed to be called from managed threads */ + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + + if (is_not_internal_port(id)) + return NULL; + + prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port, + internal_port_index(id)); + + if (!prt || prt->common.id != id) + return NULL; + +#ifdef ERTS_SMP + if (no_proc_locks) + erts_smp_port_lock(prt); + else if (erts_smp_port_trylock(prt) == EBUSY) { + /* Unlock process locks, and acquire locks in lock order... */ + erts_smp_proc_unlock(c_p, c_p_locks); + erts_smp_port_lock(prt); + erts_smp_proc_lock(c_p, c_p_locks); + } +#endif + state = erts_atomic32_read_nob(&prt->state); + if (state & invalid_sflgs) { +#ifdef ERTS_SMP + erts_smp_port_unlock(prt); +#endif + return NULL; + } + + return prt; +} + +ERTS_GLB_INLINE void +erts_port_release(Port *prt) +{ + /* Only allowed to be called from managed threads */ + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); +#ifdef ERTS_SMP + erts_smp_port_unlock(prt); +#else + if (prt->cleanup) { + prt->cleanup = 0; + erts_port_cleanup(prt); + } +#endif +} + +#ifdef ERTS_SMP + +/* + * erts_thr_id2port_sflgs() and erts_thr_port_release() can + * be used by unmanaged threads in the SMP case. + */ +ERTS_GLB_INLINE Port * +erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs) +{ + Port *prt; + ErtsThrPrgrDelayHandle dhndl; + + if (is_not_internal_port(id)) + return NULL; + + dhndl = erts_thr_progress_unmanaged_delay(); + + prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port, + internal_port_index(id)); + + if (!prt || prt->common.id != id) { + erts_thr_progress_unmanaged_continue(dhndl); + prt = NULL; + } + else { + erts_aint32_t state; + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { + erts_port_inc_refc(prt); + erts_thr_progress_unmanaged_continue(dhndl); + } + + erts_mtx_lock(prt->lock); + state = erts_atomic32_read_nob(&prt->state); + if (state & invalid_sflgs) { + erts_mtx_unlock(prt->lock); + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_port_dec_refc(prt); + prt = NULL; + } + } + + return prt; +} + +ERTS_GLB_INLINE void +erts_thr_port_release(Port *prt) +{ + erts_mtx_unlock(prt->lock); +#ifdef ERTS_SMP + if (!erts_thr_progress_is_managed_thread()) + erts_port_dec_refc(prt); +#endif +} + +#endif + +ERTS_GLB_INLINE Port * +erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl) +{ + Port *prt = ERTS_ErlDrvPort2Port(drvport); + ASSERT(prt != NULL); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return ERTS_INVALID_ERL_DRV_PORT; + + if (lock_pdl && prt->port_data_lock) + driver_pdl_lock(prt->port_data_lock); + +#if ERTS_ENABLE_LOCK_CHECK + if (!ERTS_IS_CRASH_DUMPING) { + if (erts_lc_is_emu_thr()) { + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(!prt->port_data_lock + || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx)); + } + else { + ERTS_LC_ASSERT(prt->port_data_lock); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx)); + } + } +#endif + + if (erts_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + if (lock_pdl && prt->port_data_lock) + driver_pdl_unlock(prt->port_data_lock); + return ERTS_INVALID_ERL_DRV_PORT; + } + return prt; +} + +ERTS_GLB_INLINE Port * +erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep) +{ + Port *prt = ERTS_ErlDrvPort2Port(drvport); + erts_aint32_t state; + ASSERT(prt); + ERTS_LC_ASSERT(erts_lc_is_emu_thr()); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return ERTS_INVALID_ERL_DRV_PORT; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) + || ERTS_IS_CRASH_DUMPING); + /* + * This state check is only needed since a driver callback + * might terminate the port, and then call back into the + * emulator. Drivers should preferably have been forbidden + * to call into the emulator after terminating the port, + * but it has been like this for ages. Perhaps forbid this + * in some future major release? + */ + state = erts_atomic32_read_nob(&prt->state); + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + return ERTS_INVALID_ERL_DRV_PORT; + if (statep) + *statep = state; + return prt; +} + +ERTS_GLB_INLINE Eterm +erts_drvport2id(ErlDrvPort drvport) +{ + Port *prt = erts_drvport2port(drvport); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return am_undefined; + else + return prt->common.id; +} + +ERTS_GLB_INLINE Uint32 +erts_portid2status(Eterm id) +{ + Port *prt = erts_port_lookup_raw(id); + if (prt) + return (Uint32) erts_atomic32_read_acqb(&prt->state); + else + return ERTS_PORT_SFLG_INVALID; +} + +ERTS_GLB_INLINE int +erts_is_port_alive(Eterm id) +{ + return !(erts_portid2status(id) & (ERTS_PORT_SFLG_INVALID + | ERTS_PORT_SFLGS_DEAD)); +} + +ERTS_GLB_INLINE int +erts_is_valid_tracer_port(Eterm id) +{ + return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); +} + +ERTS_GLB_INLINE int +erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep) +{ + int reds = 0; + erts_aint32_t state; + + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + + state = erts_atomic32_read_nob(&prt->state); + if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) { + reds += ERTS_PORT_REDS_TERMINATE; + erts_terminate_port(prt); + state = erts_atomic32_read_nob(&prt->state); + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + } + +#ifdef ERTS_SMP + if (prt->xports) { + reds += erts_port_handle_xports(prt); + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ASSERT(!prt->xports); + } +#endif + + if (statep) + *statep = state; + + return reds; +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +void erts_port_resume_procs(Port *); + +struct binary; + +#define ERTS_P2P_SIG_TYPE_BAD 0 +#define ERTS_P2P_SIG_TYPE_OUTPUT 1 +#define ERTS_P2P_SIG_TYPE_OUTPUTV 2 +#define ERTS_P2P_SIG_TYPE_CONNECT 3 +#define ERTS_P2P_SIG_TYPE_EXIT 4 +#define ERTS_P2P_SIG_TYPE_CONTROL 5 +#define ERTS_P2P_SIG_TYPE_CALL 6 +#define ERTS_P2P_SIG_TYPE_INFO 7 +#define ERTS_P2P_SIG_TYPE_LINK 8 +#define ERTS_P2P_SIG_TYPE_UNLINK 9 + +#define ERTS_P2P_SIG_TYPE_BITS 4 +#define ERTS_P2P_SIG_TYPE_MASK \ + ((1 << ERTS_P2P_SIG_TYPE_BITS) - 1) + +#define ERTS_P2P_SIG_DATA_FLG(N) \ + (1 << (ERTS_P2P_SIG_TYPE_BITS + (N))) +#define ERTS_P2P_SIG_DATA_FLG_BANG_OP ERTS_P2P_SIG_DATA_FLG(0) +#define ERTS_P2P_SIG_DATA_FLG_REPLY ERTS_P2P_SIG_DATA_FLG(1) +#define ERTS_P2P_SIG_DATA_FLG_NOSUSPEND ERTS_P2P_SIG_DATA_FLG(2) +#define ERTS_P2P_SIG_DATA_FLG_FORCE ERTS_P2P_SIG_DATA_FLG(3) +#define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT ERTS_P2P_SIG_DATA_FLG(4) +#define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK ERTS_P2P_SIG_DATA_FLG(5) +#define ERTS_P2P_SIG_DATA_FLG_SCHED ERTS_P2P_SIG_DATA_FLG(6) +#define ERTS_P2P_SIG_DATA_FLG_ASYNC ERTS_P2P_SIG_DATA_FLG(7) + +struct ErtsProc2PortSigData_ { + int flags; + Eterm caller; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + union { + struct { + Eterm from; + ErlIOVec *evp; + ErlDrvBinary *cbinp; + } outputv; + struct { + Eterm from; + char *bufp; + ErlDrvSizeT size; + } output; + struct { + Eterm from; + Eterm connected; + } connect; + struct { + Eterm from; + Eterm reason; + ErlHeapFragment *bp; + } exit; + struct { + struct binary *binp; + unsigned int command; + char *bufp; + ErlDrvSizeT size; + } control; + struct { + unsigned int command; + char *bufp; + ErlDrvSizeT size; + } call; + struct { + Eterm item; + } info; + struct { + Eterm port; + Eterm to; + } link; + struct { + Eterm from; + } unlink; + } u; +} ; + +ERTS_GLB_INLINE int +erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp); +ERTS_GLB_INLINE ErlDrvSizeT +erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int +erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp) +{ + switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) { + case ERTS_P2P_SIG_TYPE_OUTPUT: return !0; + case ERTS_P2P_SIG_TYPE_OUTPUTV: return !0; + default: return 0; + } +} + +ERTS_GLB_INLINE ErlDrvSizeT +erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp) +{ + switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) { + case ERTS_P2P_SIG_TYPE_OUTPUT: return sigdp->u.output.size; + case ERTS_P2P_SIG_TYPE_OUTPUTV: return sigdp->u.outputv.evp->size; + default: return (ErlDrvSizeT) 0; + } +} + +#endif + +#define ERTS_PROC2PORT_SIG_EXEC 0 +#define ERTS_PROC2PORT_SIG_ABORT 1 +#define ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND 2 +#define ERTS_PROC2PORT_SIG_ABORT_CLOSED 3 + +typedef int (*ErtsProc2PortSigCallback)(Port *, + erts_aint32_t, + int, + ErtsProc2PortSigData *); + +typedef enum { + ERTS_PORT_OP_BADARG, + ERTS_PORT_OP_CALLER_EXIT, + ERTS_PORT_OP_BUSY, + ERTS_PORT_OP_BUSY_SCHEDULED, + ERTS_PORT_OP_SCHEDULED, + ERTS_PORT_OP_DROPPED, + ERTS_PORT_OP_DONE +} ErtsPortOpResult; + +ErtsPortOpResult +erts_schedule_proc2port_signal(Process *, + Port *, + Eterm, + Eterm *, + ErtsProc2PortSigData *, + int, + ErtsPortTaskHandle *, + ErtsProc2PortSigCallback); + +int erts_deliver_port_exit(Port *, Eterm, Eterm, int); + +/* + * Port signal flags + */ +#define ERTS_PORT_SIG_FLG_BANG_OP ERTS_P2P_SIG_DATA_FLG_BANG_OP +#define ERTS_PORT_SIG_FLG_NOSUSPEND ERTS_P2P_SIG_DATA_FLG_NOSUSPEND +#define ERTS_PORT_SIG_FLG_FORCE ERTS_P2P_SIG_DATA_FLG_FORCE +#define ERTS_PORT_SIG_FLG_BROKEN_LINK ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK +#define ERTS_PORT_SIG_FLG_BAD_OUTPUT ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT +#define ERTS_PORT_SIG_FLG_FORCE_SCHED ERTS_P2P_SIG_DATA_FLG_SCHED +#define ERTS_PORT_SIG_FLG_ASYNC ERTS_P2P_SIG_DATA_FLG_ASYNC +/* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */ +#define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT + +/* + * Port ! {Owner, {command, Data}} + * Port ! {Owner, {connect, NewOwner}} + * Port ! {Owner, close} + */ +ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *); + +/* + * Signals from processes to ports. + */ +ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *); +ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *); +ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *); +ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *); +ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *); +ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *); +ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *); +ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *); + +#endif diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index d25af94099..547a42beb2 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2012. All Rights Reserved. + * Copyright Ericsson AB 2006-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,42 +33,34 @@ #include "erl_port_task.h" #include "dist.h" #include "dtrace-wrapper.h" - -#if defined(DEBUG) && 0 -#define HARD_DEBUG -#endif +#include <stdarg.h> /* - * Costs in reductions for some port operations. + * ERTS_PORT_CALLBACK_VREDS: Limit the amount of callback calls we do... */ -#define ERTS_PORT_REDS_EXECUTE 0 -#define ERTS_PORT_REDS_FREE 50 -#define ERTS_PORT_REDS_TIMEOUT 200 -#define ERTS_PORT_REDS_INPUT 200 -#define ERTS_PORT_REDS_OUTPUT 200 -#define ERTS_PORT_REDS_EVENT 200 -#define ERTS_PORT_REDS_TERMINATE 100 +#define ERTS_PORT_CALLBACK_VREDS (CONTEXT_REDS/20) +#if defined(DEBUG) && 0 +#define ERTS_HARD_DEBUG_TASK_QUEUES +#else +#undef ERTS_HARD_DEBUG_TASK_QUEUES +#endif -#define ERTS_PORT_TASK_INVALID_PORT(P, ID) \ - ((erts_port_status_get((P)) & ERTS_PORT_SFLGS_DEAD) || (P)->id != (ID)) - -#define ERTS_PORT_IS_IN_RUNQ(RQ, P) \ - ((P)->sched.next || (P)->sched.prev || (RQ)->ports.start == (P)) - -#define ERTS_PORT_NOT_IN_RUNQ(P) \ -do { \ - (P)->sched.prev = NULL; \ - (P)->sched.next = NULL; \ -} while (0) +#ifdef ERTS_HARD_DEBUG_TASK_QUEUES +static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue); +#define ERTS_PT_DBG_CHK_TASK_QS(PP, EQ, PBQ) \ + chk_task_queues((PP), (EQ), (PBQ)) +#else +#define ERTS_PT_DBG_CHK_TASK_QS(PP, EQ, PBQ) +#endif #ifdef USE_VM_PROBES #define DTRACE_DRIVER(PROBE_NAME, PP) \ - if (DTRACE_ENABLED(driver_ready_input)) { \ + if (DTRACE_ENABLED(PROBE_NAME)) { \ DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); \ DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); \ \ - dtrace_pid_str(PP->connected, process_str); \ + dtrace_pid_str(ERTS_PORT_GET_CONNECTED(PP), process_str); \ dtrace_port_str(PP, port_str); \ DTRACE3(PROBE_NAME, process_str, port_str, PP->name); \ } @@ -78,83 +70,788 @@ do { \ erts_smp_atomic_t erts_port_task_outstanding_io_tasks; -struct ErtsPortTaskQueue_ { - ErtsPortTask *first; - ErtsPortTask *last; - Port *port; -}; +#define ERTS_PT_STATE_SCHEDULED 0 +#define ERTS_PT_STATE_ABORTED 1 +#define ERTS_PT_STATE_EXECUTING 2 + +typedef union { + struct { /* I/O tasks */ + ErlDrvEvent event; + ErlDrvEventData event_data; + } io; + struct { + ErtsProc2PortSigCallback callback; + ErtsProc2PortSigData data; + } psig; +} ErtsPortTaskTypeData; struct ErtsPortTask_ { - ErtsPortTask *prev; - ErtsPortTask *next; - ErtsPortTaskQueue *queue; - ErtsPortTaskHandle *handle; + erts_smp_atomic32_t state; ErtsPortTaskType type; - ErlDrvEvent event; - ErlDrvEventData event_data; + union { + struct { + ErtsPortTask *next; + ErtsPortTaskHandle *handle; + int flags; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + ErtsPortTaskTypeData td; + } alive; + ErtsThrPrgrLaterOp release; + } u; }; -#ifdef HARD_DEBUG -#define ERTS_PT_CHK_PORTQ(RQ) check_port_queue((RQ), NULL, 0) -#define ERTS_PT_CHK_PRES_PORTQ(RQ, PP) check_port_queue((RQ), (PP), -1) -#define ERTS_PT_CHK_IN_PORTQ(RQ, PP) check_port_queue((RQ), (PP), 1) -#define ERTS_PT_CHK_NOT_IN_PORTQ(RQ, PP) check_port_queue((RQ), (PP), 0) -#define ERTS_PT_CHK_TASKQ(Q) check_task_queue((Q), NULL, 0) -#define ERTS_PT_CHK_IN_TASKQ(Q, T) check_task_queue((Q), (T), 1) -#define ERTS_PT_CHK_NOT_IN_TASKQ(Q, T) check_task_queue((Q), (T), 0) -static void -check_port_queue(Port *chk_pp, int inq); -static void -check_task_queue(ErtsPortTaskQueue *ptqp, - ErtsPortTask *chk_ptp, - int inq); -#else -#define ERTS_PT_CHK_PORTQ(RQ) -#define ERTS_PT_CHK_PRES_PORTQ(RQ, PP) -#define ERTS_PT_CHK_IN_PORTQ(RQ, PP) -#define ERTS_PT_CHK_NOT_IN_PORTQ(RQ, PP) -#define ERTS_PT_CHK_TASKQ(Q) -#define ERTS_PT_CHK_IN_TASKQ(Q, T) -#define ERTS_PT_CHK_NOT_IN_TASKQ(Q, T) +struct ErtsPortTaskHandleList_ { + ErtsPortTaskHandle handle; + union { + ErtsPortTaskHandleList *next; +#ifdef ERTS_SMP + ErtsThrPrgrLaterOp release; #endif + } u; +}; + +typedef struct ErtsPortTaskBusyCaller_ ErtsPortTaskBusyCaller; +struct ErtsPortTaskBusyCaller_ { + ErtsPortTaskBusyCaller *next; + Eterm caller; + SWord count; + ErtsPortTask *last; +}; + +#define ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS 17 +struct ErtsPortTaskBusyCallerTable_ { + ErtsPortTaskBusyCaller *bucket[ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS]; + ErtsPortTaskBusyCaller pre_alloc_busy_caller; +}; + -static void handle_remaining_tasks(ErtsRunQueue *runq, Port *pp); +static void begin_port_cleanup(Port *pp, + ErtsPortTask **execq, + int *processing_busy_q_p); ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_task, ErtsPortTask, - 200, + 1000, ERTS_ALC_T_PORT_TASK) -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_taskq, - ErtsPortTaskQueue, + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(busy_caller_table, + ErtsPortTaskBusyCallerTable, 50, - ERTS_ALC_T_PORT_TASKQ) + ERTS_ALC_T_BUSY_CALLER_TAB) + +#ifdef ERTS_SMP +static void +call_port_task_free(void *vptp) +{ + port_task_free((ErtsPortTask *) vptp); +} +#endif + +static ERTS_INLINE void +schedule_port_task_free(ErtsPortTask *ptp) +{ +#ifdef ERTS_SMP + erts_schedule_thr_prgr_later_cleanup_op(call_port_task_free, + (void *) ptp, + &ptp->u.release, + sizeof(ErtsPortTask)); +#else + port_task_free(ptp); +#endif +} + +static ERTS_INLINE ErtsPortTask * +p2p_sig_data_to_task(ErtsProc2PortSigData *sigdp) +{ + ErtsPortTask *ptp; + char *ptr = (char *) sigdp; + ptr -= offsetof(ErtsPortTask, u.alive.td.psig.data); + ptp = (ErtsPortTask *) ptr; + ASSERT(ptp->type == ERTS_PORT_TASK_PROC_SIG); + return ptp; +} + +ErtsProc2PortSigData * +erts_port_task_alloc_p2p_sig_data(void) +{ + ErtsPortTask *ptp = port_task_alloc(); + + ptp->type = ERTS_PORT_TASK_PROC_SIG; + ptp->u.alive.flags = ERTS_PT_FLG_SIG_DEP; + erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED); + + ASSERT(ptp == p2p_sig_data_to_task(&ptp->u.alive.td.psig.data)); + + return &ptp->u.alive.td.psig.data; +} + +static ERTS_INLINE Eterm +task_caller(ErtsPortTask *ptp) +{ + Eterm caller; + + ASSERT(ptp->type == ERTS_PORT_TASK_PROC_SIG); + + caller = ptp->u.alive.td.psig.data.caller; + + ASSERT(is_internal_pid(caller) || is_internal_port(caller)); + + return caller; +} + +/* + * Busy queue management + */ + +static ERTS_INLINE int +caller2bix(Eterm caller) +{ + ASSERT(is_internal_pid(caller) || is_internal_port(caller)); + return (int) (_GET_PID_DATA(caller) % ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS); +} + + +static void +popped_from_busy_queue(Port *pp, ErtsPortTask *ptp, int last) +{ + ErtsPortTaskBusyCaller **prev_bcpp = NULL, *bcp; + ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table; + Eterm caller = task_caller(ptp); + int bix = caller2bix(caller); + + ASSERT(is_internal_pid(caller)); + + ASSERT(tabp); + bcp = tabp->bucket[bix]; + prev_bcpp = &tabp->bucket[bix]; + ASSERT(bcp); + while (bcp->caller != caller) { + prev_bcpp = &bcp->next; + bcp = bcp->next; + ASSERT(bcp); + } + ASSERT(bcp->count > 0); + if (--bcp->count != 0) { + ASSERT(!last); + } + else { + *prev_bcpp = bcp->next; + if (bcp == &tabp->pre_alloc_busy_caller) + bcp->caller = am_undefined; + else + erts_free(ERTS_ALC_T_BUSY_CALLER, bcp); + if (last) { +#ifdef DEBUG + erts_aint32_t flags = +#endif + erts_smp_atomic32_read_band_nob( + &pp->sched.flags, + ~ERTS_PTS_FLG_HAVE_BUSY_TASKS); + ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS); +#ifdef DEBUG + for (bix = 0; bix < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; bix++) { + ASSERT(!tabp->bucket[bix]); + } +#endif + busy_caller_table_free(tabp); + pp->sched.taskq.local.busy.first = NULL; + pp->sched.taskq.local.busy.last = NULL; + pp->sched.taskq.local.busy.table = NULL; + } + } +} + +static void +busy_wait_move_to_busy_queue(Port *pp, ErtsPortTask *ptp) +{ + ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table; + Eterm caller = task_caller(ptp); + ErtsPortTaskBusyCaller *bcp; + int bix; + + ASSERT(is_internal_pid(caller)); + /* + * Port is busy and this task type needs to wait until not busy. + */ + + ASSERT(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY); + + ptp->u.alive.next = NULL; + if (pp->sched.taskq.local.busy.last) { + ASSERT(pp->sched.taskq.local.busy.first); + pp->sched.taskq.local.busy.last->u.alive.next = ptp; + } + else { + int i; +#ifdef DEBUG + erts_aint32_t flags; +#endif + pp->sched.taskq.local.busy.first = ptp; + +#ifdef DEBUG + flags = +#endif + erts_smp_atomic32_read_bor_nob(&pp->sched.flags, + ERTS_PTS_FLG_HAVE_BUSY_TASKS); + ASSERT(!(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS)); + + ASSERT(!tabp); + + tabp = busy_caller_table_alloc(); + pp->sched.taskq.local.busy.table = tabp; + for (i = 0; i < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; i++) + tabp->bucket[i] = NULL; + tabp->pre_alloc_busy_caller.caller = am_undefined; + } + pp->sched.taskq.local.busy.last = ptp; + + bix = caller2bix(caller); + ASSERT(tabp); + bcp = tabp->bucket[bix]; + + while (bcp && bcp->caller != caller) + bcp = bcp->next; + + if (bcp) + bcp->count++; + else { + if (tabp->pre_alloc_busy_caller.caller == am_undefined) + bcp = &tabp->pre_alloc_busy_caller; + else + bcp = erts_alloc(ERTS_ALC_T_BUSY_CALLER, + sizeof(ErtsPortTaskBusyCaller)); + bcp->caller = caller; + bcp->count = 1; + bcp->next = tabp->bucket[bix]; + tabp->bucket[bix] = bcp; + } + + bcp->last = ptp; +} + +static ERTS_INLINE int +check_sig_dep_move_to_busy_queue(Port *pp, ErtsPortTask *ptp) +{ + ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table; + ErtsPortTask *last_ptp; + ErtsPortTaskBusyCaller *bcp; + int bix; + Eterm caller; + + ASSERT(ptp->u.alive.flags & ERTS_PT_FLG_SIG_DEP); + ASSERT(pp->sched.taskq.local.busy.last); + ASSERT(tabp); + + + /* + * We are either not busy, or the task does not imply wait on busy port. + * However, due to the signaling order requirements the task might depend + * on other tasks in the busy queue. + */ + + caller = task_caller(ptp); + bix = caller2bix(caller); + bcp = tabp->bucket[bix]; + while (bcp && bcp->caller != caller) + bcp = bcp->next; + + if (!bcp) + return 0; + + /* + * There are other tasks that we depend on in the busy queue; + * move into busy queue. + */ + + bcp->count++; + last_ptp = bcp->last; + ptp->u.alive.next = last_ptp->u.alive.next; + if (!ptp->u.alive.next) { + ASSERT(pp->sched.taskq.local.busy.last == last_ptp); + pp->sched.taskq.local.busy.last = ptp; + } + last_ptp->u.alive.next = ptp; + bcp->last = ptp; + + return 1; +} + +static void +no_sig_dep_move_from_busyq(Port *pp) +{ + ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table; + ErtsPortTask *first_ptp, *last_ptp, *ptp; + ErtsPortTaskBusyCaller **prev_bcpp = NULL, *bcp = NULL; + + /* + * Move tasks at the head of the busy queue that no longer + * have any dependencies to busy wait tasks into the ordinary + * queue. + */ + + first_ptp = ptp = pp->sched.taskq.local.busy.first; + + ASSERT(ptp && !(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY)); + ASSERT(tabp); + + do { + Eterm caller = task_caller(ptp); + + if (!bcp || bcp->caller != caller) { + int bix = caller2bix(caller); + + prev_bcpp = &tabp->bucket[bix]; + bcp = tabp->bucket[bix]; + ASSERT(bcp); + while (bcp->caller != caller) { + ASSERT(bcp); + prev_bcpp = &bcp->next; + bcp = bcp->next; + } + } + + ASSERT(bcp->caller == caller); + ASSERT(bcp->count > 0); + + if (--bcp->count == 0) { + *prev_bcpp = bcp->next; + if (bcp == &tabp->pre_alloc_busy_caller) + bcp->caller = am_undefined; + else + erts_free(ERTS_ALC_T_BUSY_CALLER, bcp); + } + + last_ptp = ptp; + ptp = ptp->u.alive.next; + } while (ptp && !(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY)); + + pp->sched.taskq.local.busy.first = last_ptp->u.alive.next; + if (!pp->sched.taskq.local.busy.first) { +#ifdef DEBUG + int bix; + erts_aint32_t flags = +#endif + erts_smp_atomic32_read_band_nob( + &pp->sched.flags, + ~ERTS_PTS_FLG_HAVE_BUSY_TASKS); + ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS); +#ifdef DEBUG + for (bix = 0; bix < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; bix++) { + ASSERT(!tabp->bucket[bix]); + } +#endif + busy_caller_table_free(tabp); + pp->sched.taskq.local.busy.last = NULL; + pp->sched.taskq.local.busy.table = NULL; + } + last_ptp->u.alive.next = pp->sched.taskq.local.first; + pp->sched.taskq.local.first = first_ptp; +} + +#ifdef ERTS_HARD_DEBUG_TASK_QUEUES + +static void +chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue) +{ + Sint tot_count, tot_table_count; + int bix; + ErtsPortTask *ptp, *last; + ErtsPortTask *first = processing_busy_queue ? execq : pp->sched.taskq.local.busy.first; + ErtsPortTask *nb_task_queue = processing_busy_queue ? pp->sched.taskq.local.first : execq; + ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table; + ErtsPortTaskBusyCaller *bcp; + + if (!first) { + ASSERT(!tabp); + ASSERT(!pp->sched.taskq.local.busy.last); + ASSERT(!(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS)); + return; + } + + ASSERT(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS); + ASSERT(tabp); + + tot_count = 0; + ptp = first; + while (ptp) { + Sint count = 0; + Eterm caller = task_caller(ptp); + int bix = caller2bix(caller); + for (bcp = tabp->bucket[bix]; bcp; bcp = bcp->next) + if (bcp->caller == caller) + break; + ASSERT(bcp && bcp->caller == caller); + + ASSERT(bcp->last); + while (1) { + ErtsPortTask *ptp2; + + ASSERT(caller == task_caller(ptp)); + count++; + tot_count++; + last = ptp; + + for (ptp2 = nb_task_queue; ptp2; ptp2 = ptp2->u.alive.next) { + ASSERT(ptp != ptp2); + } + + if (ptp == bcp->last) + break; + ptp = ptp->u.alive.next; + } + + ASSERT(count == bcp->count); + ptp = ptp->u.alive.next; + } + + tot_table_count = 0; + for (bix = 0; bix < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; bix++) { + for (bcp = tabp->bucket[bix]; bcp; bcp = bcp->next) + tot_table_count += bcp->count; + } + + ASSERT(tot_count == tot_table_count); + + ASSERT(last == pp->sched.taskq.local.busy.last); +} + +#endif /* ERTS_HARD_DEBUG_TASK_QUEUES */ /* * Task handle manipulation. */ +static ERTS_INLINE void +reset_port_task_handle(ErtsPortTaskHandle *pthp) +{ + erts_smp_atomic_set_relb(pthp, (erts_aint_t) NULL); +} + static ERTS_INLINE ErtsPortTask * handle2task(ErtsPortTaskHandle *pthp) { - return (ErtsPortTask *) erts_smp_atomic_read_nob(pthp); + return (ErtsPortTask *) erts_smp_atomic_read_acqb(pthp); } static ERTS_INLINE void reset_handle(ErtsPortTask *ptp) { - if (ptp->handle) { - ASSERT(ptp == handle2task(ptp->handle)); - erts_smp_atomic_set_nob(ptp->handle, (erts_aint_t) NULL); + if (ptp->u.alive.handle) { + ASSERT(ptp == handle2task(ptp->u.alive.handle)); + reset_port_task_handle(ptp->u.alive.handle); } } static ERTS_INLINE void set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { - ptp->handle = pthp; + ptp->u.alive.handle = pthp; + if (pthp) { + erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp); + ASSERT(ptp == handle2task(ptp->u.alive.handle)); + } +} + +static ERTS_INLINE void +set_tmp_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) +{ + ptp->u.alive.handle = NULL; if (pthp) { - erts_smp_atomic_set_nob(pthp, (erts_aint_t) ptp); - ASSERT(ptp == handle2task(ptp->handle)); + /* + * IMPORTANT! Task either need to be aborted, or task handle + * need to be detached before thread progress has been made. + */ + erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp); + } +} + + +/* + * Busy port queue management + */ + +static erts_aint32_t +check_unset_busy_port_q(Port *pp, + erts_aint32_t flags, + ErtsPortTaskBusyPortQ *bpq) +{ + ErlDrvSizeT qsize, low; + int resume_procs = 0; + + ASSERT(bpq); + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + + erts_port_task_sched_lock(&pp->sched); + qsize = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->size); + low = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low); + if (qsize < low) { + erts_aint32_t mask = ~(ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q + | ERTS_PTS_FLG_BUSY_PORT_Q); + flags = erts_smp_atomic32_read_band_relb(&pp->sched.flags, mask); + if ((flags & ERTS_PTS_FLGS_BUSY) == ERTS_PTS_FLG_BUSY_PORT_Q) + resume_procs = 1; + } + else if (flags & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) { + flags = erts_smp_atomic32_read_band_relb(&pp->sched.flags, + ~ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); + flags &= ~ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q; + } + erts_port_task_sched_unlock(&pp->sched); + if (resume_procs) + erts_port_resume_procs(pp); + + return flags; +} + +static ERTS_INLINE void +aborted_proc2port_data(Port *pp, ErlDrvSizeT size) +{ + ErtsPortTaskBusyPortQ *bpq; + erts_aint32_t flags; + ErlDrvSizeT qsz; + + ASSERT(pp->sched.taskq.bpq); + + if (size == 0) + return; + + bpq = pp->sched.taskq.bpq; + + qsz = (ErlDrvSizeT) erts_smp_atomic_add_read_acqb(&bpq->size, + (erts_aint_t) -size); + ASSERT(qsz + size > qsz); + flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + ASSERT(pp->sched.taskq.bpq); + if ((flags & (ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q + | ERTS_PTS_FLG_BUSY_PORT_Q)) != ERTS_PTS_FLG_BUSY_PORT_Q) + return; + if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low)) + erts_smp_atomic32_read_bor_nob(&pp->sched.flags, + ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); +} + +static ERTS_INLINE void +dequeued_proc2port_data(Port *pp, ErlDrvSizeT size) +{ + ErtsPortTaskBusyPortQ *bpq; + erts_aint32_t flags; + ErlDrvSizeT qsz; + + ASSERT(pp->sched.taskq.bpq); + + if (size == 0) + return; + + bpq = pp->sched.taskq.bpq; + + qsz = (ErlDrvSizeT) erts_smp_atomic_add_read_acqb(&bpq->size, + (erts_aint_t) -size); + ASSERT(qsz + size > qsz); + flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + if (!(flags & ERTS_PTS_FLG_BUSY_PORT_Q)) + return; + if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_acqb(&bpq->low)) + check_unset_busy_port_q(pp, flags, bpq); +} + +static ERTS_INLINE erts_aint32_t +enqueue_proc2port_data(Port *pp, + ErtsProc2PortSigData *sigdp, + erts_aint32_t flags) +{ + ErtsPortTaskBusyPortQ *bpq = pp->sched.taskq.bpq; + if (sigdp && bpq) { + ErlDrvSizeT size = erts_proc2port_sig_command_data_size(sigdp); + if (size) { + erts_aint_t asize = erts_smp_atomic_add_read_acqb(&bpq->size, + (erts_aint_t) size); + ErlDrvSizeT qsz = (ErlDrvSizeT) asize; + + ASSERT(qsz - size < qsz); + + if (!(flags & ERTS_PTS_FLG_BUSY_PORT_Q) && qsz > bpq->high) { + flags = erts_smp_atomic32_read_bor_acqb(&pp->sched.flags, + ERTS_PTS_FLG_BUSY_PORT_Q); + flags |= ERTS_PTS_FLG_BUSY_PORT_Q; + qsz = (ErlDrvSizeT) erts_smp_atomic_read_acqb(&bpq->size); + if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low)) { + flags = (erts_smp_atomic32_read_bor_relb( + &pp->sched.flags, + ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q)); + flags |= ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q; + } + } + ASSERT(!(flags & ERTS_PTS_FLG_EXIT)); + } + } + return flags; +} + +/* + * erl_drv_busy_msgq_limits() is called by drivers either reading or + * writing the limits. + * + * A limit of zero is interpreted as a read only request (using a + * limit of zero would not be useful). Other values are interpreted + * as a write-read request. + */ + +void +erl_drv_busy_msgq_limits(ErlDrvPort dport, ErlDrvSizeT *lowp, ErlDrvSizeT *highp) +{ + Port *pp = erts_drvport2port(dport); + ErtsPortTaskBusyPortQ *bpq; + int written = 0, resume_procs = 0; + ErlDrvSizeT low, high; + + if (pp == ERTS_INVALID_ERL_DRV_PORT || !(bpq = pp->sched.taskq.bpq)) { + if (lowp) + *lowp = ERL_DRV_BUSY_MSGQ_DISABLED; + if (highp) + *highp = ERL_DRV_BUSY_MSGQ_DISABLED; + return; + } + + low = lowp ? *lowp : 0; + high = highp ? *highp : 0; + + erts_port_task_sched_lock(&pp->sched); + + if (low == ERL_DRV_BUSY_MSGQ_DISABLED + || high == ERL_DRV_BUSY_MSGQ_DISABLED) { + /* Disable busy msgq feature */ + erts_aint32_t flags; + pp->sched.taskq.bpq = NULL; + flags = ~(ERTS_PTS_FLG_BUSY_PORT_Q|ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); + flags = erts_smp_atomic32_read_band_acqb(&pp->sched.flags, flags); + if ((flags & ERTS_PTS_FLGS_BUSY) == ERTS_PTS_FLG_BUSY_PORT_Q) + resume_procs = 1; + } + else { + + if (!low) + low = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low); + else { + if (bpq->high < low) + bpq->high = low; + erts_smp_atomic_set_relb(&bpq->low, (erts_aint_t) low); + written = 1; + } + + if (!high) + high = bpq->high; + else { + if (low > high) { + low = high; + erts_smp_atomic_set_relb(&bpq->low, (erts_aint_t) low); + } + bpq->high = high; + written = 1; + } + + if (written) { + ErlDrvSizeT size = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->size); + if (size > high) + erts_smp_atomic32_read_bor_relb(&pp->sched.flags, + ERTS_PTS_FLG_BUSY_PORT_Q); + else if (size < low) + erts_smp_atomic32_read_bor_relb(&pp->sched.flags, + ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); + } + } + + erts_port_task_sched_unlock(&pp->sched); + + if (resume_procs) + erts_port_resume_procs(pp); + if (lowp) + *lowp = low; + if (highp) + *highp = high; +} + +/* + * No-suspend handles. + */ + +#ifdef ERTS_SMP +static void +free_port_task_handle_list(void *vpthlp) +{ + erts_free(ERTS_ALC_T_PT_HNDL_LIST, vpthlp); +} +#endif + +static void +schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp) +{ +#ifdef ERTS_SMP + erts_schedule_thr_prgr_later_cleanup_op(free_port_task_handle_list, + (void *) pthlp, + &pthlp->u.release, + sizeof(ErtsPortTaskHandleList)); +#else + erts_free(ERTS_ALC_T_PT_HNDL_LIST, pthlp); +#endif +} + +static ERTS_INLINE void +abort_nosuspend_task(Port *pp, + ErtsPortTaskType type, + ErtsPortTaskTypeData *tdp) +{ + + ASSERT(type == ERTS_PORT_TASK_PROC_SIG); + + if (!pp->sched.taskq.bpq) + tdp->psig.callback(NULL, + ERTS_PORT_SFLG_INVALID, + ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, + &tdp->psig.data); + else { + ErlDrvSizeT size = erts_proc2port_sig_command_data_size(&tdp->psig.data); + tdp->psig.callback(NULL, + ERTS_PORT_SFLG_INVALID, + ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, + &tdp->psig.data); + aborted_proc2port_data(pp, size); + } +} + +static ErtsPortTaskHandleList * +get_free_nosuspend_handles(Port *pp) +{ + ErtsPortTaskHandleList *nshp, *last_nshp = NULL; + + ERTS_SMP_LC_ASSERT(erts_port_task_sched_lock_is_locked(&pp->sched)); + + nshp = pp->sched.taskq.local.busy.nosuspend; + + while (nshp && !erts_port_task_is_scheduled(&nshp->handle)) { + last_nshp = nshp; + nshp = nshp->u.next; + } + + if (!last_nshp) + nshp = NULL; + else { + nshp = pp->sched.taskq.local.busy.nosuspend; + pp->sched.taskq.local.busy.nosuspend = last_nshp->u.next; + last_nshp->u.next = NULL; + if (!pp->sched.taskq.local.busy.nosuspend) + erts_smp_atomic32_read_band_nob(&pp->sched.flags, + ~ERTS_PTS_FLG_HAVE_NS_TASKS); + } + return nshp; +} + +static void +free_nosuspend_handles(ErtsPortTaskHandleList *free_nshp) +{ + while (free_nshp) { + ErtsPortTaskHandleList *nshp = free_nshp; + free_nshp = free_nshp->u.next; + schedule_port_task_handle_list_free(nshp); } } @@ -167,7 +864,6 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); pp->sched.next = NULL; - pp->sched.prev = runq->ports.end; if (runq->ports.end) { ASSERT(runq->ports.start); runq->ports.end->sched.next = pp; @@ -177,39 +873,10 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) runq->ports.start = pp; } - runq->ports.info.len++; - if (runq->ports.info.max_len < runq->ports.info.len) - runq->ports.info.max_len = runq->ports.info.len; - runq->len++; - if (runq->max_len < runq->len) - runq->max_len = runq->len; runq->ports.end = pp; ASSERT(runq->ports.start && runq->ports.end); -} - -static ERTS_INLINE void -dequeue_port(ErtsRunQueue *runq, Port *pp) -{ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - if (pp->sched.next) - pp->sched.next->sched.prev = pp->sched.prev; - else { - ASSERT(runq->ports.end == pp); - runq->ports.end = pp->sched.prev; - } - if (pp->sched.prev) - pp->sched.prev->sched.next = pp->sched.next; - else { - ASSERT(runq->ports.start == pp); - runq->ports.start = pp->sched.next; - } - ASSERT(runq->ports.info.len > 0); - runq->ports.info.len--; - ASSERT(runq->len > 0); - runq->len--; - ASSERT(runq->ports.start || !runq->ports.end); - ASSERT(runq->ports.end || !runq->ports.start); + erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); } static ERTS_INLINE Port * @@ -222,16 +889,11 @@ pop_port(ErtsRunQueue *runq) } else { runq->ports.start = runq->ports.start->sched.next; - if (runq->ports.start) - runq->ports.start->sched.prev = NULL; - else { + if (!runq->ports.start) { ASSERT(runq->ports.end == pp); runq->ports.end = NULL; } - ASSERT(runq->ports.info.len > 0); - runq->ports.info.len--; - ASSERT(runq->len > 0); - runq->len--; + erts_smp_dec_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); } ASSERT(runq->ports.start || !runq->ports.end); @@ -239,294 +901,447 @@ pop_port(ErtsRunQueue *runq) return pp; } +/* + * Task queue operations + */ -#ifdef HARD_DEBUG +static ERTS_INLINE int +enqueue_task(Port *pp, + ErtsPortTask *ptp, + ErtsProc2PortSigData *sigdp, + ErtsPortTaskHandleList *ns_pthlp, + erts_aint32_t *flagsp) -static void -check_port_queue(ErtsRunQueue *runq, Port *chk_pp, int inq) { - Port *pp; - Port *last_pp; - Port *first_pp = runq->ports.start; - int no_forward = 0, no_backward = 0; - int found_forward = 0, found_backward = 0; - if (!first_pp) { - ASSERT(!runq->ports.end); - } + int res; + erts_aint32_t fail_flags = ERTS_PTS_FLG_EXIT; + erts_aint32_t flags; + ptp->u.alive.next = NULL; + if (ns_pthlp) + fail_flags |= ERTS_PTS_FLG_BUSY_PORT; + erts_port_task_sched_lock(&pp->sched); + flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + if (flags & fail_flags) + res = 0; else { - ASSERT(!first_pp->sched.prev); - for (pp = first_pp; pp; pp = pp->sched.next) { - ASSERT(pp->sched.taskq); - if (pp->sched.taskq->first) - no_forward++; - if (chk_pp == pp) - found_forward = 1; - if (!pp->sched.prev) { - ASSERT(first_pp == pp); - } - if (!pp->sched.next) { - ASSERT(runq->ports.end == pp); - last_pp = pp; - } - } - for (pp = last_pp; pp; pp = pp->sched.prev) { - ASSERT(pp->sched.taskq); - if (pp->sched.taskq->last) - no_backward++; - if (chk_pp == pp) - found_backward = 1; - if (!pp->sched.prev) { - ASSERT(first_pp == pp); - } - if (!pp->sched.next) { - ASSERT(runq->ports.end == pp); - } - check_task_queue(pp->sched.taskq, NULL, 0); - } - ASSERT(no_forward == no_backward); - } - ASSERT(no_forward == runq->ports.info.len); - if (chk_pp) { - if (chk_pp->sched.taskq || chk_pp->sched.exe_taskq) { - ASSERT(chk_pp->sched.taskq != chk_pp->sched.exe_taskq); + if (ns_pthlp) { + ns_pthlp->u.next = pp->sched.taskq.local.busy.nosuspend; + pp->sched.taskq.local.busy.nosuspend = ns_pthlp; } - ASSERT(!chk_pp->sched.taskq || chk_pp->sched.taskq->first); - if (inq < 0) - inq = chk_pp->sched.taskq && !chk_pp->sched.exe_taskq; - if (inq) { - ASSERT(found_forward && found_backward); + if (pp->sched.taskq.in.last) { + ASSERT(pp->sched.taskq.in.first); + ASSERT(!pp->sched.taskq.in.last->u.alive.next); + + pp->sched.taskq.in.last->u.alive.next = ptp; } else { - ASSERT(!found_forward && !found_backward); - } - } -} - -#endif - -/* - * Task queue operations - */ + ASSERT(!pp->sched.taskq.in.first); -static ERTS_INLINE ErtsPortTaskQueue * -port_taskq_init(ErtsPortTaskQueue *ptqp, Port *pp) -{ - if (ptqp) { - ptqp->first = NULL; - ptqp->last = NULL; - ptqp->port = pp; + pp->sched.taskq.in.first = ptp; + } + pp->sched.taskq.in.last = ptp; + flags = enqueue_proc2port_data(pp, sigdp, flags); + res = 1; } - return ptqp; + erts_port_task_sched_unlock(&pp->sched); + *flagsp = flags; + return res; } static ERTS_INLINE void -enqueue_task(ErtsPortTaskQueue *ptqp, ErtsPortTask *ptp) +prepare_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) { - ERTS_PT_CHK_NOT_IN_TASKQ(ptqp, ptp); - ptp->next = NULL; - ptp->prev = ptqp->last; - ptp->queue = ptqp; - if (ptqp->last) { - ASSERT(ptqp->first); - ptqp->last->next = ptp; + erts_aint32_t act = erts_smp_atomic32_read_nob(&pp->sched.flags); + + if (!pp->sched.taskq.local.busy.first || (act & ERTS_PTS_FLG_BUSY_PORT)) { + *execqp = pp->sched.taskq.local.first; + *processing_busy_q_p = 0; } else { - ASSERT(!ptqp->first); - ptqp->first = ptp; + *execqp = pp->sched.taskq.local.busy.first; + *processing_busy_q_p = 1; + } + + ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p); + + while (1) { + erts_aint32_t new, exp; + + new = exp = act; + + new &= ~ERTS_PTS_FLG_IN_RUNQ; + new |= ERTS_PTS_FLG_EXEC; + + act = erts_smp_atomic32_cmpxchg_nob(&pp->sched.flags, new, exp); + + ASSERT(act & ERTS_PTS_FLG_IN_RUNQ); + + if (exp == act) + break; } - ptqp->last = ptp; - ERTS_PT_CHK_IN_TASKQ(ptqp, ptp); } -static ERTS_INLINE void -push_task(ErtsPortTaskQueue *ptqp, ErtsPortTask *ptp) +/* finalize_exec() return value != 0 if port should remain active */ +static ERTS_INLINE int +finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) { - ERTS_PT_CHK_NOT_IN_TASKQ(ptqp, ptp); - ptp->next = ptqp->first; - ptp->prev = NULL; - ptp->queue = ptqp; - if (ptqp->first) { - ASSERT(ptqp->last); - ptqp->first->prev = ptp; - } + erts_aint32_t act; + + if (!processing_busy_q) + pp->sched.taskq.local.first = *execq; else { - ASSERT(!ptqp->last); - ptqp->last = ptp; + pp->sched.taskq.local.busy.first = *execq; + ASSERT(*execq); + } + + ERTS_PT_DBG_CHK_TASK_QS(pp, *execq, processing_busy_q); + + *execq = NULL; + + act = erts_smp_atomic32_read_nob(&pp->sched.flags); + if (act & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) + act = check_unset_busy_port_q(pp, act, pp->sched.taskq.bpq); + + while (1) { + erts_aint32_t new, exp; + + new = exp = act; + + new &= ~ERTS_PTS_FLG_EXEC; + if (act & ERTS_PTS_FLG_HAVE_TASKS) + new |= ERTS_PTS_FLG_IN_RUNQ; + + act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); + + ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ)); + + if (exp == act) + break; } - ptqp->first = ptp; - ERTS_PT_CHK_IN_TASKQ(ptqp, ptp); + + return (act & ERTS_PTS_FLG_HAVE_TASKS) != 0; } -static ERTS_INLINE void -dequeue_task(ErtsPortTask *ptp) +static ERTS_INLINE erts_aint32_t +select_queue_for_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) { - ASSERT(ptp); - ASSERT(ptp->queue); - ERTS_PT_CHK_IN_TASKQ(ptp->queue, ptp); - if (ptp->next) - ptp->next->prev = ptp->prev; - else { - ASSERT(ptp->queue->last == ptp); - ptp->queue->last = ptp->prev; + erts_aint32_t flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + + if (flags & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) + flags = check_unset_busy_port_q(pp, flags, pp->sched.taskq.bpq); + + ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p); + + if (flags & ERTS_PTS_FLG_BUSY_PORT) { + if (*processing_busy_q_p) { + ErtsPortTask *ptp; + + ptp = pp->sched.taskq.local.busy.first = *execqp; + if (!ptp) + pp->sched.taskq.local.busy.last = NULL; + else if (!(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY)) + no_sig_dep_move_from_busyq(pp); + + *execqp = pp->sched.taskq.local.first; + *processing_busy_q_p = 0; + + ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p); + } + + return flags; } - if (ptp->prev) - ptp->prev->next = ptp->next; - else { - ASSERT(ptp->queue->first == ptp); - ptp->queue->first = ptp->next; + + /* Not busy */ + + if (!*processing_busy_q_p && pp->sched.taskq.local.busy.first) { + pp->sched.taskq.local.first = *execqp; + *execqp = pp->sched.taskq.local.busy.first; + *processing_busy_q_p = 1; + + ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p); } - ASSERT(ptp->queue->first || !ptp->queue->last); - ASSERT(ptp->queue->last || !ptp->queue->first); - ERTS_PT_CHK_NOT_IN_TASKQ(ptp->queue, ptp); + return flags; } -static ERTS_INLINE ErtsPortTask * -pop_task(ErtsPortTaskQueue *ptqp) +/* + * check_task_for_exec() returns a value !0 if the task + * is ok to execute; otherwise 0. + */ +static ERTS_INLINE int +check_task_for_exec(Port *pp, + erts_aint32_t flags, + ErtsPortTask **execqp, + int *processing_busy_q_p, + ErtsPortTask *ptp) { - ErtsPortTask *ptp = ptqp->first; - if (!ptp) { - ASSERT(!ptqp->last); + + if (!*processing_busy_q_p) { + /* Processing normal queue */ + + ERTS_PT_DBG_CHK_TASK_QS(pp, ptp, *processing_busy_q_p); + + if ((flags & ERTS_PTS_FLG_BUSY_PORT) + && (ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY)) { + + busy_wait_move_to_busy_queue(pp, ptp); + ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p); + + return 0; + } + + if (pp->sched.taskq.local.busy.last + && (ptp->u.alive.flags & ERTS_PT_FLG_SIG_DEP)) { + + int res = !check_sig_dep_move_to_busy_queue(pp, ptp); + ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p); + + return res; + } + } else { - ERTS_PT_CHK_IN_TASKQ(ptqp, ptp); - ASSERT(!ptp->prev); - ptqp->first = ptp->next; - if (ptqp->first) - ptqp->first->prev = NULL; - else { - ASSERT(ptqp->last == ptp); - ptqp->last = NULL; + /* Processing busy queue */ + + ASSERT(!(flags & ERTS_PTS_FLG_BUSY_PORT)); + + ERTS_PT_DBG_CHK_TASK_QS(pp, ptp, *processing_busy_q_p); + + popped_from_busy_queue(pp, ptp, !*execqp); + + if (!*execqp) { + *execqp = pp->sched.taskq.local.first; + *processing_busy_q_p = 0; } - ASSERT(ptp->queue->first || !ptp->queue->last); - ASSERT(ptp->queue->last || !ptp->queue->first); + + ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p); + } - ERTS_PT_CHK_NOT_IN_TASKQ(ptqp, ptp); - return ptp; + + return 1; } -#ifdef HARD_DEBUG +static ErtsPortTask * +fetch_in_queue(Port *pp, ErtsPortTask **execqp) +{ + ErtsPortTask *ptp; + ErtsPortTaskHandleList *free_nshp = NULL; -static void -check_task_queue(ErtsPortTaskQueue *ptqp, - ErtsPortTask *chk_ptp, - int inq) + erts_port_task_sched_lock(&pp->sched); + + ptp = pp->sched.taskq.in.first; + pp->sched.taskq.in.first = NULL; + pp->sched.taskq.in.last = NULL; + if (ptp) + *execqp = ptp->u.alive.next; + else + erts_smp_atomic32_read_band_nob(&pp->sched.flags, + ~ERTS_PTS_FLG_HAVE_TASKS); + + + if (pp->sched.taskq.local.busy.nosuspend) + free_nshp = get_free_nosuspend_handles(pp); + + erts_port_task_sched_unlock(&pp->sched); + + if (free_nshp) + free_nosuspend_handles(free_nshp); + + return ptp; +} + +static ERTS_INLINE ErtsPortTask * +select_task_for_exec(Port *pp, + ErtsPortTask **execqp, + int *processing_busy_q_p) { ErtsPortTask *ptp; - ErtsPortTask *last_ptp; - ErtsPortTask *first_ptp = ptqp->first; - int found_forward = 0, found_backward = 0; - if (!first_ptp) { - ASSERT(!ptqp->last); - } - else { - ASSERT(!first_ptp->prev); - for (ptp = first_ptp; ptp; ptp = ptp->next) { - ASSERT(ptp->queue == ptqp); - if (chk_ptp == ptp) - found_forward = 1; - if (!ptp->prev) { - ASSERT(first_ptp == ptp); - } - if (!ptp->next) { - ASSERT(ptqp->last == ptp); - last_ptp = ptp; - } - } - for (ptp = last_ptp; ptp; ptp = ptp->prev) { - ASSERT(ptp->queue == ptqp); - if (chk_ptp == ptp) - found_backward = 1; - if (!ptp->prev) { - ASSERT(first_ptp == ptp); - } - if (!ptp->next) { - ASSERT(ptqp->last == ptp); - } - } - } - if (chk_ptp) { - if (inq) { - ASSERT(found_forward && found_backward); - } + erts_aint32_t flags; + + flags = select_queue_for_exec(pp, execqp, processing_busy_q_p); + + while (1) { + ptp = *execqp; + if (ptp) + *execqp = ptp->u.alive.next; else { - ASSERT(!found_forward && !found_backward); + ptp = fetch_in_queue(pp, execqp); + if (!ptp) + return NULL; } + if (check_task_for_exec(pp, flags, execqp, processing_busy_q_p, ptp)) + return ptp; } } -#endif + +/* + * Cut time slice + */ + +int +erl_drv_consume_timeslice(ErlDrvPort dprt, int percent) +{ + Port *pp = erts_drvport2port(dprt); + if (pp == ERTS_INVALID_ERL_DRV_PORT) + return -1; + if (percent < 1) + percent = 1; + else if (100 < percent) + percent = 100; + pp->reds += percent*((CONTEXT_REDS+99)/100); + if (pp->reds < CONTEXT_REDS) + return 0; + pp->reds = CONTEXT_REDS; + return 1; +} + +void +erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *pthp) +{ + ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); + reset_port_task_handle(pthp); +} /* * Abort a scheduled task. */ int -erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp) +erts_port_task_abort(ErtsPortTaskHandle *pthp) { - ErtsRunQueue *runq; - ErtsPortTaskQueue *ptqp; + int res; ErtsPortTask *ptp; - Port *pp; - int port_is_dequeued = 0; - - pp = &erts_port[internal_port_index(id)]; - runq = erts_port_runq(pp); +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); +#endif ptp = handle2task(pthp); + if (!ptp) + res = -1; + else { + erts_aint32_t old_state; + +#ifdef DEBUG + ErtsPortTaskHandle *saved_pthp = ptp->u.alive.handle; + ERTS_SMP_READ_MEMORY_BARRIER; + old_state = erts_smp_atomic32_read_nob(&ptp->state); + if (old_state == ERTS_PT_STATE_SCHEDULED) { + ASSERT(!saved_pthp || saved_pthp == pthp); + } +#endif - if (!ptp) { - erts_smp_runq_unlock(runq); - return 1; + old_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + ERTS_PT_STATE_ABORTED, + ERTS_PT_STATE_SCHEDULED); + if (old_state != ERTS_PT_STATE_SCHEDULED) + res = - 1; /* Task already aborted, executing, or executed */ + else { + + reset_port_task_handle(pthp); + + switch (ptp->type) { + case ERTS_PORT_TASK_INPUT: + case ERTS_PORT_TASK_OUTPUT: + case ERTS_PORT_TASK_EVENT: + ASSERT(erts_smp_atomic_read_nob( + &erts_port_task_outstanding_io_tasks) > 0); + erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); + break; + default: + break; + } + + res = 0; + } } - ASSERT(ptp->handle == pthp); - ptqp = ptp->queue; - ASSERT(pp == ptqp->port); +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif - ERTS_PT_CHK_PRES_PORTQ(runq, pp); - ASSERT(ptqp); - ASSERT(ptqp->first); + return res; +} - dequeue_task(ptp); - reset_handle(ptp); +void +erts_port_task_abort_nosuspend_tasks(Port *pp) +{ + ErtsPortTaskHandleList *abort_list; +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl = ERTS_THR_PRGR_DHANDLE_INVALID; +#endif - switch (ptp->type) { - case ERTS_PORT_TASK_INPUT: - case ERTS_PORT_TASK_OUTPUT: - case ERTS_PORT_TASK_EVENT: - ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) > 0); - erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); - break; - default: - break; - } + erts_port_task_sched_lock(&pp->sched); + erts_smp_atomic32_read_band_nob(&pp->sched.flags, + ~ERTS_PTS_FLG_HAVE_NS_TASKS); + abort_list = pp->sched.taskq.local.busy.nosuspend; + pp->sched.taskq.local.busy.nosuspend = NULL; + erts_port_task_sched_unlock(&pp->sched); + + while (abort_list) { +#ifdef DEBUG + ErtsPortTaskHandle *saved_pthp; +#endif + ErtsPortTaskType type; + ErtsPortTaskTypeData td; + ErtsPortTaskHandle *pthp; + ErtsPortTask *ptp; + ErtsPortTaskHandleList *pthlp; + erts_aint32_t old_state; - ASSERT(ptqp == pp->sched.taskq || ptqp == pp->sched.exe_taskq); + pthlp = abort_list; + abort_list = pthlp->u.next; - if (ptqp->first || pp->sched.taskq != ptqp) - ptqp = NULL; - else { - pp->sched.taskq = NULL; - if (!pp->sched.exe_taskq) { - dequeue_port(runq, pp); - ERTS_PORT_NOT_IN_RUNQ(pp); - port_is_dequeued = 1; +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + + pthp = &pthlp->handle; + ptp = handle2task(pthp); + if (!ptp) { +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_thr_progress_unmanaged_continue(dhndl); +#endif + schedule_port_task_handle_list_free(pthlp); + continue; } - } - ERTS_PT_CHK_PRES_PORTQ(runq, pp); +#ifdef DEBUG + saved_pthp = ptp->u.alive.handle; + ERTS_SMP_READ_MEMORY_BARRIER; + old_state = erts_smp_atomic32_read_nob(&ptp->state); + if (old_state == ERTS_PT_STATE_SCHEDULED) { + ASSERT(saved_pthp == pthp); + } +#endif - erts_smp_runq_unlock(runq); - - if (erts_system_profile_flags.runnable_ports && port_is_dequeued) { - profile_runnable_port(pp, am_inactive); - } + old_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + ERTS_PT_STATE_ABORTED, + ERTS_PT_STATE_SCHEDULED); + if (old_state != ERTS_PT_STATE_SCHEDULED) { + /* Task already aborted, executing, or executed */ +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_thr_progress_unmanaged_continue(dhndl); +#endif + schedule_port_task_handle_list_free(pthlp); + continue; + } - port_task_free(ptp); - if (ptqp) - port_taskq_free(ptqp); + reset_port_task_handle(pthp); - return 0; + type = ptp->type; + td = ptp->u.alive.td; + +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_thr_progress_unmanaged_continue(dhndl); +#endif + schedule_port_task_handle_list_free(pthlp); + + abort_nosuspend_task(pp, type, &td); + } } /* @@ -537,244 +1352,261 @@ int erts_port_task_schedule(Eterm id, ErtsPortTaskHandle *pthp, ErtsPortTaskType type, - ErlDrvEvent event, - ErlDrvEventData event_data) + ...) { + ErtsProc2PortSigData *sigdp = NULL; + ErtsPortTaskHandleList *ns_pthlp = NULL; +#ifdef ERTS_SMP + ErtsRunQueue *xrunq; + ErtsThrPrgrDelayHandle dhndl; +#endif ErtsRunQueue *runq; Port *pp; - ErtsPortTask *ptp; - int enq_port = 0; - - /* - * NOTE: We might not have the port lock here. We are only - * allowed to access the 'sched', 'tab_status', - * and 'id' fields of the port struct while - * tasks_lock is held. - */ + ErtsPortTask *ptp = NULL; + erts_aint32_t act, add_flags; if (pthp && erts_port_task_is_scheduled(pthp)) { ASSERT(0); - erts_port_task_abort(id, pthp); + erts_port_task_abort(pthp); } - ptp = port_task_alloc(); - ASSERT(is_internal_port(id)); - pp = &erts_port[internal_port_index(id)]; - runq = erts_port_runq(pp); - - if (!runq || ERTS_PORT_TASK_INVALID_PORT(pp, id)) { - if (runq) - erts_smp_runq_unlock(runq); - return -1; - } - ASSERT(!erts_port_task_is_scheduled(pthp)); +#ifdef ERTS_SMP + dhndl = erts_thr_progress_unmanaged_delay(); +#endif - ERTS_PT_CHK_PRES_PORTQ(runq, pp); + pp = erts_port_lookup_raw(id); - if (!pp->sched.taskq && !pp->sched.exe_taskq) { #ifdef ERTS_SMP - ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); - if (xrunq) { - /* Port emigrated ... */ - erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); - erts_smp_runq_unlock(runq); - runq = xrunq; - } - enq_port = !pp->sched.taskq && !pp->sched.exe_taskq; -#else - enq_port = 1; -#endif + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { + if (pp) + erts_port_inc_refc(pp); + erts_thr_progress_unmanaged_continue(dhndl); } +#endif - ASSERT(!enq_port || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + if (!pp) + goto fail; - if (!pp->sched.taskq) - pp->sched.taskq = port_taskq_init(port_taskq_alloc(), pp); + if (type != ERTS_PORT_TASK_PROC_SIG) { + ptp = port_task_alloc(); - ASSERT(ptp); + ptp->type = type; + ptp->u.alive.flags = 0; - ptp->type = type; - ptp->event = event; - ptp->event_data = event_data; + erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED); - set_handle(ptp, pthp); + set_handle(ptp, pthp); + } switch (type) { - case ERTS_PORT_TASK_FREE: - erl_exit(ERTS_ABORT_EXIT, - "erts_port_task_schedule(): Cannot schedule free task\n"); - break; case ERTS_PORT_TASK_INPUT: - case ERTS_PORT_TASK_OUTPUT: - case ERTS_PORT_TASK_EVENT: + case ERTS_PORT_TASK_OUTPUT: { + va_list argp; + va_start(argp, type); + ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent); + va_end(argp); erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); - /* Fall through... */ + break; + } + case ERTS_PORT_TASK_EVENT: { + va_list argp; + va_start(argp, type); + ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent); + ptp->u.alive.td.io.event_data = va_arg(argp, ErlDrvEventData); + va_end(argp); + erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); + break; + } + case ERTS_PORT_TASK_PROC_SIG: { + va_list argp; + va_start(argp, type); + sigdp = va_arg(argp, ErtsProc2PortSigData *); + ptp = p2p_sig_data_to_task(sigdp); + ptp->u.alive.td.psig.callback = va_arg(argp, ErtsProc2PortSigCallback); + ptp->u.alive.flags |= va_arg(argp, int); + va_end(argp); + if (!(ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND)) + set_tmp_handle(ptp, pthp); + else { + ns_pthlp = erts_alloc(ERTS_ALC_T_PT_HNDL_LIST, + sizeof(ErtsPortTaskHandleList)); + set_handle(ptp, &ns_pthlp->handle); + } + break; + } default: - enqueue_task(pp->sched.taskq, ptp); break; } -#ifndef ERTS_SMP - /* - * When (!enq_port && !pp->sched.exe_taskq) is true in the smp case, - * the port might not be in the run queue. If this is the case, another - * thread is in the process of enqueueing the port. This very seldom - * occur, but do occur and is a valid scenario. Debug info showing this - * enqueue in progress must be introduced before we can enable (modified - * versions of these) assertions in the smp case again. - */ -#if defined(HARD_DEBUG) - if (pp->sched.exe_taskq || enq_port) - ERTS_PT_CHK_NOT_IN_PORTQ(runq, pp); - else - ERTS_PT_CHK_IN_PORTQ(runq, pp); -#elif defined(DEBUG) - if (!enq_port && !pp->sched.exe_taskq) { - /* We should be in port run q */ - ASSERT(pp->sched.prev || runq->ports.start == pp); + if (!enqueue_task(pp, ptp, sigdp, ns_pthlp, &act)) { + reset_handle(ptp); + if (ns_pthlp && !(act & ERTS_PTS_FLG_EXIT)) + goto abort_nosuspend; + else + goto fail; } -#endif -#endif - if (!enq_port) { - ERTS_PT_CHK_PRES_PORTQ(runq, pp); - erts_smp_runq_unlock(runq); - } - else { - enqueue_port(runq, pp); - ERTS_PT_CHK_PRES_PORTQ(runq, pp); - - if (erts_system_profile_flags.runnable_ports) { - profile_runnable_port(pp, am_active); + add_flags = ERTS_PTS_FLG_HAVE_TASKS; + if (ns_pthlp) + add_flags |= ERTS_PTS_FLG_HAVE_NS_TASKS; + + while (1) { + erts_aint32_t new, exp; + + if ((act & add_flags) == add_flags + && (act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + goto done; /* Done */ + + new = exp = act; + new |= add_flags; + if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + new |= ERTS_PTS_FLG_IN_RUNQ; + + act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); + + if (exp == act) { + if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + break; /* Need to enqueue port */ + goto done; /* Done */ } + if (act & ERTS_PTS_FLG_EXIT) + goto done; /* Died after our task insert... */ + } + + /* Enqueue port on run-queue */ + + runq = erts_port_runq(pp); + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + +#ifdef ERTS_SMP + xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); + if (xrunq) { + /* Port emigrated ... */ + erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); + runq = erts_port_runq(pp); + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + } +#endif - erts_smp_notify_inc_runq(runq); + enqueue_port(runq, pp); + + if (erts_system_profile_flags.runnable_ports) { + profile_runnable_port(pp, am_active); } + + erts_smp_runq_unlock(runq); + + erts_smp_notify_inc_runq(runq); + +done: + +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_port_dec_refc(pp); +#endif + + return 0; + +abort_nosuspend: + +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_port_dec_refc(pp); +#endif + + abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td); + + ASSERT(ns_pthlp); + erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp); + if (ptp) + port_task_free(ptp); + return 0; + +fail: + +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_port_dec_refc(pp); +#endif + + if (ns_pthlp) + erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp); + + if (ptp) + port_task_free(ptp); + + return -1; } void erts_port_task_free_port(Port *pp) { + erts_aint32_t flags; ErtsRunQueue *runq; - int port_is_dequeued = 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); - ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD)); + ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); + runq = erts_port_runq(pp); - ASSERT(runq); - ERTS_PT_CHK_PRES_PORTQ(runq, pp); - if (pp->sched.exe_taskq) { - /* I (this thread) am currently executing this port, free it - when scheduled out... */ - ErtsPortTask *ptp = port_task_alloc(); - erts_smp_port_state_lock(pp); - pp->status &= ~ERTS_PORT_SFLG_CLOSING; - pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; - erts_may_save_closed_port(pp); - erts_smp_port_state_unlock(pp); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1); - ptp->type = ERTS_PORT_TASK_FREE; - ptp->event = (ErlDrvEvent) -1; - ptp->event_data = NULL; - set_handle(ptp, NULL); - push_task(pp->sched.exe_taskq, ptp); - ERTS_PT_CHK_PRES_PORTQ(runq, pp); - erts_smp_runq_unlock(runq); - } - else { - ErtsPortTaskQueue *ptqp = pp->sched.taskq; - if (ptqp) { - dequeue_port(runq, pp); - ERTS_PORT_NOT_IN_RUNQ(pp); - port_is_dequeued = 1; - } - erts_smp_port_state_lock(pp); - pp->status &= ~ERTS_PORT_SFLG_CLOSING; - pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; - erts_may_save_closed_port(pp); - erts_smp_port_state_unlock(pp); -#ifdef ERTS_SMP - erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ -#endif - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ - handle_remaining_tasks(runq, pp); /* May release runq lock */ - ASSERT(!pp->sched.exe_taskq && (!ptqp || !ptqp->first)); - pp->sched.taskq = NULL; - ERTS_PT_CHK_PRES_PORTQ(runq, pp); -#ifndef ERTS_SMP - ASSERT(pp->status & ERTS_PORT_SFLG_PORT_DEBUG); - erts_port_status_set(pp, ERTS_PORT_SFLG_FREE); -#endif - erts_smp_runq_unlock(runq); + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + erts_port_task_sched_lock(&pp->sched); + flags = erts_smp_atomic32_read_bor_relb(&pp->sched.flags, + ERTS_PTS_FLG_EXIT); + erts_port_task_sched_unlock(&pp->sched); + erts_atomic32_read_bset_relb(&pp->state, + (ERTS_PORT_SFLG_CONNECTED + | ERTS_PORT_SFLG_EXITING + | ERTS_PORT_SFLG_CLOSING + | ERTS_PORT_SFLG_FREE), + ERTS_PORT_SFLG_FREE); - if (erts_system_profile_flags.runnable_ports && port_is_dequeued) { - profile_runnable_port(pp, am_inactive); - } + erts_smp_runq_unlock(runq); - if (ptqp) - port_taskq_free(ptqp); - } + if (!(flags & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + begin_port_cleanup(pp, NULL, NULL); } -typedef struct { - ErtsRunQueue *runq; - int *resp; -} ErtsPortTaskExeBlockData; - /* - * Run all scheduled tasks for the first port in run queue. If - * new tasks appear while running reschedule port (free task is - * an exception; it is always handled instantly). + * Execute scheduled tasks of a port. * * erts_port_task_execute() is called by scheduler threads between - * scheduleing of processes. Sched lock should be held by caller. + * scheduling of processes. Run-queue lock should be held by caller. */ int erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) { - int port_was_enqueued = 0; Port *pp; - ErtsPortTaskQueue *ptqp; - ErtsPortTask *ptp; + ErtsPortTask *execq; + int processing_busy_q; int res = 0; - int reds = ERTS_PORT_REDS_EXECUTE; + int vreds = 0; + int reds = 0; erts_aint_t io_tasks_executed = 0; int fpe_was_unmasked; + erts_aint32_t state; + int active; Uint64 start_time = 0; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - ERTS_PT_CHK_PORTQ(runq); - pp = pop_port(runq); if (!pp) { res = 0; goto done; } - ERTS_PORT_NOT_IN_RUNQ(pp); + erts_smp_runq_unlock(runq); *curr_port_pp = pp; - - ASSERT(pp->sched.taskq); - ASSERT(pp->sched.taskq->first); - ptqp = pp->sched.taskq; - pp->sched.taskq = NULL; - - ASSERT(!pp->sched.exe_taskq); - pp->sched.exe_taskq = ptqp; - - if (erts_smp_port_trylock(pp) == EBUSY) { - erts_smp_runq_unlock(runq); - erts_smp_port_lock(pp); - erts_smp_runq_lock(runq); - } if (erts_sched_stat.enabled) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); @@ -791,23 +1623,38 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_smp_spin_unlock(&erts_sched_stat.lock); } + prepare_exec(pp, &execq, &processing_busy_q); + + erts_smp_port_lock(pp); + /* trace port scheduling, in */ if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { trace_sched_ports(pp, am_in); } - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + fpe_was_unmasked = erts_block_fpe(); - ERTS_PT_CHK_PRES_PORTQ(runq, pp); - ptp = pop_task(ptqp); + state = erts_atomic32_read_nob(&pp->state); + pp->reds = ERTS_PORT_REDS_EXECUTE; + goto begin_handle_tasks; - fpe_was_unmasked = erts_block_fpe(); + while (1) { + erts_aint32_t task_state; + ErtsPortTask *ptp; - while (ptp) { - ASSERT(pp->sched.taskq != pp->sched.exe_taskq); + ptp = select_task_for_exec(pp, &execq, &processing_busy_q); + if (!ptp) + break; + + task_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + ERTS_PT_STATE_EXECUTING, + ERTS_PT_STATE_SCHEDULED); + if (task_state != ERTS_PT_STATE_SCHEDULED) { + ASSERT(task_state == ERTS_PT_STATE_ABORTED); + goto aborted_port_task; + } reset_handle(ptp); - erts_smp_runq_unlock(runq); if (erts_system_monitor_long_schedule != 0) { start_time = erts_timestamp_millis(); @@ -818,60 +1665,59 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ASSERT(pp->drv_ptr); switch (ptp->type) { - case ERTS_PORT_TASK_FREE: /* May be pushed in q at any time */ - reds += ERTS_PORT_REDS_FREE; - erts_smp_runq_lock(runq); - - erts_unblock_fpe(fpe_was_unmasked); - ASSERT(pp->status & ERTS_PORT_SFLG_FREE_SCHEDULED); - if (ptqp->first || (pp->sched.taskq && pp->sched.taskq->first)) - handle_remaining_tasks(runq, pp); - ASSERT(!ptqp->first - && (!pp->sched.taskq || !pp->sched.taskq->first)); -#ifdef ERTS_SMP - erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ -#else - erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE); -#endif - - port_task_free(ptp); - if (pp->sched.taskq) - port_taskq_free(pp->sched.taskq); - pp->sched.taskq = NULL; - - goto tasks_done; case ERTS_PORT_TASK_TIMEOUT: - reds += ERTS_PORT_REDS_TIMEOUT; - if (!(pp->status & ERTS_PORT_SFLGS_DEAD)) { + reds = ERTS_PORT_REDS_TIMEOUT; + if (!(state & ERTS_PORT_SFLGS_DEAD)) { DTRACE_DRIVER(driver_timeout, pp); (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); } break; case ERTS_PORT_TASK_INPUT: - reds += ERTS_PORT_REDS_INPUT; - ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + reds = ERTS_PORT_REDS_INPUT; + ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_input, pp); /* NOTE some windows drivers use ->ready_input for input and output */ - (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->event); + (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, + ptp->u.alive.td.io.event); io_tasks_executed++; break; case ERTS_PORT_TASK_OUTPUT: - reds += ERTS_PORT_REDS_OUTPUT; - ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + reds = ERTS_PORT_REDS_OUTPUT; + ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_output, pp); - (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->event); + (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, + ptp->u.alive.td.io.event); io_tasks_executed++; break; case ERTS_PORT_TASK_EVENT: - reds += ERTS_PORT_REDS_EVENT; - ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + reds = ERTS_PORT_REDS_EVENT; + ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_event, pp); - (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->event, ptp->event_data); + (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, + ptp->u.alive.td.io.event, + ptp->u.alive.td.io.event_data); io_tasks_executed++; break; + case ERTS_PORT_TASK_PROC_SIG: { + ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data; + ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); + if (!pp->sched.taskq.bpq) + reds = ptp->u.alive.td.psig.callback(pp, + state, + ERTS_PROC2PORT_SIG_EXEC, + sigdp); + else { + ErlDrvSizeT size = erts_proc2port_sig_command_data_size(sigdp); + reds = ptp->u.alive.td.psig.callback(pp, + state, + ERTS_PROC2PORT_SIG_EXEC, + sigdp); + dequeued_proc2port_data(pp, size); + } + break; + } case ERTS_PORT_TASK_DIST_CMD: - reds += erts_dist_command(pp, CONTEXT_REDS-reds); + reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds); break; default: erl_exit(ERTS_ABORT_EXIT, @@ -880,11 +1726,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) break; } - if ((pp->status & ERTS_PORT_SFLG_CLOSING) - && erts_is_port_ioq_empty(pp)) { - reds += ERTS_PORT_REDS_TERMINATE; - erts_terminate_port(pp); - } + reds += erts_port_driver_callback_epilogue(pp, &state); if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; @@ -893,27 +1735,35 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } } start_time = 0; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); -#ifdef ERTS_SMP - if (pp->xports) - erts_smp_xports_unlock(pp); - ASSERT(!pp->xports); -#endif + aborted_port_task: + schedule_port_task_free(ptp); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + begin_handle_tasks: + if (state & ERTS_PORT_SFLG_FREE) { + reds += ERTS_PORT_REDS_FREE; + begin_port_cleanup(pp, &execq, &processing_busy_q); - port_task_free(ptp); + break; + } - erts_smp_runq_lock(runq); + vreds += ERTS_PORT_CALLBACK_VREDS; + reds += ERTS_PORT_CALLBACK_VREDS; - ptp = pop_task(ptqp); - } + pp->reds += reds; + reds = 0; - tasks_done: + if (pp->reds >= CONTEXT_REDS) + break; + } erts_unblock_fpe(fpe_was_unmasked); + /* trace port scheduling, out */ + if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { + trace_sched_ports(pp, am_out); + } + if (io_tasks_executed) { ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) >= io_tasks_executed); @@ -921,81 +1771,59 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) -1*io_tasks_executed); } - *curr_port_pp = NULL; - #ifdef ERTS_SMP ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); #endif - if (!pp->sched.taskq) { - ASSERT(pp->sched.exe_taskq); - pp->sched.exe_taskq = NULL; + active = finalize_exec(pp, &execq, processing_busy_q); + + reds = pp->reds - vreds; + + erts_port_release(pp); + + *curr_port_pp = NULL; + + erts_smp_runq_lock(runq); + + if (!active) { + if (erts_system_profile_flags.runnable_ports) + profile_runnable_port(pp, am_inactive); } else { #ifdef ERTS_SMP ErtsRunQueue *xrunq; #endif - ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD)); - ASSERT(pp->sched.taskq->first); + ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); #ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); if (!xrunq) { #endif enqueue_port(runq, pp); - ASSERT(pp->sched.exe_taskq); - pp->sched.exe_taskq = NULL; /* No need to notify ourselves about inc in runq. */ #ifdef ERTS_SMP } else { /* Port emigrated ... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_smp_runq_unlock(runq); + + xrunq = erts_port_runq(pp); + ASSERT(xrunq); enqueue_port(xrunq, pp); - ASSERT(pp->sched.exe_taskq); - pp->sched.exe_taskq = NULL; erts_smp_runq_unlock(xrunq); erts_smp_notify_inc_runq(xrunq); + + erts_smp_runq_lock(runq); } #endif - port_was_enqueued = 1; } + done: res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); - ERTS_PT_CHK_PRES_PORTQ(runq, pp); - - port_taskq_free(ptqp); - - if (erts_system_profile_flags.runnable_ports && (port_was_enqueued != 1)) { - profile_runnable_port(pp, am_inactive); - } - - /* trace port scheduling, out */ - if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { - trace_sched_ports(pp, am_out); - } -#ifndef ERTS_SMP - erts_port_release(pp); -#else - { - erts_aint_t refc; - erts_smp_mtx_unlock(pp->lock); - refc = erts_smp_atomic_dec_read_nob(&pp->refc); - ASSERT(refc >= 0); - if (refc == 0) { - erts_smp_runq_unlock(runq); - erts_port_cleanup(pp); /* Might aquire runq lock */ - erts_smp_runq_lock(runq); - res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) - != (erts_aint_t) 0); - } - } -#endif - - done: runq->scheduler->reductions += reds; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -1004,121 +1832,260 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) return res; } -/* - * Handle remaining tasks after a free task. - */ +#ifdef ERTS_SMP +static void +release_port(void *vport) +{ + erts_port_dec_refc((Port *) vport); +} static void -handle_remaining_tasks(ErtsRunQueue *runq, Port *pp) +schedule_release_port(void *vport) { + Port *pp = (Port*)vport; + /* This is only used when a port release was ordered from a non-scheduler */ + erts_schedule_thr_prgr_later_op(release_port, + (void *) pp, + &pp->common.u.release); +} + +#endif + +static void +begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) { - int i; - ErtsPortTask *ptp; - ErtsPortTaskQueue *ptqps[] = {pp->sched.exe_taskq, pp->sched.taskq}; + int i, max; + ErtsPortTaskBusyCallerTable *tabp; + ErtsPortTask *qs[3]; + ErtsPortTaskHandleList *free_nshp = NULL; + ErtsProcList *plp; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); - for (i = 0; i < sizeof(ptqps)/sizeof(ErtsPortTaskQueue *); i++) { - if (!ptqps[i]) - continue; + /* + * Abort remaining tasks... + * + * We want to process queues in the following order in order + * to preserve signal ordering guarantees: + * 1. Local busy queue + * 2. Local queue + * 3. In queue + */ + + max = 0; + if (!execqp) { + if (pp->sched.taskq.local.busy.first) + qs[max++] = pp->sched.taskq.local.busy.first; + if (pp->sched.taskq.local.first) + qs[max++] = pp->sched.taskq.local.first; + } + else { + if (*processing_busy_q_p) { + if (*execqp) + qs[max++] = *execqp; + if (pp->sched.taskq.local.first) + qs[max++] = pp->sched.taskq.local.first; + } + else { + if (pp->sched.taskq.local.busy.first) + qs[max++] = pp->sched.taskq.local.busy.first; + if (*execqp) + qs[max++] = *execqp; + } + *execqp = NULL; + *processing_busy_q_p = 0; + } + pp->sched.taskq.local.busy.first = NULL; + pp->sched.taskq.local.busy.last = NULL; + pp->sched.taskq.local.first = NULL; + tabp = pp->sched.taskq.local.busy.table; + if (tabp) { + int bix; + for (bix = 0; bix < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; bix++) { + ErtsPortTaskBusyCaller *bcp = tabp->bucket[bix]; + while (bcp) { + ErtsPortTaskBusyCaller *free_bcp = bcp; + bcp = bcp->next; + if (free_bcp != &tabp->pre_alloc_busy_caller) + erts_free(ERTS_ALC_T_BUSY_CALLER, free_bcp); + } + } + + busy_caller_table_free(tabp); + pp->sched.taskq.local.busy.table = NULL; + } + + erts_port_task_sched_lock(&pp->sched); + qs[max] = pp->sched.taskq.in.first; + pp->sched.taskq.in.first = NULL; + pp->sched.taskq.in.last = NULL; + erts_port_task_sched_unlock(&pp->sched); + if (qs[max]) + max++; + + for (i = 0; i < max; i++) { + while (1) { + erts_aint32_t state; + ErtsPortTask *ptp = qs[i]; + if (!ptp) + break; + + qs[i] = ptp->u.alive.next; + + /* Normal case here is aborted tasks... */ + state = erts_smp_atomic32_read_nob(&ptp->state); + if (state == ERTS_PT_STATE_ABORTED) + goto aborted_port_task; + + state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + ERTS_PT_STATE_EXECUTING, + ERTS_PT_STATE_SCHEDULED); + if (state != ERTS_PT_STATE_SCHEDULED) { + ASSERT(state == ERTS_PT_STATE_ABORTED); + goto aborted_port_task; + } - ptp = pop_task(ptqps[i]); - while (ptp) { reset_handle(ptp); - erts_smp_runq_unlock(runq); switch (ptp->type) { - case ERTS_PORT_TASK_FREE: case ERTS_PORT_TASK_TIMEOUT: break; case ERTS_PORT_TASK_INPUT: - erts_stale_drv_select(pp->id, ptp->event, DO_READ, 1); + erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), + ptp->u.alive.td.io.event, + DO_READ, + 1); break; case ERTS_PORT_TASK_OUTPUT: - erts_stale_drv_select(pp->id, ptp->event, DO_WRITE, 1); + erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), + ptp->u.alive.td.io.event, + DO_WRITE, + 1); break; case ERTS_PORT_TASK_EVENT: - erts_stale_drv_select(pp->id, ptp->event, 0, 1); + erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), + ptp->u.alive.td.io.event, + 0, + 1); break; case ERTS_PORT_TASK_DIST_CMD: break; + case ERTS_PORT_TASK_PROC_SIG: { + ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data; + if (!pp->sched.taskq.bpq) + ptp->u.alive.td.psig.callback(NULL, + ERTS_PORT_SFLG_INVALID, + ERTS_PROC2PORT_SIG_ABORT_CLOSED, + sigdp); + else { + ErlDrvSizeT size = erts_proc2port_sig_command_data_size(sigdp); + ptp->u.alive.td.psig.callback(NULL, + ERTS_PORT_SFLG_INVALID, + ERTS_PROC2PORT_SIG_ABORT_CLOSED, + sigdp); + aborted_proc2port_data(pp, size); + } + break; + } default: erl_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); } - port_task_free(ptp); + aborted_port_task: + schedule_port_task_free(ptp); + } + } + + erts_smp_atomic32_read_band_nob(&pp->sched.flags, + ~(ERTS_PTS_FLG_HAVE_BUSY_TASKS + |ERTS_PTS_FLG_HAVE_TASKS + |ERTS_PTS_FLGS_BUSY)); - erts_smp_runq_lock(runq); - ptp = pop_task(ptqps[i]); + erts_port_task_sched_lock(&pp->sched); + + /* Cleanup nosuspend handles... */ + free_nshp = (pp->sched.taskq.local.busy.nosuspend + ? get_free_nosuspend_handles(pp) + : NULL); + ASSERT(!pp->sched.taskq.local.busy.nosuspend); + + /* Make sure not to leave any processes suspended on the port... */ + plp = pp->suspended; + pp->suspended = NULL; + + erts_port_task_sched_unlock(&pp->sched); + + if (free_nshp) + free_nosuspend_handles(free_nshp); + + if (erts_proclist_fetch(&plp, NULL)) { +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_port_unblocked)) { + DTRACE_CHARBUF(port_str, 16); + DTRACE_CHARBUF(pid_str, 16); + ErtsProcList* plp2 = plp; + + erts_snprintf(port_str, sizeof(port_str), "%T", pp->common.id); + while (plp2 != NULL) { + erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + DTRACE2(process_port_unblocked, pid_str, port_str); + } } +#endif + erts_resume_processes(plp); } - ASSERT(!pp->sched.taskq || !pp->sched.taskq->first); + /* + * Schedule cleanup of port structure... + */ +#ifdef ERTS_SMP + /* We might not be a scheduler, eg. traceing to port we are sys_msg_dispatcher */ + if (!erts_get_scheduler_data()) { + erts_schedule_misc_aux_work(1, schedule_release_port, (void*)pp); + } else { + /* Has to be more or less immediate to release any driver */ + erts_schedule_thr_prgr_later_op(release_port, + (void *) pp, + &pp->common.u.release); + } +#else + pp->cleanup = 1; +#endif } int erts_port_is_scheduled(Port *pp) { - int res; - ErtsRunQueue *runq = erts_port_runq(pp); - res = pp->sched.taskq || pp->sched.exe_taskq; - erts_smp_runq_unlock(runq); - return res; + erts_aint32_t flags = erts_smp_atomic32_read_acqb(&pp->sched.flags); + return (flags & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)) != 0; } #ifdef ERTS_SMP -ErtsMigrateResult -erts_port_migrate(Port *prt, int *prt_locked, - ErtsRunQueue *from_rq, int *from_locked, - ErtsRunQueue *to_rq, int *to_locked) +void +erts_enqueue_port(ErtsRunQueue *rq, Port *pp) { - ERTS_SMP_LC_ASSERT(*from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - if (!*from_locked || !*to_locked) { - if (from_rq < to_rq) { - if (!*to_locked) { - if (!*from_locked) - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - else if (erts_smp_runq_trylock(from_rq) == EBUSY) { - erts_smp_runq_unlock(to_rq); - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - } - else { - if (!*from_locked) { - if (!*to_locked) - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - else if (erts_smp_runq_trylock(to_rq) == EBUSY) { - erts_smp_runq_unlock(from_rq); - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - } - *to_locked = *from_locked = 1; - } - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - /* Refuse to migrate to a suspended run queue */ - if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue)) - return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; - if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) - return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; - dequeue_port(from_rq, prt); - erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) to_rq); - enqueue_port(to_rq, prt); - return ERTS_MIGRATE_SUCCESS; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); + ASSERT(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ); + enqueue_port(rq, pp); +} + +Port * +erts_dequeue_port(ErtsRunQueue *rq) +{ + Port *pp; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + pp = pop_port(rq); + ASSERT(!pp + || rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); + ASSERT(!pp || (erts_smp_atomic32_read_nob(&pp->sched.flags) + & ERTS_PTS_FLG_IN_RUNQ)); + return pp; } #endif @@ -1132,5 +2099,5 @@ erts_port_task_init(void) erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0); init_port_task_alloc(); - init_port_taskq_alloc(); + init_busy_caller_table_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index d7104e1143..e4d964146e 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2011. All Rights Reserved. + * Copyright Ericsson AB 2006-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -27,6 +27,9 @@ #define ERTS_PORT_TASK_H_BASIC_TYPES__ #include "erl_sys_driver.h" #include "erl_smp.h" +#define ERL_PORT_GET_PORT_TYPE_ONLY__ +#include "erl_port.h" +#undef ERL_PORT_GET_PORT_TYPE_ONLY__ typedef erts_smp_atomic_t ErtsPortTaskHandle; #endif @@ -43,13 +46,19 @@ typedef erts_smp_atomic_t ErtsPortTaskHandle; #define ERTS_INCLUDE_SCHEDULER_INTERNALS #endif +#define ERTS_PT_FLG_WAIT_BUSY (1 << 0) +#define ERTS_PT_FLG_SIG_DEP (1 << 1) +#define ERTS_PT_FLG_NOSUSPEND (1 << 2) +#define ERTS_PT_FLG_REF (1 << 3) +#define ERTS_PT_FLG_BAD_OUTPUT (1 << 4) + typedef enum { - ERTS_PORT_TASK_FREE, ERTS_PORT_TASK_INPUT, ERTS_PORT_TASK_OUTPUT, ERTS_PORT_TASK_EVENT, ERTS_PORT_TASK_TIMEOUT, - ERTS_PORT_TASK_DIST_CMD + ERTS_PORT_TASK_DIST_CMD, + ERTS_PORT_TASK_PROC_SIG } ErtsPortTaskType; #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS @@ -57,19 +66,76 @@ typedef enum { extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks; #endif +#define ERTS_PTS_FLG_IN_RUNQ (((erts_aint32_t) 1) << 0) +#define ERTS_PTS_FLG_EXEC (((erts_aint32_t) 1) << 1) +#define ERTS_PTS_FLG_HAVE_TASKS (((erts_aint32_t) 1) << 2) +#define ERTS_PTS_FLG_EXIT (((erts_aint32_t) 1) << 3) +#define ERTS_PTS_FLG_BUSY_PORT (((erts_aint32_t) 1) << 4) +#define ERTS_PTS_FLG_BUSY_PORT_Q (((erts_aint32_t) 1) << 5) +#define ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q (((erts_aint32_t) 1) << 6) +#define ERTS_PTS_FLG_HAVE_BUSY_TASKS (((erts_aint32_t) 1) << 7) +#define ERTS_PTS_FLG_HAVE_NS_TASKS (((erts_aint32_t) 1) << 8) +#define ERTS_PTS_FLG_PARALLELISM (((erts_aint32_t) 1) << 9) +#define ERTS_PTS_FLG_FORCE_SCHED (((erts_aint32_t) 1) << 10) + +#define ERTS_PTS_FLGS_BUSY \ + (ERTS_PTS_FLG_BUSY_PORT | ERTS_PTS_FLG_BUSY_PORT_Q) + +#define ERTS_PTS_FLGS_FORCE_SCHEDULE_OP \ + (ERTS_PTS_FLG_EXIT \ + | ERTS_PTS_FLG_HAVE_BUSY_TASKS \ + | ERTS_PTS_FLG_HAVE_TASKS \ + | ERTS_PTS_FLG_EXEC \ + | ERTS_PTS_FLG_FORCE_SCHED) + +#define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH 8192 +#define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW 4096 + +typedef struct { + ErlDrvSizeT high; + erts_smp_atomic_t low; + erts_smp_atomic_t size; +} ErtsPortTaskBusyPortQ; + typedef struct ErtsPortTask_ ErtsPortTask; -typedef struct ErtsPortTaskQueue_ ErtsPortTaskQueue; +typedef struct ErtsPortTaskBusyCallerTable_ ErtsPortTaskBusyCallerTable; +typedef struct ErtsPortTaskHandleList_ ErtsPortTaskHandleList; typedef struct { Port *next; - Port *prev; - ErtsPortTaskQueue *taskq; - ErtsPortTaskQueue *exe_taskq; + struct { + struct { + struct { + ErtsPortTask *first; + ErtsPortTask *last; + ErtsPortTaskBusyCallerTable *table; + ErtsPortTaskHandleList *nosuspend; + } busy; + ErtsPortTask *first; + } local; + struct { + ErtsPortTask *first; + ErtsPortTask *last; + } in; + ErtsPortTaskBusyPortQ *bpq; + } taskq; + erts_smp_atomic32_t flags; +#ifdef ERTS_SMP + erts_mtx_t mtx; +#endif } ErtsPortTaskSched; ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp); ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp); -ERTS_GLB_INLINE void erts_port_task_init_sched(ErtsPortTaskSched *ptsp); +ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, + ErtsPortTaskBusyPortQ *bpq); +ERTS_GLB_INLINE void erts_port_task_init_sched(ErtsPortTaskSched *ptsp, + Eterm id); +ERTS_GLB_INLINE void erts_port_task_fini_sched(ErtsPortTaskSched *ptsp); +ERTS_GLB_INLINE void erts_port_task_sched_lock(ErtsPortTaskSched *ptsp); +ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp); +ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp); + #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); #endif @@ -88,13 +154,75 @@ erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp) return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL; } +ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, + ErtsPortTaskBusyPortQ *bpq) +{ + if (bpq) { + erts_aint_t low = (erts_aint_t) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW; + erts_smp_atomic_init_nob(&bpq->low, low); + bpq->high = (ErlDrvSizeT) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH; + erts_smp_atomic_init_nob(&bpq->size, (erts_aint_t) 0); + } + ptsp->taskq.bpq = bpq; +} + ERTS_GLB_INLINE void -erts_port_task_init_sched(ErtsPortTaskSched *ptsp) +erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) { +#ifdef ERTS_SMP + char *lock_str = "port_sched_lock"; +#endif ptsp->next = NULL; - ptsp->prev = NULL; - ptsp->taskq = NULL; - ptsp->exe_taskq = NULL; + ptsp->taskq.local.busy.first = NULL; + ptsp->taskq.local.busy.last = NULL; + ptsp->taskq.local.busy.table = NULL; + ptsp->taskq.local.busy.nosuspend = NULL; + ptsp->taskq.local.first = NULL; + ptsp->taskq.in.first = NULL; + ptsp->taskq.in.last = NULL; + erts_smp_atomic32_init_nob(&ptsp->flags, 0); +#ifdef ERTS_SMP +#ifdef ERTS_ENABLE_LOCK_COUNT + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; +#endif + erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id); +#endif +} + +ERTS_GLB_INLINE void +erts_port_task_sched_lock(ErtsPortTaskSched *ptsp) +{ +#ifdef ERTS_SMP + erts_mtx_lock(&ptsp->mtx); +#endif +} + +ERTS_GLB_INLINE void +erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp) +{ +#ifdef ERTS_SMP + erts_mtx_unlock(&ptsp->mtx); +#endif +} + +ERTS_GLB_INLINE int +erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp) +{ +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + return erts_lc_mtx_is_locked(&ptsp->mtx); +#else + return 0; +#endif +} + + +ERTS_GLB_INLINE void +erts_port_task_fini_sched(ErtsPortTaskSched *ptsp) +{ +#ifdef ERTS_SMP + erts_mtx_destroy(&ptsp->mtx); +#endif } #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS @@ -115,21 +243,22 @@ int erts_port_task_execute(ErtsRunQueue *, Port **); void erts_port_task_init(void); #endif -int erts_port_task_abort(Eterm id, ErtsPortTaskHandle *); +void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *); +int erts_port_task_abort(ErtsPortTaskHandle *); + +void erts_port_task_abort_nosuspend_tasks(Port *); + int erts_port_task_schedule(Eterm, ErtsPortTaskHandle *, ErtsPortTaskType, - ErlDrvEvent, - ErlDrvEventData); + ...); void erts_port_task_free_port(Port *); int erts_port_is_scheduled(Port *); +ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void); + #ifdef ERTS_SMP -ErtsMigrateResult erts_port_migrate(Port *, - int *, - ErtsRunQueue *, - int *, - ErtsRunQueue *, - int *); +void erts_enqueue_port(ErtsRunQueue *rq, Port *pp); +Port *erts_dequeue_port(ErtsRunQueue *rq); #endif #undef ERTS_INCLUDE_SCHEDULER_INTERNALS #endif /* ERL_PORT_TASK_H__ */ diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 34da9cab84..436147749e 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2011. All Rights Reserved. + * Copyright Ericsson AB 2005-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -57,17 +57,17 @@ do { \ (CNT) += res__; \ } while (0) -#define PRINT_ULONG(CNT, FN, ARG, C, P, W, I) \ +#define PRINT_UWORD(CNT, FN, ARG, C, P, W, I) \ do { \ - int res__ = erts_printf_ulong((FN), (ARG), (C), (P), (W), (I)); \ + int res__ = erts_printf_uword((FN), (ARG), (C), (P), (W), (I)); \ if (res__ < 0) \ return res__; \ (CNT) += res__; \ } while (0) -#define PRINT_SLONG(CNT, FN, ARG, C, P, W, I) \ +#define PRINT_SWORD(CNT, FN, ARG, C, P, W, I) \ do { \ - int res__ = erts_printf_slong((FN), (ARG), (C), (P), (W), (I)); \ + int res__ = erts_printf_sword((FN), (ARG), (C), (P), (W), (I)); \ if (res__ < 0) \ return res__; \ (CNT) += res__; \ @@ -153,7 +153,7 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) if ((i < 0) || (i >= atom_table_size()) || (atom_tab(i) == NULL)) { PRINT_STRING(res, fn, arg, "<bad atom index: "); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) i); + PRINT_SWORD(res, fn, arg, 'd', 0, 1, (ErlPfSWord) i); PRINT_CHAR(res, fn, arg, '>'); return res; } @@ -203,7 +203,7 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) default: if (IS_CNTRL(c)) { PRINT_CHAR(res, fn, arg, '\\'); - PRINT_ULONG(res, fn, arg, 'o', 1, 3, (unsigned long) c); + PRINT_UWORD(res, fn, arg, 'o', 1, 3, (ErlPfUWord) c); } else PRINT_CHAR(res, fn, arg, (char) c); @@ -334,7 +334,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, break; } case SMALL_DEF: - PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj)); + PRINT_SWORD(res, fn, arg, 'd', 0, 1, (ErlPfSWord) signed_val(obj)); break; case BIG_DEF: { int print_res; @@ -360,36 +360,36 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, case REF_DEF: case EXTERNAL_REF_DEF: PRINT_STRING(res, fn, arg, "#Ref<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) ref_channel_no(wobj)); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, + (ErlPfUWord) ref_channel_no(wobj)); ref_num = ref_numbers(wobj); for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) { PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) ref_num[i]); } PRINT_CHAR(res, fn, arg, '>'); break; case PID_DEF: case EXTERNAL_PID_DEF: PRINT_CHAR(res, fn, arg, '<'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_channel_no(wobj)); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, + (ErlPfUWord) pid_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_number(wobj)); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, + (ErlPfUWord) pid_number(wobj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_serial(wobj)); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, + (ErlPfUWord) pid_serial(wobj)); PRINT_CHAR(res, fn, arg, '>'); break; case PORT_DEF: case EXTERNAL_PORT_DEF: PRINT_STRING(res, fn, arg, "#Port<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_channel_no(wobj)); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, + (ErlPfUWord) port_channel_no(wobj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_number(wobj)); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, + (ErlPfUWord) port_number(wobj)); PRINT_CHAR(res, fn, arg, '>'); break; case LIST_DEF: @@ -437,13 +437,16 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } break; case BINARY_DEF: - { + if (header_is_bin_matchstate(*boxed_val(wobj))) { + PRINT_STRING(res, fn, arg, "#MatchState"); + } + else { ProcBin* pb = (ProcBin *) binary_val(wobj); if (pb->size == 1) PRINT_STRING(res, fn, arg, "<<1 byte>>"); else { PRINT_STRING(res, fn, arg, "<<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) pb->size); PRINT_STRING(res, fn, arg, " bytes>>"); } } @@ -459,8 +462,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '.'); PRINT_BUF(res, fn, arg, name->name, name->len); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) ep->code[2]); + PRINT_SWORD(res, fn, arg, 'd', 0, 1, + (ErlPfSWord) ep->code[2]); PRINT_CHAR(res, fn, arg, '>'); } break; @@ -472,11 +475,11 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_STRING(res, fn, arg, "#Fun<"); PRINT_BUF(res, fn, arg, ap->name, ap->len); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_index); + PRINT_SWORD(res, fn, arg, 'd', 0, 1, + (ErlPfSWord) funp->fe->old_index); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_uniq); + PRINT_SWORD(res, fn, arg, 'd', 0, 1, + (ErlPfSWord) funp->fe->old_uniq); PRINT_CHAR(res, fn, arg, '>'); } break; @@ -495,10 +498,13 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } int -erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, - unsigned long* term_base) +erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, + ErlPfEterm* term_base) { - int res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); + int res; + ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); + + res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); if (res < 0) return res; if (precision <= 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index a48a3de34c..f92c99d713 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2011. All Rights Reserved. + * Copyright Ericsson AB 2005-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,6 @@ #define ERL_PRINTF_TERM_H__ #include "erl_printf_format.h" -int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision, - unsigned long* term_base); +int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, + ErlPfEterm* term_base); #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index d7d440d81a..434d5ca147 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -42,6 +42,7 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "dtrace-wrapper.h" +#include "erl_ptab.h" #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2) @@ -97,34 +98,63 @@ #define HIGH_BIT (1 << PRIORITY_HIGH) #define NORMAL_BIT (1 << PRIORITY_NORMAL) #define LOW_BIT (1 << PRIORITY_LOW) +#define PORT_BIT (1 << ERTS_PORT_PRIO_LEVEL) -#define ERTS_MAYBE_SAVE_TERMINATING_PROCESS(P) \ -do { \ - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); \ - if (saved_term_procs.end) \ - save_terminating_process((P)); \ -} while (0) +#define ERTS_EMPTY_RUNQ(RQ) \ + ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0 \ + && (RQ)->misc.start == NULL) + +#undef RUNQ_READ_RQ +#undef RUNQ_SET_RQ +#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_smp_atomic_read_nob((X))) +#define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ)) -#define ERTS_EMPTY_RUNQ(RQ) \ - ((RQ)->len == 0 && (RQ)->misc.start == NULL) +#ifdef DEBUG +# if defined(ARCH_64) && !HALFWORD_HEAP +# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ + (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) +# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ +do { \ + ASSERT((RQP) != NULL); \ + ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \ + ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000LL));\ +} while (0) +# else +# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ + (RUNQ_SET_RQ((RQP), (0xdead0003 | ((N) << 4)))) +# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ +do { \ + ASSERT((RQP) != NULL); \ + ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ + ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ +} while (0) +# endif +#else +# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) +# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) +#endif #define ERTS_EMPTY_RUNQ_PORTS(RQ) \ - ((RQ)->ports.info.len == 0 && (RQ)->misc.start == NULL) + (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL) + +const Process erts_invalid_process = {{ERTS_INVALID_PID}}; extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -static Sint p_last; -static Sint p_next; -static Sint p_serial; -static Uint p_serial_mask; -static Uint p_serial_shift; - int erts_sched_compact_load; Uint erts_no_schedulers; -Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; -Uint erts_process_tab_index_mask; + +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY (4*1024*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY (512*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM (64*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_EAGER (16*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_EAGER (1024) + +static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM; + +ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); int erts_sched_thread_suggested_stack_size = -1; @@ -191,7 +221,7 @@ static struct { struct { int active_runqs; int reds; - int max_len; + erts_aint32_t max_len; } prev_rise; Uint n; } balance_info; @@ -211,8 +241,6 @@ erts_sched_stat_t erts_sched_stat; static erts_tsd_key_t sched_data_key; #endif -static erts_smp_mtx_t proc_tab_mtx; - static erts_smp_atomic32_t function_calls; #ifdef ERTS_SMP @@ -238,7 +266,6 @@ typedef union { static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -Process** process_tab; static Uint last_reductions; static Uint last_exact_reductions; Uint erts_default_process_flags; @@ -255,29 +282,6 @@ struct erts_system_profile_flags_t erts_system_profile_flags; #if ERTS_MAX_PROCESSES > 0x7fffffff #error "Need to store process_count in another type" #endif -static erts_smp_atomic32_t process_count; - -typedef struct ErtsTermProcElement_ ErtsTermProcElement; -struct ErtsTermProcElement_ { - ErtsTermProcElement *next; - ErtsTermProcElement *prev; - int ix; - union { - struct { - Eterm pid; - SysTimeval spawned; - SysTimeval exited; - } process; - struct { - SysTimeval time; - } bif_invocation; - } u; -}; - -static struct { - ErtsTermProcElement *start; - ErtsTermProcElement *end; -} saved_term_procs; ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_op_list, ErtsMiscOpList, @@ -339,8 +343,6 @@ do { \ * Local functions. */ -static void init_processes_bif(void); -static void save_terminating_process(Process *p); static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, @@ -371,6 +373,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; + valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; #endif #if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; @@ -407,6 +410,31 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) } #endif + +static ERTS_INLINE Uint64 +ensure_later_proc_interval(Uint64 interval) +{ + return erts_smp_ensure_later_interval_nob(erts_ptab_interval(&erts_proc), interval); +} + +Uint64 +erts_get_proc_interval(void) +{ + return erts_smp_current_interval_nob(erts_ptab_interval(&erts_proc)); +} + +Uint64 +erts_ensure_later_proc_interval(Uint64 interval) +{ + return ensure_later_proc_interval(interval); +} + +Uint64 +erts_step_proc_interval(void) +{ + return erts_smp_step_interval_nob(erts_ptab_interval(&erts_proc)); +} + void erts_pre_init_process(void) { @@ -452,11 +480,18 @@ erts_pre_init_process(void) #endif } +#ifdef ERTS_SMP +static void +release_process(void *vproc) +{ + erts_smp_proc_dec_refc((Process *) vproc); +} +#endif + /* initialize the scheduler */ void -erts_init_process(int ncpu) +erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) { - Uint proc_bits = ERTS_PROC_BITS; #ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; @@ -465,25 +500,19 @@ erts_init_process(int ncpu) init_proclist_alloc(); - erts_smp_atomic32_init_nob(&process_count, 0); - - if (erts_use_r9_pids_ports) { - proc_bits = ERTS_R9_PROC_BITS; - ASSERT(erts_max_processes <= (1 << ERTS_R9_PROC_BITS)); - } - - process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE, - erts_max_processes*sizeof(Process*)); - sys_memzero(process_tab, erts_max_processes * sizeof(Process*)); - - erts_smp_mtx_init(&proc_tab_mtx, "proc_tab"); - p_last = -1; - p_next = 0; - p_serial = 0; + erts_ptab_init_table(&erts_proc, + ERTS_ALC_T_PROC_TABLE, +#ifdef ERTS_SMP + release_process, +#else + NULL, +#endif + (ErtsPTabElementCommon *) &erts_invalid_process.common, + proc_tab_size, + sizeof(Process), + "process_table", + legacy_proc_tab); - p_serial_shift = erts_fit_in_bits(erts_max_processes - 1); - p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift); - erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift); last_reductions = 0; last_exact_reductions = 0; erts_default_process_flags = 0; @@ -493,7 +522,6 @@ void erts_late_init_process(void) { int ix; - init_processes_bif(); erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat"); for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) { @@ -741,8 +769,9 @@ static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { ErtsProcList *plp = proclist_alloc(); - plp->pid = p->id; - plp->started = p->started; + ensure_later_proc_interval(p->common.u.alive.started_interval); + plp->pid = p->common.id; + plp->started_interval = p->common.u.alive.started_interval; return plp; } @@ -752,13 +781,6 @@ proclist_destroy(ErtsProcList *plp) proclist_free(plp); } -static ERTS_INLINE int -proclist_same(ErtsProcList *plp, Process *p) -{ - return (plp->pid == p->id - && erts_cmp_timeval(&plp->started, &p->started) == 0); -} - ErtsProcList * erts_proclist_create(Process *p) { @@ -771,12 +793,6 @@ erts_proclist_destroy(ErtsProcList *plp) proclist_destroy(plp); } -int -erts_proclist_same(ErtsProcList *plp, Process *p) -{ - return proclist_same(plp, p); -} - void * erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) { @@ -908,6 +924,53 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) #ifdef ERTS_SMP static ERTS_INLINE void +haw_chk_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) +{ + if (awdp->later_op.first + && erts_thr_progress_cmp(val, awdp->later_op.thr_prgr) >= 0) { + awdp->later_op.size = thr_prgr_later_cleanup_op_threshold; + } +} + +static ERTS_INLINE void +haw_thr_prgr_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) +{ + int cmp = erts_thr_progress_cmp(val, awdp->latest_wakeup); + if (cmp != 0) { + if (cmp > 0) { + awdp->latest_wakeup = val; + haw_chk_later_cleanup_op_wakeup(awdp, val); + } + erts_thr_progress_wakeup(awdp->esdp, val); + } +} + +static ERTS_INLINE void +haw_thr_prgr_soft_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) +{ + if (erts_thr_progress_cmp(val, awdp->latest_wakeup) > 0) { + awdp->latest_wakeup = val; + haw_chk_later_cleanup_op_wakeup(awdp, val); + erts_thr_progress_wakeup(awdp->esdp, val); + } +} + +static ERTS_INLINE void +haw_thr_prgr_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val, UWord size) +{ + if (erts_thr_progress_cmp(val, awdp->latest_wakeup) > 0) { + awdp->later_op.thr_prgr = val; + if (awdp->later_op.size > size) + awdp->later_op.size -= size; + else { + awdp->latest_wakeup = val; + awdp->later_op.size = thr_prgr_later_cleanup_op_threshold; + erts_thr_progress_wakeup(awdp->esdp, val); + } + } +} + +static ERTS_INLINE void haw_thr_prgr_current_reset(ErtsAuxWorkData *awdp) { awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID; @@ -1065,8 +1128,7 @@ misc_aux_work_clean(ErtsThrQ_t *q, case ERTS_THR_Q_NEED_THR_PRGR: #ifdef ERTS_SMP set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); - erts_thr_progress_wakeup(awdp->esdp, - erts_thr_q_need_thr_progress(q)); + haw_thr_prgr_soft_wakeup(awdp, erts_thr_q_need_thr_progress(q)); #endif case ERTS_THR_Q_CLEAN: break; @@ -1224,8 +1286,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #ifdef ERTS_SMP case ERTS_ASYNC_READY_NEED_THR_PRGR: - erts_thr_progress_wakeup(awdp->esdp, - awdp->async_ready.thr_prgr); + haw_thr_prgr_soft_wakeup(awdp, awdp->async_ready.thr_prgr); awdp->async_ready.need_thr_prgr = 1; return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #endif @@ -1234,7 +1295,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, } } -#endif +#endif /* ERTS_USE_ASYNC_READY_Q */ + static ERTS_INLINE erts_aint32_t handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) @@ -1270,6 +1332,17 @@ erts_alloc_notify_delayed_dealloc(int ix) ERTS_SSI_AUX_WORK_DD); } +void +erts_alloc_ensure_handle_delayed_dealloc_call(int ix) +{ +#ifdef DEBUG + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(!esdp || ix == (int) esdp->no); +#endif + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_DD); +} + static ERTS_INLINE erts_aint32_t handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { @@ -1298,7 +1371,7 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin awdp->dd.thr_prgr = wakeup; set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); awdp->dd.thr_prgr = wakeup; - erts_thr_progress_wakeup(awdp->esdp, wakeup); + haw_thr_prgr_soft_wakeup(awdp, wakeup); } else if (awdp->dd.completed_callback) { awdp->dd.completed_callback(awdp->dd.completed_arg); @@ -1339,7 +1412,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i if (wakeup == ERTS_THR_PRGR_INVALID) wakeup = erts_thr_progress_later(awdp->esdp); awdp->dd.thr_prgr = wakeup; - erts_thr_progress_wakeup(awdp->esdp, wakeup); + haw_thr_prgr_soft_wakeup(awdp, wakeup); } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); @@ -1353,6 +1426,94 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; } +/* + * Handle scheduled thread progress later operations. + */ +#define ERTS_MAX_THR_PRGR_LATER_OPS 50 + +static ERTS_INLINE erts_aint32_t +handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + int lops; + ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); + + for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) { + ErtsThrPrgrLaterOp *lop = awdp->later_op.first; + if (!erts_thr_progress_has_reached_this(current, lop->later)) + return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; + awdp->later_op.first = lop->next; + if (!awdp->later_op.first) { + awdp->later_op.last = NULL; + } + lop->func(lop->data); + if (!awdp->later_op.first) { + awdp->later_op.size = thr_prgr_later_cleanup_op_threshold; + awdp->later_op.last = NULL; + unset_aux_work_flags(awdp->ssi, + ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); + return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; + } + } + + return aux_work; +} + +static ERTS_INLINE ErtsThrPrgrVal +enqueue_later_op(ErtsSchedulerData *esdp, + void (*later_func)(void *), + void *later_data, + ErtsThrPrgrLaterOp *lop) +{ + ErtsThrPrgrVal later = erts_thr_progress_later(esdp); + ASSERT(esdp); + + lop->func = later_func; + lop->data = later_data; + lop->later = later; + lop->next = NULL; + if (!esdp->aux_work_data.later_op.last) + esdp->aux_work_data.later_op.first = lop; + else + esdp->aux_work_data.later_op.last->next = lop; + esdp->aux_work_data.later_op.last = lop; + set_aux_work_flags_wakeup_nob(esdp->ssi, + ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); + return later; +} + +#endif /* ERTS_SMP */ + +void +erts_schedule_thr_prgr_later_op(void (*later_func)(void *), + void *later_data, + ErtsThrPrgrLaterOp *lop) +{ +#ifndef ERTS_SMP + later_func(later_data); +#else + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop); + haw_thr_prgr_wakeup(&esdp->aux_work_data, later); +#endif +} + +void +erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *), + void *later_data, + ErtsThrPrgrLaterOp *lop, + UWord size) +{ +#ifndef ERTS_SMP + later_func(later_data); +#else + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop); + haw_thr_prgr_later_cleanup_op_wakeup(&esdp->aux_work_data, later, size); +#endif +} + +#ifdef ERTS_SMP + static erts_atomic32_t completed_dealloc_count; static void @@ -1465,37 +1626,32 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); awdp->esdp->run_queue->halt_in_progress = 1; if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { - int i; + int i, max = erts_ptab_max(&erts_port); erts_smp_atomic32_set_nob(&erts_halt_progress, 1); - for (i = 0; i < erts_max_ports; i++) { - Port *prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - | ERTS_PORT_SFLG_HALT))) { - erts_smp_port_state_unlock(prt); - continue; - } - /* We need to set the halt flag - get the port lock */ -#ifdef ERTS_SMP - erts_smp_atomic_inc_nob(&prt->refc); -#endif - erts_smp_port_state_unlock(prt); -#ifdef ERTS_SMP - erts_smp_mtx_lock(prt->lock); -#endif - if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - | ERTS_PORT_SFLG_HALT))) { - erts_port_release(prt); + for (i = 0; i < max; i++) { + erts_aint32_t state; + Port *prt = erts_pix2port(i); + if (!prt) continue; - } - erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT); - erts_smp_atomic32_inc_nob(&erts_halt_progress); - if (prt->status & (ERTS_PORT_SFLG_EXITING - | ERTS_PORT_SFLG_CLOSING)) { - erts_port_release(prt); + state = erts_atomic32_read_acqb(&prt->state); + if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT)) continue; + + /* We need to set the halt flag - get the port lock */ + + erts_smp_port_lock(prt); + + state = erts_atomic32_read_nob(&prt->state); + if (!(state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT))) { + state = erts_atomic32_read_bor_relb(&prt->state, + ERTS_PORT_SFLG_HALT); + erts_smp_atomic32_inc_nob(&erts_halt_progress); + if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING))) + erts_deliver_port_exit(prt, prt->common.id, am_killed, 0); } - erts_do_exit_port(prt, prt->id, am_killed); + erts_port_release(prt); } if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { @@ -1578,6 +1734,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), handle_fix_alloc); +#ifdef ERTS_SMP + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP, + handle_thr_prgr_later_op); +#endif + #if ERTS_USE_ASYNC_READY_Q HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY, handle_async_ready); @@ -1738,7 +1899,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) ERTS_DBG_CHK_AUX_WORK_VAL(type); ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); -// erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); + /* erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); */ if (!enable) { old = erts_atomic32_read_band_mb(&aux_work_tmo->type[ix], ~type); @@ -1768,8 +1929,8 @@ sched_waiting_sys(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); ASSERT(rq->waiting >= 0); - rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); + (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); rq->waiting++; rq->waiting *= -1; rq->woken = 0; @@ -1845,8 +2006,8 @@ static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - rq->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); + (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); if (rq->waiting < 0) rq->waiting--; else @@ -1878,9 +2039,8 @@ ongoing_multi_scheduling_block(void) static ERTS_INLINE void empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_read_band_nob(&rq->info_flags, - ~ERTS_RUNQ_IFLG_NONEMPTY); - if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { + Uint32 old_flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED); + if (old_flags & ERTS_RUNQ_FLG_NONEMPTY) { #ifdef DEBUG erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* @@ -1902,9 +2062,8 @@ empty_runq(ErtsRunQueue *rq) static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { - erts_aint32_t oifls = erts_smp_atomic32_read_bor_nob(&rq->info_flags, - ERTS_RUNQ_IFLG_NONEMPTY); - if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { + Uint32 old_flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_NONEMPTY); + if (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY)) { #ifdef DEBUG erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* @@ -2496,6 +2655,8 @@ set_no_active_runqs(int active) erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; + if ((exp & ERTS_NO_RUNQS_MASK) == active) + break; new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); new |= active & ERTS_NO_RUNQS_MASK; act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); @@ -2525,19 +2686,16 @@ try_inc_no_active_runqs(int active) static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { - erts_aint32_t iflgs; + Uint32 flags; ErtsRunQueue *wrq; if (crq->ix == ix) return 0; wrq = ERTS_RUNQ_IX(ix); - iflgs = erts_smp_atomic32_read_nob(&wrq->info_flags); - if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { + flags = ERTS_RUNQ_FLGS_GET(wrq); + if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) { if (activate) { - if (try_inc_no_active_runqs(ix+1)) { - erts_smp_xrunq_lock(crq, wrq); - wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; - erts_smp_xrunq_unlock(crq, wrq); - } + if (try_inc_no_active_runqs(ix+1)) + (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); } wake_scheduler(wrq, 0); return 1; @@ -2604,9 +2762,7 @@ erts_sched_notify_check_cpu_bind(void) int ix; for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - erts_smp_runq_unlock(rq); + (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); wake_scheduler(rq, 0); } #else @@ -2615,409 +2771,544 @@ erts_sched_notify_check_cpu_bind(void) } -#ifdef ERTS_SMP +static ERTS_INLINE void +enqueue_process(ErtsRunQueue *runq, int prio, Process *p) +{ + ErtsRunPrioQueue *rpq; -ErtsRunQueue * -erts_prepare_emigrate(ErtsRunQueue *c_rq, ErtsRunQueueInfo *c_rqi, int prio) + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + + erts_smp_inc_runq_len(runq, &runq->procs.prio_info[prio], prio); + + if (prio == PRIORITY_LOW) { + p->schedule_count = RESCHEDULE_LOW; + rpq = &runq->procs.prio[PRIORITY_NORMAL]; + } + else { + p->schedule_count = 1; + rpq = &runq->procs.prio[prio]; + } + + p->next = NULL; + if (rpq->last) + rpq->last->next = p; + else + rpq->first = p; + rpq->last = p; +} + + +static ERTS_INLINE void +unqueue_process(ErtsRunQueue *runq, + ErtsRunPrioQueue *rpq, + ErtsRunQueueInfo *rqi, + int prio, + Process *prev_proc, + Process *proc) { - ASSERT(ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)); - ASSERT(ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) - || c_rqi->len >= c_rqi->migrate.limit.this); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - while (1) { - ErtsRunQueue *n_rq = c_rqi->migrate.runq; - ERTS_DBG_VERIFY_VALID_RUNQP(n_rq); - erts_smp_xrunq_lock(c_rq, n_rq); - - /* - * erts_smp_xrunq_lock() may release lock on c_rq! We have - * to check that we still want to emigrate and emigrate - * to the same run queue as before. - */ + if (prev_proc) + prev_proc->next = proc->next; + else + rpq->first = proc->next; + if (!proc->next) + rpq->last = prev_proc; - if (ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)) { - Uint32 force = (ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) - | (c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)); - if (force || c_rqi->len > c_rqi->migrate.limit.this) { - ErtsRunQueueInfo *n_rqi; - /* We still want to emigrate */ - - if (n_rq != c_rqi->migrate.runq) { - /* Ahh... run queue changed; need to do it all over again... */ - erts_smp_runq_unlock(n_rq); - continue; - } - else { + if (!rpq->first) + rpq->last = NULL; - if (prio == ERTS_PORT_PRIO_LEVEL) - n_rqi = &n_rq->ports.info; - else - n_rqi = &n_rq->procs.prio_info[prio]; + erts_smp_dec_runq_len(runq, rqi, prio); +} - if (force || (n_rqi->len < c_rqi->migrate.limit.other)) { - /* emigrate ... */ - return n_rq; - } - } - } - } - ASSERT(n_rq != c_rq); - erts_smp_runq_unlock(n_rq); - if (!(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { - /* No more emigrations to this runq */ - ERTS_UNSET_RUNQ_FLG_EMIGRATE(c_rq->flags, prio); - ERTS_DBG_SET_INVALID_RUNQP(c_rqi->migrate.runq, 0x3); - } +static ERTS_INLINE Process * +dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep) +{ + erts_aint32_t state; + int prio; + ErtsRunPrioQueue *rpq; + ErtsRunQueueInfo *rqi; + Process *p; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + + ASSERT(PRIORITY_NORMAL == prio_q + || PRIORITY_HIGH == prio_q + || PRIORITY_MAX == prio_q); + + rpq = &runq->procs.prio[prio_q]; + p = rpq->first; + if (!p) return NULL; + + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + + state = erts_smp_atomic32_read_nob(&p->state); + if (statep) + *statep = state; + + prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + + rqi = &runq->procs.prio_info[prio]; + + if (p) + unqueue_process(runq, rpq, rqi, prio, NULL, p); + + return p; +} + +static ERTS_INLINE int +check_requeue_process(ErtsRunQueue *rq, int prio_q) +{ + ErtsRunPrioQueue *rpq = &rq->procs.prio[prio_q]; + Process *p = rpq->first; + if (--p->schedule_count > 0 && p != rpq->last) { + /* reschedule */ + rpq->first = p->next; + rpq->last->next = p; + rpq->last = p; + p->next = NULL; + return 1; } + return 0; +} + +#ifdef ERTS_SMP + +static ErtsRunQueue * +check_immigration_need(ErtsRunQueue *c_rq, ErtsMigrationPath *mp, int prio) +{ + int len; + Uint32 f_flags, f_rq_flags; + ErtsRunQueue *f_rq; + + f_flags = mp->prio[prio].flags; + + ASSERT(ERTS_CHK_RUNQ_FLG_IMMIGRATE(mp->flags, prio)); + + f_rq = mp->prio[prio].runq; + if (!f_rq) + return NULL; + + f_rq_flags = ERTS_RUNQ_FLGS_GET(f_rq); + if (f_rq_flags & ERTS_RUNQ_FLG_PROTECTED) + return NULL; + + if (ERTS_CHK_RUNQ_FLG_EVACUATE(f_flags, prio)) + return f_rq; + + if (f_rq_flags & ERTS_RUNQ_FLG_INACTIVE) + return f_rq; + + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&c_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len); + + if (len < mp->prio[prio].limit.this) { + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&f_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&f_rq->procs.prio_info[prio].len); + + if (len > mp->prio[prio].limit.other) + return f_rq; + } + return NULL; } static void -immigrate(ErtsRunQueue *rq) +immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) { - int prio; + Uint32 iflags, iflag; + erts_smp_runq_unlock(c_rq); - ASSERT(rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK); + ASSERT(erts_thr_progress_is_managed_thread()); - for (prio = 0; prio < ERTS_NO_PRIO_LEVELS; prio++) { - if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio)) { - ErtsRunQueueInfo *rqi = (prio == ERTS_PORT_PRIO_LEVEL - ? &rq->ports.info - : &rq->procs.prio_info[prio]); - ErtsRunQueue *from_rq = rqi->migrate.runq; - int rq_locked, from_rq_locked; + iflags = mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; - ERTS_DBG_VERIFY_VALID_RUNQP(from_rq); + iflag = iflags & -iflags; - rq_locked = 1; - from_rq_locked = 1; - erts_smp_xrunq_lock(rq, from_rq); - /* - * erts_smp_xrunq_lock() may release lock on rq! We have - * to check that we still want to immigrate from the same - * run queue as before. - */ - if (ERTS_CHK_RUNQ_FLG_IMMIGRATE(rq->flags, prio) - && from_rq == rqi->migrate.runq) { - ErtsRunQueueInfo *from_rqi = (prio == ERTS_PORT_PRIO_LEVEL - ? &from_rq->ports.info - : &from_rq->procs.prio_info[prio]); - if ((ERTS_CHK_RUNQ_FLG_EVACUATE(rq->flags, prio) - && ERTS_CHK_RUNQ_FLG_EVACUATE(from_rq->flags, prio) - && from_rqi->len) - || (from_rqi->len > rqi->migrate.limit.other - && rqi->len < rqi->migrate.limit.this)) { - if (prio == ERTS_PORT_PRIO_LEVEL) { - Port *prt = from_rq->ports.start; - if (prt) { - int prt_locked = 0; - (void) erts_port_migrate(prt, &prt_locked, - from_rq, &from_rq_locked, - rq, &rq_locked); - if (prt_locked) - erts_smp_port_unlock(prt); - } - } - else { - Process *proc; - ErtsRunPrioQueue *from_rpq; - from_rpq = (prio == PRIORITY_LOW - ? &from_rq->procs.prio[PRIORITY_NORMAL] - : &from_rq->procs.prio[prio]); - for (proc = from_rpq->first; proc; proc = proc->next) - if (proc->prio == prio && !proc->bound_runq) - break; - if (proc) { - ErtsProcLocks proc_locks = 0; - (void) erts_proc_migrate(proc, &proc_locks, - from_rq, &from_rq_locked, - rq, &rq_locked); - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - } + while (iflag) { + ErtsRunQueue *rq; + int prio; + + switch (iflag) { + case (MAX_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_MAX; + break; + case (HIGH_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_HIGH; + break; + case (NORMAL_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_NORMAL; + break; + case (LOW_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = PRIORITY_LOW; + break; + case (PORT_BIT << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT): + prio = ERTS_PORT_PRIO_LEVEL; + break; + default: + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Invalid immigrate queue mask", + __FILE__, __LINE__, __func__); + prio = 0; + break; + } + + iflags &= ~iflag; + iflag = iflags & -iflags; + + rq = check_immigration_need(c_rq, mp, prio); + if (rq) { + erts_smp_runq_lock(rq); + if (prio == ERTS_PORT_PRIO_LEVEL) { + Port *prt; + prt = erts_dequeue_port(rq); + if (prt) + RUNQ_SET_RQ(&prt->run_queue, c_rq); + erts_smp_runq_unlock(rq); + if (prt) { + /* port might terminate while we have no lock... */ + rq = erts_port_runq(prt); + if (rq) { + if (rq != c_rq) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s(): Internal error", + __FILE__, __LINE__, __func__); + erts_enqueue_port(c_rq, prt); + if (!iflag) + return; /* done */ + erts_smp_runq_unlock(c_rq); } } - else { - ERTS_UNSET_RUNQ_FLG_IMMIGRATE(rq->flags, prio); - ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x1); + } + else { + ErtsRunPrioQueue *rpq = &rq->procs.prio[prio == PRIORITY_LOW + ? PRIORITY_NORMAL + : prio]; + Process *prev_proc = NULL; + Process *proc = rpq->first; + int rq_locked = 1; + + while (proc) { + erts_aint32_t state; + state = erts_smp_atomic32_read_acqb(&proc->state); + if (!(ERTS_PSFLG_BOUND & state) + && (prio == (int) (ERTS_PSFLG_PRIO_MASK & state))) { + ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio]; + unqueue_process(rq, rpq, rqi, prio, prev_proc, proc); + erts_smp_runq_unlock(rq); + RUNQ_SET_RQ(&proc->run_queue, c_rq); + rq_locked = 0; + + erts_smp_runq_lock(c_rq); + enqueue_process(c_rq, prio, proc); + if (!iflag) + return; /* done */ + erts_smp_runq_unlock(c_rq); + break; + } + prev_proc = proc; + proc = proc->next; } + if (rq_locked) + erts_smp_runq_unlock(rq); } - if (from_rq_locked) - erts_smp_runq_unlock(from_rq); - if (!rq_locked) - erts_smp_runq_lock(rq); } } + + erts_smp_runq_lock(c_rq); +} + +static ERTS_INLINE void +suspend_run_queue(ErtsRunQueue *rq) +{ + erts_smp_atomic32_read_bor_nob(&rq->scheduler->ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_SUSPENDED); + + wake_scheduler(rq, 0); } +static void scheduler_ix_resume_wake(Uint ix); + +static ERTS_INLINE void +resume_run_queue(ErtsRunQueue *rq) +{ + int pix; + + erts_smp_runq_lock(rq); + + (void) ERTS_RUNQ_FLGS_READ_BSET(rq, + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK + | ERTS_RUNQ_FLG_SUSPENDED), + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + rq->procs.prio_info[pix].max_len = 0; + rq->procs.prio_info[pix].reds = 0; + } + rq->ports.info.max_len = 0; + rq->ports.info.reds = 0; + rq->max_len = 0; + + erts_smp_runq_unlock(rq); + + scheduler_ix_resume_wake(rq->ix); +} + +typedef struct { + Process *first; + Process *last; +} ErtsStuckBoundProcesses; + static void -evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) +schedule_bound_processes(ErtsRunQueue *rq, + ErtsStuckBoundProcesses *sbpp) { - Port *prt; - int notify_to_rq = 0; - int prio; - int prt_locked = 0; - int rq_locked = 0; - int evac_rq_locked = 1; - ErtsMigrateResult mres; + Process *proc, *next; + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - erts_smp_runq_lock(evac_rq); + proc = sbpp->first; + while (proc) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + next = proc->next; + enqueue_process(rq, (int) (ERTS_PSFLG_PRIO_MASK & state), proc); + proc = next; + } +} - erts_smp_atomic32_read_bor_nob(&evac_rq->scheduler->ssi->flags, - ERTS_SSI_FLG_SUSPENDED); +static void +evacuate_run_queue(ErtsRunQueue *rq, + ErtsStuckBoundProcesses *sbpp) +{ + int prio_q; + ErtsRunQueue *to_rq; + ErtsMigrationPaths *mps; + ErtsMigrationPath *mp; - evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; - evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK - | ERTS_RUNQ_FLGS_EVACUATE_QMASK - | ERTS_RUNQ_FLG_SUSPENDED); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - erts_smp_atomic32_read_bor_nob(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); - /* - * Need to set up evacuation paths first since we - * may release the run queue lock on evac_rq - * when evacuating. - */ - evac_rq->misc.evac_runq = rq; - evac_rq->ports.info.migrate.runq = rq; - for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) - evac_rq->procs.prio_info[prio].migrate.runq = rq; + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; /* Evacuate scheduled misc ops */ - if (evac_rq->misc.start) { - rq_locked = 1; - erts_smp_xrunq_lock(evac_rq, rq); - if (rq->misc.end) - rq->misc.end->next = evac_rq->misc.start; + if (rq->misc.start) { + ErtsMiscOpList *start, *end; + + to_rq = mp->misc_evac_runq; + if (!to_rq) + return; + + start = rq->misc.start; + end = rq->misc.end; + rq->misc.start = NULL; + rq->misc.end = NULL; + erts_smp_runq_unlock(rq); + + erts_smp_runq_lock(to_rq); + if (to_rq->misc.end) + to_rq->misc.end->next = start; else - rq->misc.start = evac_rq->misc.start; - rq->misc.end = evac_rq->misc.end; - evac_rq->misc.start = NULL; - evac_rq->misc.end = NULL; + to_rq->misc.start = start; + + to_rq->misc.end = end; + erts_smp_runq_unlock(to_rq); + smp_notify_inc_runq(to_rq); + erts_smp_runq_lock(to_rq); } - /* Evacuate scheduled ports */ - prt = evac_rq->ports.start; - while (prt) { - mres = erts_port_migrate(prt, &prt_locked, - evac_rq, &evac_rq_locked, - rq, &rq_locked); - if (mres == ERTS_MIGRATE_SUCCESS) - notify_to_rq = 1; - if (prt_locked) - erts_smp_port_unlock(prt); - if (!evac_rq_locked) { - evac_rq_locked = 1; - erts_smp_runq_lock(evac_rq); + if (rq->ports.start) { + Port *prt; + + to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq; + if (!to_rq) + return; + + /* Evacuate scheduled ports */ + prt = rq->ports.start; + while (prt) { + ErtsRunQueue *prt_rq; + prt = erts_dequeue_port(rq); + RUNQ_SET_RQ(&prt->run_queue, to_rq); + erts_smp_runq_unlock(rq); + /* + * The port might terminate while + * we have no lock on it... + */ + prt_rq = erts_port_runq(prt); + if (prt_rq) { + if (prt_rq != to_rq) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s() internal error\n", + __FILE__, __LINE__, __func__); + erts_enqueue_port(to_rq, prt); + erts_smp_runq_unlock(to_rq); + } + erts_smp_runq_lock(rq); + prt = rq->ports.start; } - prt = evac_rq->ports.start; + smp_notify_inc_runq(to_rq); } /* Evacuate scheduled processes */ - for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) { + for (prio_q = 0; prio_q < ERTS_NO_PROC_PRIO_QUEUES; prio_q++) { + erts_aint32_t state; Process *proc; + int notify = 0; + to_rq = NULL; - switch (prio) { - case PRIORITY_MAX: - case PRIORITY_HIGH: - case PRIORITY_NORMAL: - proc = evac_rq->procs.prio[prio].first; - while (proc) { - ErtsProcLocks proc_locks = 0; - - /* Bound processes are stuck... */ - while (proc->bound_runq) { - proc = proc->next; - if (!proc) - goto end_of_proc; - } - - mres = erts_proc_migrate(proc, &proc_locks, - evac_rq, &evac_rq_locked, - rq, &rq_locked); - if (mres == ERTS_MIGRATE_SUCCESS) - notify_to_rq = 1; - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - if (!evac_rq_locked) { - erts_smp_runq_lock(evac_rq); - evac_rq_locked = 1; - } + if (!mp->prio[prio_q].runq) + return; + if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) + return; - proc = evac_rq->procs.prio[prio].first; + proc = dequeue_process(rq, prio_q, &state); + while (proc) { + if (ERTS_PSFLG_BOUND & state) { + /* Bound processes get stuck here... */ + proc->next = NULL; + if (sbpp->last) + sbpp->last->next = proc; + else + sbpp->first = proc; + sbpp->last = proc; } + else { + int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + erts_smp_runq_unlock(rq); - end_of_proc: + to_rq = mp->prio[prio].runq; + RUNQ_SET_RQ(&proc->run_queue, to_rq); -#ifdef DEBUG - for (proc = evac_rq->procs.prio[prio].first; - proc; - proc = proc->next) { - ASSERT(proc->bound_runq); + erts_smp_runq_lock(to_rq); + enqueue_process(to_rq, prio, proc); + erts_smp_runq_unlock(to_rq); + notify = 1; + + erts_smp_runq_lock(rq); } -#endif - break; - case PRIORITY_LOW: - break; - default: - ASSERT(!"Invalid process priority"); - break; + proc = dequeue_process(rq, prio_q, &state); } + if (notify) + smp_notify_inc_runq(to_rq); } - - if (rq_locked) - erts_smp_runq_unlock(rq); - - if (evac_rq_locked) - erts_smp_runq_unlock(evac_rq); - - if (notify_to_rq) - smp_notify_inc_runq(rq); - - wake_scheduler(evac_rq, 0); } static int -try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq) +try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, Uint32 flags) { - Process *proc; - int vrq_locked; + Uint32 procs_qmask = flags & ERTS_RUNQ_FLGS_PROCS_QMASK; + int max_prio_bit; + ErtsRunPrioQueue *rpq; - if (*rq_lockedp) - erts_smp_xrunq_lock(rq, vrq); - else - erts_smp_runq_lock(vrq); - vrq_locked = 1; + if (*rq_lockedp) { + erts_smp_runq_unlock(rq); + *rq_lockedp = 0; + } - ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); - ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + + erts_smp_runq_lock(vrq); if (rq->halt_in_progress) - goto try_steal_port; + goto no_procs; /* * Check for a runnable process to steal... */ - switch (vrq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) { - case MAX_BIT: - case MAX_BIT|HIGH_BIT: - case MAX_BIT|NORMAL_BIT: - case MAX_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT: - case MAX_BIT|HIGH_BIT|LOW_BIT: - case MAX_BIT|NORMAL_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT: - for (proc = vrq->procs.prio[PRIORITY_MAX].last; - proc; - proc = proc->prev) { - if (!proc->bound_runq) - break; - } - if (proc) + while (procs_qmask) { + Process *prev_proc; + Process *proc; + + max_prio_bit = procs_qmask & -procs_qmask; + switch (max_prio_bit) { + case MAX_BIT: + rpq = &vrq->procs.prio[PRIORITY_MAX]; break; - case HIGH_BIT: - case HIGH_BIT|NORMAL_BIT: - case HIGH_BIT|LOW_BIT: - case HIGH_BIT|NORMAL_BIT|LOW_BIT: - for (proc = vrq->procs.prio[PRIORITY_HIGH].last; - proc; - proc = proc->prev) { - if (!proc->bound_runq) - break; - } - if (proc) + case HIGH_BIT: + rpq = &vrq->procs.prio[PRIORITY_HIGH]; break; - case NORMAL_BIT: - case LOW_BIT: - case NORMAL_BIT|LOW_BIT: - for (proc = vrq->procs.prio[PRIORITY_NORMAL].last; - proc; - proc = proc->prev) { - if (!proc->bound_runq) - break; - } - if (proc) + case NORMAL_BIT: + case LOW_BIT: + rpq = &vrq->procs.prio[PRIORITY_NORMAL]; break; - case 0: - proc = NULL; - break; - default: - ASSERT(!"Invalid queue mask"); - proc = NULL; - break; - } + case 0: + goto no_procs; + default: + ASSERT(!"Invalid queue mask"); + goto no_procs; + } - if (proc) { - ErtsProcLocks proc_locks = 0; - int res; - ErtsMigrateResult mres; - mres = erts_proc_migrate(proc, &proc_locks, - vrq, &vrq_locked, - rq, rq_lockedp); - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - res = !0; - switch (mres) { - case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED: - res = 0; - case ERTS_MIGRATE_SUCCESS: - if (vrq_locked) + prev_proc = NULL; + proc = rpq->first; + + while (proc) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + if (!(ERTS_PSFLG_BOUND & state)) { + /* Steal process */ + int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio]; + unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc); erts_smp_runq_unlock(vrq); - return res; - default: /* Other failures */ - break; - } - } + RUNQ_SET_RQ(&proc->run_queue, rq); - ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); - ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + erts_smp_runq_lock(rq); + *rq_lockedp = 1; + enqueue_process(rq, prio, proc); + return !0; + } + prev_proc = proc; + proc = proc->next; + } - if (!vrq_locked) { - if (*rq_lockedp) - erts_smp_xrunq_lock(rq, vrq); - else - erts_smp_runq_lock(vrq); - vrq_locked = 1; + procs_qmask &= ~max_prio_bit; } - try_steal_port: +no_procs: - ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); - ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(vrq)); /* * Check for a runnable port to steal... */ - if (vrq->ports.info.len) { - Port *prt = vrq->ports.end; - int prt_locked = 0; - int res; - ErtsMigrateResult mres; - - mres = erts_port_migrate(prt, &prt_locked, - vrq, &vrq_locked, - rq, rq_lockedp); - if (prt_locked) - erts_smp_port_unlock(prt); - res = !0; - switch (mres) { - case ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED: - res = 0; - case ERTS_MIGRATE_SUCCESS: - if (vrq_locked) - erts_smp_runq_unlock(vrq); - return res; - default: /* Other failures */ - break; + if (vrq->ports.start) { + ErtsRunQueue *prt_rq; + Port *prt = erts_dequeue_port(vrq); + RUNQ_SET_RQ(&prt->run_queue, rq); + erts_smp_runq_unlock(vrq); + + /* + * The port might terminate while + * we have no lock on it... + */ + + prt_rq = erts_port_runq(prt); + if (!prt_rq) + return 0; + else { + if (prt_rq != rq) + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:%s() internal error\n", + __FILE__, __LINE__, __func__); + *rq_lockedp = 1; + erts_enqueue_port(rq, prt); + return !0; } } - if (vrq_locked) - erts_smp_runq_unlock(vrq); + erts_smp_runq_unlock(vrq); return 0; } @@ -3027,9 +3318,10 @@ static ERTS_INLINE int check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); - erts_aint32_t iflgs = erts_smp_atomic32_read_nob(&vrq->info_flags); - if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) - return try_steal_task_from_victim(rq, rq_lockedp, vrq); + Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq); + if ((flags & (ERTS_RUNQ_FLG_NONEMPTY + | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY) + return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags); else return 0; } @@ -3039,15 +3331,12 @@ static int try_steal_task(ErtsRunQueue *rq) { int res, rq_locked, vix, active_rqs, blnc_rqs; + Uint32 flags; - /* - * We are not allowed to steal jobs to this run queue - * if it is suspended. Note that it might get suspended - * at any time when we don't have the lock on the run - * queue. - */ - if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - return 0; + /* Protect jobs we steal from getting stolen from us... */ + flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_PROTECTED); + if (flags & ERTS_RUNQ_FLG_SUSPENDED) + return 0; /* go suspend instead... */ res = 0; rq_locked = 1; @@ -3166,12 +3455,123 @@ do { \ ASSERT(sum__ == (RQ)->full_reds_history_sum); \ } while (0); +#define ERTS_PRE_ALLOCED_MPATHS 8 + +erts_atomic_t erts_migration_paths; + +static struct { + size_t size; + ErtsMigrationPaths *freelist; + struct { + ErtsMigrationPaths *first; + ErtsMigrationPaths *last; + } retired; +} mpaths; + +static void +init_migration_paths(void) +{ + int qix, i; + char *p; + ErtsMigrationPaths *mps; + + mpaths.size = sizeof(ErtsMigrationPaths); + mpaths.size += sizeof(ErtsMigrationPath)*(erts_no_schedulers-1); + mpaths.size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(mpaths.size); + + p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_LL_MPATHS, + (mpaths.size + * ERTS_PRE_ALLOCED_MPATHS)); + mpaths.freelist = NULL; + for (i = 0; i < ERTS_PRE_ALLOCED_MPATHS-1; i++) { + mps = (ErtsMigrationPaths *) p; + mps->next = mpaths.freelist; + mpaths.freelist = mps; + p += mpaths.size; + } + + mps = (ErtsMigrationPaths *) p; + mps->block = NULL; + for (qix = 0; qix < erts_no_run_queues; qix++) { + int pix; + mps->mpath[qix].flags = 0; + mps->mpath[qix].misc_evac_runq = NULL; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + mps->mpath[qix].prio[pix].limit.this = -1; + mps->mpath[qix].prio[pix].limit.other = -1; + mps->mpath[qix].prio[pix].runq = NULL; + mps->mpath[qix].prio[pix].flags = 0; + } + } + erts_atomic_init_wb(&erts_migration_paths, (erts_aint_t) mps); +} + +static ERTS_INLINE ErtsMigrationPaths * +alloc_mpaths(void) +{ + void *block; + ErtsMigrationPaths *res; + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx)); + + res = mpaths.freelist; + if (res) { + mpaths.freelist = res->next; + res->block = NULL; + return res; + } + res = erts_alloc(ERTS_ALC_T_SL_MPATHS, + mpaths.size+ERTS_CACHE_LINE_SIZE); + block = (void *) res; + if (((UWord) res) & ERTS_CACHE_LINE_MASK) + res = (ErtsMigrationPaths *) ((((UWord) res) & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE); + res->block = block; + return res; +} + +static ERTS_INLINE void +retire_mpaths(ErtsMigrationPaths *mps) +{ + ErtsThrPrgrVal current; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx)); + + current = erts_thr_progress_current(); + + while (mpaths.retired.first) { + ErtsMigrationPaths *tmp = mpaths.retired.first; + if (!erts_thr_progress_has_reached_this(current, tmp->thr_prgr)) + break; + mpaths.retired.first = tmp->next; + if (tmp->block) { + erts_free(ERTS_ALC_T_SL_MPATHS, tmp->block); + } + else { + tmp->next = mpaths.freelist; + mpaths.freelist = tmp; + } + } + + if (!mpaths.retired.first) + mpaths.retired.last = NULL; + + mps->thr_prgr = erts_thr_progress_later(NULL); + mps->next = NULL; + + if (mpaths.retired.last) + mpaths.retired.last->next = mps; + else + mpaths.retired.first = mps; + mpaths.retired.last = mps; +} + static void check_balance(ErtsRunQueue *c_rq) { #if ERTS_MAX_PROCESSES >= (1 << 27) # error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27) #endif + ErtsMigrationPaths *new_mpaths, *old_mpaths; ErtsRunQueueBalance avg = {0}; Sint64 scheds_reds, full_scheds_reds; int forced, active, current_active, oowc, half_full_scheds, full_scheds, @@ -3197,9 +3597,9 @@ check_balance(ErtsRunQueue *c_rq) ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) - rq->flags |= ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK; + (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); else - rq->flags &= ~ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK; + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; }); @@ -3241,7 +3641,7 @@ check_balance(ErtsRunQueue *c_rq) ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); erts_smp_runq_lock(rq); - run_queue_info[qix].flags = rq->flags; + run_queue_info[qix].flags = ERTS_RUNQ_FLGS_GET_NOB(rq); for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { run_queue_info[qix].prio[pix].max_len = rq->procs.prio_info[pix].max_len; @@ -3575,47 +3975,27 @@ erts_fprintf(stderr, "--------------------------------\n"); set_no_active_runqs(active); balance_info.halftime = 1; - erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + new_mpaths = alloc_mpaths(); + + /* Write migration paths */ - /* Write migration paths and reset balance statistics in all queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { int mqix; - Uint32 flags; - ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); - ErtsRunQueueInfo *rqi; - flags = run_queue_info[qix].flags; - erts_smp_runq_lock(rq); - flags |= (rq->flags & ~ERTS_RUNQ_FLGS_MIGRATION_INFO); - ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK)); - if (rq->waiting) - flags |= ERTS_RUNQ_FLG_OUT_OF_WORK; - - rq->full_reds_history_sum - = run_queue_info[qix].full_reds_history_sum; - rq->full_reds_history[freds_hist_ix] - = run_queue_info[qix].full_reds_history_change; + Uint32 flags = run_queue_info[qix].flags; + ErtsMigrationPath *mp = &new_mpaths->mpath[qix]; - ERTS_DBG_CHK_FULL_REDS_HISTORY(rq); + mp->flags = flags; + mp->misc_evac_runq = NULL; - rq->out_of_work_count = 0; - rq->flags = flags; - rq->max_len = rq->len; for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { - rqi = (pix == ERTS_PORT_PRIO_LEVEL - ? &rq->ports.info - : &rq->procs.prio_info[pix]); - rqi->max_len = rqi->len; - rqi->reds = 0; if (!(ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix) | ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix))) { ASSERT(run_queue_info[qix].prio[pix].immigrate_from < 0); ASSERT(run_queue_info[qix].prio[pix].emigrate_to < 0); -#ifdef DEBUG - rqi->migrate.limit.this = -1; - rqi->migrate.limit.other = -1; - ERTS_DBG_SET_INVALID_RUNQP(rqi->migrate.runq, 0x2); -#endif - + mp->prio[pix].limit.this = -1; + mp->prio[pix].limit.other = -1; + mp->prio[pix].runq = NULL; + mp->prio[pix].flags = 0; } else if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, pix)) { ASSERT(!ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix)); @@ -3623,11 +4003,12 @@ erts_fprintf(stderr, "--------------------------------\n"); ASSERT(run_queue_info[qix].prio[pix].emigrate_to >= 0); mqix = run_queue_info[qix].prio[pix].emigrate_to; - rqi->migrate.limit.this + mp->prio[pix].limit.this = run_queue_info[qix].prio[pix].migration_limit; - rqi->migrate.limit.other + mp->prio[pix].limit.other = run_queue_info[mqix].prio[pix].migration_limit; - rqi->migrate.runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].flags = run_queue_info[mqix].flags; } else { ASSERT(ERTS_CHK_RUNQ_FLG_IMMIGRATE(flags, pix)); @@ -3635,24 +4016,141 @@ erts_fprintf(stderr, "--------------------------------\n"); ASSERT(run_queue_info[qix].prio[pix].immigrate_from >= 0); mqix = run_queue_info[qix].prio[pix].immigrate_from; - rqi->migrate.limit.this + mp->prio[pix].limit.this = run_queue_info[qix].prio[pix].migration_limit; - rqi->migrate.limit.other + mp->prio[pix].limit.other = run_queue_info[mqix].prio[pix].migration_limit; - rqi->migrate.runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].runq = ERTS_RUNQ_IX(mqix); + mp->prio[pix].flags = run_queue_info[mqix].flags; } } + } + + old_mpaths = erts_get_migration_paths_managed(); + + /* Keep offline run-queues as is */ + for (qix = blnc_no_rqs; qix < erts_no_schedulers; qix++) { + ErtsMigrationPath *nmp = &new_mpaths->mpath[qix]; + ErtsMigrationPath *omp = &old_mpaths->mpath[qix]; + + nmp->flags = omp->flags; + nmp->misc_evac_runq = omp->misc_evac_runq; + + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + nmp->prio[pix].limit.this = omp->prio[pix].limit.this; + nmp->prio[pix].limit.other = omp->prio[pix].limit.other; + nmp->prio[pix].runq = omp->prio[pix].runq; + nmp->prio[pix].flags = omp->prio[pix].flags; + } + } + + + /* Publish new migration paths... */ + erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths); + + /* Reset balance statistics in all online queues */ + for (qix = 0; qix < blnc_no_rqs; qix++) { + Uint32 flags = run_queue_info[qix].flags; + ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); + + erts_smp_runq_lock(rq); + ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK)); + if (rq->waiting) + flags |= ERTS_RUNQ_FLG_OUT_OF_WORK; + + rq->full_reds_history_sum + = run_queue_info[qix].full_reds_history_sum; + rq->full_reds_history[freds_hist_ix] + = run_queue_info[qix].full_reds_history_change; + + ERTS_DBG_CHK_FULL_REDS_HISTORY(rq); + + rq->out_of_work_count = 0; + (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); + + rq->max_len = rq->len; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + ErtsRunQueueInfo *rqi; + rqi = (pix == ERTS_PORT_PRIO_LEVEL + ? &rq->ports.info + : &rq->procs.prio_info[pix]); + erts_smp_reset_max_len(rq, rqi); + rqi->reds = 0; + } rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; erts_smp_runq_unlock(rq); } + erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + balance_info.n++; + retire_mpaths(old_mpaths); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); } +static void +change_no_used_runqs(int used) +{ + ErtsMigrationPaths *new_mpaths, *old_mpaths; + int qix; + erts_smp_mtx_lock(&balance_info.update_mtx); + set_no_used_runqs(used); + + old_mpaths = erts_get_migration_paths_managed(); + new_mpaths = alloc_mpaths(); + + /* Write migration paths... */ + + for (qix = 0; qix < used; qix++) { + int pix; + ErtsMigrationPath *omp = &old_mpaths->mpath[qix]; + ErtsMigrationPath *nmp = &new_mpaths->mpath[qix]; + + nmp->flags = omp->flags & ~ERTS_RUNQ_FLGS_MIGRATION_QMASKS; + nmp->misc_evac_runq = NULL; + + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + nmp->prio[pix].limit.this = -1; + nmp->prio[pix].limit.other = -1; + nmp->prio[pix].runq = NULL; + nmp->prio[pix].flags = 0; + } + } + for (qix = used; qix < erts_no_run_queues; qix++) { + int pix; + ErtsRunQueue *to_rq = ERTS_RUNQ_IX(qix % used); + ErtsMigrationPath *nmp = &new_mpaths->mpath[qix]; + + nmp->flags = (ERTS_RUNQ_FLGS_EMIGRATE_QMASK + | ERTS_RUNQ_FLGS_EVACUATE_QMASK); + nmp->misc_evac_runq = to_rq; + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + nmp->prio[pix].limit.this = -1; + nmp->prio[pix].limit.other = -1; + nmp->prio[pix].runq = to_rq; + nmp->prio[pix].flags = 0; + } + } + + /* ... and publish them. */ + erts_atomic_set_wb(&erts_migration_paths, (erts_aint_t) new_mpaths); + + retire_mpaths(old_mpaths); + + /* Make sure that we balance soon... */ + balance_info.forced_check_balance = 1; + + erts_smp_mtx_unlock(&balance_info.update_mtx); + + erts_smp_runq_lock(ERTS_RUNQ_IX(0)); + ERTS_RUNQ_IX(0)->check_balance_reds = 0; + erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); +} + + #endif /* #ifdef ERTS_SMP */ Uint @@ -3680,11 +4178,11 @@ typedef enum { } ErtsSchedWakeupOtherThreshold; typedef enum { - ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL, + ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT, ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY } ErtsSchedWakeupOtherType; -/* First proposal */ +/* Default */ #define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS) #define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS) @@ -3701,7 +4199,7 @@ typedef enum { #define ERTS_WAKEUP_OTHER_DEC_SHIFT 2 #define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10) -/* To be legacy */ +/* Legacy */ #define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY (200*CONTEXT_REDS) #define ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY (50*CONTEXT_REDS) @@ -3720,11 +4218,11 @@ static struct { int limit; int dec_shift; int dec_mask; - void (*check)(ErtsRunQueue *rq); + void (*check)(ErtsRunQueue *rq, Uint32 flags); } wakeup_other; static void -wakeup_other_check(ErtsRunQueue *rq) +wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { @@ -3742,6 +4240,8 @@ wakeup_other_check(ErtsRunQueue *rq) if (rq->wakeup_other > wakeup_other.limit) { int empty_rqs = erts_smp_atomic32_read_acqb(&no_empty_run_queues); + if (flags & ERTS_RUNQ_FLG_PROTECTED) + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); if (empty_rqs != 0) wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; @@ -3786,18 +4286,21 @@ wakeup_other_set_limit(void) } static void -wakeup_other_check_legacy(ErtsRunQueue *rq) +wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - if (rq->len < 2) { + erts_aint32_t len = rq->len; + if (len < 2) { rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; if (rq->wakeup_other < 0) rq->wakeup_other = 0; } else if (rq->wakeup_other < wakeup_other.limit) - rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY; + rq->wakeup_other += len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY; else { + if (flags & ERTS_RUNQ_FLG_PROTECTED) + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; @@ -3834,7 +4337,7 @@ static void set_wakeup_other_data(void) { switch (wakeup_other.type) { - case ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL: + case ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT: wakeup_other.check = wakeup_other_check; wakeup_other_set_limit(); break; @@ -3881,7 +4384,7 @@ runq_supervisor(void *unused) for (ix = 0; ix < no_rqs; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - if (erts_smp_atomic32_read_acqb(&rq->info_flags) & ERTS_RUNQ_IFLG_NONEMPTY) { + if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) { erts_smp_runq_lock(rq); if (rq->len != 0) wake_scheduler_on_empty_runq(rq); /* forced wakeup... */ @@ -3900,7 +4403,7 @@ erts_early_init_scheduling(int no_schedulers) aux_work_timeout_early_init(no_schedulers); #ifdef ERTS_SMP wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; - wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; + wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; #endif sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM @@ -3912,6 +4415,7 @@ erts_early_init_scheduling(int no_schedulers) int erts_sched_set_wakeup_other_thresold(char *str) { +#ifdef ERTS_SMP ErtsSchedWakeupOtherThreshold threshold; if (sys_strcmp(str, "very_high") == 0) threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH; @@ -3925,29 +4429,38 @@ erts_sched_set_wakeup_other_thresold(char *str) threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW; else return EINVAL; -#ifdef ERTS_SMP wakeup_other.threshold = threshold; set_wakeup_other_data(); -#endif return 0; +#else + if (sys_strcmp(str, "very_high") == 0 || sys_strcmp(str, "high") == 0 || + sys_strcmp(str, "medium") == 0 || sys_strcmp(str, "low") == 0 || + sys_strcmp(str, "very_low") == 0) { + return 0; + } + return EINVAL; +#endif } int erts_sched_set_wakeup_other_type(char *str) { +#ifdef ERTS_SMP ErtsSchedWakeupOtherType type; - if (sys_strcmp(str, "proposal") == 0) - type = ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL; - else if (sys_strcmp(str, "default") == 0) - type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; + if (sys_strcmp(str, "default") == 0) + type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; else if (sys_strcmp(str, "legacy") == 0) type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; else return EINVAL; -#ifdef ERTS_SMP wakeup_other.type = type; -#endif return 0; +#else + if (sys_strcmp(str, "default") == 0 || sys_strcmp(str, "legacy") == 0) { + return 0; + } + return EINVAL; +#endif } int @@ -3990,6 +4503,25 @@ erts_sched_set_busy_wait_threshold(char *str) return 0; } + +int +erts_sched_set_wake_cleanup_threshold(char *str) +{ + if (sys_strcmp(str, "very_lazy") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY; + else if (sys_strcmp(str, "lazy") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY; + else if (sys_strcmp(str, "medium") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM; + else if (sys_strcmp(str, "eager") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_EAGER; + else if (sys_strcmp(str, "very_eager") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_EAGER; + else + return EINVAL; + return 0; +} + static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) { @@ -3997,10 +4529,15 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->esdp = esdp; awdp->ssi = esdp ? esdp->ssi : NULL; #ifdef ERTS_SMP + awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST; awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.completed_callback = NULL; awdp->dd.completed_arg = NULL; + awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST; + awdp->later_op.size = 0; + awdp->later_op.first = NULL; + awdp->later_op.last = NULL; #endif #ifdef ERTS_USE_ASYNC_READY_Q #ifdef ERTS_SMP @@ -4065,7 +4602,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); rq->ix = ix; - erts_smp_atomic32_init_nob(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); /* make sure that the "extra" id correponds to the schedulers * id if the esdp->no <-> ix+1 mapping change. @@ -4076,7 +4612,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->waiting = 0; rq->woken = 0; - rq->flags = 0; + ERTS_RUNQ_FLGS_INIT(rq, ERTS_RUNQ_FLG_NONEMPTY); rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; rq->full_reds_history_sum = 0; for (rix = 0; rix < ERTS_FULL_REDS_HISTORY_SIZE; rix++) { @@ -4090,19 +4626,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->wakeup_other_reds = 0; rq->halt_in_progress = 0; - rq->procs.len = 0; rq->procs.pending_exiters = NULL; rq->procs.context_switches = 0; rq->procs.reductions = 0; for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - rq->procs.prio_info[pix].len = 0; + erts_smp_atomic32_init_nob(&rq->procs.prio_info[pix].len, 0); rq->procs.prio_info[pix].max_len = 0; rq->procs.prio_info[pix].reds = 0; - rq->procs.prio_info[pix].migrate.limit.this = 0; - rq->procs.prio_info[pix].migrate.limit.other = 0; - ERTS_DBG_SET_INVALID_RUNQP(rq->procs.prio_info[pix].migrate.runq, - 0x0); if (pix < ERTS_NO_PROC_PRIO_LEVELS - 1) { rq->procs.prio[pix].first = NULL; rq->procs.prio[pix].last = NULL; @@ -4111,14 +4642,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->misc.start = NULL; rq->misc.end = NULL; - rq->misc.evac_runq = NULL; - rq->ports.info.len = 0; + erts_smp_atomic32_init_nob(&rq->ports.info.len, 0); rq->ports.info.max_len = 0; rq->ports.info.reds = 0; - rq->ports.info.migrate.limit.this = 0; - rq->ports.info.migrate.limit.other = 0; - rq->ports.info.migrate.runq = NULL; rq->ports.start = NULL; rq->ports.end = NULL; } @@ -4219,6 +4746,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) esdp->reductions = 0; init_sched_wall_time(&esdp->sched_wall_time); + erts_port_task_handle_init(&esdp->nosuspend_port_task_handle); + } init_misc_aux_work(); @@ -4244,7 +4773,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) schdlr_sspnd.msb.ongoing = 0; erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - init_no_runqs(no_schedulers, no_schedulers_online); + init_no_runqs(no_schedulers_online, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; @@ -4256,10 +4785,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) balance_info.prev_rise.reds = 0; balance_info.n = 0; + init_migration_paths(); + if (no_schedulers_online < no_schedulers) { + change_no_used_runqs(no_schedulers_online); for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % no_schedulers_online)); + suspend_run_queue(ERTS_RUNQ_IX(ix)); } schdlr_sspnd.wait_curr_online = no_schedulers_online; @@ -4301,6 +4832,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) erts_smp_atomic32_init_relb(&erts_halt_progress, -1); erts_halt_code = 0; + +#if !defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + erts_lc_set_thread_name("scheduler 1"); +#endif + } ErtsRunQueue * @@ -4322,91 +4858,197 @@ erts_get_scheduler_data(void) #endif -static int remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive); +/* + * scheduler_out_process() return with c_rq locked. + */ +static ERTS_INLINE int +schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p) +{ + erts_aint32_t a, e, n; + int res = 0; + + a = state; + + while (1) { + n = e = a; + + ASSERT(a & ERTS_PSFLG_RUNNING); + ASSERT(!(a & ERTS_PSFLG_IN_RUNQ)); + + n &= ~ERTS_PSFLG_RUNNING; + if ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) + n |= ERTS_PSFLG_IN_RUNQ; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + } + + if (!(n & ERTS_PSFLG_IN_RUNQ)) { + if (erts_system_profile_flags.runnable_procs) + profile_runnable_proc(p, am_inactive); + } + else { + int prio = (int) (ERTS_PSFLG_PRIO_MASK & n); + ErtsRunQueue *runq = erts_get_runq_proc(p); + + ASSERT(!(n & ERTS_PSFLG_SUSPENDED)); + +#ifdef ERTS_SMP + if (!(ERTS_PSFLG_BOUND & n)) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio); + if (new_runq) { + RUNQ_SET_RQ(&p->run_queue, new_runq); + runq = new_runq; + } + } +#endif + ASSERT(runq); + res = 1; + + erts_smp_runq_lock(runq); + + /* Enqueue the process */ + enqueue_process(runq, prio, p); + + if (runq == c_rq) + return res; + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); + } + erts_smp_runq_lock(c_rq); + return res; +} static ERTS_INLINE void -suspend_process(ErtsRunQueue *rq, Process *p) +add2runq(Process *p, erts_aint32_t state) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - p->rcount++; /* count number of suspend */ -#ifdef ERTS_SMP - ASSERT(!(p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) - || p == erts_get_current_process()); - ASSERT(p->status != P_RUNNING - || p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING); - if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) - goto runable; -#endif - switch(p->status) { - case P_SUSPENDED: - break; - case P_RUNABLE: + int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + ErtsRunQueue *runq = erts_get_runq_proc(p); + #ifdef ERTS_SMP - runable: - if (!ERTS_PROC_PENDING_EXIT(p)) + if (!(ERTS_PSFLG_BOUND & state)) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio); + if (new_runq) { + RUNQ_SET_RQ(&p->run_queue, new_runq); + runq = new_runq; + } + } #endif - remove_proc_from_runq(rq, p, 1); - /* else: - * leave process in schedq so it will discover the pending exit - */ - p->rstatus = P_RUNABLE; /* wakeup as runnable */ - break; - case P_RUNNING: - p->rstatus = P_RUNABLE; /* wakeup as runnable */ - break; - case P_WAITING: - p->rstatus = P_WAITING; /* wakeup as waiting */ - break; - case P_EXITING: - return; /* ignore this */ - case P_GARBING: - case P_FREE: - erl_exit(1, "bad state in suspend_process()\n"); + ASSERT(runq); + + erts_smp_runq_lock(runq); + + /* Enqueue the process */ + enqueue_process(runq, prio, p); + + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); + +} + +static ERTS_INLINE void +schedule_process(Process *p, erts_aint32_t state, int active_enq) +{ + erts_aint32_t a = state, n; + + while (1) { + erts_aint32_t e; + n = e = a; + + if (a & ERTS_PSFLG_FREE) + return; /* We don't want to schedule free processes... */ + n |= ERTS_PSFLG_ACTIVE; + if (!(a & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_RUNNING))) + n |= ERTS_PSFLG_IN_RUNQ; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + if (!active_enq && (a & ERTS_PSFLG_ACTIVE)) + return; /* Someone else activated process ... */ } - if ((erts_system_profile_flags.runnable_procs) && (p->rcount == 1) && (p->status != P_WAITING)) { - profile_runnable_proc(p, am_inactive); + if (erts_system_profile_flags.runnable_procs + && !(a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) { + profile_runnable_proc(p, am_active); } - p->status = P_SUSPENDED; - + if ((n & ERTS_PSFLG_IN_RUNQ) && !(a & ERTS_PSFLG_IN_RUNQ)) + add2runq(p, n); } -static ERTS_INLINE void -resume_process(Process *p) +void +erts_schedule_process(Process *p, erts_aint32_t state) { - Uint32 *statusp; + schedule_process(p, state, 0); +} + +static ERTS_INLINE int +suspend_process(Process *c_p, Process *p) +{ + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + int suspended = 0; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - switch (p->status) { - case P_SUSPENDED: - statusp = &p->status; - break; - case P_GARBING: - if (p->gcstatus == P_SUSPENDED) { - statusp = &p->gcstatus; - break; + + if ((state & ERTS_PSFLG_SUSPENDED)) + suspended = -1; + else { + if (c_p == p) { + state = erts_smp_atomic32_read_bor_relb(&p->state, + ERTS_PSFLG_SUSPENDED); + state |= ERTS_PSFLG_SUSPENDED; + ASSERT(state & ERTS_PSFLG_RUNNING); + suspended = 1; + } + else { + while (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_EXITING))) { + erts_aint32_t e, n; + n = e = state; + n |= ERTS_PSFLG_SUSPENDED; + state = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e); + if (state == e) { + state = n; + suspended = 1; + break; + } + } } - /* Fall through */ - default: - return; } + if (state & ERTS_PSFLG_SUSPENDED) { + + ASSERT(!(ERTS_PSFLG_RUNNING & state) + || p == erts_get_current_process()); + + if (erts_system_profile_flags.runnable_procs + && (p->rcount == 0) + && (state & ERTS_PSFLG_ACTIVE)) { + profile_runnable_proc(p, am_inactive); + } + + p->rcount++; /* count number of suspend */ + } + return suspended; +} + +static ERTS_INLINE void +resume_process(Process *p) +{ + erts_aint32_t state; + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ASSERT(p->rcount > 0); if (--p->rcount > 0) /* multiple suspend */ return; - switch(p->rstatus) { - case P_RUNABLE: - erts_add_to_runq(p); - break; - case P_WAITING: - *statusp = P_WAITING; - break; - default: - erl_exit(1, "bad state in resume_process()\n"); + + state = erts_smp_atomic32_read_band_mb(&p->state, ~ERTS_PSFLG_SUSPENDED); + state &= ~ERTS_PSFLG_SUSPENDED; + if ((state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_IN_RUNQ + | ERTS_PSFLG_RUNNING)) == ERTS_PSFLG_ACTIVE) { + schedule_process(p, state, 1); } - p->rstatus = P_FREE; } int @@ -4530,6 +5172,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) int wake = 0; erts_aint32_t aux_work; int thr_prgr_active = 1; + ErtsStuckBoundProcesses sbp = {NULL, NULL}; /* * Schedulers may be suspended in two different ways: @@ -4544,6 +5187,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) ASSERT(no != 1); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); erts_sched_check_cpu_bind_prep_suspend(esdp); @@ -4607,19 +5252,28 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_mtx_unlock(&schdlr_sspnd.mtx); while (1) { + erts_aint32_t qmask; erts_aint32_t flgs; + qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) + & ERTS_RUNQ_FLGS_QMASK); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { + if (aux_work|qmask) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); + if (qmask) { + erts_smp_runq_lock(esdp->run_queue); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + } } if (!aux_work) { @@ -4689,56 +5343,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); + schedule_bound_processes(esdp->run_queue, &sbp); + erts_sched_check_cpu_bind_post_suspend(esdp); } -#define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID) \ -do { \ - int pix__; \ - (RQ)->misc.evac_runq = NULL; \ - (RQ)->ports.info.migrate.runq = NULL; \ - (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EVACUATE_QMASK \ - | ERTS_RUNQ_FLG_SUSPENDED); \ - (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ - (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ - erts_smp_atomic32_read_band_nob(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ - for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ - (RQ)->procs.prio_info[pix__].max_len = 0; \ - (RQ)->procs.prio_info[pix__].reds = 0; \ - ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\ - (DBG_ID)); \ - } \ - (RQ)->ports.info.max_len = 0; \ - (RQ)->ports.info.reds = 0; \ -} while (0) - -#define ERTS_RUNQ_RESET_MIGRATION_PATHS__(RQ) \ -do { \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ))); \ - (RQ)->misc.evac_runq = NULL; \ - (RQ)->ports.info.migrate.runq = NULL; \ - (RQ)->flags &= ~(ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ - | ERTS_RUNQ_FLGS_EVACUATE_QMASK); \ -} while (0) - -#ifdef DEBUG -#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \ -do { \ - int pix__; \ - ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ)); \ - for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) \ - ERTS_DBG_SET_INVALID_RUNQP((RQ)->procs.prio_info[pix__].migrate.runq,\ - (DBG_ID)); \ -} while (0) -#else -#define ERTS_RUNQ_RESET_MIGRATION_PATHS(RQ, DBG_ID) \ - ERTS_RUNQ_RESET_MIGRATION_PATHS__((RQ)) -#endif - ErtsSchedSuspendResult erts_schedulers_state(Uint *total, Uint *online, @@ -4808,27 +5417,13 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - for (ix = online; ix < no; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); - erts_smp_runq_unlock(rq); - scheduler_ix_resume_wake(ix); - } - /* - * Spread evacuation paths among all online - * run queues. - */ - for (ix = no; ix < erts_no_run_queues; ix++) { - ErtsRunQueue *from_rq = ERTS_RUNQ_IX(ix); - ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); - evacuate_run_queue(from_rq, to_rq); - } - set_no_used_runqs(no); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + change_no_used_runqs(no); + + for (ix = online; ix < no; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); + + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -4855,25 +5450,11 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - - for (ix = 0; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x6); - erts_smp_runq_unlock(rq); - } - /* - * Evacutation order important! Newly suspended run queues - * has to be evacuated last. - */ - for (ix = erts_no_run_queues-1; ix >= no; ix--) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % no)); - set_no_used_runqs(no); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + change_no_used_runqs(no); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + for (ix = no; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); wake_scheduler(rq, 0); @@ -4933,8 +5514,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) else if (on) { /* ------ BLOCK ------ */ if (schdlr_sspnd.msb.procs) { plp = proclist_create(p); - plp->next = schdlr_sspnd.msb.procs; - schdlr_sspnd.msb.procs = plp; + erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); p->flags |= F_HAVE_BLCKD_MSCHED; ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); @@ -4970,25 +5550,14 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - set_no_used_runqs(1); - for (ix = 0; ix < online; ix++) { + change_no_used_runqs(1); + for (ix = 1; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + + for (ix = 1; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); - ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); - erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0); } - /* - * Evacuate all activities in all other run queues - * into the first run queue. Note order is important, - * online run queues has to be evacuated last. - */ - for (ix = erts_no_run_queues-1; ix >= 1; ix--) - evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0)); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) != schdlr_sspnd.msb.wait_active) { @@ -5030,17 +5599,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); - plp->next = schdlr_sspnd.msb.procs; - schdlr_sspnd.msb.procs = plp; -#ifdef DEBUG - ERTS_FOREACH_RUNQ(srq, - { - if (srq != ERTS_RUNQ_IX(0)) { - ASSERT(ERTS_EMPTY_RUNQ(srq)); - ASSERT(srq->flags & ERTS_RUNQ_FLG_SUSPENDED); - } - }); -#endif + erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); ASSERT(p->scheduler_data); } } @@ -5051,20 +5610,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } else { /* ------ UNBLOCK ------ */ if (p->flags & F_HAVE_BLCKD_MSCHED) { - ErtsProcList **plpp = &schdlr_sspnd.msb.procs; - plp = schdlr_sspnd.msb.procs; + ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs); while (plp) { - if (!proclist_same(plp, p)){ - plpp = &plp->next; - plp = plp->next; - } - else { - *plpp = plp->next; - proclist_destroy(plp); + ErtsProcList *tmp_plp = plp; + plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp); + if (erts_proclist_same(tmp_plp, p)) { + erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp); + proclist_destroy(tmp_plp); if (!all) break; - plp = *plpp; } } } @@ -5072,27 +5627,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else { ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); -#ifdef DEBUG - ERTS_FOREACH_RUNQ(rq, - { - if (rq != p->scheduler_data->run_queue) { - if (!ERTS_EMPTY_RUNQ(rq)) { - Process *rp; - int pix; - ASSERT(rq->ports.info.len == 0); - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - for (rp = rq->procs.prio[pix].first; - rp; - rp = rp->next) { - ASSERT(rp->bound_runq); - } - } - } - - ASSERT(rq->flags & ERTS_RUNQ_FLG_SUSPENDED); - } - }); -#endif p->flags &= ~F_HAVE_BLCKD_MSCHED; schdlr_sspnd.msb.ongoing = 0; if (schdlr_sspnd.online == 1) { @@ -5102,35 +5636,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } else { int online = schdlr_sspnd.online; - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_mtx_lock(&balance_info.update_mtx); + + change_no_used_runqs(online); /* Resume all online run queues */ - for (ix = 1; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); - erts_smp_runq_unlock(rq); - scheduler_ix_resume_wake(ix); - } + for (ix = 1; ix < online; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); - /* Spread evacuation paths among all online run queues */ for (ix = online; ix < erts_no_run_queues; ix++) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % online)); - - set_no_used_runqs(online); - /* Make sure that we balance soon... */ - balance_info.forced_check_balance = 1; - erts_smp_runq_lock(ERTS_RUNQ_IX(0)); - ERTS_RUNQ_IX(0)->check_balance_reds = 0; - erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + suspend_run_queue(ERTS_RUNQ_IX(ix)); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -5170,23 +5688,25 @@ erts_multi_scheduling_blockers(Process *p) Eterm res = NIL; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (schdlr_sspnd.msb.procs) { + if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; - Uint max_size; - ASSERT(schdlr_sspnd.msb.procs); - for (max_size = 0, plp1 = schdlr_sspnd.msb.procs; + Uint max_size = 0; + + for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); plp1; - plp1 = plp1->next) { + plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { max_size += 2; } ASSERT(max_size); hp = HAlloc(p, max_size); hp_end = hp + max_size; - for (plp1 = schdlr_sspnd.msb.procs; plp1; plp1 = plp1->next) { - for (plp2 = schdlr_sspnd.msb.procs; + for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + plp1; + plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { + for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); plp2->pid != plp1->pid; - plp2 = plp2->next); + plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2)); if (plp2 == plp1) { res = CONS(hp, plp1->pid, res); hp += 2; @@ -5453,11 +5973,8 @@ handle_pend_sync_suspend(Process *suspendee, if (suspender) { ASSERT(is_nil(suspender->suspendee)); if (suspendee_alive) { - ErtsRunQueue *rq = erts_get_runq_proc(suspendee); - erts_smp_runq_lock(rq); - suspend_process(rq, suspendee); - erts_smp_runq_unlock(rq); - suspender->suspendee = suspendee->id; + erts_suspend(suspendee, suspendee_locks, NULL); + suspender->suspendee = suspendee->common.id; } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ @@ -5478,7 +5995,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ERTS_SMP_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN); ERTS_SMP_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)); - if (c_p->id == pid) + if (c_p->common.id == pid) return erts_pid2proc(c_p, c_p_locks, pid, pid_locks); if (c_p_locks & ERTS_PROC_LOCK_STATUS) @@ -5499,10 +6016,9 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, resume_process(rp); } else { - ErtsRunQueue *cp_rq, *rp_rq; rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, ERTS_PROC_LOCK_STATUS); + pid, pid_locks|ERTS_PROC_LOCK_STATUS); if (!rp) { c_p->flags &= ~F_P2PNR_RESCHED; @@ -5511,58 +6027,36 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); - cp_rq = erts_get_runq_proc(c_p); - rp_rq = erts_get_runq_proc(rp); - erts_smp_runqs_lock(cp_rq, rp_rq); - if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - running: - /* Phiu... */ - - /* - * If we got pending suspenders and suspend ourselves waiting - * to suspend another process we might deadlock. - * In this case we have to yield, be suspended by - * someone else and then do it all over again. - */ - if (!c_p->pending_suspenders) { - /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->id, handle_pend_sync_suspend); - ASSERT(is_nil(c_p->suspendee)); - - /* Suspend c_p; when rp is suspended c_p will be resumed. */ - suspend_process(cp_rq, c_p); - c_p->flags |= F_P2PNR_RESCHED; - } - /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - rp = ERTS_PROC_LOCK_BUSY; + if (suspend) { + if (suspend_process(c_p, rp)) + goto done; } else { - ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; - if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { - erts_smp_runqs_unlock(cp_rq, rp_rq); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, pid_locks|ERTS_PROC_LOCK_STATUS); - if (!rp) - goto done; - /* run-queues may have changed */ - cp_rq = erts_get_runq_proc(c_p); - rp_rq = erts_get_runq_proc(rp); - erts_smp_runqs_lock(cp_rq, rp_rq); - if (rp->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - /* Ahh... */ - erts_smp_proc_unlock(rp, - pid_locks & ~ERTS_PROC_LOCK_STATUS); - goto running; - } - } + if (!(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&rp->state))) + goto done; - /* rp is not running and we got the locks we want... */ - if (suspend) - suspend_process(rp_rq, rp); } - erts_smp_runqs_unlock(cp_rq, rp_rq); + + /* Other process running */ + + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(c_p, c_p); + c_p->flags |= F_P2PNR_RESCHED; + } + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; } done: @@ -5619,36 +6113,26 @@ erts_pid2proc_nropt(Process *c_p, ErtsProcLocks c_p_locks, return erts_pid2proc_not_running(c_p, c_p_locks, pid, pid_locks); } -static ERTS_INLINE void -do_bif_suspend_process(ErtsSuspendMonitor *smon, - Process *suspendee, - ErtsRunQueue *locked_runq) +static ERTS_INLINE int +do_bif_suspend_process(Process *c_p, + ErtsSuspendMonitor *smon, + Process *suspendee) { ASSERT(suspendee); - ASSERT(!suspendee->is_exiting); + ASSERT(!ERTS_PROC_IS_EXITING(suspendee)); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(suspendee)); if (smon) { if (!smon->active) { - ErtsRunQueue *rq; - - if (locked_runq) - rq = locked_runq; - else { - rq = erts_get_runq_proc(suspendee); - erts_smp_runq_lock(rq); - } - - suspend_process(rq, suspendee); - - if (!locked_runq) - erts_smp_runq_unlock(rq); + if (!suspend_process(c_p, suspendee)) + return 0; } smon->active += smon->pending; ASSERT(smon->active); smon->pending = 0; + return 1; } - + return 0; } static void @@ -5669,13 +6153,20 @@ handle_pend_bif_sync_suspend(Process *suspendee, ASSERT(is_nil(suspender->suspendee)); if (!suspendee_alive) erts_delete_suspend_monitor(&suspender->suspend_monitors, - suspendee->id); + suspendee->common.id); else { +#ifdef DEBUG + int res; +#endif ErtsSuspendMonitor *smon; smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, - suspendee->id); - do_bif_suspend_process(smon, suspendee, NULL); - suspender->suspendee = suspendee->id; + suspendee->common.id); +#ifdef DEBUG + res = +#endif + do_bif_suspend_process(suspendee, smon, suspendee); + ASSERT(!smon || res != 0); + suspender->suspendee = suspendee->common.id; } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ @@ -5704,12 +6195,19 @@ handle_pend_bif_async_suspend(Process *suspendee, ASSERT(is_nil(suspender->suspendee)); if (!suspendee_alive) erts_delete_suspend_monitor(&suspender->suspend_monitors, - suspendee->id); + suspendee->common.id); else { +#ifdef DEBUG + int res; +#endif ErtsSuspendMonitor *smon; smon = erts_lookup_suspend_monitor(suspender->suspend_monitors, - suspendee->id); - do_bif_suspend_process(smon, suspendee, NULL); + suspendee->common.id); +#ifdef DEBUG + res = +#endif + do_bif_suspend_process(suspendee, smon, suspendee); + ASSERT(!smon || res != 0); } erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK); } @@ -5749,7 +6247,7 @@ suspend_process_2(BIF_ALIST_2) int unless_suspending = 0; - if (BIF_P->id == BIF_ARG_1) + if (BIF_P->common.id == BIF_ARG_1) goto badarg; /* We are not allowed to suspend ourselves */ if (is_not_nil(BIF_ARG_2)) { @@ -5794,7 +6292,8 @@ suspend_process_2(BIF_ALIST_2) /* This is really a piece of cake without SMP support... */ if (!smon->active) { - suspend_process(ERTS_RUNQ_IX(0), suspendee); + erts_smp_atomic32_read_bor_nob(&suspendee->state, ERTS_PSFLG_SUSPENDED); + suspend_process(BIF_P, suspendee); smon->active++; res = am_true; } @@ -5837,21 +6336,15 @@ suspend_process_2(BIF_ALIST_2) if (smon->pending && unless_suspending) res = am_false; else { - ErtsRunQueue *rq; if (smon->pending == INT_MAX) goto system_limit; smon->pending++; - rq = erts_get_runq_proc(suspendee); - erts_smp_runq_lock(rq); - if (suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) + if (!do_bif_suspend_process(BIF_P, smon, suspendee)) add_pend_suspend(suspendee, - BIF_P->id, + BIF_P->common.id, handle_pend_bif_async_suspend); - else - do_bif_suspend_process(smon, suspendee, rq); - erts_smp_runq_unlock(rq); res = am_true; } @@ -5888,7 +6381,6 @@ suspend_process_2(BIF_ALIST_2) /* done */ } else { - ErtsRunQueue *cp_rq, *s_rq; /* We haven't got any active suspends on the suspendee */ /* @@ -5905,12 +6397,7 @@ suspend_process_2(BIF_ALIST_2) if (!unless_suspending || smon->pending == 0) smon->pending++; - cp_rq = erts_get_runq_proc(BIF_P); - s_rq = erts_get_runq_proc(suspendee); - erts_smp_runqs_lock(cp_rq, s_rq); - if (!(suspendee->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING)) { - do_bif_suspend_process(smon, suspendee, s_rq); - erts_smp_runqs_unlock(cp_rq, s_rq); + if (do_bif_suspend_process(BIF_P, smon, suspendee)) { res = (!unless_suspending || smon->active == 1 ? am_true : am_false); @@ -5919,7 +6406,7 @@ suspend_process_2(BIF_ALIST_2) else { /* Mark suspendee pending for suspend by BIF_P */ add_pend_suspend(suspendee, - BIF_P->id, + BIF_P->common.id, handle_pend_bif_sync_suspend); ASSERT(is_nil(BIF_P->suspendee)); @@ -5930,8 +6417,7 @@ suspend_process_2(BIF_ALIST_2) * This time with BIF_P->suspendee == BIF_ARG_1 (see * above). */ - suspend_process(cp_rq, BIF_P); - erts_smp_runqs_unlock(cp_rq, s_rq); + suspend_process(BIF_P, BIF_P); goto yield; } } @@ -5939,9 +6425,15 @@ suspend_process_2(BIF_ALIST_2) } #endif /* ERTS_SMP */ - - ASSERT(suspendee->status == P_SUSPENDED || (asynchronous && smon->pending)); - ASSERT(suspendee->status == P_SUSPENDED || !smon->active); +#ifdef DEBUG + { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&suspendee->state); + ASSERT((state & ERTS_PSFLG_SUSPENDED) + || (asynchronous && smon->pending)); + ASSERT((state & ERTS_PSFLG_SUSPENDED) + || !smon->active); + } +#endif erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(BIF_P, xlocks); @@ -5988,7 +6480,7 @@ resume_process_1(BIF_ALIST_1) Process *suspendee; int is_active; - if (BIF_P->id == BIF_ARG_1) + if (BIF_P->common.id == BIF_ARG_1) BIF_ERROR(BIF_P, BADARG); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); @@ -6036,9 +6528,8 @@ resume_process_1(BIF_ALIST_1) if (!suspendee) goto no_suspendee; - ASSERT(suspendee->status == P_SUSPENDED - || (suspendee->status == P_GARBING - && suspendee->gcstatus == P_SUSPENDED)); + ASSERT(ERTS_PSFLG_SUSPENDED + & erts_smp_atomic32_read_nob(&suspendee->state)); resume_process(suspendee); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); @@ -6067,449 +6558,46 @@ erts_run_queues_len(Uint *qlen) Uint len = 0; ERTS_ATOMIC_FOREACH_RUNQ(rq, { - if (qlen) - qlen[i++] = rq->procs.len; - len += rq->procs.len; + Sint pqlen = 0; + int pix; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) + pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len); + + if (pqlen < 0) + pqlen = 0; + if (qlen) + qlen[i++] = pqlen; + len += pqlen; } ); return len; } -#ifdef HARDDEBUG_RUNQS -static void -check_procs_runq(ErtsRunQueue *runq, Process *p_in_q, Process *p_not_in_q) -{ - int len[ERTS_NO_PROC_PRIO_LEVELS] = {0}; - int tot_len; - int prioq, prio; - int found_p_in_q; - Process *p, *prevp; - - found_p_in_q = 0; - for (prioq = 0; prioq < ERTS_NO_PROC_PRIO_LEVELS - 1; prioq++) { - prevp = NULL; - for (p = runq->procs.prio[prioq].first; p; p = p->next) { - ASSERT(p != p_not_in_q); - if (p == p_in_q) - found_p_in_q = 1; - switch (p->prio) { - case PRIORITY_MAX: - case PRIORITY_HIGH: - case PRIORITY_NORMAL: - ASSERT(prioq == p->prio); - break; - case PRIORITY_LOW: - ASSERT(prioq == PRIORITY_NORMAL); - break; - default: - ASSERT(!"Bad prio on process"); - } - len[p->prio]++; - ASSERT(prevp == p->prev); - if (p->prev) { - ASSERT(p->prev->next == p); - } - else { - ASSERT(runq->procs.prio[prioq].first == p); - } - if (p->next) { - ASSERT(p->next->prev == p); - } - else { - ASSERT(runq->procs.prio[prioq].last == p); - } - ASSERT(p->run_queue == runq); - prevp = p; - } - } - - ASSERT(!p_in_q || found_p_in_q); - - tot_len = 0; - for (prio = 0; prio < ERTS_NO_PROC_PRIO_LEVELS; prio++) { - ASSERT(len[prio] == runq->procs.prio_info[prio].len); - if (len[prio]) { - ASSERT(runq->flags & (1 << prio)); - } - else { - ASSERT(!(runq->flags & (1 << prio))); - } - tot_len += len[prio]; - } - ASSERT(runq->procs.len == tot_len); -} -# define ERTS_DBG_CHK_PROCS_RUNQ(RQ) check_procs_runq((RQ), NULL, NULL) -# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P) check_procs_runq((RQ), (P), NULL) -# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P) check_procs_runq((RQ), NULL, (P)) -#else -# define ERTS_DBG_CHK_PROCS_RUNQ(RQ) -# define ERTS_DBG_CHK_PROCS_RUNQ_PROC(RQ, P) -# define ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(RQ, P) -#endif - - -static ERTS_INLINE void -enqueue_process(ErtsRunQueue *runq, Process *p) -{ - ErtsRunPrioQueue *rpq; - ErtsRunQueueInfo *rqi; - - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - - ASSERT(p->bound_runq || !(runq->flags & ERTS_RUNQ_FLG_SUSPENDED)); - - rqi = &runq->procs.prio_info[p->prio]; - rqi->len++; - if (rqi->max_len < rqi->len) - rqi->max_len = rqi->len; - - runq->procs.len++; - runq->len++; - if (runq->max_len < runq->len) - runq->max_len = runq->len; - - runq->flags |= (1 << p->prio); - - rpq = (p->prio == PRIORITY_LOW - ? &runq->procs.prio[PRIORITY_NORMAL] - : &runq->procs.prio[p->prio]); - - p->next = NULL; - p->prev = rpq->last; - if (rpq->last) - rpq->last->next = p; - else - rpq->first = p; - rpq->last = p; - - switch (p->status) { - case P_EXITING: - break; - case P_GARBING: - p->gcstatus = P_RUNABLE; - break; - default: - p->status = P_RUNABLE; - break; - } - -#ifdef ERTS_SMP - p->status_flags |= ERTS_PROC_SFLG_INRUNQ; -#endif - - ERTS_DBG_CHK_PROCS_RUNQ_PROC(runq, p); -} - - -static ERTS_INLINE int -dequeue_process(ErtsRunQueue *runq, Process *p) -{ - ErtsRunPrioQueue *rpq; - int res = 1; - - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - - ERTS_DBG_CHK_PROCS_RUNQ(runq); - - rpq = &runq->procs.prio[p->prio == PRIORITY_LOW ? PRIORITY_NORMAL : p->prio]; - if (p->prev) { - p->prev->next = p->next; - } - else if (rpq->first == p) { - rpq->first = p->next; - } - else { - res = 0; - } - if (p->next) { - p->next->prev = p->prev; - } - else if (rpq->last == p) { - rpq->last = p->prev; - } - else { - ASSERT(res == 0); - } - - if (res) { - - if (--runq->procs.prio_info[p->prio].len == 0) - runq->flags &= ~(1 << p->prio); - runq->procs.len--; - runq->len--; - -#ifdef ERTS_SMP - p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ; -#endif - } - - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); - return res; -} - -/* schedule a process */ -static ERTS_INLINE ErtsRunQueue * -internal_add_to_runq(ErtsRunQueue *runq, Process *p) -{ - Uint32 prev_status = p->status; - ErtsRunQueue *add_runq; -#ifdef ERTS_SMP - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - - if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) - return NULL; - else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { - ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); - p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - return NULL; - } - ASSERT(!p->scheduler_data); -#endif - - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); -#ifndef ERTS_SMP - /* Never schedule a suspended process (ok in smp case) */ - ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0); - add_runq = runq; -#else - ASSERT(!p->bound_runq || p->bound_runq == p->run_queue); - if (p->bound_runq) { - if (p->bound_runq == runq) - add_runq = runq; - else { - add_runq = p->bound_runq; - erts_smp_xrunq_lock(runq, add_runq); - } - } - else { - add_runq = erts_check_emigration_need(runq, p->prio); - if (!add_runq) - add_runq = runq; - else /* Process emigrated */ - p->run_queue = add_runq; - } -#endif - - /* Enqueue the process */ - enqueue_process(add_runq, p); - - if ((erts_system_profile_flags.runnable_procs) - && (prev_status == P_WAITING - || prev_status == P_SUSPENDED)) { - profile_runnable_proc(p, am_active); - } - - if (add_runq != runq) - erts_smp_runq_unlock(add_runq); - - return add_runq; -} - - -void -erts_add_to_runq(Process *p) -{ - ErtsRunQueue *notify_runq; - ErtsRunQueue *runq = erts_get_runq_proc(p); - erts_smp_runq_lock(runq); - notify_runq = internal_add_to_runq(runq, p); - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(notify_runq); - -} - -/* Possibly remove a scheduled process we need to suspend */ - -static int -remove_proc_from_runq(ErtsRunQueue *rq, Process *p, int to_inactive) -{ - int res; - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - -#ifdef ERTS_SMP - if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { - p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - ASSERT(!remove_proc_from_runq(rq, p, 0)); - return 1; - } -#endif - - res = dequeue_process(rq, p); - - if (res && erts_system_profile_flags.runnable_procs && to_inactive) - profile_runnable_proc(p, am_inactive); - -#ifdef ERTS_SMP - ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ)); -#endif - - return res; -} - -#ifdef ERTS_SMP - -ErtsMigrateResult -erts_proc_migrate(Process *p, ErtsProcLocks *plcks, - ErtsRunQueue *from_rq, int *from_locked, - ErtsRunQueue *to_rq, int *to_locked) -{ - ERTS_SMP_LC_ASSERT(*plcks == erts_proc_lc_my_proc_locks(p)); - ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_STATUS & *plcks) - || from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - /* - * If we have the lock on the run queue to migrate to, - * check that it isn't suspended. If it is suspended, - * we will refuse to migrate to it anyway. - */ - if (*to_locked && (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)) - return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - - /* We need status lock on process and locks on both run queues */ - - if (!(ERTS_PROC_LOCK_STATUS & *plcks)) { - if (erts_smp_proc_trylock(p, ERTS_PROC_LOCK_STATUS) == EBUSY) { - ErtsProcLocks lcks = *plcks; - Eterm pid = p->id; - Process *proc = *plcks ? p : NULL; - - if (*from_locked) { - *from_locked = 0; - erts_smp_runq_unlock(from_rq); - } - if (*to_locked) { - *to_locked = 0; - erts_smp_runq_unlock(to_rq); - } - - proc = erts_pid2proc_opt(proc, - lcks, - pid, - lcks|ERTS_PROC_LOCK_STATUS, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (!proc) { - *plcks = 0; - return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; - } - ASSERT(proc == p); - } - *plcks |= ERTS_PROC_LOCK_STATUS; - } - - ASSERT(!p->bound_runq); - - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - if (p->run_queue != from_rq) - return ERTS_MIGRATE_FAILED_RUNQ_CHANGED; - - if (!*from_locked || !*to_locked) { - if (from_rq < to_rq) { - if (!*to_locked) { - if (!*from_locked) - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - else if (erts_smp_runq_trylock(from_rq) == EBUSY) { - erts_smp_runq_unlock(to_rq); - erts_smp_runq_lock(from_rq); - erts_smp_runq_lock(to_rq); - } - } - else { - if (!*from_locked) { - if (!*to_locked) - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - else if (erts_smp_runq_trylock(to_rq) == EBUSY) { - erts_smp_runq_unlock(from_rq); - erts_smp_runq_lock(to_rq); - erts_smp_runq_lock(from_rq); - } - } - *to_locked = *from_locked = 1; - } - - ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); - ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - /* Ok we now got all locks we need; do it... */ - - /* Refuse to migrate to a suspended run queue */ - if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED; - - if ((p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) - || !(p->status_flags & ERTS_PROC_SFLG_INRUNQ)) - return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; - - dequeue_process(from_rq, p); - p->run_queue = to_rq; - enqueue_process(to_rq, p); - - return ERTS_MIGRATE_SUCCESS; -} -#endif /* ERTS_SMP */ - Eterm erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, Process *rp, Eterm rpid) { Eterm res = am_undefined; - Process *p; - - if (rp) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS - & erts_proc_lc_my_proc_locks(rp)); - p = rp; - } - else { - p = erts_pid2proc_opt(c_p, c_p_locks, - rpid, ERTS_PROC_LOCK_STATUS, - ERTS_P2P_FLG_ALLOW_OTHER_X); - } + Process *p = rp ? rp : erts_proc_lookup_raw(rpid); if (p) { - switch (p->status) { - case P_RUNABLE: - res = am_runnable; - break; - case P_WAITING: - res = am_waiting; - break; - case P_RUNNING: - res = am_running; - break; - case P_EXITING: + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (state & ERTS_PSFLG_FREE) + res = am_free; + else if (state & ERTS_PSFLG_EXITING) res = am_exiting; - break; - case P_GARBING: + else if (state & ERTS_PSFLG_GC) res = am_garbage_collecting; - break; - case P_SUSPENDED: + else if (state & ERTS_PSFLG_SUSPENDED) res = am_suspended; - break; - case P_FREE: /* We cannot look up a process in P_FREE... */ - default: /* Not a valid status... */ - erl_exit(1, "Bad status (%b32u) found for process %T\n", - p->status, p->id); - break; - } - -#ifdef ERTS_SMP - if (!rp && (p != c_p || !(ERTS_PROC_LOCK_STATUS & c_p_locks))) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + else if (state & ERTS_PSFLG_RUNNING) + res = am_running; + else if (state & ERTS_PSFLG_ACTIVE) + res = am_runnable; + else + res = am_waiting; } +#ifdef ERTS_SMP else { int i; ErtsSchedulerData *esdp; @@ -6517,50 +6605,53 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, for (i = 0; i < erts_no_schedulers; i++) { esdp = ERTS_SCHEDULER_IX(i); erts_smp_runq_lock(esdp->run_queue); - if (esdp->free_process && esdp->free_process->id == rpid) { + if (esdp->free_process + && esdp->free_process->common.id == rpid) { res = am_free; erts_smp_runq_unlock(esdp->run_queue); break; } erts_smp_runq_unlock(esdp->run_queue); } - -#endif - } - +#endif return res; } /* -** Suspend a process +** Suspend a currently executing process ** If we are to suspend on a port the busy_port is the thing ** otherwise busy_port is NIL */ void -erts_suspend(Process* process, ErtsProcLocks process_locks, Port *busy_port) +erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port) { - ErtsRunQueue *rq; - - ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); - if (!(process_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); - - rq = erts_get_runq_proc(process); - - erts_smp_runq_lock(rq); - - suspend_process(rq, process); + int suspend; - erts_smp_runq_unlock(rq); + ASSERT(c_p == erts_get_current_process()); + ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); + if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); if (busy_port) - erts_wake_process_later(busy_port, process); + suspend = erts_save_suspend_process_on_port(busy_port, c_p); + else + suspend = 1; - if (!(process_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); + if (suspend) { +#ifdef DEBUG + int res = +#endif + suspend_process(c_p, c_p); + ASSERT(res); + } + + if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + if (suspend && busy_port && erts_system_monitor_flags.busy_port) + monitor_generic(c_p, am_busy_port, busy_port->common.id); } void @@ -6575,16 +6666,19 @@ erts_resume(Process* process, ErtsProcLocks process_locks) } int -erts_resume_processes(ErtsProcList *plp) +erts_resume_processes(ErtsProcList *list) { + /* 'list' is expected to have been fetched (i.e. not a ring anymore) */ int nresumed = 0; + ErtsProcList *plp = list; + while (plp) { Process *proc; ErtsProcList *fplp; ASSERT(is_internal_pid(plp->pid)); proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS); if (proc) { - if (proclist_same(plp, proc)) { + if (erts_proclist_same(plp, proc)) { resume_process(proc); nresumed++; } @@ -6600,57 +6694,54 @@ erts_resume_processes(ErtsProcList *plp) Eterm erts_get_process_priority(Process *p) { - ErtsRunQueue *rq; - Eterm value; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = erts_get_runq_proc(p); - erts_smp_runq_lock(rq); - switch(p->prio) { - case PRIORITY_MAX: value = am_max; break; - case PRIORITY_HIGH: value = am_high; break; - case PRIORITY_NORMAL: value = am_normal; break; - case PRIORITY_LOW: value = am_low; break; - default: ASSERT(0); value = am_undefined; break; + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + switch (state & ERTS_PSFLG_PRIO_MASK) { + case PRIORITY_MAX: return am_max; + case PRIORITY_HIGH: return am_high; + case PRIORITY_NORMAL: return am_normal; + case PRIORITY_LOW: return am_low; + default: ASSERT(0); return am_undefined; } - erts_smp_runq_unlock(rq); - return value; } Eterm -erts_set_process_priority(Process *p, Eterm new_value) +erts_set_process_priority(Process *p, Eterm value) { - ErtsRunQueue *rq; - Eterm old_value; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = erts_get_runq_proc(p); -#ifdef ERTS_SMP - ASSERT(!(p->status_flags & ERTS_PROC_SFLG_INRUNQ)); -#endif - erts_smp_runq_lock(rq); - switch(p->prio) { - case PRIORITY_MAX: old_value = am_max; break; - case PRIORITY_HIGH: old_value = am_high; break; - case PRIORITY_NORMAL: old_value = am_normal; break; - case PRIORITY_LOW: old_value = am_low; break; - default: ASSERT(0); old_value = am_undefined; break; - } - switch (new_value) { - case am_max: p->prio = PRIORITY_MAX; break; - case am_high: p->prio = PRIORITY_HIGH; break; - case am_normal: p->prio = PRIORITY_NORMAL; break; - case am_low: p->prio = PRIORITY_LOW; break; - default: old_value = THE_NON_VALUE; break; + erts_aint32_t a, oprio, nprio; + + switch (value) { + case am_max: nprio = (erts_aint32_t) PRIORITY_MAX; break; + case am_high: nprio = (erts_aint32_t) PRIORITY_HIGH; break; + case am_normal: nprio = (erts_aint32_t) PRIORITY_NORMAL; break; + case am_low: nprio = (erts_aint32_t) PRIORITY_LOW; break; + default: return THE_NON_VALUE; break; } - erts_smp_runq_unlock(rq); - return old_value; -} -/* note that P_RUNNING is only set so that we don't try to remove -** running processes from the schedule queue if they exit - a running -** process not being in the schedule queue!! -** Schedule for up to INPUT_REDUCTIONS context switches, -** return 1 if more to do. -*/ + a = erts_smp_atomic32_read_nob(&p->state); + if (nprio == (a & ERTS_PSFLG_PRIO_MASK)) + oprio = nprio; + else { + erts_aint32_t e, n; + do { + oprio = a & ERTS_PSFLG_PRIO_MASK; + n = e = a; + + ASSERT(!(a & ERTS_PSFLG_IN_RUNQ)); + + n &= ~ERTS_PSFLG_PRIO_MASK; + n |= nprio; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + } while (a != e); + } + + switch (oprio) { + case PRIORITY_MAX: return am_max; + case PRIORITY_HIGH: return am_high; + case PRIORITY_NORMAL: return am_normal; + case PRIORITY_LOW: return am_low; + default: ASSERT(0); return am_undefined; + } +} /* * schedule() is called from BEAM (process_main()) or HiPE @@ -6673,7 +6764,6 @@ erts_set_process_priority(Process *p, Eterm new_value) Process *schedule(Process *p, int calls) { ErtsRunQueue *rq; - ErtsRunPrioQueue *rpq; erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; @@ -6681,6 +6771,8 @@ Process *schedule(Process *p, int calls) int input_reductions; int actual_reds; int reds; + Uint32 flags; + erts_aint32_t state = 0; /* Supress warning... */ #ifdef USE_VM_PROBES if (p != NULL && DTRACE_ENABLED(process_unscheduled)) { @@ -6736,95 +6828,61 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - if ((erts_system_profile_flags.runnable_procs) - && (p->status == P_WAITING)) { - profile_runnable_proc(p, am_inactive); - } + state = erts_smp_atomic32_read_acqb(&p->state); if (IS_TRACED(p)) { - if (IS_TRACED_FL(p, F_TRACE_CALLS) && p->status != P_FREE) { + if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); - } - switch (p->status) { - case P_EXITING: + if (state & (ERTS_PSFLG_FREE|ERTS_PSFLG_EXITING)) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, am_out_exiting); - break; - case P_FREE: - if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) - trace_sched(p, am_out_exited); - break; - default: + trace_sched(p, ((state & ERTS_PSFLG_FREE) + ? am_out_exited + : am_out_exiting)); + } + else { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) trace_sched(p, am_out); else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(p, am_out); - break; } - } - -#ifdef ERTS_SMP - if (ERTS_PROC_PENDING_EXIT(p)) { - erts_handle_pending_exit(p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; } - if (p->pending_suspenders) { - handle_pending_suspend(p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) - || p->rcount == 0); - } +#ifdef ERTS_SMP + if (state & ERTS_PSFLG_PENDING_EXIT) + erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS)); + if (p->pending_suspenders) + handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS)); #endif esdp->reductions += reds; - erts_smp_runq_lock(rq); + schedule_out_process(rq, state, p); /* Returns with rq locked! */ - ERTS_PROC_REDUCTIONS_EXECUTED(rq, p->prio, reds, actual_reds); + ERTS_PROC_REDUCTIONS_EXECUTED(rq, + (int) (state & ERTS_PSFLG_PRIO_MASK), + reds, + actual_reds); esdp->current_process = NULL; #ifdef ERTS_SMP p->scheduler_data = NULL; - p->runq_flags &= ~ERTS_PROC_RUNQ_FLG_RUNNING; - p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; - - if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { - ErtsRunQueue *notify_runq; - p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - notify_runq = internal_add_to_runq(rq, p); - if (notify_runq != rq) - smp_notify_inc_runq(notify_runq); - } #endif - if (p->status == P_FREE) { + if (state & ERTS_PSFLG_FREE) { #ifdef ERTS_SMP ASSERT(esdp->free_process == p); esdp->free_process = NULL; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - erts_smp_proc_dec_refc(p); #else erts_free_proc(p); #endif - } else { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); } -#ifdef ERTS_SMP - { - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - rq->procs.pending_exiters = NULL; + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - if (pnd_xtrs) { - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } - - } +#ifdef ERTS_SMP ASSERT(!esdp->free_process); #endif ASSERT(!esdp->current_process); @@ -6844,8 +6902,22 @@ Process *schedule(Process *p, int calls) ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); check_activities_to_run: { +#ifdef ERTS_SMP + ErtsMigrationPaths *mps; + ErtsMigrationPath *mp; #ifdef ERTS_SMP + { + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + if (erts_proclist_fetch(&pnd_xtrs, NULL)) { + rq->procs.pending_exiters = NULL; + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); + } + + } +#endif if (rq->check_balance_reds <= 0) check_balance(rq); @@ -6853,20 +6925,28 @@ Process *schedule(Process *p, int calls) ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) - immigrate(rq); + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; + + if (mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) + immigrate(rq, mp); + + continue_check_activities_to_run: + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + continue_check_activities_to_run_known_flags: - continue_check_activities_to_run: - if (rq->flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND - | ERTS_RUNQ_FLG_SUSPENDED)) { - if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { - ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED); + if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { + + if (flags & ERTS_RUNQ_FLG_SUSPENDED) { suspend_scheduler(esdp); + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); } - if (rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) + if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { + flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); + flags &= ~ ERTS_RUNQ_FLG_CHK_CPU_BIND; erts_sched_check_cpu_bind(esdp); + } } { @@ -6895,14 +6975,12 @@ Process *schedule(Process *p, int calls) } #endif /* ERTS_SMP */ - ASSERT(rq->len == rq->procs.len + rq->ports.info.len); - - if ((rq->len == 0 && !rq->misc.start) - || (rq->halt_in_progress - && rq->ports.info.len == 0 && !rq->misc.start)) { + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) + || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { + /* Prepare for scheduler wait */ #ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); rq->wakeup_other = 0; @@ -6910,21 +6988,27 @@ Process *schedule(Process *p, int calls) empty_runq(rq); - if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { - ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED); + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if (flags & ERTS_RUNQ_FLG_SUSPENDED) { non_empty_runq(rq); - goto continue_check_activities_to_run; + goto continue_check_activities_to_run_known_flags; } - else if (!(rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { + else if (!(flags & ERTS_RUNQ_FLG_INACTIVE)) { + if (try_steal_task(rq)) { + non_empty_runq(rq); + goto continue_check_activities_to_run; + } + + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + /* * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done * after trying to steal a task. */ - if (try_steal_task(rq) - || (rq->flags & ERTS_RUNQ_FLG_SUSPENDED)) { + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if (flags & ERTS_RUNQ_FLG_SUSPENDED) { non_empty_runq(rq); - goto continue_check_activities_to_run; + goto continue_check_activities_to_run_known_flags; } } @@ -6955,6 +7039,7 @@ Process *schedule(Process *p, int calls) erl_sys_schedule(1); dt = erts_do_time_read_and_reset(); if (dt) erts_bump_timer(dt); + #ifdef ERTS_SMP erts_smp_runq_lock(rq); clear_sys_scheduling(); @@ -6968,14 +7053,14 @@ Process *schedule(Process *p, int calls) exec_misc_ops(rq); #ifdef ERTS_SMP - wakeup_other.check(rq); + wakeup_other.check(rq, flags); #endif /* * Find a new port to run. */ - if (rq->ports.info.len) { + if (RUNQ_READ_LEN(&rq->ports.info.len)) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); if ((have_outstanding_io && fcalls > 2*input_reductions) @@ -7002,167 +7087,130 @@ Process *schedule(Process *p, int calls) /* * Find a new process to run. */ - pick_next_process: - - ERTS_DBG_CHK_PROCS_RUNQ(rq); - - switch (rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) { - case MAX_BIT: - case MAX_BIT|HIGH_BIT: - case MAX_BIT|NORMAL_BIT: - case MAX_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT: - case MAX_BIT|HIGH_BIT|LOW_BIT: - case MAX_BIT|NORMAL_BIT|LOW_BIT: - case MAX_BIT|HIGH_BIT|NORMAL_BIT|LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_MAX]; - break; - case HIGH_BIT: - case HIGH_BIT|NORMAL_BIT: - case HIGH_BIT|LOW_BIT: - case HIGH_BIT|NORMAL_BIT|LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_HIGH]; - break; - case NORMAL_BIT: - rpq = &rq->procs.prio[PRIORITY_NORMAL]; - break; - case LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_NORMAL]; - break; - case NORMAL_BIT|LOW_BIT: - rpq = &rq->procs.prio[PRIORITY_NORMAL]; - ASSERT(rpq->first != NULL); - p = rpq->first; - if (p->prio == PRIORITY_LOW) { - if (p == rpq->last || p->skipped >= RESCHEDULE_LOW-1) - p->skipped = 0; - else { - /* skip it */ - p->skipped++; - rpq->first = p->next; - rpq->first->prev = NULL; - rpq->last->next = p; - p->prev = rpq->last; - p->next = NULL; - rpq->last = p; + pick_next_process: { + int prio_q; + int qmask; + + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + qmask = (int) (flags & ERTS_RUNQ_FLGS_PROCS_QMASK); + switch (qmask & -qmask) { + case MAX_BIT: + prio_q = PRIORITY_MAX; + break; + case HIGH_BIT: + prio_q = PRIORITY_HIGH; + break; + case NORMAL_BIT: + case LOW_BIT: + prio_q = PRIORITY_NORMAL; + if (check_requeue_process(rq, PRIORITY_NORMAL)) goto pick_next_process; - } + break; + case 0: /* No process at all */ + default: + ASSERT(qmask == 0); + goto check_activities_to_run; } - break; - case 0: /* No process at all */ - default: - ASSERT((rq->flags & ERTS_RUNQ_FLGS_PROCS_QMASK) == 0); - ASSERT(rq->procs.len == 0); - goto check_activities_to_run; - } - BM_START_TIMER(system); + BM_START_TIMER(system); - /* - * Take the chosen process out of the queue. - */ - ASSERT(rpq->first); /* Wrong qmask in rq->flags? */ - p = rpq->first; -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(rq == p->run_queue); -#endif - rpq->first = p->next; - if (!rpq->first) - rpq->last = NULL; - else - rpq->first->prev = NULL; - - p->next = p->prev = NULL; + /* + * Take the chosen process out of the queue. + */ + p = dequeue_process(rq, prio_q, &state); - if (--rq->procs.prio_info[p->prio].len == 0) - rq->flags &= ~(1 << p->prio); - ASSERT(rq->procs.len > 0); - rq->procs.len--; - ASSERT(rq->len > 0); - rq->len--; + ASSERT(p); /* Wrong qmask in rq->flags? */ - { - Uint32 ee_flgs = (ERTS_RUNQ_FLG_EVACUATE(p->prio) - | ERTS_RUNQ_FLG_EMIGRATE(p->prio)); - - if ((rq->flags & (ERTS_RUNQ_FLG_SUSPENDED|ee_flgs)) == ee_flgs) - ERTS_UNSET_RUNQ_FLG_EVACUATE(rq->flags, p->prio); - } + while (1) { + erts_aint32_t exp, new, tmp; + tmp = new = exp = state; + new &= ~ERTS_PSFLG_IN_RUNQ; + tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); + if (tmp != ERTS_PSFLG_SUSPENDED) + new |= ERTS_PSFLG_RUNNING; + state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp); + if (state == exp) { + tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); + if (tmp == ERTS_PSFLG_SUSPENDED) + goto pick_next_process; + state = new; + break; + } + } - ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(rq, p); + rq->procs.context_switches++; - rq->procs.context_switches++; + esdp->current_process = p; - esdp->current_process = p; + } #ifdef ERTS_SMP - p->runq_flags |= ERTS_PROC_RUNQ_FLG_RUNNING; erts_smp_runq_unlock(rq); + if (flags & ERTS_RUNQ_FLG_PROTECTED) + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + ERTS_SMP_CHK_NO_PROC_LOCKS; erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); if (erts_sched_stat.enabled) { + int prio; UWord old = ERTS_PROC_SCHED_ID(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_STATUS), (UWord) esdp->no); int migrated = old && old != esdp->no; + prio = (int) (state & ERTS_PSFLG_PRIO_MASK); + erts_smp_spin_lock(&erts_sched_stat.lock); - erts_sched_stat.prio[p->prio].total_executed++; - erts_sched_stat.prio[p->prio].executed++; + erts_sched_stat.prio[prio].total_executed++; + erts_sched_stat.prio[prio].executed++; if (migrated) { - erts_sched_stat.prio[p->prio].total_migrated++; - erts_sched_stat.prio[p->prio].migrated++; + erts_sched_stat.prio[prio].total_migrated++; + erts_sched_stat.prio[prio].migrated++; } erts_smp_spin_unlock(&erts_sched_stat.lock); } - p->status_flags |= ERTS_PROC_SFLG_RUNNING; - p->status_flags &= ~ERTS_PROC_SFLG_INRUNQ; if (ERTS_PROC_PENDING_EXIT(p)) { erts_handle_pending_exit(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_nob(&p->state); } ASSERT(!p->scheduler_data); p->scheduler_data = esdp; - #endif - ASSERT(p->status != P_SUSPENDED); /* Never run a suspended process */ + /* Never run a suspended process */ + ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); reds = context_reds; if (IS_TRACED(p)) { - switch (p->status) { - case P_EXITING: + if (state & ERTS_PSFLG_EXITING) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) trace_sched(p, am_in_exiting); - break; - default: + } + else { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED)) trace_sched(p, am_in); else if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(p, am_in); - break; } if (IS_TRACED_FL(p, F_TRACE_CALLS)) { erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN); } } - if (p->status != P_EXITING) - p->status = P_RUNNING; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #ifdef ERTS_SMP - if (is_not_nil(p->tracer_proc)) + if (is_not_nil(ERTS_TRACER_PROC(p))) erts_check_my_tracer_proc(p); #endif - if (!ERTS_PROC_IS_EXITING(p) + if (!(state & ERTS_PSFLG_EXITING) && ((FLAGS(p) & F_FORCE_GC) || (MSO(p).overhead > BIN_VHEAP_SZ(p)))) { reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); @@ -7189,7 +7237,7 @@ erts_sched_stat_modify(int what) break; case ERTS_SCHED_STAT_MODIFY_DISABLE: erts_smp_thr_progress_block(); - erts_sched_stat.enabled = 1; + erts_sched_stat.enabled = 0; erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_CLEAR: @@ -7253,17 +7301,19 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0); ErtsMiscOpList *molp = misc_op_list_alloc(); +#ifdef ERTS_SMP + ErtsMigrationPaths *mpaths = erts_get_migration_paths(); - erts_smp_runq_lock(rq); - - while (rq->misc.evac_runq) { - ErtsRunQueue *tmp_rq = rq->misc.evac_runq; - erts_smp_runq_unlock(rq); - rq = tmp_rq; - erts_smp_runq_lock(rq); + if (!mpaths) + rq = ERTS_RUNQ_IX(0); + else { + ErtsRunQueue *erq = mpaths->mpath[rq->ix].misc_evac_runq; + if (erq) + rq = erq; } +#endif - ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + erts_smp_runq_lock(rq); molp->next = NULL; molp->func = func; @@ -7273,7 +7323,9 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) else rq->misc.start = molp; rq->misc.end = molp; + erts_smp_runq_unlock(rq); + smp_notify_inc_runq(rq); } @@ -7357,156 +7409,75 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } -/* - * erts_test_next_pid() is only used for testing. - */ -Sint -erts_test_next_pid(int set, Uint next) +void +erts_free_proc(Process *p) { - Sint res; - Sint p_prev; - - - erts_smp_mtx_lock(&proc_tab_mtx); - - if (!set) { - res = p_next < 0 ? -1 : (p_serial << p_serial_shift | p_next); - } - else { - - p_serial = (Sint) ((next >> p_serial_shift) & p_serial_mask); - p_next = (Sint) (erts_process_tab_index_mask & next); - - if (p_next >= erts_max_processes) { - p_next = 0; - p_serial++; - p_serial &= p_serial_mask; - } - - p_prev = p_next; - - do { - if (!process_tab[p_next]) - break; - p_next++; - if(p_next >= erts_max_processes) { - p_next = 0; - p_serial++; - p_serial &= p_serial_mask; - } - } while (p_prev != p_next); - - res = process_tab[p_next] ? -1 : (p_serial << p_serial_shift | p_next); - - } +#ifdef ERTS_SMP + erts_proc_lock_fin(p); +#endif + erts_free(ERTS_ALC_T_PROC, (void *) p); +} - erts_smp_mtx_unlock(&proc_tab_mtx); +typedef struct { + Process *proc; + erts_aint32_t state; + ErtsRunQueue *run_queue; +} ErtsEarlyProcInit; - return res; +static void early_init_process_struct(void *varg, Eterm data) +{ + ErtsEarlyProcInit *arg = (ErtsEarlyProcInit *) varg; + Process *proc = arg->proc; -} + proc->common.id = make_internal_pid(data); + erts_smp_atomic32_init_relb(&proc->state, arg->state); -Uint erts_process_count(void) -{ - erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count); - ASSERT(res >= 0); - return (Uint) res; -} +#ifdef ERTS_SMP + RUNQ_SET_RQ(&proc->run_queue, arg->run_queue); -void -erts_free_proc(Process *p) -{ -#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) - erts_lcnt_proc_lock_destroy(p); + erts_proc_lock_init(proc); /* All locks locked */ #endif - erts_free(ERTS_ALC_T_PROC, (void *) p); -} +} /* ** Allocate process and find out where to place next process. */ static Process* -alloc_process(void) +alloc_process(ErtsRunQueue *rq, erts_aint32_t state) { -#ifdef ERTS_SMP - erts_pix_lock_t *pix_lock; -#endif - Process* p; - int p_prev; - - erts_smp_mtx_lock(&proc_tab_mtx); - - if (p_next == -1) { - p = NULL; - goto error; /* Process table full! */ - } + ErtsEarlyProcInit init_arg; + Process *p; - p = (Process*) erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process)); + p = erts_alloc_fnf(ERTS_ALC_T_PROC, sizeof(Process)); if (!p) - goto error; /* ENOMEM */ - - p_last = p_next; + return NULL; - erts_get_emu_time(&p->started); + init_arg.proc = (Process *) p; + init_arg.run_queue = rq; + init_arg.state = state; -#ifdef ERTS_SMP - pix_lock = ERTS_PIX2PIXLOCK(p_next); - erts_pix_lock(pix_lock); -#endif - ASSERT(!process_tab[p_next]); + ASSERT(((char *) p) == ((char *) &p->common)); - process_tab[p_next] = p; - erts_smp_atomic32_inc_nob(&process_count); - p->id = make_internal_pid(p_serial << p_serial_shift | p_next); - if (p->id == ERTS_INVALID_PID) { - /* Do not use the invalid pid; change serial */ - p_serial++; - p_serial &= p_serial_mask; - p->id = make_internal_pid(p_serial << p_serial_shift | p_next); - ASSERT(p->id != ERTS_INVALID_PID); + if (!erts_ptab_new_element(&erts_proc, + &p->common, + (void *) &init_arg, + early_init_process_struct)) { + erts_free(ERTS_ALC_T_PROC, p); + return NULL; } - ASSERT(internal_pid_serial(p->id) <= (erts_use_r9_pids_ports - ? ERTS_MAX_PID_R9_SERIAL - : ERTS_MAX_PID_SERIAL)); -#ifdef ERTS_SMP - erts_proc_lock_init(p); /* All locks locked */ - erts_pix_unlock(pix_lock); -#endif - - p->rstatus = P_FREE; + ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL); + + p->approx_started = erts_get_approx_time(); p->rcount = 0; - /* - * set p_next to the next available slot - */ - - p_prev = p_next; - while (1) { - p_next++; - if(p_next >= erts_max_processes) { - p_serial++; - p_serial &= p_serial_mask; - p_next = 0; - } - - if (p_prev == p_next) { - p_next = -1; - break; /* Table full! */ - } - - if (!process_tab[p_next]) - break; /* found a free slot */ - } - - error: - - erts_smp_mtx_unlock(&proc_tab_mtx); + ASSERT(p == (Process *) (erts_ptab_pix2intptr_nob( + &erts_proc, + internal_pid_index(p->common.id)))); return p; - } Eterm @@ -7516,13 +7487,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { - ErtsRunQueue *rq, *notify_runq; + ErtsRunQueue *rq = NULL; Process *p; Sint arity; /* Number of arguments. */ Uint arg_size; /* Size of arguments. */ Uint sz; /* Needed words on heap. */ Uint heap_need; /* Size needed on heap. */ Eterm res = THE_NON_VALUE; + erts_aint32_t state = 0; + erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; #ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -7536,8 +7509,24 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). so->error_code = BADARG; goto error; } - p = alloc_process(); /* All proc locks are locked by this thread - on success */ + + if (so->flags & SPO_USE_ARGS) { + if (so->scheduler) { + int ix = so->scheduler-1; + ASSERT(0 <= ix && ix < erts_no_run_queues); + rq = ERTS_RUNQ_IX(ix); + state |= ERTS_PSFLG_BOUND; + } + prio = (erts_aint32_t) so->priority; + } + + state |= (prio & ERTS_PSFLG_PRIO_MASK); + + if (!rq) + rq = erts_get_runq_proc(parent); + + p = alloc_process(rq, state); /* All proc locks are locked by this thread + on success */ if (!p) { erts_send_error_to_logger_str(parent->group_leader, "Too many processes\n"); @@ -7557,22 +7546,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->flags = erts_default_process_flags; - /* Scheduler queue mutex should be locked when changeing - * prio. In this case we don't have to lock it, since - * noone except us has access to the process. - */ if (so->flags & SPO_USE_ARGS) { p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; - p->prio = so->priority; p->max_gen_gcs = so->max_gen_gcs; } else { p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; - p->prio = PRIORITY_NORMAL; p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); } - p->skipped = 0; + p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); p->initial[INITIAL_MOD] = mod; @@ -7609,6 +7592,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->htop = p->heap; p->heap_sz = sz; p->catches = 0; + p->extra_root = NULL; p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; @@ -7641,21 +7625,21 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->reds = 0; #ifdef ERTS_SMP - p->u.ptimer = NULL; + p->common.u.alive.ptimer = NULL; #else - sys_memset(&p->u.tm, 0, sizeof(ErlTimer)); + sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer)); #endif - p->reg = NULL; - p->nlinks = NULL; - p->monitors = NULL; + p->common.u.alive.reg = NULL; + ERTS_P_LINKS(p) = NULL; + ERTS_P_MONITORS(p) = NULL; p->nodes_monitors = NULL; p->suspend_monitors = NULL; ASSERT(is_pid(parent->group_leader)); if (parent->group_leader == ERTS_INVALID_PID) - p->group_leader = p->id; + p->group_leader = p->common.id; else { /* Needs to be done after the heap has been set up */ p->group_leader = @@ -7664,7 +7648,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } - erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); + erts_get_default_tracing(&ERTS_TRACE_FLAGS(p), &ERTS_TRACER_PROC(p)); p->msg.first = NULL; p->msg.last = &p->msg.first; @@ -7674,9 +7658,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.first = NULL; p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; - p->bound_runq = NULL; #endif - p->bif_timers = NULL; + p->u.bif_timers = NULL; p->mbuf = NULL; p->mbuf_sz = 0; p->psd = NULL; @@ -7688,7 +7671,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). DT_UTAG(p) = NIL; DT_UTAG_FLAGS(p) = 0; #endif - p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id; + p->parent = (parent->common.id == ERTS_INVALID_PID + ? NIL + : parent->common.id); INIT_HOLE_CHECK(p); #ifdef DEBUG @@ -7696,18 +7681,19 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #endif if (IS_TRACED(parent)) { - if (parent->trace_flags & F_TRACE_SOS) { - p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); - p->tracer_proc = parent->tracer_proc; + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) { + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); + ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); } if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { - trace_proc_spawn(parent, p->id, mod, func, args); + trace_proc_spawn(parent, p->common.id, mod, func, args); } - if (parent->trace_flags & F_TRACE_SOS1) { /* Overrides TRACE_CHILDREN */ - p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); - p->tracer_proc = parent->tracer_proc; - p->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS); - parent->trace_flags &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS1) { + /* Overrides TRACE_CHILDREN */ + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); + ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); + ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOS1 | F_TRACE_SOS); } } @@ -7720,27 +7706,27 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). int ret; #endif if (IS_TRACED_FL(parent, F_TRACE_PROCS)) { - trace_proc(parent, parent, am_link, p->id); + trace_proc(parent, parent, am_link, p->common.id); } #ifdef DEBUG - ret = erts_add_link(&(parent->nlinks), LINK_PID, p->id); + ret = erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id); ASSERT(ret == 0); - ret = erts_add_link(&(p->nlinks), LINK_PID, parent->id); + ret = erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id); ASSERT(ret == 0); #else - erts_add_link(&(parent->nlinks), LINK_PID, p->id); - erts_add_link(&(p->nlinks), LINK_PID, parent->id); + erts_add_link(&ERTS_P_LINKS(parent), LINK_PID, p->common.id); + erts_add_link(&ERTS_P_LINKS(p), LINK_PID, parent->common.id); #endif if (IS_TRACED(parent)) { - if (parent->trace_flags & (F_TRACE_SOL|F_TRACE_SOL1)) { - p->trace_flags |= (parent->trace_flags & TRACEE_FLAGS); - p->tracer_proc = parent->tracer_proc; /* maybe steal */ + if (ERTS_TRACE_FLAGS(parent) & (F_TRACE_SOL|F_TRACE_SOL1)) { + ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent)&TRACEE_FLAGS); + ERTS_TRACER_PROC(p) = ERTS_TRACER_PROC(parent); /*maybe steal*/ - if (parent->trace_flags & F_TRACE_SOL1) { /* maybe override */ - p ->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); - parent->trace_flags &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOL1) {/*maybe override*/ + ERTS_TRACE_FLAGS(p) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); + ERTS_TRACE_FLAGS(parent) &= ~(F_TRACE_SOL1 | F_TRACE_SOL); } } } @@ -7753,16 +7739,13 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm mref; mref = erts_make_ref(parent); - erts_add_monitor(&(parent->monitors), MON_ORIGIN, mref, p->id, NIL); - erts_add_monitor(&(p->monitors), MON_TARGET, mref, parent->id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(parent), MON_ORIGIN, mref, p->common.id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(p), MON_TARGET, mref, parent->common.id, NIL); so->mref = mref; } #ifdef ERTS_SMP p->scheduler_data = NULL; - p->is_exiting = 0; - p->status_flags = 0; - p->runq_flags = 0; p->suspendee = NIL; p->pending_suspenders = NULL; p->pending_exit.reason = THE_NON_VALUE; @@ -7773,36 +7756,17 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->fp_exception = 0; #endif + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + + res = p->common.id; + /* * Schedule process for execution. */ - if (!((so->flags & SPO_USE_ARGS) && so->scheduler)) - rq = erts_get_runq_proc(parent); - else { - int ix = so->scheduler-1; - ASSERT(0 <= ix && ix < erts_no_run_queues); - rq = ERTS_RUNQ_IX(ix); - p->bound_runq = rq; - } - - erts_smp_runq_lock(rq); - -#ifdef ERTS_SMP - p->run_queue = rq; -#endif - - p->status = P_WAITING; - notify_runq = internal_add_to_runq(rq, p); - - erts_smp_runq_unlock(rq); + schedule_process(p, state, 0); - smp_notify_inc_runq(notify_runq); - - res = p->id; - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - - VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id)); + VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id)); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_spawn)) { @@ -7836,15 +7800,11 @@ void erts_init_empty_process(Process *p) p->max_gen_gcs = 0; p->min_heap_size = 0; p->min_vheap_size = 0; - p->status = P_RUNABLE; - p->gcstatus = P_RUNABLE; - p->rstatus = P_RUNABLE; p->rcount = 0; - p->id = ERTS_INVALID_PID; - p->prio = PRIORITY_NORMAL; + p->common.id = ERTS_INVALID_PID; p->reds = 0; - p->tracer_proc = NIL; - p->trace_flags = F_INITIAL_TRACE_FLAGS; + ERTS_TRACER_PROC(p) = NIL; + ERTS_TRACE_FLAGS(p) = F_INITIAL_TRACE_FLAGS; p->group_leader = ERTS_INVALID_PID; p->flags = 0; p->fvalue = NIL; @@ -7857,15 +7817,14 @@ void erts_init_empty_process(Process *p) p->bin_old_vheap = 0; p->bin_vheap_mature = 0; #ifdef ERTS_SMP - p->u.ptimer = NULL; - p->bound_runq = NULL; + p->common.u.alive.ptimer = NULL; #else - memset(&(p->u.tm), 0, sizeof(ErlTimer)); + memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer)); #endif p->next = NULL; p->off_heap.first = NULL; p->off_heap.overhead = 0; - p->reg = NULL; + p->common.u.alive.reg = NULL; p->heap_sz = 0; p->high_water = NULL; p->old_hend = NULL; @@ -7874,15 +7833,15 @@ void erts_init_empty_process(Process *p) p->mbuf = NULL; p->mbuf_sz = 0; p->psd = NULL; - p->monitors = NULL; - p->nlinks = NULL; /* List of links */ + ERTS_P_MONITORS(p) = NULL; + ERTS_P_LINKS(p) = NULL; /* List of links */ p->nodes_monitors = NULL; p->suspend_monitors = NULL; p->msg.first = NULL; p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; p->msg.len = 0; - p->bif_timers = NULL; + p->u.bif_timers = NULL; p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; @@ -7909,8 +7868,8 @@ void erts_init_empty_process(Process *p) p->def_arg_reg[5] = 0; p->parent = NIL; - p->started.tv_sec = 0; - p->started.tv_usec = 0; + p->approx_started = 0; + p->common.u.alive.started_interval = 0; #ifdef HIPE hipe_init_process(&p->hipe); @@ -7924,12 +7883,10 @@ void erts_init_empty_process(Process *p) p->last_old_htop = NULL; #endif + erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); #ifdef ERTS_SMP p->scheduler_data = NULL; - p->is_exiting = 0; - p->status_flags = 0; - p->runq_flags = 0; p->msg_inq.first = NULL; p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; @@ -7939,7 +7896,7 @@ void erts_init_empty_process(Process *p) p->pending_exit.bp = NULL; erts_proc_lock_init(p); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - p->run_queue = ERTS_RUNQ_IX(0); + RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); #endif #if !defined(NO_FPE_SIGNALS) || defined(HIPE) @@ -7958,25 +7915,25 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->stop == NULL); ASSERT(p->hend == NULL); ASSERT(p->heap == NULL); - ASSERT(p->id == ERTS_INVALID_PID); - ASSERT(p->tracer_proc == NIL); - ASSERT(p->trace_flags == F_INITIAL_TRACE_FLAGS); + ASSERT(p->common.id == ERTS_INVALID_PID); + ASSERT(ERTS_TRACER_PROC(p) == NIL); + ASSERT(ERTS_TRACE_FLAGS(p) == F_INITIAL_TRACE_FLAGS); ASSERT(p->group_leader == ERTS_INVALID_PID); ASSERT(p->next == NULL); - ASSERT(p->reg == NULL); + ASSERT(p->common.u.alive.reg == NULL); ASSERT(p->heap_sz == 0); ASSERT(p->high_water == NULL); ASSERT(p->old_hend == NULL); ASSERT(p->old_htop == NULL); ASSERT(p->old_heap == NULL); - ASSERT(p->monitors == NULL); - ASSERT(p->nlinks == NULL); + ASSERT(ERTS_P_MONITORS(p) == NULL); + ASSERT(ERTS_P_LINKS(p) == NULL); ASSERT(p->nodes_monitors == NULL); ASSERT(p->suspend_monitors == NULL); ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); - ASSERT(p->bif_timers == NULL); + ASSERT(p->u.bif_timers == NULL); ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -8017,8 +7974,8 @@ erts_cleanup_empty_process(Process* p) free_message_buffer(p->mbuf); p->mbuf = NULL; } -#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) - erts_lcnt_proc_lock_destroy(p); +#ifdef ERTS_SMP + erts_proc_lock_fin(p); #endif #ifdef DEBUG erts_debug_verify_clean_empty_process(p); @@ -8033,7 +7990,7 @@ delete_process(Process* p) { ErlMessage* mp; - VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id)); + VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); /* Cleanup psd */ @@ -8107,34 +8064,40 @@ delete_process(Process* p) mp = next_mp; } - ASSERT(!p->monitors); - ASSERT(!p->nlinks); ASSERT(!p->nodes_monitors); ASSERT(!p->suspend_monitors); p->fvalue = NIL; } +static ERTS_INLINE erts_aint32_t +set_proc_exiting_state(Process *p, erts_aint32_t state) +{ + erts_aint32_t a, n, e; + a = state; + while (1) { + n = e = a; + n &= ~(ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); + n |= ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE; + if (!(a & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING))) + n |= ERTS_PSFLG_IN_RUNQ; + a = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e); + if (a == e) + break; + } + return a; +} + static ERTS_INLINE void -set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) +set_proc_exiting(Process *p, + erts_aint32_t state, + Eterm reason, + ErlHeapFragment *bp) { -#ifdef ERTS_SMP - erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); - /* - * You are required to have all proc locks and the pix lock when going - * to status P_EXITING. This makes it is enough to take any lock when - * looking up a process (pid2proc()) to prevent the looked up process - * from exiting until the lock has been released. - */ - erts_pix_lock(pix_lock); - p->is_exiting = 1; -#endif - p->status = P_EXITING; -#ifdef ERTS_SMP - erts_pix_unlock(pix_lock); -#endif + state = set_proc_exiting_state(p, state); + p->fvalue = reason; if (bp) erts_link_mbuf_to_proc(p, bp); @@ -8147,6 +8110,14 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp) KILL_CATCHES(p); cancel_timer(p); p->i = (BeamInstr *) beam_exit; + + if (erts_system_profile_flags.runnable_procs + && !(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) { + profile_runnable_proc(p, am_active); + } + + if (!(state & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING))) + add2runq(p, state); } @@ -8159,8 +8130,8 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) ASSERT(is_value(c_p->pending_exit.reason)); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks); ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN); - ERTS_SMP_LC_ASSERT(c_p->status != P_EXITING); - ERTS_SMP_LC_ASSERT(c_p->status != P_FREE); + ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE) + & erts_smp_atomic32_read_nob(&c_p->state))); /* Ensure that all locks on c_p are locked before proceeding... */ if (locks == ERTS_PROC_LOCKS_ALL) @@ -8173,7 +8144,10 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) } } - set_proc_exiting(c_p, c_p->pending_exit.reason, c_p->pending_exit.bp); + set_proc_exiting(c_p, + erts_smp_atomic32_read_acqb(&c_p->state), + c_p->pending_exit.reason, + c_p->pending_exit.bp); c_p->pending_exit.reason = THE_NON_VALUE; c_p->pending_exit.bp = NULL; @@ -8184,16 +8158,19 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) static void handle_pending_exiters(ErtsProcList *pnd_xtrs) { + /* 'list' is expected to have been fetched (i.e. not a ring anymore) */ ErtsProcList *plp = pnd_xtrs; - ErtsProcList *free_plp; + while (plp) { + ErtsProcList *free_plp; Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL); if (p) { - if (proclist_same(plp, p) - && !(p->status_flags & ERTS_PROC_SFLG_RUNNING)) { - ASSERT(p->status_flags & ERTS_PROC_SFLG_INRUNQ); - ASSERT(ERTS_PROC_PENDING_EXIT(p)); - erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); + if (erts_proclist_same(plp, p)) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & ERTS_PSFLG_RUNNING)) { + ASSERT(state & ERTS_PSFLG_PENDING_EXIT); + erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); + } } erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); } @@ -8217,11 +8194,10 @@ save_pending_exiter(Process *p) erts_smp_runq_lock(rq); - plp->next = rq->procs.pending_exiters; - rq->procs.pending_exiters = plp; + erts_proclist_store_last(&rq->procs.pending_exiters, plp); erts_smp_runq_unlock(rq); - + wake_scheduler(rq, 1); } #endif @@ -8265,7 +8241,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, hp = bp->mem; mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap); /* the trace token must in this case be updated by the caller */ - seq_trace_output(token, mess, SEQ_TRACE_SEND, to->id, NULL); + seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); erts_queue_message(to, to_locksp, bp, mess, temp_token #ifdef USE_VM_PROBES @@ -8283,7 +8259,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, * SMP emulator). When the signal is received the receiver receives an * 'EXIT' message if it is trapping exits; otherwise, it will either * ignore the signal if the exit reason is normal, or go into an - * exiting state (status P_EXITING). When a process has gone into the + * exiting state (ERTS_PSFLG_EXITING). When a process has gone into the * exiting state it will not execute any more Erlang code, but it might * take a while before it actually exits. The exit signal is being * received when the 'EXIT' message is put in the message queue, the @@ -8356,6 +8332,7 @@ send_exit_signal(Process *c_p, /* current process if and only Uint32 flags /* flags */ ) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); Eterm rsn = reason == am_kill ? am_killed : reason; ERTS_SMP_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp)); @@ -8377,7 +8354,7 @@ send_exit_signal(Process *c_p, /* current process if and only } #endif - if (ERTS_PROC_IS_TRAPPING_EXITS(rp) + if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { if (is_not_nil(token) #ifdef USE_VM_PROBES @@ -8393,9 +8370,7 @@ send_exit_signal(Process *c_p, /* current process if and only } else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) { #ifdef ERTS_SMP - if (!ERTS_PROC_PENDING_EXIT(rp) && !rp->is_exiting) { - ASSERT(rp->status != P_EXITING); - ASSERT(rp->status != P_FREE); + if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) { ASSERT(!rp->pending_exit.bp); if (rp == c_p && (*rp_locks & ERTS_PROC_LOCK_MAIN)) { @@ -8411,9 +8386,9 @@ send_exit_signal(Process *c_p, /* current process if and only } *rp_locks = ERTS_PROC_LOCKS_ALL; } - set_proc_exiting(c_p, rsn, NULL); + set_proc_exiting(c_p, state, rsn, NULL); } - else if (!(rp->status_flags & ERTS_PROC_SFLG_RUNNING)) { + else if (!(state & ERTS_PSFLG_RUNNING)) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; if (need_locks @@ -8430,6 +8405,7 @@ send_exit_signal(Process *c_p, /* current process if and only /* ...and we have all locks on it... */ *rp_locks = ERTS_PROC_LOCKS_ALL; set_proc_exiting(rp, + state, (is_immed(rsn) ? rsn : copy_object(rsn, rp)), @@ -8459,11 +8435,9 @@ send_exit_signal(Process *c_p, /* current process if and only &bp->off_heap); rp->pending_exit.bp = bp; } - ASSERT(ERTS_PROC_PENDING_EXIT(rp)); + erts_smp_atomic32_read_bor_relb(&rp->state, + ERTS_PSFLG_PENDING_EXIT); } - if (!(rp->status_flags - & (ERTS_PROC_SFLG_INRUNQ|ERTS_PROC_SFLG_RUNNING))) - erts_add_to_runq(rp); } /* else: * @@ -8475,17 +8449,14 @@ send_exit_signal(Process *c_p, /* current process if and only * exit or by itself before seeing the pending exit. */ #else /* !ERTS_SMP */ - if (c_p == rp) { - rp->status = P_EXITING; - c_p->fvalue = rsn; - } - else if (rp->status != P_EXITING) { /* No recursive process exits /PaN */ - Eterm old_status = rp->status; + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + if (!(state & ERTS_PSFLG_EXITING)) { set_proc_exiting(rp, - is_immed(rsn) ? rsn : copy_object(rsn, rp), + state, + (is_immed(rsn) || c_p == rp + ? rsn + : copy_object(rsn, rp)), NULL); - if (old_status != P_RUNABLE && old_status != P_RUNNING) - erts_add_to_runq(rp); } #endif return -1; /* Receiver will exit */ @@ -8561,7 +8532,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) if (!rp) { goto done; } - rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon == NULL) { goto done; @@ -8596,7 +8567,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ASSERT(mon->type == MON_TARGET); ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); if (is_internal_port(mon->pid)) { - Port *prt = erts_id2port(mon->pid, NULL, 0); + Port *prt = erts_id2port(mon->pid); if (prt == NULL) { goto done; } @@ -8612,13 +8583,13 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) goto done; } UseTmpHeapNoproc(3); - rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); if (rmon) { erts_destroy_monitor(rmon); watched = (is_atom(mon->name) ? TUPLE2(lhp, mon->name, erts_this_dist_entry->sysname) - : pcontext->p->id); + : pcontext->p->common.id); erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, watched, pcontext->reason); } @@ -8683,21 +8654,22 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) switch(lnk->type) { case LINK_PID: if(is_internal_port(item)) { - Port *prt = erts_id2port(item, NULL, 0); - if (prt) { - rlnk = erts_remove_link(&prt->nlinks, p->id); - if (rlnk) - erts_destroy_link(rlnk); - erts_do_exit_port(prt, p->id, reason); - erts_port_release(prt); - } + Port *prt = erts_port_lookup(item, ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (prt) + erts_port_exit(NULL, + (ERTS_PORT_SIG_FLG_FORCE_SCHED + | ERTS_PORT_SIG_FLG_BROKEN_LINK), + prt, + p->common.id, + reason, + NULL); } else if(is_external_port(item)) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Erroneous link between %T and external port %T " "found\n", - p->id, + p->common.id, item); erts_send_error_to_logger_nogl(dsbufp); ASSERT(0); /* It isn't possible to setup such a link... */ @@ -8707,14 +8679,14 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) | ERTS_PROC_LOCKS_XSIG_SEND); rp = erts_pid2proc(NULL, 0, item, rp_locks); if (rp) { - rlnk = erts_remove_link(&(rp->nlinks), p->id); + rlnk = erts_remove_link(&ERTS_P_LINKS(rp), p->common.id); /* If rlnk == NULL, we got unlinked while exiting, i.e., do nothing... */ if (rlnk) { int xres; erts_destroy_link(rlnk); xres = send_exit_signal(NULL, - p->id, + p->common.id, rp, &rp_locks, reason, @@ -8726,7 +8698,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { /* We didn't exit the process and it is traced */ if (IS_TRACED_FL(rp, F_TRACE_PROCS)) { - trace_proc(p, rp, am_getting_unlinked, p->id); + trace_proc(p, rp, am_getting_unlinked, p->common.id); } } } @@ -8740,12 +8712,12 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) ErtsDSigData dsd; int code; ErtsDistLinkData dld; - erts_remove_dist_link(&dld, p->id, item, dep); + erts_remove_dist_link(&dld, p->common.id, item, dep); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_exit_tt(&dsd, p->id, item, reason, - SEQ_TRACE_TOKEN(p)); + code = erts_dsig_send_exit_tt(&dsd, p->common.id, item, + reason, SEQ_TRACE_TOKEN(p)); ASSERT(code == ERTS_DSIG_SEND_OK); } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); @@ -8760,7 +8732,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) /* dist entries have node links in a separate structure to avoid confusion */ erts_smp_de_links_lock(dep); - rlnk = erts_remove_link(&(dep->node_links), p->id); + rlnk = erts_remove_link(&(dep->node_links), p->common.id); erts_smp_de_links_unlock(dep); if (rlnk) erts_destroy_link(rlnk); @@ -8788,22 +8760,14 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) erts_destroy_suspend_monitor(smon); } -static void -continue_exit_process(Process *p -#ifdef ERTS_SMP - , erts_pix_lock_t *pix_lock -#endif - ); - /* this function fishishes a process and propagates exit messages - called by process_main when a process dies */ void erts_do_exit_process(Process* p, Eterm reason) { #ifdef ERTS_SMP - erts_pix_lock_t *pix_lock = ERTS_PID2PIXLOCK(p->id); + erts_aint32_t state; #endif - p->arity = 0; /* No live registers */ p->fvalue = reason; @@ -8821,27 +8785,17 @@ erts_do_exit_process(Process* p, Eterm reason) #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going - to status P_EXITING, it is enough to take any lock when + to exiting state (ERTS_PSFLG_EXITING), it is enough to take any lock when looking up a process (erts_pid2proc()) to prevent the looked up process from exiting until the lock has been released. */ erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif - - if (erts_system_profile_flags.runnable_procs && (p->status != P_WAITING)) { - profile_runnable_proc(p, am_inactive); - } -#ifdef ERTS_SMP - erts_pix_lock(pix_lock); - p->is_exiting = 1; -#endif - - p->status = P_EXITING; - -#ifdef ERTS_SMP - erts_pix_unlock(pix_lock); - - if (ERTS_PROC_PENDING_EXIT(p)) { +#ifndef ERTS_SMP + set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state)); +#else + state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state)); + if (state & ERTS_PSFLG_PENDING_EXIT) { /* Process exited before pending exit was received... */ p->pending_exit.reason = THE_NON_VALUE; if (p->pending_exit.bp) { @@ -8863,46 +8817,30 @@ erts_do_exit_process(Process* p, Eterm reason) trace_proc(p, p, am_exit, reason); } - erts_trace_check_exiting(p->id); + erts_trace_check_exiting(p->common.id); - ASSERT((p->trace_flags & F_INITIAL_TRACE_FLAGS) == F_INITIAL_TRACE_FLAGS); + ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS) + == F_INITIAL_TRACE_FLAGS); cancel_timer(p); /* Always cancel timer just in case */ - /* - * The timer of this process can *not* be used anymore. The field used - * for the timer is now used for misc exiting data. - */ - p->u.exit_data = NULL; - - if (p->bif_timers) + if (p->u.bif_timers) erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#ifdef ERTS_SMP - continue_exit_process(p, pix_lock); -#else - continue_exit_process(p); -#endif -} + /* + * The p->u.bif_timers of this process can *not* be used anymore; + * will be overwritten by misc termination data. + */ + p->u.terminate = NULL; -void -erts_continue_exit_process(Process *c_p) -{ -#ifdef ERTS_SMP - continue_exit_process(c_p, ERTS_PID2PIXLOCK(c_p->id)); -#else - continue_exit_process(c_p); -#endif + + erts_continue_exit_process(p); } -static void -continue_exit_process(Process *p -#ifdef ERTS_SMP - , erts_pix_lock_t *pix_lock -#endif - ) +void +erts_continue_exit_process(Process *p) { ErtsLink* lnk; ErtsMonitor *mon; @@ -8918,11 +8856,7 @@ continue_exit_process(Process *p ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); -#ifdef DEBUG - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - ASSERT(p->status == P_EXITING); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); -#endif + ASSERT(ERTS_PROC_IS_EXITING(p)); #ifdef ERTS_SMP if (p->flags & F_HAVE_BLCKD_MSCHED) { @@ -8973,9 +8907,9 @@ continue_exit_process(Process *p * The registered name *should* be the last "erlang resource" to * cleanup. */ - if (p->reg) { + if (p->common.u.alive.reg) { (void) erts_unregister_name(p, ERTS_PROC_LOCK_MAIN, NULL, THE_NON_VALUE); - ASSERT(!p->reg); + ASSERT(!p->common.u.alive.reg); } erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); @@ -8989,49 +8923,33 @@ continue_exit_process(Process *p yield_allowed = 0; #endif + /* + * Note! The monitor and link fields will be overwritten + * by erts_ptab_delete_element() below. + */ + mon = ERTS_P_MONITORS(p); + lnk = ERTS_P_LINKS(p); + { - int pix; /* Do *not* use erts_get_runq_proc() */ ErtsRunQueue *rq; rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p)); - ASSERT(internal_pid_index(p->id) < erts_max_processes); - pix = internal_pid_index(p->id); - - erts_smp_mtx_lock(&proc_tab_mtx); erts_smp_runq_lock(rq); #ifdef ERTS_SMP - erts_pix_lock(pix_lock); - ASSERT(p->scheduler_data); ASSERT(p->scheduler_data->current_process == p); ASSERT(p->scheduler_data->free_process == NULL); p->scheduler_data->current_process = NULL; p->scheduler_data->free_process = p; - p->status_flags = 0; #endif - process_tab[pix] = NULL; /* Time of death! */ - ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0); - erts_smp_atomic32_dec_nob(&process_count); -#ifdef ERTS_SMP - erts_pix_unlock(pix_lock); -#endif - erts_smp_runq_unlock(rq); + /* Time of death! */ + erts_ptab_delete_element(&erts_proc, &p->common); - if (p_next < 0) { - if (p_last >= p_next) { - p_serial++; - p_serial &= p_serial_mask; - } - p_next = pix; - } - - ERTS_MAYBE_SAVE_TERMINATING_PROCESS(p); - - erts_smp_mtx_unlock(&proc_tab_mtx); + erts_smp_runq_unlock(rq); } /* @@ -9041,12 +8959,20 @@ continue_exit_process(Process *p * when the monitors and/or links hit. */ - mon = p->monitors; - p->monitors = NULL; /* to avoid recursive deletion during traversal */ + { + /* Inactivate and notify free */ + erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state); + while (1) { + n = e = a; + ASSERT(a & ERTS_PSFLG_EXITING); + n |= ERTS_PSFLG_FREE; + n &= ~ERTS_PSFLG_ACTIVE; + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + } + } - lnk = p->nlinks; - p->nlinks = NULL; - p->status = P_FREE; dep = ((p->flags & F_DISTRIBUTION) ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) : NULL); @@ -9076,7 +9002,7 @@ continue_exit_process(Process *p UseTmpHeap(4,p); hp = &tmp_heap[0]; - exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason); + exit_tuple = TUPLE3(hp, am_EXIT, p->common.id, reason); exit_tuple_sz = size_object(exit_tuple); @@ -9099,10 +9025,18 @@ continue_exit_process(Process *p if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); + if (p->extra_root != NULL) { + (p->extra_root->cleanup)(p->extra_root); /* Should deallocate + whole structure */ + p->extra_root = NULL; + } + delete_process(p); +#ifdef ERTS_SMP erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); +#endif return; @@ -9115,8 +9049,6 @@ continue_exit_process(Process *p ERTS_SMP_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p)); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks); - ASSERT(p->status == P_EXITING); - p->i = (BeamInstr *) beam_continue_exit; if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) { @@ -9124,8 +9056,6 @@ continue_exit_process(Process *p curr_locks |= ERTS_PROC_LOCK_STATUS; } - erts_add_to_runq(p); - if (curr_locks != ERTS_PROC_LOCK_MAIN) erts_smp_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks); @@ -9137,33 +9067,15 @@ continue_exit_process(Process *p static void timeout_proc(Process* p) { + erts_aint32_t state; BeamInstr** pi = (BeamInstr **) p->def_arg_reg; p->i = *pi; p->flags |= F_TIMO; p->flags &= ~F_INSLPQUEUE; - switch (p->status) { - case P_GARBING: - switch (p->gcstatus) { - case P_SUSPENDED: - goto suspended; - case P_WAITING: - goto waiting; - default: - break; - } - break; - case P_WAITING: - waiting: - erts_add_to_runq(p); - break; - case P_SUSPENDED: - suspended: - p->rstatus = P_RUNABLE; /* MUST set resume status to runnable */ - break; - default: - break; - } + state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & ERTS_PSFLG_ACTIVE)) + schedule_process(p, state, 0); } @@ -9173,9 +9085,9 @@ cancel_timer(Process* p) ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); p->flags &= ~(F_INSLPQUEUE|F_TIMO); #ifdef ERTS_SMP - erts_cancel_smp_ptimer(p->u.ptimer); + erts_cancel_smp_ptimer(p->common.u.alive.ptimer); #else - erts_cancel_timer(&p->u.tm); + erts_cancel_timer(&p->common.u.alive.tm); #endif } @@ -9196,12 +9108,12 @@ set_timer(Process* p, Uint timeout) p->flags &= ~F_TIMO; #ifdef ERTS_SMP - erts_create_smp_ptimer(&p->u.ptimer, - p->id, + erts_create_smp_ptimer(&p->common.u.alive.ptimer, + p->common.id, (ErlTimeoutProc) timeout_proc, timeout); #else - erts_set_timer(&p->u.tm, + erts_set_timer(&p->common.u.alive.tm, (ErlTimeoutProc) timeout_proc, NULL, (void*) p, @@ -9219,7 +9131,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) Eterm* sp; int yreg = -1; - if (p->trace_flags & F_SENSITIVE) { + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) { return; } erts_program_counter_info(to, to_arg, p); @@ -9231,6 +9143,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) void erts_program_counter_info(int to, void *to_arg, Process *p) { + erts_aint32_t state; int i; erts_print(to, to_arg, "Program counter: %p (", p->i); @@ -9239,7 +9152,8 @@ erts_program_counter_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "CP: %p (", p->cp); print_function_from_pc(to, to_arg, p->cp); erts_print(to, to_arg, ")\n"); - if (!((p->status == P_RUNNING) || (p->status == P_GARBING))) { + state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_GC))) { erts_print(to, to_arg, "arity = %d\n",p->arity); if (!ERTS_IS_CRASH_DUMPING) { /* @@ -9285,7 +9199,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) erts_print(to, to_arg, "\n%p ", sp); } else { char sbuf[16]; - sprintf(sbuf, "y(%d)", yreg); + erts_snprintf(sbuf, sizeof(sbuf), "y(%d)", yreg); erts_print(to, to_arg, "%-8s ", sbuf); yreg++; } @@ -9305,1085 +9219,6 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) return yreg; } -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * The processes/0 BIF implementation. * -\* */ - - -#define ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED 25 -#define ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE 1000 -#define ERTS_PROCESSES_BIF_MIN_START_REDS \ - (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \ - / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED) - -#define ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS 1 - -#define ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED 10 - -#define ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS \ - (ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE \ - / ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED) - - -#define ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED 75 - -#define ERTS_PROCS_DBG_DO_TRACE 0 - -#ifdef DEBUG -# define ERTS_PROCESSES_BIF_DEBUGLEVEL 100 -#else -# define ERTS_PROCESSES_BIF_DEBUGLEVEL 0 -#endif - -#define ERTS_PROCS_DBGLVL_CHK_HALLOC 1 -#define ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS 5 -#define ERTS_PROCS_DBGLVL_CHK_PIDS 10 -#define ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST 20 -#define ERTS_PROCS_DBGLVL_CHK_RESLIST 20 - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0 -# define ERTS_PROCS_ASSERT(EXP) -#else -# define ERTS_PROCS_ASSERT(EXP) \ - ((void) ((EXP) \ - ? 1 \ - : (debug_processes_assert_error(#EXP, __FILE__, __LINE__), 0))) -#endif - - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC -# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ) \ -do { \ - ERTS_PROCS_ASSERT(!(PBDP)->debug.heap); \ - ERTS_PROCS_ASSERT(!(PBDP)->debug.heap_size); \ - (PBDP)->debug.heap = (HP); \ - (PBDP)->debug.heap_size = (SZ); \ -} while (0) -# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP) \ -do { \ - ERTS_PROCS_ASSERT((PBDP)->debug.heap); \ - ERTS_PROCS_ASSERT((PBDP)->debug.heap_size); \ - ERTS_PROCS_ASSERT((PBDP)->debug.heap + (PBDP)->debug.heap_size == (HP));\ - (PBDP)->debug.heap = NULL; \ - (PBDP)->debug.heap_size = 0; \ -} while (0) -# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP) \ -do { \ - (PBDP)->debug.heap = NULL; \ - (PBDP)->debug.heap_size = 0; \ -} while (0) -#else -# define ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(PBDP, HP, SZ) -# define ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(PBDP, HP) -# define ERTS_PROCS_DBG_HEAP_ALLOC_INIT(PBDP) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST -# define ERTS_PROCS_DBG_CHK_RESLIST(R) debug_processes_check_res_list((R)) -#else -# define ERTS_PROCS_DBG_CHK_RESLIST(R) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS -# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP) debug_processes_save_all_pids((PBDP)) -# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP) \ -do { \ - if (!(PBDP)->debug.correct_pids_verified) \ - debug_processes_verify_all_pids((PBDP)); \ -} while (0) -# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP) \ -do { \ - if ((PBDP)->debug.correct_pids) { \ - erts_free(ERTS_ALC_T_PROCS_PIDS, \ - (PBDP)->debug.correct_pids); \ - (PBDP)->debug.correct_pids = NULL; \ - } \ -} while(0) -# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP) \ -do { \ - (PBDP)->debug.correct_pids_verified = 0; \ - (PBDP)->debug.correct_pids = NULL; \ -} while (0) -#else -# define ERTS_PROCS_DBG_SAVE_PIDS(PBDP) -# define ERTS_PROCS_DBG_VERIFY_PIDS(PBDP) -# define ERTS_PROCS_DBG_CLEANUP_CHK_PIDS(PBDP) -# define ERTS_PROCS_DBG_CHK_PIDS_INIT(PBDP) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) \ - debug_processes_check_found_pid((PBDP), (PID), (TVP), 1) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) \ - debug_processes_check_found_pid((PBDP), (PID), (TVP), 0) -#else -# define ERTS_PROCS_DBG_CHK_PID_FOUND(PBDP, PID, TVP) -# define ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(PBDP, PID, TVP) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -# define ERTS_PROCS_DBG_CHK_TPLIST() \ - debug_processes_check_term_proc_list() -# define ERTS_PROCS_DBG_CHK_FREELIST(FL) \ - debug_processes_check_term_proc_free_list(FL) -#else -# define ERTS_PROCS_DBG_CHK_TPLIST() -# define ERTS_PROCS_DBG_CHK_FREELIST(FL) -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL == 0 -#if ERTS_PROCS_DBG_DO_TRACE -# define ERTS_PROCS_DBG_INIT(P, PBDP) (PBDP)->debug.caller = (P)->id -# else -# define ERTS_PROCS_DBG_INIT(P, PBDP) -# endif -# define ERTS_PROCS_DBG_CLEANUP(PBDP) -#else -# define ERTS_PROCS_DBG_INIT(P, PBDP) \ -do { \ - (PBDP)->debug.caller = (P)->id; \ - ERTS_PROCS_DBG_HEAP_ALLOC_INIT((PBDP)); \ - ERTS_PROCS_DBG_CHK_PIDS_INIT((PBDP)); \ -} while (0) -# define ERTS_PROCS_DBG_CLEANUP(PBDP) \ -do { \ - ERTS_PROCS_DBG_CLEANUP_CHK_PIDS((PBDP)); \ -} while (0) -#endif - -#if ERTS_PROCS_DBG_DO_TRACE -# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT) \ - erts_fprintf(stderr, "%T %s:%d:%s(): %s\n", \ - (PID), __FILE__, __LINE__, #FUNC, #WHAT) -#else -# define ERTS_PROCS_DBG_TRACE(PID, FUNC, WHAT) -#endif - -static Uint processes_bif_tab_chunks; -static Export processes_trap_export; - -typedef struct { - SysTimeval time; -} ErtsProcessesBifChunkInfo; - -typedef enum { - INITIALIZING, - INSPECTING_TABLE, - INSPECTING_TERMINATED_PROCESSES, - BUILDING_RESULT, - RETURN_RESULT -} ErtsProcessesBifState; - -typedef struct { - ErtsProcessesBifState state; - Eterm caller; - ErtsProcessesBifChunkInfo *chunk; - int tix; - int pid_ix; - int pid_sz; - Eterm *pid; - ErtsTermProcElement *bif_invocation; /* Only used when > 1 chunk */ - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 || ERTS_PROCS_DBG_DO_TRACE - struct { - Eterm caller; -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - SysTimeval *pid_started; -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_HALLOC - Eterm *heap; - Uint heap_size; -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS - int correct_pids_verified; - Eterm *correct_pids; -#endif - } debug; -#endif - -} ErtsProcessesBifData; - - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 -static void debug_processes_assert_error(char* expr, char* file, int line); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST -static void debug_processes_check_res_list(Eterm list); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS -static void debug_processes_save_all_pids(ErtsProcessesBifData *pbdp); -static void debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -static void debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, - Eterm pid, - SysTimeval *started, - int pid_should_be_found); -#endif -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -static SysTimeval debug_tv_start; -static void debug_processes_check_term_proc_list(void); -static void debug_processes_check_term_proc_free_list(ErtsTermProcElement *tpep); -#endif - -static void -save_terminating_process(Process *p) -{ - ErtsTermProcElement *tpep = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, - sizeof(ErtsTermProcElement)); - ERTS_PROCS_ASSERT(saved_term_procs.start && saved_term_procs.end); - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); - - ERTS_PROCS_DBG_CHK_TPLIST(); - - tpep->prev = saved_term_procs.end; - tpep->next = NULL; - tpep->ix = internal_pid_index(p->id); - tpep->u.process.pid = p->id; - tpep->u.process.spawned = p->started; - erts_get_emu_time(&tpep->u.process.exited); - - saved_term_procs.end->next = tpep; - saved_term_procs.end = tpep; - - ERTS_PROCS_DBG_CHK_TPLIST(); - - ERTS_PROCS_ASSERT((tpep->prev->ix >= 0 - ? erts_cmp_timeval(&tpep->u.process.exited, - &tpep->prev->u.process.exited) - : erts_cmp_timeval(&tpep->u.process.exited, - &tpep->prev->u.bif_invocation.time)) > 0); -} - -static void -cleanup_processes_bif_data(Binary *bp) -{ - ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(bp); - - ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, cleanup_processes_bif_data, call); - - if (pbdp->state != INITIALIZING) { - - if (pbdp->chunk) { - erts_free(ERTS_ALC_T_PROCS_CNKINF, pbdp->chunk); - pbdp->chunk = NULL; - } - if (pbdp->pid) { - erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->pid); - pbdp->pid = NULL; - } - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - if (pbdp->debug.pid_started) { - erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.pid_started); - pbdp->debug.pid_started = NULL; - } -#endif - - if (pbdp->bif_invocation) { - ErtsTermProcElement *tpep; - - erts_smp_mtx_lock(&proc_tab_mtx); - - ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, - cleanup_processes_bif_data, - term_proc_cleanup); - - tpep = pbdp->bif_invocation; - pbdp->bif_invocation = NULL; - - ERTS_PROCS_DBG_CHK_TPLIST(); - - if (tpep->prev) { - /* - * Only remove this bif invokation when we - * have preceding invokations. - */ - tpep->prev->next = tpep->next; - if (tpep->next) - tpep->next->prev = tpep->prev; - else { - /* - * At the time of writing this branch cannot be - * reached. I don't want to remove this code though - * since it may be possible to reach this line - * in the future if the cleanup order in - * erts_do_exit_process() is changed. The ASSERT(0) - * is only here to make us aware that the reorder - * has happened. /rickard - */ - ASSERT(0); - saved_term_procs.end = tpep->prev; - } - erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep); - } - else { - /* - * Free all elements until next bif invokation - * is found. - */ - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - do { - ErtsTermProcElement *ftpep = tpep; - tpep = tpep->next; - erts_free(ERTS_ALC_T_PROCS_TPROC_EL, ftpep); - } while (tpep && tpep->ix >= 0); - saved_term_procs.start = tpep; - if (tpep) - tpep->prev = NULL; - else - saved_term_procs.end = NULL; - } - - ERTS_PROCS_DBG_CHK_TPLIST(); - - erts_smp_mtx_unlock(&proc_tab_mtx); - - } - } - - ERTS_PROCS_DBG_TRACE(pbdp->debug.caller, - cleanup_processes_bif_data, - return); - ERTS_PROCS_DBG_CLEANUP(pbdp); -} - -static int -processes_bif_engine(Process *p, Eterm *res_accp, Binary *mbp) -{ - ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp); - int have_reds; - int reds; - int locked = 0; - - do { - switch (pbdp->state) { - case INITIALIZING: - pbdp->chunk = erts_alloc(ERTS_ALC_T_PROCS_CNKINF, - (sizeof(ErtsProcessesBifChunkInfo) - * processes_bif_tab_chunks)); - pbdp->tix = 0; - pbdp->pid_ix = 0; - - erts_smp_mtx_lock(&proc_tab_mtx); - locked = 1; - - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, init); - - pbdp->pid_sz = erts_process_count(); - pbdp->pid = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(Eterm)*pbdp->pid_sz); - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(SysTimeval)*pbdp->pid_sz); -#endif - - ERTS_PROCS_DBG_SAVE_PIDS(pbdp); - - if (processes_bif_tab_chunks == 1) - pbdp->bif_invocation = NULL; - else { - /* - * We will have to access the table multiple times - * releasing the table lock in between chunks. - */ - pbdp->bif_invocation = erts_alloc(ERTS_ALC_T_PROCS_TPROC_EL, - sizeof(ErtsTermProcElement)); - pbdp->bif_invocation->ix = -1; - erts_get_emu_time(&pbdp->bif_invocation->u.bif_invocation.time); - ERTS_PROCS_DBG_CHK_TPLIST(); - - pbdp->bif_invocation->next = NULL; - if (saved_term_procs.end) { - pbdp->bif_invocation->prev = saved_term_procs.end; - saved_term_procs.end->next = pbdp->bif_invocation; - ERTS_PROCS_ASSERT(saved_term_procs.start); - } - else { - pbdp->bif_invocation->prev = NULL; - saved_term_procs.start = pbdp->bif_invocation; - } - saved_term_procs.end = pbdp->bif_invocation; - - ERTS_PROCS_DBG_CHK_TPLIST(); - - } - - pbdp->state = INSPECTING_TABLE; - /* Fall through */ - - case INSPECTING_TABLE: { - int ix = pbdp->tix; - int indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - int cix = ix / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - int end_ix = ix + indices; - SysTimeval *invocation_timep; - - invocation_timep = (pbdp->bif_invocation - ? &pbdp->bif_invocation->u.bif_invocation.time - : NULL); - - ERTS_PROCS_ASSERT(is_nil(*res_accp)); - if (!locked) { - erts_smp_mtx_lock(&proc_tab_mtx); - locked = 1; - } - - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_table); - - if (cix != 0) - erts_get_emu_time(&pbdp->chunk[cix].time); - else if (pbdp->bif_invocation) - pbdp->chunk[0].time = *invocation_timep; - /* else: Time is irrelevant */ - - if (end_ix >= erts_max_processes) { - ERTS_PROCS_ASSERT(cix+1 == processes_bif_tab_chunks); - end_ix = erts_max_processes; - indices = end_ix - ix; - /* What to do when done with this chunk */ - pbdp->state = (processes_bif_tab_chunks == 1 - ? BUILDING_RESULT - : INSPECTING_TERMINATED_PROCESSES); - } - - for (; ix < end_ix; ix++) { - Process *rp = process_tab[ix]; - if (rp - && (!invocation_timep - || erts_cmp_timeval(&rp->started, - invocation_timep) < 0)) { - ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); - pbdp->pid[pbdp->pid_ix] = rp->id; - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] = rp->started; -#endif - - pbdp->pid_ix++; - ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); - } - } - - pbdp->tix = end_ix; - - erts_smp_mtx_unlock(&proc_tab_mtx); - locked = 0; - - reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; - BUMP_REDS(p, reds); - - have_reds = ERTS_BIF_REDS_LEFT(p); - - if (have_reds && pbdp->state == INSPECTING_TABLE) { - ix = pbdp->tix; - indices = ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - end_ix = ix + indices; - if (end_ix > erts_max_processes) { - end_ix = erts_max_processes; - indices = end_ix - ix; - } - - reds = indices/ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED; - - /* Pretend we have no reds left if we haven't got enough - reductions to complete next chunk */ - if (reds > have_reds) - have_reds = 0; - } - - break; - } - - case INSPECTING_TERMINATED_PROCESSES: { - int i; - int max_reds; - int free_term_procs = 0; - SysTimeval *invocation_timep; - ErtsTermProcElement *tpep; - ErtsTermProcElement *free_list = NULL; - - tpep = pbdp->bif_invocation; - ERTS_PROCS_ASSERT(tpep); - invocation_timep = &tpep->u.bif_invocation.time; - - max_reds = have_reds = ERTS_BIF_REDS_LEFT(p); - if (max_reds > ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS) - max_reds = ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS; - - reds = 0; - erts_smp_mtx_lock(&proc_tab_mtx); - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, insp_term_procs); - - ERTS_PROCS_DBG_CHK_TPLIST(); - - if (tpep->prev) - tpep->prev->next = tpep->next; - else { - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - saved_term_procs.start = tpep->next; - - if (saved_term_procs.start && saved_term_procs.start->ix >= 0) { - free_list = saved_term_procs.start; - free_term_procs = 1; - } - } - - if (tpep->next) - tpep->next->prev = tpep->prev; - else - saved_term_procs.end = tpep->prev; - - tpep = tpep->next; - - i = 0; - while (reds < max_reds && tpep) { - if (tpep->ix < 0) { - if (free_term_procs) { - ERTS_PROCS_ASSERT(free_list); - ERTS_PROCS_ASSERT(tpep->prev); - - tpep->prev->next = NULL; /* end of free_list */ - saved_term_procs.start = tpep; - tpep->prev = NULL; - free_term_procs = 0; - } - } - else { - int cix = tpep->ix/ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE; - SysTimeval *chunk_timep = &pbdp->chunk[cix].time; - Eterm pid = tpep->u.process.pid; - ERTS_PROCS_ASSERT(is_internal_pid(pid)); - - if (erts_cmp_timeval(&tpep->u.process.spawned, - invocation_timep) < 0) { - if (erts_cmp_timeval(&tpep->u.process.exited, - chunk_timep) < 0) { - ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, - pid, - &tpep->u.process.spawned); - pbdp->pid[pbdp->pid_ix] = pid; -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started[pbdp->pid_ix] = tpep->u.process.spawned; -#endif - pbdp->pid_ix++; - ERTS_PROCS_ASSERT(pbdp->pid_ix <= pbdp->pid_sz); - } - else { - ERTS_PROCS_DBG_CHK_PID_FOUND(pbdp, - pid, - &tpep->u.process.spawned); - } - } - else { - ERTS_PROCS_DBG_CHK_PID_NOT_FOUND(pbdp, - pid, - &tpep->u.process.spawned); - } - - i++; - if (i == ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED) { - reds++; - i = 0; - } - if (free_term_procs) - reds += ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS; - } - tpep = tpep->next; - } - - if (free_term_procs) { - ERTS_PROCS_ASSERT(free_list); - saved_term_procs.start = tpep; - if (!tpep) - saved_term_procs.end = NULL; - else { - ERTS_PROCS_ASSERT(tpep->prev); - tpep->prev->next = NULL; /* end of free_list */ - tpep->prev = NULL; - } - } - - if (!tpep) { - /* Done */ - ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz); - pbdp->state = BUILDING_RESULT; - pbdp->bif_invocation->next = free_list; - free_list = pbdp->bif_invocation; - pbdp->bif_invocation = NULL; - } - else { - /* Link in bif_invocation again where we left off */ - pbdp->bif_invocation->prev = tpep->prev; - pbdp->bif_invocation->next = tpep; - tpep->prev = pbdp->bif_invocation; - if (pbdp->bif_invocation->prev) - pbdp->bif_invocation->prev->next = pbdp->bif_invocation; - else { - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - saved_term_procs.start = pbdp->bif_invocation; - } - } - - ERTS_PROCS_DBG_CHK_TPLIST(); - ERTS_PROCS_DBG_CHK_FREELIST(free_list); - erts_smp_mtx_unlock(&proc_tab_mtx); - - /* - * We do the actual free of term proc structures now when we - * have released the table lock instead of when we encountered - * them. This since free() isn't for free and we don't want to - * unnecessarily block other schedulers. - */ - while (free_list) { - tpep = free_list; - free_list = tpep->next; - erts_free(ERTS_ALC_T_PROCS_TPROC_EL, tpep); - } - - have_reds -= reds; - if (have_reds < 0) - have_reds = 0; - BUMP_REDS(p, reds); - break; - } - - case BUILDING_RESULT: { - int conses, ix, min_ix; - Eterm *hp; - Eterm res = *res_accp; - - ERTS_PROCS_DBG_VERIFY_PIDS(pbdp); - ERTS_PROCS_DBG_CHK_RESLIST(res); - - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, begin_build_res); - - have_reds = ERTS_BIF_REDS_LEFT(p); - conses = ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED*have_reds; - min_ix = pbdp->pid_ix - conses; - if (min_ix < 0) { - min_ix = 0; - conses = pbdp->pid_ix; - } - - hp = HAlloc(p, conses*2); - ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, conses*2); - - for (ix = pbdp->pid_ix - 1; ix >= min_ix; ix--) { - ERTS_PROCS_ASSERT(is_internal_pid(pbdp->pid[ix])); - res = CONS(hp, pbdp->pid[ix], res); - hp += 2; - } - - ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp); - - pbdp->pid_ix = min_ix; - if (min_ix == 0) - pbdp->state = RETURN_RESULT; - else { - pbdp->pid_sz = min_ix; - pbdp->pid = erts_realloc(ERTS_ALC_T_PROCS_PIDS, - pbdp->pid, - sizeof(Eterm)*pbdp->pid_sz); -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS - pbdp->debug.pid_started = erts_realloc(ERTS_ALC_T_PROCS_PIDS, - pbdp->debug.pid_started, - sizeof(SysTimeval)*pbdp->pid_sz); -#endif - } - reds = conses/ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED; - BUMP_REDS(p, reds); - have_reds -= reds; - - ERTS_PROCS_DBG_CHK_RESLIST(res); - ERTS_PROCS_DBG_TRACE(p->id, processes_bif_engine, end_build_res); - *res_accp = res; - break; - } - case RETURN_RESULT: - cleanup_processes_bif_data(mbp); - return 1; - - default: - erl_exit(ERTS_ABORT_EXIT, - "erlang:processes/0: Invalid state: %d\n", - (int) pbdp->state); - } - - - } while (have_reds || pbdp->state == RETURN_RESULT); - - return 0; -} - -/* - * processes_trap/2 is a hidden BIF that processes/0 traps to. - */ - -static BIF_RETTYPE processes_trap(BIF_ALIST_2) -{ - Eterm res_acc; - Binary *mbp; - - /* - * This bif cannot be called from erlang code. It can only be - * trapped to from processes/0; therefore, a bad argument - * is a processes/0 internal error. - */ - - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, call); - ERTS_PROCS_ASSERT(is_nil(BIF_ARG_1) || is_list(BIF_ARG_1)); - - res_acc = BIF_ARG_1; - - ERTS_PROCS_ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_2)); - - mbp = ((ProcBin *) binary_val(BIF_ARG_2))->val; - - ERTS_PROCS_ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) - == cleanup_processes_bif_data); - ERTS_PROCS_ASSERT( - ((ErtsProcessesBifData *) ERTS_MAGIC_BIN_DATA(mbp))->debug.caller - == BIF_P->id); - - if (processes_bif_engine(BIF_P, &res_acc, mbp)) { - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, return); - BIF_RET(res_acc); - } - else { - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_trap, trap); - ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, BIF_ARG_2); - } -} - - - -/* - * The actual processes/0 BIF. - */ - -BIF_RETTYPE processes_0(BIF_ALIST_0) -{ - /* - * A requirement: The list of pids returned should be a consistent - * snapshot of all processes existing at some point - * in time during the execution of processes/0. Since - * processes might terminate while processes/0 is - * executing, we have to keep track of terminated - * processes and add them to the result. We also - * ignore processes created after processes/0 has - * begun executing. - */ - Eterm res_acc = NIL; - Binary *mbp = erts_create_magic_binary(sizeof(ErtsProcessesBifData), - cleanup_processes_bif_data); - ErtsProcessesBifData *pbdp = ERTS_MAGIC_BIN_DATA(mbp); - - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, call); - pbdp->state = INITIALIZING; - ERTS_PROCS_DBG_INIT(BIF_P, pbdp); - - if (ERTS_BIF_REDS_LEFT(BIF_P) >= ERTS_PROCESSES_BIF_MIN_START_REDS - && processes_bif_engine(BIF_P, &res_acc, mbp)) { - erts_bin_free(mbp); - ERTS_PROCS_DBG_CHK_RESLIST(res_acc); - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, return); - BIF_RET(res_acc); - } - else { - Eterm *hp; - Eterm magic_bin; - ERTS_PROCS_DBG_CHK_RESLIST(res_acc); - hp = HAlloc(BIF_P, PROC_BIN_SIZE); - ERTS_PROCS_DBG_SAVE_HEAP_ALLOC(pbdp, hp, PROC_BIN_SIZE); - magic_bin = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mbp); - ERTS_PROCS_DBG_VERIFY_HEAP_ALLOC_USED(pbdp, hp); - ERTS_PROCS_DBG_TRACE(BIF_P->id, processes_0, trap); - ERTS_BIF_YIELD2(&processes_trap_export, BIF_P, res_acc, magic_bin); - } -} - -static void -init_processes_bif(void) -{ - saved_term_procs.start = NULL; - saved_term_procs.end = NULL; - processes_bif_tab_chunks = (((erts_max_processes - 1) - / ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE) - + 1); - - /* processes_trap/2 is a hidden BIF that the processes/0 BIF traps to. */ - sys_memset((void *) &processes_trap_export, 0, sizeof(Export)); - processes_trap_export.address = &processes_trap_export.code[3]; - processes_trap_export.code[0] = am_erlang; - processes_trap_export.code[1] = am_processes_trap; - processes_trap_export.code[2] = 2; - processes_trap_export.code[3] = (BeamInstr) em_apply_bif; - processes_trap_export.code[4] = (BeamInstr) &processes_trap; - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST - erts_get_emu_time(&debug_tv_start); -#endif - -} - -/* - * Debug stuff - */ - -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) -int -erts_dbg_check_halloc_lock(Process *p) -{ - if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) - return 1; - if (p->id == ERTS_INVALID_PID) - return 1; - if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) - return 1; - if (erts_thr_progress_is_blocking()) - return 1; - return 0; -} -#endif - -Eterm -erts_debug_processes(Process *c_p) -{ - /* This is the old processes/0 BIF. */ - int i; - Uint need; - Eterm res; - Eterm* hp; - Process *p; -#ifdef DEBUG - Eterm *hp_end; -#endif - - erts_smp_mtx_lock(&proc_tab_mtx); - - res = NIL; - need = erts_process_count() * 2; - hp = HAlloc(c_p, need); /* we need two heap words for each pid */ -#ifdef DEBUG - hp_end = hp + need; -#endif - - /* make the list by scanning bakward */ - - - for (i = erts_max_processes-1; i >= 0; i--) { - if ((p = process_tab[i]) != NULL) { - res = CONS(hp, process_tab[i]->id, res); - hp += 2; - } - } - ASSERT(hp == hp_end); - - erts_smp_mtx_unlock(&proc_tab_mtx); - - return res; -} - -Eterm -erts_debug_processes_bif_info(Process *c_p) -{ - ERTS_DECL_AM(processes_bif_info); - Eterm elements[] = { - AM_processes_bif_info, - make_small((Uint) ERTS_PROCESSES_BIF_MIN_START_REDS), - make_small((Uint) processes_bif_tab_chunks), - make_small((Uint) ERTS_PROCESSES_BIF_TAB_CHUNK_SIZE), - make_small((Uint) ERTS_PROCESSES_BIF_TAB_INSPECT_INDICES_PER_RED), - make_small((Uint) ERTS_PROCESSES_BIF_TAB_FREE_TERM_PROC_REDS), - make_small((Uint) ERTS_PROCESSES_BIF_INSPECT_TERM_PROC_PER_RED), - make_small((Uint) ERTS_PROCESSES_INSPECT_TERM_PROC_MAX_REDS), - make_small((Uint) ERTS_PROCESSES_BIF_BUILD_RESULT_CONSES_PER_RED), - make_small((Uint) ERTS_PROCESSES_BIF_DEBUGLEVEL) - }; - Uint sz = 0; - Eterm *hp; - (void) erts_bld_tuplev(NULL, &sz, sizeof(elements)/sizeof(Eterm), elements); - hp = HAlloc(c_p, sz); - return erts_bld_tuplev(&hp, NULL, sizeof(elements)/sizeof(Eterm), elements); -} - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_FOUND_PIDS -static void -debug_processes_check_found_pid(ErtsProcessesBifData *pbdp, - Eterm pid, - SysTimeval *tvp, - int pid_should_be_found) -{ - int i; - for (i = 0; i < pbdp->pid_ix; i++) { - if (pbdp->pid[i] == pid - && pbdp->debug.pid_started[i].tv_sec == tvp->tv_sec - && pbdp->debug.pid_started[i].tv_usec == tvp->tv_usec) { - ERTS_PROCS_ASSERT(pid_should_be_found); - return; - } - } - ERTS_PROCS_ASSERT(!pid_should_be_found); -} -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_RESLIST -static void -debug_processes_check_res_list(Eterm list) -{ - while (is_list(list)) { - Eterm* consp = list_val(list); - Eterm hd = CAR(consp); - ERTS_PROCS_ASSERT(is_internal_pid(hd)); - list = CDR(consp); - } - - ERTS_PROCS_ASSERT(is_nil(list)); -} -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS - -static void -debug_processes_save_all_pids(ErtsProcessesBifData *pbdp) -{ - int ix, tix, cpix; - pbdp->debug.correct_pids_verified = 0; - pbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PROCS_PIDS, - sizeof(Eterm)*pbdp->pid_sz); - - for (tix = 0, cpix = 0; tix < erts_max_processes; tix++) { - Process *rp = process_tab[tix]; - if (rp) { - ERTS_PROCS_ASSERT(is_internal_pid(rp->id)); - pbdp->debug.correct_pids[cpix++] = rp->id; - ERTS_PROCS_ASSERT(cpix <= pbdp->pid_sz); - } - } - ERTS_PROCS_ASSERT(cpix == pbdp->pid_sz); - - for (ix = 0; ix < pbdp->pid_sz; ix++) - pbdp->pid[ix] = make_small(ix); -} - -static void -debug_processes_verify_all_pids(ErtsProcessesBifData *pbdp) -{ - int ix, cpix; - - ERTS_PROCS_ASSERT(pbdp->pid_ix == pbdp->pid_sz); - - for (ix = 0; ix < pbdp->pid_sz; ix++) { - int found = 0; - Eterm pid = pbdp->pid[ix]; - ERTS_PROCS_ASSERT(is_internal_pid(pid)); - for (cpix = ix; cpix < pbdp->pid_sz; cpix++) { - if (pbdp->debug.correct_pids[cpix] == pid) { - pbdp->debug.correct_pids[cpix] = NIL; - found = 1; - break; - } - } - if (!found) { - for (cpix = 0; cpix < ix; cpix++) { - if (pbdp->debug.correct_pids[cpix] == pid) { - pbdp->debug.correct_pids[cpix] = NIL; - found = 1; - break; - } - } - } - ERTS_PROCS_ASSERT(found); - } - pbdp->debug.correct_pids_verified = 1; - - erts_free(ERTS_ALC_T_PROCS_PIDS, pbdp->debug.correct_pids); - pbdp->debug.correct_pids = NULL; -} -#endif /* ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_PIDS */ - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST -static void -debug_processes_check_term_proc_list(void) -{ - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&proc_tab_mtx)); - if (!saved_term_procs.start) - ERTS_PROCS_ASSERT(!saved_term_procs.end); - else { - SysTimeval tv_now; - SysTimeval *prev_xtvp = NULL; - ErtsTermProcElement *tpep; - erts_get_emu_time(&tv_now); - - for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) { - if (!tpep->prev) - ERTS_PROCS_ASSERT(saved_term_procs.start == tpep); - else - ERTS_PROCS_ASSERT(tpep->prev->next == tpep); - if (!tpep->next) - ERTS_PROCS_ASSERT(saved_term_procs.end == tpep); - else - ERTS_PROCS_ASSERT(tpep->next->prev == tpep); - if (tpep->ix < 0) { - SysTimeval *tvp = &tpep->u.bif_invocation.time; - ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, tvp) < 0 - && erts_cmp_timeval(tvp, &tv_now) < 0); - } - else { - SysTimeval *stvp = &tpep->u.process.spawned; - SysTimeval *xtvp = &tpep->u.process.exited; - - ERTS_PROCS_ASSERT(erts_cmp_timeval(&debug_tv_start, - stvp) < 0); - ERTS_PROCS_ASSERT(erts_cmp_timeval(stvp, xtvp) < 0); - if (prev_xtvp) - ERTS_PROCS_ASSERT(erts_cmp_timeval(prev_xtvp, xtvp) < 0); - prev_xtvp = xtvp; - ERTS_PROCS_ASSERT(is_internal_pid(tpep->u.process.pid)); - ERTS_PROCS_ASSERT(tpep->ix - == internal_pid_index(tpep->u.process.pid)); - } - } - - } -} - -static void -debug_processes_check_term_proc_free_list(ErtsTermProcElement *free_list) -{ - if (saved_term_procs.start) { - ErtsTermProcElement *ftpep; - ErtsTermProcElement *tpep; - - for (ftpep = free_list; ftpep; ftpep = ftpep->next) { - for (tpep = saved_term_procs.start; tpep; tpep = tpep->next) - ERTS_PROCS_ASSERT(ftpep != tpep); - } - } -} - -#endif - -#if ERTS_PROCESSES_BIF_DEBUGLEVEL != 0 - -static void -debug_processes_assert_error(char* expr, char* file, int line) -{ - fflush(stdout); - erts_fprintf(stderr, "%s:%d: Assertion failed: %s\n", file, line, expr); - fflush(stderr); - abort(); -} - -#endif - -/* *\ - * End of the processes/0 BIF implementation. * -\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - /* * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS @@ -10410,3 +9245,19 @@ void erl_halt(int code) notify_reap_ports_relb(); } } + +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int +erts_dbg_check_halloc_lock(Process *p) +{ + if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) + return 1; + if (p->common.id == ERTS_INVALID_PID) + return 1; + if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) + return 1; + if (erts_thr_progress_is_blocking()) + return 1; + return 0; +} +#endif diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 9b0e9f1ccd..8e5467f196 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -42,6 +42,9 @@ typedef struct process Process; #include "erl_process_lock.h" /* Only pull out important types... */ #undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ +#define ERL_PORT_GET_PORT_TYPE_ONLY__ +#include "erl_port.h" +#undef ERL_PORT_GET_PORT_TYPE_ONLY__ #include "erl_vm.h" #include "erl_smp.h" #include "erl_message.h" @@ -66,11 +69,10 @@ typedef struct process Process; #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY struct ErtsNodesMonitor_; -struct port; #define ERTS_MAX_NO_OF_SCHEDULERS 1024 -#define ERTS_DEFAULT_MAX_PROCESSES (1 << 15) +#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18) #define ERTS_HEAP_ALLOC(Type, Size) \ erts_alloc((Type), (Size)) @@ -112,28 +114,29 @@ extern int erts_sched_thread_suggested_stack_size; #define PRIORITY_NORMAL 2 #define PRIORITY_LOW 3 #define ERTS_NO_PROC_PRIO_LEVELS 4 +#define ERTS_NO_PROC_PRIO_QUEUES 3 #define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS +#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1) #define ERTS_RUNQ_FLGS_PROCS_QMASK \ ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1) -#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1) -#define ERTS_RUNQ_FLGS_MIGRATE_QMASK \ +#define ERTS_RUNQ_FLGS_QMASK \ ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1) #define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \ - ERTS_NO_PROC_PRIO_LEVELS + ERTS_NO_PRIO_LEVELS #define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \ (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS) #define ERTS_RUNQ_FLGS_EVACUATE_SHFT \ (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS) #define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ - (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT) + (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT) #define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ - (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT) + (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT) #define ERTS_RUNQ_FLGS_EVACUATE_QMASK \ - (ERTS_RUNQ_FLGS_MIGRATE_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT) + (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT) #define ERTS_RUNQ_FLG_BASE2 \ (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS) @@ -148,14 +151,18 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3)) #define ERTS_RUNQ_FLG_INACTIVE \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4)) +#define ERTS_RUNQ_FLG_NONEMPTY \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5)) +#define ERTS_RUNQ_FLG_PROTECTED \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6)) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \ | ERTS_RUNQ_FLGS_EVACUATE_QMASK) + #define ERTS_RUNQ_FLGS_MIGRATION_INFO \ - (ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ - | ERTS_RUNQ_FLG_INACTIVE \ + (ERTS_RUNQ_FLG_INACTIVE \ | ERTS_RUNQ_FLG_OUT_OF_WORK \ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK) @@ -186,34 +193,24 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \ ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO))) -#define ERTS_RUNQ_IFLG_SUSPENDED (((erts_aint32_t) 1) << 0) -#define ERTS_RUNQ_IFLG_NONEMPTY (((erts_aint32_t) 1) << 1) - - -#ifdef DEBUG -# if defined(ARCH_64) && !HALFWORD_HEAP -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (*((char **) &(RQP)) = (char *) (0xdeadbeefdead0003 | ((N) << 4))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \ - ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000));\ -} while (0) -# else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (*((char **) &(RQP)) = (char *) (0xdead0003 | ((N) << 4))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ - ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ -} while (0) -# endif -#else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) -#endif +#define ERTS_RUNQ_FLGS_INIT(RQ, INIT) \ + erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT)) +#define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags, \ + (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags, \ + (erts_aint32_t) ~(FLGS))) +#define ERTS_RUNQ_FLGS_GET(RQ) \ + ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags)) +#define ERTS_RUNQ_FLGS_GET_NOB(RQ) \ + ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags)) +#define ERTS_RUNQ_FLGS_GET_MB(RQ) \ + ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags)) +#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \ + (erts_aint32_t) (MSK), \ + (erts_aint32_t) (FLGS))) typedef enum { ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED, @@ -258,14 +255,15 @@ typedef enum { #define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 2) #define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 3) #define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 7) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 9) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 10) -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 11) -#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 12) +#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 8) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 9) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 10) +#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 11) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 12) +#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 13) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -291,8 +289,9 @@ struct ErtsSchedulerSleepInfo_ { typedef struct ErtsProcList_ ErtsProcList; struct ErtsProcList_ { Eterm pid; - SysTimeval started; + Uint64 started_interval; ErtsProcList* next; + ErtsProcList* prev; }; typedef struct ErtsMiscOpList_ ErtsMiscOpList; @@ -312,21 +311,39 @@ typedef struct ErtsSchedulerData_ ErtsSchedulerData; typedef struct ErtsRunQueue_ ErtsRunQueue; typedef struct { - int len; - int max_len; + erts_smp_atomic32_t len; + erts_aint32_t max_len; int reds; +} ErtsRunQueueInfo; + +#ifdef ERTS_SMP + +typedef struct { + Uint32 flags; + ErtsRunQueue *misc_evac_runq; struct { struct { int this; int other; } limit; ErtsRunQueue *runq; - } migrate; -} ErtsRunQueueInfo; + Uint32 flags; + } prio[ERTS_NO_PRIO_LEVELS]; +} ErtsMigrationPath; + +typedef struct ErtsMigrationPaths_ ErtsMigrationPaths; + +struct ErtsMigrationPaths_ { + void *block; + ErtsMigrationPaths *next; + ErtsThrPrgrVal thr_prgr; + ErtsMigrationPath mpath[1]; +}; + +#endif /* ERTS_SMP */ struct ErtsRunQueue_ { int ix; - erts_smp_atomic32_t info_flags; erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; @@ -334,19 +351,18 @@ struct ErtsRunQueue_ { ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ int woken; - Uint32 flags; + erts_smp_atomic32_t flags; int check_balance_reds; int full_reds_history_sum; int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE]; int out_of_work_count; - int max_len; - int len; + erts_aint32_t max_len; + erts_aint32_t len; int wakeup_other; int wakeup_other_reds; int halt_in_progress; struct { - int len; ErtsProcList *pending_exiters; Uint context_switches; Uint reductions; @@ -361,13 +377,13 @@ struct ErtsRunQueue_ { struct { ErtsMiscOpList *start; ErtsMiscOpList *end; - ErtsRunQueue *evac_runq; + erts_smp_atomic_t evac_runq; } misc; struct { ErtsRunQueueInfo info; - struct port *start; - struct port *end; + Port *start; + Port *end; } ports; }; @@ -385,7 +401,7 @@ extern ErtsAlignedRunQueue *erts_aligned_run_queues; #define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS) \ do { \ (RQ)->procs.reductions += (AREDS); \ - (RQ)->procs.prio_info[p->prio].reds += (REDS); \ + (RQ)->procs.prio_info[(PRIO)].reds += (REDS); \ (RQ)->check_balance_reds -= (REDS); \ (RQ)->wakeup_other_reds += (AREDS); \ } while (0) @@ -408,6 +424,11 @@ typedef struct { } ErtsSchedWallTime; typedef struct { + Uint64 reclaimed; + Uint64 garbage_cols; +} ErtsGCInfo; + +typedef struct { int sched; erts_aint32_t aux_work; } ErtsDelayedAuxWorkWakeupJob; @@ -418,6 +439,7 @@ typedef struct { ErtsSchedulerSleepInfo *ssi; #ifdef ERTS_SMP ErtsThrPrgrVal current_thr_prgr; + ErtsThrPrgrVal latest_wakeup; #endif struct { int ix; @@ -431,6 +453,12 @@ typedef struct { void (*completed_callback)(void *); void (*completed_arg)(void *); } dd; + struct { + ErtsThrPrgrVal thr_prgr; + UWord size; + ErtsThrPrgrLaterOp *first; + ErtsThrPrgrLaterOp *last; + } later_op; #endif #ifdef ERTS_USE_ASYNC_READY_Q struct { @@ -477,7 +505,7 @@ struct ErtsSchedulerData_ { ErtsSchedulerSleepInfo *ssi; Process *current_process; Uint no; /* Scheduler number */ - struct port *current_port; + Port *current_port; ErtsRunQueue *run_queue; int virtual_reds; int cpu_id; /* >= 0 when bound */ @@ -488,6 +516,8 @@ struct ErtsSchedulerData_ { Uint64 reductions; ErtsSchedWallTime sched_wall_time; + ErtsGCInfo gc_info; + ErtsPortTaskHandle nosuspend_port_task_handle; #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; @@ -506,6 +536,90 @@ extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; extern ErtsSchedulerData *erts_scheduler_data; #endif +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int erts_smp_lc_runq_is_locked(ErtsRunQueue *); +#endif + +#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS + +/* + * Run queue locked during modifications. We use atomic ops since + * other threads peek at values without run queue lock. + */ + +ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); +ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); +ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) +{ + erts_aint32_t len; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + len = erts_smp_atomic32_read_nob(&rqi->len); + ASSERT(len >= 0); + if (len == 0) { + ASSERT((erts_smp_atomic32_read_nob(&rq->flags) + & ((erts_aint32_t) (1 << prio))) == 0); + erts_smp_atomic32_read_bor_nob(&rq->flags, + (erts_aint32_t) (1 << prio)); + } + len++; + if (rqi->max_len < len) + rqi->max_len = len; + + erts_smp_atomic32_set_relb(&rqi->len, len); + + rq->len++; + if (rq->max_len < rq->len) + rq->max_len = len; + ASSERT(rq->len > 0); +} + +ERTS_GLB_INLINE void +erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) +{ + erts_aint32_t len; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + len = erts_smp_atomic32_read_nob(&rqi->len); + len--; + ASSERT(len >= 0); + if (len == 0) { + ASSERT((erts_smp_atomic32_read_nob(&rq->flags) + & ((erts_aint32_t) (1 << prio)))); + erts_smp_atomic32_read_band_nob(&rq->flags, + ~((erts_aint32_t) (1 << prio))); + } + erts_smp_atomic32_set_relb(&rqi->len, len); + + rq->len--; + ASSERT(rq->len >= 0); +} + +ERTS_GLB_INLINE void +erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) +{ + erts_aint32_t len; + + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + + len = erts_smp_atomic32_read_nob(&rqi->len); + ASSERT(rqi->max_len >= len); + rqi->max_len = len; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X)) + +#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ + /* * Process Specific Data. * @@ -589,6 +703,14 @@ struct ErtsPendingSuspend_ { #endif + +typedef struct ErlExtraRootSet_ ErlExtraRootSet; +struct ErlExtraRootSet_ { + Eterm *objv; + Uint sz; + void (*cleanup)(ErlExtraRootSet *); +}; + /* Defines to ease the change of memory architecture */ # define HEAP_START(p) (p)->heap # define HEAP_TOP(p) (p)->htop @@ -618,6 +740,8 @@ struct ErtsPendingSuspend_ { # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap struct process { + ErtsPTabElementCommon common; /* *Need* to be first in struct */ + /* All fields in the PCB that differs between different heap * architectures, have been moved to the end of this struct to * make sure that as few offsets as possible differ. Different @@ -660,17 +784,9 @@ struct process { * Number of reductions left to execute. * Only valid for the current process. */ - Uint32 status; /* process STATE */ - Uint32 gcstatus; /* process gc STATE */ - Uint32 rstatus; /* process resume STATE */ Uint32 rcount; /* suspend count */ - Eterm id; /* The pid of this process */ - int prio; /* Priority of process */ - int skipped; /* Times a low prio process has been rescheduled */ + int schedule_count; /* Times left to reschedule a low prio process */ Uint reds; /* No of reductions for this process */ - Eterm tracer_proc; /* If proc is traced, this is the tracer - (can NOT be boxed) */ - Uint trace_flags; /* Trace flags (used to be in flags) */ Eterm group_leader; /* Pid in charge (can be boxed) */ Uint flags; /* Trap exit, etc (no trace flags anymore) */ @@ -679,11 +795,6 @@ struct process { Eterm ftrace; /* Latest exception stack trace dump */ Process *next; /* Pointer to next process in run queue */ - Process *prev; /* Pointer to prev process in run queue */ - - struct reg_proc *reg; /* NULL iff not registered */ - ErtsLink *nlinks; - ErtsMonitor *monitors; /* The process monitors, both ends */ struct ErtsNodesMonitor_ *nodes_monitors; @@ -693,7 +804,12 @@ struct process { ErlMessageQueue msg; /* Message queue */ - ErtsBifTimer *bif_timers; /* Bif timers aiming at this process */ + ErlExtraRootSet *extra_root; /* Used by trapping BIF's */ + + union { + ErtsBifTimer *bif_timers; /* Bif timers aiming at this process */ + void *terminate; + } u; ProcDict *dictionary; /* Process dictionary, may be NULL */ @@ -717,8 +833,7 @@ struct process { * Information mainly for post-mortem use (erl crash dump). */ Eterm parent; /* Pid of process that created this process. */ - SysTimeval started; /* Time when started. */ - + erts_approx_time_t approx_started; /* Time when started. */ /* This is the place, where all fields that differs between memory * architectures, have gone to. @@ -740,28 +855,16 @@ struct process { Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ - union { -#ifdef ERTS_SMP - ErtsSmpPTimer *ptimer; -#else - ErlTimer tm; /* Timer entry */ -#endif - void *exit_data; /* Misc data referred during termination */ - } u; - - ErtsRunQueue *bound_runq; + erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ #ifdef ERTS_SMP + ErlMessageInQueue msg_inq; + ErtsPendExit pending_exit; erts_proc_lock_t lock; ErtsSchedulerData *scheduler_data; - int is_exiting; - Uint32 runq_flags; - Uint32 status_flags; - ErlMessageInQueue msg_inq; Eterm suspendee; ErtsPendingSuspend *pending_suspenders; - ErtsPendExit pending_exit; - ErtsRunQueue *run_queue; + erts_smp_atomic_t run_queue; #ifdef HIPE struct hipe_process_state_smp hipe_smp; #endif @@ -786,6 +889,8 @@ struct process { #endif }; +extern const Process erts_invalid_process; + #ifdef CHECK_FOR_HOLES # define INIT_HOLE_CHECK(p) \ do { \ @@ -815,6 +920,29 @@ void erts_check_for_holes(Process* p); #define SEQ_TRACE_TOKEN(p) ((p)->seq_trace_token) +#if ERTS_NO_PROC_PRIO_LEVELS > 4 +# error "Need to increase ERTS_PSFLG_PRIO_SHIFT" +#endif + +#define ERTS_PSFLG_PRIO_SHIFT 2 + +#define ERTS_PSFLG_BIT(N) \ + (((erts_aint32_t) 1) << (ERTS_PSFLG_PRIO_SHIFT + (N))) + +#define ERTS_PSFLG_PRIO_MASK (ERTS_PSFLG_BIT(0) - 1) + +#define ERTS_PSFLG_FREE ERTS_PSFLG_BIT(0) +#define ERTS_PSFLG_EXITING ERTS_PSFLG_BIT(1) +#define ERTS_PSFLG_PENDING_EXIT ERTS_PSFLG_BIT(2) +#define ERTS_PSFLG_ACTIVE ERTS_PSFLG_BIT(3) +#define ERTS_PSFLG_IN_RUNQ ERTS_PSFLG_BIT(4) +#define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(5) +#define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(6) +#define ERTS_PSFLG_GC ERTS_PSFLG_BIT(7) +#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(8) +#define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(9) + + /* The sequential tracing token is a tuple of size 5: * * {Flags, Label, Serial, Sender} @@ -887,9 +1015,6 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif -extern Process** process_tab; -extern Uint erts_max_processes; -extern Uint erts_process_tab_index_mask; extern Uint erts_default_process_flags; extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; /* If any of the erts_system_monitor_* variables are set (enabled), @@ -919,16 +1044,8 @@ struct erts_system_profile_flags_t { }; extern struct erts_system_profile_flags_t erts_system_profile_flags; -#define INVALID_PID(p, pid) ((p) == NULL \ - || (p)->id != (pid) \ - || (p)->status == P_EXITING) - -#define IS_TRACED(p) ( (p)->tracer_proc != NIL ) -#define ARE_TRACE_FLAGS_ON(p,tf) ( ((p)->trace_flags & (tf|F_SENSITIVE)) == (tf) ) -#define IS_TRACED_FL(p,tf) ( IS_TRACED(p) && ARE_TRACE_FLAGS_ON(p,tf) ) - /* process flags */ -#define F_TRAPEXIT (1 << 0) +#define F_HIBERNATE_SCHED (1 << 0) /* Schedule out after hibernate op */ #define F_INSLPQUEUE (1 << 1) /* Set if in timer queue */ #define F_TIMO (1 << 2) /* Set if timeout */ #define F_HEAP_GROW (1 << 3) @@ -939,7 +1056,6 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */ #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ -#define F_HIBERNATE_SCHED (1 << 11) /* Schedule out after hibernate op */ /* process trace_flags */ #define F_SENSITIVE (1 << 0) @@ -1006,67 +1122,10 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define DT_UTAG_FLAGS(P) ((P)->dt_utag_flags) #endif - -#ifdef ERTS_SMP -/* Status flags ... */ -#define ERTS_PROC_SFLG_PENDADD2SCHEDQ (((Uint32) 1) << 0) /* Pending - add to - schedule q */ -#define ERTS_PROC_SFLG_INRUNQ (((Uint32) 1) << 1) /* Process is - in run q */ -#define ERTS_PROC_SFLG_TRAPEXIT (((Uint32) 1) << 2) /* Process is - trapping - exit */ -#define ERTS_PROC_SFLG_RUNNING (((Uint32) 1) << 3) /* Process is - running */ -/* Scheduler flags in process struct... */ -#define ERTS_PROC_RUNQ_FLG_RUNNING (((Uint32) 1) << 0) /* Process is - running */ - -#endif - - -#ifdef ERTS_SMP -#define ERTS_PROC_IS_TRAPPING_EXITS(P) \ - (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) \ - & ERTS_PROC_LOCK_STATUS), \ - (P)->status_flags & ERTS_PROC_SFLG_TRAPEXIT) - -#define ERTS_PROC_SET_TRAP_EXIT(P) \ - (ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS) \ - & erts_proc_lc_my_proc_locks((P))) \ - == (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)), \ - (P)->status_flags |= ERTS_PROC_SFLG_TRAPEXIT, \ - (P)->flags |= F_TRAPEXIT, \ - 1) - -#define ERTS_PROC_UNSET_TRAP_EXIT(P) \ - (ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS) \ - & erts_proc_lc_my_proc_locks((P))) \ - == (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)), \ - (P)->status_flags &= ~ERTS_PROC_SFLG_TRAPEXIT, \ - (P)->flags &= ~F_TRAPEXIT, \ - 0) -#else -#define ERTS_PROC_IS_TRAPPING_EXITS(P) ((P)->flags & F_TRAPEXIT) -#define ERTS_PROC_SET_TRAP_EXIT(P) ((P)->flags |= F_TRAPEXIT, 1) -#define ERTS_PROC_UNSET_TRAP_EXIT(P) ((P)->flags &= ~F_TRAPEXIT, 0) -#endif - /* Option flags to erts_send_exit_signal() */ #define ERTS_XSIG_FLG_IGN_KILL (((Uint32) 1) << 0) #define ERTS_XSIG_FLG_NO_IGN_NORMAL (((Uint32) 1) << 1) - -/* Process status values */ -#define P_FREE 0 -#define P_RUNABLE 1 -#define P_WAITING 2 -#define P_RUNNING 3 -#define P_EXITING 4 -#define P_GARBING 5 -#define P_SUSPENDED 6 - #define CANCEL_TIMER(p) \ do { \ if ((p)->flags & (F_INSLPQUEUE)) \ @@ -1088,14 +1147,192 @@ void erts_early_init_scheduling(int); void erts_init_scheduling(int, int); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_gc_info_request(Process *c_p); +Uint64 erts_get_proc_interval(void); +Uint64 erts_ensure_later_proc_interval(Uint64); +Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); -int erts_proclist_same(ErtsProcList *, Process *); + +ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); +ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *); +ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *); +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *); +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *); +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *); +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *); +ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **); +ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **); +ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **); +ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *); +ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *); +ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *); +ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int +erts_proclist_same(ErtsProcList *plp, Process *p) +{ + return (plp->pid == p->common.id + && (plp->started_interval + == p->common.u.alive.started_interval)); +} + +ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list, + ErtsProcList *element) +{ + if (!*list) + element->next = element->prev = element; + else { + element->prev = (*list)->prev; + element->next = *list; + element->prev->next = element; + element->next->prev = element; + } + *list = element; +} + +ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list, + ErtsProcList *element) +{ + if (!*list) { + element->next = element->prev = element; + *list = element; + } + else { + element->prev = (*list)->prev; + element->next = *list; + element->prev->next = element; + element->next->prev = element; + } +} + +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list) +{ + return list; +} + +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list) +{ + if (!list) + return NULL; + else + return list->prev; +} + +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list, + ErtsProcList *element) +{ + ErtsProcList *next; + ASSERT(list && element); + next = element->next; + return list == next ? NULL : next; +} + +ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list, + ErtsProcList *element) +{ + ErtsProcList *prev; + ASSERT(list && element); + prev = element->prev; + return list == element ? NULL : prev; +} + +ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list) +{ + if (!*list) + return NULL; + else { + ErtsProcList *res = *list; + if (res == *list) + *list = NULL; + else + *list = res->next; + res->next->prev = res->prev; + res->prev->next = res->next; + return res; + } +} + +ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list) +{ + if (!*list) + return NULL; + else { + ErtsProcList *res = (*list)->prev; + if (res == *list) + *list = NULL; + res->next->prev = res->prev; + res->prev->next = res->next; + return res; + } +} + +ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first, + ErtsProcList **list_last) +{ + if (!*list_first) { + if (list_last) + *list_last = NULL; + return 0; + } + else { + if (list_last) + *list_last = (*list_first)->prev; + (*list_first)->prev->next = NULL; + (*list_first)->prev = NULL; + return !0; + } +} + +ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list, + ErtsProcList *element) +{ + ASSERT(list && *list); + if (*list == element) { + *list = element->next; + if (*list == element) + *list = NULL; + } + element->next->prev = element->prev; + element->prev->next = element->next; +} + +ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list) +{ + return list == NULL; +} + +ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list, + ErtsProcList *element) +{ + ASSERT(list && element); + return list == element; +} + +ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list, + ErtsProcList *element) +{ + ASSERT(list && element); + return list->prev == element; +} + +#endif int erts_sched_set_wakeup_other_thresold(char *str); int erts_sched_set_wakeup_other_type(char *str); int erts_sched_set_busy_wait_threshold(char *str); +int erts_sched_set_wake_cleanup_threshold(char *); + +void erts_schedule_thr_prgr_later_op(void (*)(void *), + void *, + ErtsThrPrgrLaterOp *); +void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), + void *, + ErtsThrPrgrLaterOp *, + UWord); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); @@ -1118,11 +1355,16 @@ int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); +void erts_alloc_ensure_handle_delayed_dealloc_call(int); void erts_smp_notify_check_children_needed(void); #endif #if ERTS_USE_ASYNC_READY_Q void erts_notify_check_async_ready_queue(void *); #endif +#ifdef ERTS_SMP +void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later); +void erts_notify_finish_breakpointing(Process* p); +#endif void erts_schedule_misc_aux_work(int sched_id, void (*func)(void *), void *arg); @@ -1133,7 +1375,7 @@ void erts_schedule_multi_misc_aux_work(int ignore_self, erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); -void erts_init_process(int); +void erts_init_process(int, int, int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); Uint erts_run_queues_len(Uint *); void erts_add_to_runq(Process *); @@ -1143,14 +1385,6 @@ Eterm erts_get_schedulers_binds(Process *c_p); Eterm erts_set_cpu_topology(Process *c_p, Eterm term); Eterm erts_bind_schedulers(Process *c_p, Eterm how); ErtsRunQueue *erts_schedid2runq(Uint); -#ifdef ERTS_SMP -ErtsMigrateResult erts_proc_migrate(Process *, - ErtsProcLocks *, - ErtsRunQueue *, - int *, - ErtsRunQueue *, - int *); -#endif Process *schedule(Process*, int); void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); @@ -1160,7 +1394,6 @@ void set_timer(Process*, Uint); void cancel_timer(Process*); /* Begin System profile */ Uint erts_runnable_process_count(void); -Uint erts_process_count(void); /* End System profile */ void erts_init_empty_process(Process *p); void erts_cleanup_empty_process(Process* p); @@ -1184,7 +1417,7 @@ Eterm erts_sched_stat_term(Process *p, int total); void erts_free_proc(Process *); -void erts_suspend(Process*, ErtsProcLocks, struct port*); +void erts_suspend(Process*, ErtsProcLocks, Port*); void erts_resume(Process*, ErtsProcLocks); int erts_resume_processes(ErtsProcList *); @@ -1199,8 +1432,7 @@ int erts_send_exit_signal(Process *, #ifdef ERTS_SMP void erts_handle_pending_exit(Process *, ErtsProcLocks); #define ERTS_PROC_PENDING_EXIT(P) \ - (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) & ERTS_PROC_LOCK_STATUS),\ - (P)->pending_exit.reason != THE_NON_VALUE) + (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state)) #else #define ERTS_PROC_PENDING_EXIT(P) 0 #endif @@ -1210,12 +1442,11 @@ void erts_deep_process_dump(int, void *); Eterm erts_get_reader_groups_map(Process *c_p); Eterm erts_debug_reader_groups_map(Process *c_p, int groups); -Sint erts_test_next_pid(int, Uint); -Eterm erts_debug_processes(Process *c_p); -Eterm erts_debug_processes_bif_info(Process *c_p); Uint erts_debug_nbalance(void); int erts_debug_wait_deallocations(Process *c_p); +Uint erts_process_memory(Process *c_p); + #ifdef ERTS_SMP # define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) # define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data) @@ -1252,13 +1483,26 @@ ErtsSchedulerData *erts_get_scheduler_data(void) #endif #endif +void erts_schedule_process(Process *, erts_aint32_t); + +ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void +erts_proc_notify_new_message(Process *p) +{ + /* No barrier needed, due to msg lock */ + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + if (!(state & ERTS_PSFLG_ACTIVE)) + erts_schedule_process(p, state); +} +#endif + #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) #define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__ #include "erl_process_lock.h" #undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__ -int erts_smp_lc_runq_is_locked(ErtsRunQueue *); #define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L) \ do { \ if ((L)) \ @@ -1384,13 +1628,91 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler) #endif +#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS + #ifdef ERTS_SMP -ErtsRunQueue *erts_prepare_emigrate(ErtsRunQueue *c_rq, - ErtsRunQueueInfo *c_rqi, - int prio); +#include "erl_thr_progress.h" + +extern erts_atomic_t erts_migration_paths; + +ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void); +ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void); ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq, int prio); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE ErtsMigrationPaths * +erts_get_migration_paths_managed(void) +{ + return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths); +} + +ERTS_GLB_INLINE ErtsMigrationPaths * +erts_get_migration_paths(void) +{ + if (erts_thr_progress_is_managed_thread()) + return erts_get_migration_paths_managed(); + else + return NULL; +} + +ERTS_GLB_INLINE ErtsRunQueue * +erts_check_emigration_need(ErtsRunQueue *c_rq, int prio) +{ + ErtsMigrationPaths *mps = erts_get_migration_paths(); + ErtsMigrationPath *mp; + Uint32 flags; + + if (!mps) + return NULL; + + mp = &mps->mpath[c_rq->ix]; + flags = mp->flags; + + if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) { + int len; + + if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) { + /* force emigration */ + return mp->prio[prio].runq; + } + + if (flags & ERTS_RUNQ_FLG_INACTIVE) { + /* + * Run queue was inactive at last balance. Verify that + * it still is before forcing emigration. + */ + if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE) + return mp->prio[prio].runq; + } + + + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&c_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len); + + if (len > mp->prio[prio].limit.this) { + ErtsRunQueue *n_rq = mp->prio[prio].runq; + if (n_rq) { + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&n_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len); + + if (len < mp->prio[prio].limit.other) + return n_rq; + } + } + } + return NULL; +} + +#endif + +#endif + #endif ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp); @@ -1413,29 +1735,6 @@ ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ERTS_SMP -ERTS_GLB_INLINE ErtsRunQueue * -erts_check_emigration_need(ErtsRunQueue *c_rq, int prio) -{ - ErtsRunQueueInfo *c_rqi; - - if (!ERTS_CHK_RUNQ_FLG_EMIGRATE(c_rq->flags, prio)) - return NULL; - - if (prio == ERTS_PORT_PRIO_LEVEL) - c_rqi = &c_rq->ports.info; - else - c_rqi = &c_rq->procs.prio_info[prio]; - - if (!ERTS_CHK_RUNQ_FLG_EVACUATE(c_rq->flags, prio) - && !(c_rq->flags & ERTS_RUNQ_FLG_INACTIVE) - && c_rqi->len <= c_rqi->migrate.limit.this) - return NULL; - - return erts_prepare_emigrate(c_rq, c_rqi, prio); -} -#endif - ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp) { @@ -1456,7 +1755,7 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void) { Process *proc = erts_get_current_process(); - return proc ? proc->id : THE_NON_VALUE; + return proc ? proc->common.id : THE_NON_VALUE; } ERTS_GLB_INLINE @@ -1473,10 +1772,9 @@ Uint erts_get_scheduler_id(void) ERTS_GLB_INLINE ErtsRunQueue * erts_get_runq_proc(Process *p) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); #ifdef ERTS_SMP - ASSERT(p->run_queue); - return p->run_queue; + ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue)); + return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue); #else return ERTS_RUNQ_IX(0); #endif @@ -1647,25 +1945,13 @@ extern int erts_disable_proc_not_running_opt; #ifdef DEBUG #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \ - do { ASSERT(!(P)->is_exiting); } while (0) + do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0) #else #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) #endif -/* NOTE: At least one process lock has to be held on P! */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_PROC_IS_EXITING(P) \ - (ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((P)) != 0 \ - || erts_lc_pix_lock_is_locked(ERTS_PID2PIXLOCK((P)->id))),\ - (P)->is_exiting) -#else -#define ERTS_PROC_IS_EXITING(P) ((P)->is_exiting) -#endif - #else /* !ERTS_SMP */ -#define ERTS_PROC_IS_EXITING(P) ((P)->status == P_EXITING) - #define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) #define erts_pid2proc_not_running erts_pid2proc @@ -1673,11 +1959,15 @@ extern int erts_disable_proc_not_running_opt; #endif +#define ERTS_PROC_IS_EXITING(P) \ + (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state)) + + /* Minimum NUMBER of processes for a small system to start */ -#ifdef ERTS_SMP +#define ERTS_MIN_PROCESSES 1024 +#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS +#undef ERTS_MIN_PROCESSES #define ERTS_MIN_PROCESSES ERTS_NO_OF_PIX_LOCKS -#else -#define ERTS_MIN_PROCESSES 16 #endif void erts_smp_notify_inc_runq(ErtsRunQueue *runq); @@ -1700,6 +1990,7 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) } } + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* #ifdef ERTS_SMP */ diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 93466da3aa..a611b52af2 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2009. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -360,7 +360,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) erts_fprintf(stderr, "Process dictionary for process %T is broken, trying to " "display term found in line %d:\n" - "%T\n", p->id, __LINE__, old); + "%T\n", p->common.id, __LINE__, old); #endif erl_exit(1, "Damaged process dictionary found during erase/1."); } @@ -405,7 +405,7 @@ Eterm erts_pd_hash_get(Process *p, Eterm id) erts_fprintf(stderr, "Process dictionary for process %T is broken, trying to " "display term found in line %d:\n" - "%T\n", p->id, __LINE__, tmp); + "%T\n", p->common.id, __LINE__, tmp); #endif erl_exit(1, "Damaged process dictionary found during get/1."); } @@ -614,7 +614,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) erts_fprintf(stderr, "Process dictionary for process %T is broken, trying to " "display term found in line %d:\n" - "%T\n", p->id, __LINE__, old); + "%T\n", p->common.id, __LINE__, old); #endif erl_exit(1, "Damaged process dictionary found during put/2."); diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 3550f1396c..6cd0d23b97 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -60,25 +60,59 @@ extern BeamInstr beam_continue_exit[]; void erts_deep_process_dump(int to, void *to_arg) { - int i; + int i, max = erts_ptab_max(&erts_proc); all_binaries = NULL; - for (i = 0; i < erts_max_processes; i++) { - if ((process_tab[i] != NULL) && (process_tab[i]->i != ENULL)) { - if (process_tab[i]->status != P_EXITING) { - Process* p = process_tab[i]; - - if (p->status != P_GARBING) { - dump_process_info(to, to_arg, p); - } - } + for (i = 0; i < max; i++) { + Process *p = erts_pix2proc(i); + if (p && p->i != ENULL) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) + dump_process_info(to, to_arg, p); } } dump_binaries(to, to_arg, all_binaries); } +Uint erts_process_memory(Process *p) { + ErlMessage *mp; + Uint size = 0; + struct saved_calls *scb; + size += sizeof(Process); + + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + + erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); + erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); + size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); + if (p->old_hend && p->old_heap) + size += (p->old_hend - p->old_heap) * sizeof(Eterm); + + size += p->msg.len * sizeof(ErlMessage); + + for (mp = p->msg.first; mp; mp = mp->next) + if (mp->data.attached) + size += erts_msg_attached_data_size(mp)*sizeof(Eterm); + + if (p->arg_reg != p->def_arg_reg) { + size += p->arity * sizeof(p->arg_reg[0]); + } + + if (p->psd) + size += sizeof(ErtsPSD); + + scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); + if (scb) { + size += (sizeof(struct saved_calls) + + (scb->len-1) * sizeof(scb->ct[0])); + } + + size += erts_dicts_mem_size(p); + return size; +} + static void dump_process_info(int to, void *to_arg, Process *p) { @@ -88,8 +122,8 @@ dump_process_info(int to, void *to_arg, Process *p) ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); - if ((p->trace_flags & F_SENSITIVE) == 0 && p->msg.first) { - erts_print(to, to_arg, "=proc_messages:%T\n", p->id); + if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) { + erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); for (mp = p->msg.first; mp != NULL; mp = mp->next) { Eterm mesg = ERL_MESSAGE_TERM(mp); if (is_value(mesg)) @@ -103,21 +137,21 @@ dump_process_info(int to, void *to_arg, Process *p) } } - if ((p->trace_flags & F_SENSITIVE) == 0) { + if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { if (p->dictionary) { - erts_print(to, to_arg, "=proc_dictionary:%T\n", p->id); + erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); erts_deep_dictionary_dump(to, to_arg, p->dictionary, dump_element_nl); } } - if ((p->trace_flags & F_SENSITIVE) == 0) { - erts_print(to, to_arg, "=proc_stack:%T\n", p->id); + if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { + erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); for (sp = p->stop; sp < STACK_START(p); sp++) { yreg = stack_element_dump(to, to_arg, p, sp, yreg); } - erts_print(to, to_arg, "=proc_heap:%T\n", p->id); + erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); for (sp = p->stop; sp < STACK_START(p); sp++) { Eterm term = *sp; @@ -326,7 +360,7 @@ heap_dump(int to, void *to_arg, Eterm x) int i; GET_DOUBLE_DATA((ptr+1), f); - i = sys_double_to_chars(f.fd, (char*) sbuf); + i = sys_double_to_chars(f.fd, (char*) sbuf, sizeof(sbuf)); sys_memset(sbuf+i, 0, 31-i); erts_print(to, to_arg, "F%X:%s\n", i, sbuf); *ptr = OUR_NIL; diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index f7900317cc..2db5df06b4 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -66,11 +66,12 @@ #endif #include "erl_process.h" - -const Process erts_proc_lock_busy; +#include "erl_thr_progress.h" #ifdef ERTS_SMP +#if ERTS_PROC_LOCK_OWN_IMPL + #define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000 #define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000 @@ -90,6 +91,13 @@ static void check_queue(erts_proc_lock_t *lck); #error "The size of the 'uflgs' field of the erts_tse_t type is too small" #endif +static int proc_lock_spin_count; +static int aux_thr_proc_lock_spin_count; + +static void cleanup_tse(void); + +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + #ifdef ERTS_ENABLE_LOCK_CHECK static struct { Sint16 proc_lock_main; @@ -101,10 +109,6 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; -static int proc_lock_spin_count; -static int aux_thr_proc_lock_spin_count; - -static void cleanup_tse(void); void erts_init_proc_lock(int cpus) @@ -118,13 +122,8 @@ erts_init_proc_lock(int cpus) erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); #endif } +#if ERTS_PROC_LOCK_OWN_IMPL erts_thr_install_exit_handler(cleanup_tse); -#ifdef ERTS_ENABLE_LOCK_CHECK - lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); - lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); - lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); - lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); -#endif if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC @@ -141,8 +140,17 @@ erts_init_proc_lock(int cpus) } if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; +#endif +#ifdef ERTS_ENABLE_LOCK_CHECK + lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); + lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); + lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); + lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); +#endif } +#if ERTS_PROC_LOCK_OWN_IMPL + #ifdef ERTS_ENABLE_LOCK_CHECK #define CHECK_UNUSED_TSE(W) ERTS_LC_ASSERT((W)->uflgs == 0) #else @@ -164,13 +172,6 @@ tse_return(erts_tse_t *tse) erts_tse_return(tse); } -void -erts_proc_lock_prepare_proc_lock_waiter(void) -{ - tse_return(tse_fetch(NULL)); -} - - static void cleanup_tse(void) { @@ -397,7 +398,7 @@ wait_for_locks(Process *p, ErtsProcLocks need_locks, ErtsProcLocks olflgs) { - erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); + erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->common.id); erts_tse_t *wtr; /* Acquire a waiter object on which this thread can wait. */ @@ -551,7 +552,7 @@ erts_proc_unlock_failed(Process *p, erts_pix_lock_t *pixlck, ErtsProcLocks wait_locks) { - erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); + erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->common.id); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); @@ -560,6 +561,16 @@ erts_proc_unlock_failed(Process *p, transfer_locks(p, wait_locks, pix_lock, 1); /* unlocks pix_lock */ } +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + +void +erts_proc_lock_prepare_proc_lock_waiter(void) +{ +#if ERTS_PROC_LOCK_OWN_IMPL + tse_return(tse_fetch(NULL)); +#endif +} + /* * proc_safelock() locks process locks on two processes. In order * to avoid a deadlock, proc_safelock() unlocks those locks that @@ -568,12 +579,11 @@ erts_proc_unlock_failed(Process *p, */ static void -proc_safelock(Process *a_proc, - erts_pix_lock_t *a_pix_lck, +proc_safelock(int is_managed, + Process *a_proc, ErtsProcLocks a_have_locks, ErtsProcLocks a_need_locks, Process *b_proc, - erts_pix_lock_t *b_pix_lck, ErtsProcLocks b_have_locks, ErtsProcLocks b_need_locks) { @@ -581,7 +591,6 @@ proc_safelock(Process *a_proc, #ifdef ERTS_ENABLE_LOCK_CHECK Eterm pid1, pid2; #endif - erts_pix_lock_t *pix_lck1, *pix_lck2; ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2; ErtsProcLocks unlock_mask; int lock_no, refc1 = 0, refc2 = 0; @@ -593,53 +602,47 @@ proc_safelock(Process *a_proc, * Locks with the same lock order should be locked on p1 before p2. */ if (a_proc) { - if (a_proc->id < b_proc->id) { + if (a_proc->common.id < b_proc->common.id) { p1 = a_proc; #ifdef ERTS_ENABLE_LOCK_CHECK - pid1 = a_proc->id; + pid1 = a_proc->common.id; #endif - pix_lck1 = a_pix_lck; need_locks1 = a_need_locks; have_locks1 = a_have_locks; p2 = b_proc; #ifdef ERTS_ENABLE_LOCK_CHECK - pid2 = b_proc->id; + pid2 = b_proc->common.id; #endif - pix_lck2 = b_pix_lck; need_locks2 = b_need_locks; have_locks2 = b_have_locks; } - else if (a_proc->id > b_proc->id) { + else if (a_proc->common.id > b_proc->common.id) { p1 = b_proc; #ifdef ERTS_ENABLE_LOCK_CHECK - pid1 = b_proc->id; + pid1 = b_proc->common.id; #endif - pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = a_proc; #ifdef ERTS_ENABLE_LOCK_CHECK - pid2 = a_proc->id; + pid2 = a_proc->common.id; #endif - pix_lck2 = a_pix_lck; need_locks2 = a_need_locks; have_locks2 = a_have_locks; } else { ERTS_LC_ASSERT(a_proc == b_proc); - ERTS_LC_ASSERT(a_proc->id == b_proc->id); + ERTS_LC_ASSERT(a_proc->common.id == b_proc->common.id); p1 = a_proc; #ifdef ERTS_ENABLE_LOCK_CHECK - pid1 = a_proc->id; + pid1 = a_proc->common.id; #endif - pix_lck1 = a_pix_lck; need_locks1 = a_need_locks | b_need_locks; have_locks1 = a_have_locks | b_have_locks; p2 = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; #endif - pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; } @@ -647,16 +650,14 @@ proc_safelock(Process *a_proc, else { p1 = b_proc; #ifdef ERTS_ENABLE_LOCK_CHECK - pid1 = b_proc->id; + pid1 = b_proc->common.id; #endif - pix_lck1 = b_pix_lck; need_locks1 = b_need_locks; have_locks1 = b_have_locks; p2 = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK pid2 = 0; #endif - pix_lck2 = NULL; need_locks2 = 0; have_locks2 = 0; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -704,21 +705,21 @@ proc_safelock(Process *a_proc, if (unlock_locks) { have_locks1 &= ~unlock_locks; need_locks1 |= unlock_locks; - if (!have_locks1) { + if (!is_managed && !have_locks1) { refc1 = 1; erts_smp_proc_inc_refc(p1); } - erts_smp_proc_unlock__(p1, pix_lck1, unlock_locks); + erts_smp_proc_unlock(p1, unlock_locks); } unlock_locks = unlock_mask & have_locks2; if (unlock_locks) { have_locks2 &= ~unlock_locks; need_locks2 |= unlock_locks; - if (!have_locks2) { + if (!is_managed && !have_locks2) { refc2 = 1; erts_smp_proc_inc_refc(p2); } - erts_smp_proc_unlock__(p2, pix_lck2, unlock_locks); + erts_smp_proc_unlock(p2, unlock_locks); } } @@ -749,7 +750,7 @@ proc_safelock(Process *a_proc, if (need_locks2 & lock) lock_no--; locks = need_locks1 & lock_mask; - erts_smp_proc_lock__(p1, pix_lck1, locks); + erts_smp_proc_lock(p1, locks); have_locks1 |= locks; need_locks1 &= ~locks; } @@ -760,7 +761,7 @@ proc_safelock(Process *a_proc, lock = (1 << ++lock_no); } locks = need_locks2 & lock_mask; - erts_smp_proc_lock__(p2, pix_lck2, locks); + erts_smp_proc_lock(p2, locks); have_locks2 |= locks; need_locks2 &= ~locks; } @@ -795,10 +796,12 @@ proc_safelock(Process *a_proc, } #endif - if (refc1) - erts_smp_proc_dec_refc(p1); - if (refc2) - erts_smp_proc_dec_refc(p2); + if (!is_managed) { + if (refc1) + erts_smp_proc_dec_refc(p1); + if (refc2) + erts_smp_proc_dec_refc(p2); + } } void @@ -809,77 +812,196 @@ erts_proc_safelock(Process *a_proc, ErtsProcLocks b_have_locks, ErtsProcLocks b_need_locks) { - proc_safelock(a_proc, - a_proc ? ERTS_PID2PIXLOCK(a_proc->id) : NULL, + proc_safelock(erts_get_scheduler_id() != 0, + a_proc, a_have_locks, a_need_locks, b_proc, - b_proc ? ERTS_PID2PIXLOCK(b_proc->id) : NULL, b_have_locks, b_need_locks); } -/* - * erts_pid2proc_safelock() is called from erts_pid2proc_opt() when - * it wasn't possible to trylock all locks needed. - * c_p - current process - * c_p_have_locks - locks held on c_p - * pid - process id of process we are looking up - * proc - process struct of process we are looking - * up (both in and out argument) - * need_locks - all locks we need (including have_locks) - * pix_lock - pix lock for process we are looking up - * flags - option flags - */ -void -erts_pid2proc_safelock(Process *c_p, - ErtsProcLocks c_p_have_locks, - Process **proc, - ErtsProcLocks need_locks, - erts_pix_lock_t *pix_lock, - int flags) +Process * +erts_pid2proc_opt(Process *c_p, + ErtsProcLocks c_p_have_locks, + Eterm pid, + ErtsProcLocks pid_need_locks, + int flags) { - Process *p = *proc; - ERTS_LC_ASSERT(p->lock.refc > 0); - ERTS_LC_ASSERT(process_tab[internal_pid_index(p->id)] == p); - p->lock.refc++; - erts_pix_unlock(pix_lock); - - proc_safelock(c_p, - c_p ? ERTS_PID2PIXLOCK(c_p->id) : NULL, - c_p_have_locks, - c_p_have_locks, - p, - pix_lock, - 0, - need_locks); + Process *dec_refc_proc = NULL; + ErtsThrPrgrDelayHandle dhndl; + ErtsProcLocks need_locks; + Uint pix; + Process *proc; +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + ErtsProcLocks lcnt_locks; +#endif - erts_pix_lock(pix_lock); +#ifdef ERTS_ENABLE_LOCK_CHECK + if (c_p) { + ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; + if (might_unlock) + erts_proc_lc_might_unlock(c_p, might_unlock); + } +#endif - if (!p->is_exiting - || ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && process_tab[internal_pid_index(p->id)] == p)) { - ERTS_LC_ASSERT(p->lock.refc > 1); - p->lock.refc--; + if (is_not_internal_pid(pid)) + return NULL; + pix = internal_pid_index(pid); + + ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); + need_locks = pid_need_locks; + + if (c_p && c_p->common.id == pid) { + ASSERT(c_p->common.id != ERTS_INVALID_PID); + ASSERT(c_p == erts_pix2proc(pix)); + + if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && ERTS_PROC_IS_EXITING(c_p)) + return NULL; + need_locks &= ~c_p_have_locks; + if (!need_locks) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(c_p); + return c_p; + } } - else { - /* No proc. Note, we need to keep refc until after process unlock */ - erts_pix_unlock(pix_lock); - erts_smp_proc_unlock__(p, pix_lock, need_locks); - *proc = NULL; - erts_pix_lock(pix_lock); - ERTS_LC_ASSERT(p->lock.refc > 0); - if (--p->lock.refc == 0) { - erts_pix_unlock(pix_lock); - erts_free_proc(p); - erts_pix_lock(pix_lock); + + dhndl = erts_thr_progress_unmanaged_delay(); + + proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, pix); + + if (proc) { + if (proc->common.id != pid) + proc = NULL; + else if (!need_locks) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); } + else { + int busy; + +#if ERTS_PROC_LOCK_OWN_IMPL +#ifdef ERTS_ENABLE_LOCK_COUNT + lcnt_locks = need_locks; + if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { + erts_lcnt_proc_lock(&proc->lock, need_locks); + } +#endif + +#ifdef ERTS_ENABLE_LOCK_CHECK + /* Make sure erts_pid2proc_safelock() is enough to handle + a potential lock order violation situation... */ + busy = erts_proc_lc_trylock_force_busy(proc, need_locks); + if (!busy) +#endif +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + { + /* Try a quick trylock to grab all the locks we need. */ + busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) + erts_proc_lc_trylock(proc, need_locks, !busy); +#endif +#ifdef ERTS_PROC_LOCK_DEBUG + if (!busy) + erts_proc_lock_op_debug(proc, need_locks, 1); +#endif + } + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + if (flags & ERTS_P2P_FLG_TRY_LOCK) + erts_lcnt_proc_trylock(&proc->lock, need_locks, + busy ? EBUSY : 0); +#endif + + if (!busy) { + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + /* all is great */ + if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) + erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, + __FILE__, __LINE__); +#endif + + } + else { + if (flags & ERTS_P2P_FLG_TRY_LOCK) + proc = ERTS_PROC_LOCK_BUSY; + else { + int managed; + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + erts_smp_proc_inc_refc(proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) + erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); +#endif + + managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED; + if (!managed) { + erts_smp_proc_inc_refc(proc); + erts_thr_progress_unmanaged_continue(dhndl); + dec_refc_proc = proc; + + /* + * We don't want to call + * erts_thr_progress_unmanaged_continue() + * again. + */ + dhndl = ERTS_THR_PRGR_DHANDLE_MANAGED; + } + + proc_safelock(managed, + c_p, + c_p_have_locks, + c_p_have_locks, + proc, + 0, + need_locks); + } + } + } + } + + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_thr_progress_unmanaged_continue(dhndl); + + if (need_locks + && proc + && proc != ERTS_PROC_LOCK_BUSY + && (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + ? ERTS_PROC_IS_EXITING(proc) + : (proc + != (Process *) erts_ptab_pix2intptr_nob(&erts_proc, pix)))) { + + erts_smp_proc_unlock(proc, need_locks); + + if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + dec_refc_proc = proc; + proc = NULL; + } + + if (dec_refc_proc) + erts_smp_proc_dec_refc(dec_refc_proc); + +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) + ERTS_LC_ASSERT(!proc + || proc == ERTS_PROC_LOCK_BUSY + || (pid_need_locks == + (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) + & pid_need_locks))); +#endif + + return proc; } void erts_proc_lock_init(Process *p) { +#if ERTS_PROC_LOCK_OWN_IMPL int i; /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -890,32 +1012,67 @@ erts_proc_lock_init(Process *p) #endif for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) p->lock.queue[i] = NULL; - p->lock.refc = 1; -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_proc_lock_init(p); - erts_lcnt_proc_lock(&(p->lock), ERTS_PROC_LOCKS_ALL); - erts_lcnt_proc_lock_post_x(&(p->lock), ERTS_PROC_LOCKS_ALL, __FILE__, __LINE__); -#endif - #ifdef ERTS_ENABLE_LOCK_CHECK erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1); #endif +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id); + ethr_mutex_lock(&p->lock.main.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.main.lc); +#endif + erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id); + ethr_mutex_lock(&p->lock.link.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.link.lc); +#endif + erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id); + ethr_mutex_lock(&p->lock.msgq.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.msgq.lc); +#endif + erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id); + ethr_mutex_lock(&p->lock.status.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.status.lc); +#endif +#endif + erts_atomic32_init_nob(&p->lock.refc, 1); #ifdef ERTS_PROC_LOCK_DEBUG for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); #endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_proc_lock_init(p); + erts_lcnt_proc_lock(&(p->lock), ERTS_PROC_LOCKS_ALL); + erts_lcnt_proc_lock_post_x(&(p->lock), ERTS_PROC_LOCKS_ALL, __FILE__, __LINE__); +#endif +} + +void +erts_proc_lock_fin(Process *p) +{ +#if ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_mtx_destroy(&p->lock.main); + erts_mtx_destroy(&p->lock.link); + erts_mtx_destroy(&p->lock.msgq); + erts_mtx_destroy(&p->lock.status); +#endif +#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) + erts_lcnt_proc_lock_destroy(p); +#endif } /* --- Process lock counting ----------------------------------------------- */ -#ifdef ERTS_ENABLE_LOCK_COUNT +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_proc_lock_init(Process *p) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (p->id != ERTS_INVALID_PID) { - erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, p->id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, p->id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, p->id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK, p->id); + if (p->common.id != ERTS_INVALID_PID) { + erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, p->common.id); + erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, p->common.id); + erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, p->common.id); + erts_lcnt_init_lock_x(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK, p->common.id); } else { erts_lcnt_init_lock(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK); erts_lcnt_init_lock(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK); @@ -1023,11 +1180,12 @@ void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res } -void erts_lcnt_enable_proc_lock_count(int enable) { - int i; +void erts_lcnt_enable_proc_lock_count(int enable) +{ + int i, max = erts_ptab_max(&erts_proc); - for (i = 0; i < erts_max_processes; ++i) { - Process* p = process_tab[i]; + for (i = 0; i < max; ++i) { + Process* p = erts_pix2proc(i); if (p) { if (enable) { if (!ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) { @@ -1049,11 +1207,13 @@ void erts_lcnt_enable_proc_lock_count(int enable) { #ifdef ERTS_ENABLE_LOCK_CHECK +#if ERTS_PROC_LOCK_OWN_IMPL + void erts_proc_lc_lock(Process *p, ErtsProcLocks locks) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; @@ -1077,7 +1237,7 @@ void erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; @@ -1101,7 +1261,7 @@ void erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; @@ -1121,11 +1281,14 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) } } +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; @@ -1143,13 +1306,24 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_main; erts_lc_might_unlock(&lck); } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_lc_might_unlock(&p->lock.main.lc); + if (locks & ERTS_PROC_LOCK_LINK) + erts_lc_might_unlock(&p->lock.link.lc); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_lc_might_unlock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_lc_might_unlock(&p->lock.status.lc); +#endif } void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; @@ -1167,13 +1341,24 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_require_lock(&lck); } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_lc_require_lock(&p->lock.main.lc); + if (locks & ERTS_PROC_LOCK_LINK) + erts_lc_require_lock(&p->lock.link.lc); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_lc_require_lock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_lc_require_lock(&p->lock.status.lc); +#endif } void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; @@ -1191,15 +1376,26 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_main; erts_lc_unrequire_lock(&lck); } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_lc_unrequire_lock(&p->lock.main.lc); + if (locks & ERTS_PROC_LOCK_LINK) + erts_lc_unrequire_lock(&p->lock.link.lc); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_lc_unrequire_lock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_lc_unrequire_lock(&p->lock.status.lc); +#endif } +#if ERTS_PROC_LOCK_OWN_IMPL int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) { if (locks & ERTS_PROC_LOCKS_ALL) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) @@ -1218,42 +1414,61 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) return 0; } +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ + void erts_proc_lc_chk_only_proc_main(Process *p) { +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t proc_main = ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK); erts_lc_check_exact(&proc_main, 1); +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_check_exact(&p->lock.main.lc, 1); +#endif } +#if ERTS_PROC_LOCK_OWN_IMPL #define ERTS_PROC_LC_EMPTY_LOCK_INIT \ ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK) +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) { int have_locks_len = 0; +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; if (locks & ERTS_PROC_LOCK_MAIN) { have_locks[have_locks_len].id = lc_id.proc_lock_main; - have_locks[have_locks_len++].extra = p->id; + have_locks[have_locks_len++].extra = p->common.id; } if (locks & ERTS_PROC_LOCK_LINK) { have_locks[have_locks_len].id = lc_id.proc_lock_link; - have_locks[have_locks_len++].extra = p->id; + have_locks[have_locks_len++].extra = p->common.id; } if (locks & ERTS_PROC_LOCK_MSGQ) { have_locks[have_locks_len].id = lc_id.proc_lock_msgq; - have_locks[have_locks_len++].extra = p->id; + have_locks[have_locks_len++].extra = p->common.id; } if (locks & ERTS_PROC_LOCK_STATUS) { have_locks[have_locks_len].id = lc_id.proc_lock_status; - have_locks[have_locks_len++].extra = p->id; - } - + have_locks[have_locks_len++].extra = p->common.id; + } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_lock_t have_locks[4]; + if (locks & ERTS_PROC_LOCK_MAIN) + have_locks[have_locks_len++] = p->lock.main.lc; + if (locks & ERTS_PROC_LOCK_LINK) + have_locks[have_locks_len++] = p->lock.link.lc; + if (locks & ERTS_PROC_LOCK_MSGQ) + have_locks[have_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_STATUS) + have_locks[have_locks_len++] = p->lock.status.lc; +#endif erts_lc_check(have_locks, have_locks_len, NULL, 0); } @@ -1262,6 +1477,7 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) { int have_locks_len = 0; int have_not_locks_len = 0; +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, @@ -1273,36 +1489,57 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCK_MAIN) { have_locks[have_locks_len].id = lc_id.proc_lock_main; - have_locks[have_locks_len++].extra = p->id; + have_locks[have_locks_len++].extra = p->common.id; } else { have_not_locks[have_not_locks_len].id = lc_id.proc_lock_main; - have_not_locks[have_not_locks_len++].extra = p->id; + have_not_locks[have_not_locks_len++].extra = p->common.id; } if (locks & ERTS_PROC_LOCK_LINK) { have_locks[have_locks_len].id = lc_id.proc_lock_link; - have_locks[have_locks_len++].extra = p->id; + have_locks[have_locks_len++].extra = p->common.id; } else { have_not_locks[have_not_locks_len].id = lc_id.proc_lock_link; - have_not_locks[have_not_locks_len++].extra = p->id; + have_not_locks[have_not_locks_len++].extra = p->common.id; } if (locks & ERTS_PROC_LOCK_MSGQ) { have_locks[have_locks_len].id = lc_id.proc_lock_msgq; - have_locks[have_locks_len++].extra = p->id; + have_locks[have_locks_len++].extra = p->common.id; } else { have_not_locks[have_not_locks_len].id = lc_id.proc_lock_msgq; - have_not_locks[have_not_locks_len++].extra = p->id; + have_not_locks[have_not_locks_len++].extra = p->common.id; } if (locks & ERTS_PROC_LOCK_STATUS) { have_locks[have_locks_len].id = lc_id.proc_lock_status; - have_locks[have_locks_len++].extra = p->id; + have_locks[have_locks_len++].extra = p->common.id; } else { have_not_locks[have_not_locks_len].id = lc_id.proc_lock_status; - have_not_locks[have_not_locks_len++].extra = p->id; + have_not_locks[have_not_locks_len++].extra = p->common.id; } +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_lock_t have_locks[4]; + erts_lc_lock_t have_not_locks[4]; + + if (locks & ERTS_PROC_LOCK_MAIN) + have_locks[have_locks_len++] = p->lock.main.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.main.lc; + if (locks & ERTS_PROC_LOCK_LINK) + have_locks[have_locks_len++] = p->lock.link.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.link.lc; + if (locks & ERTS_PROC_LOCK_MSGQ) + have_locks[have_locks_len++] = p->lock.msgq.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_STATUS) + have_locks[have_locks_len++] = p->lock.status.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.status.lc; +#endif erts_lc_check(have_locks, have_locks_len, have_not_locks, have_not_locks_len); @@ -1312,20 +1549,26 @@ ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p) { int resv[4]; + ErtsProcLocks res = 0; +#if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t locks[4] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_link, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_status, - p->id, + p->common.id, ERTS_LC_FLG_LT_PROCLOCK)}; - - ErtsProcLocks res = 0; +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_lc_lock_t locks[4] = {p->lock.main.lc, + p->lock.link.lc, + p->lock.msgq.lc, + p->lock.status.lc}; +#endif erts_lc_have_locks(resv, locks, 4); if (resv[0]) @@ -1358,7 +1601,7 @@ erts_proc_lc_chk_no_proc_locks(char *file, int line) #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */ -#ifdef ERTS_PROC_LOCK_HARD_DEBUG +#if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_HARD_DEBUG) void check_queue(erts_proc_lock_t *lck) { @@ -1391,4 +1634,4 @@ check_queue(erts_proc_lock_t *lck) } #endif -#endif /* ERTS_SMP (the whole file) */ +#endif /* ERTS_SMP */ diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 062b8366d3..9dd503f3cb 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -37,10 +37,21 @@ #include "erl_smp.h" +#if defined(VALGRIND) || defined(ETHR_DISABLE_NATIVE_IMPLS) +# define ERTS_PROC_LOCK_OWN_IMPL 0 +#else +# define ERTS_PROC_LOCK_OWN_IMPL 1 +#endif + #define ERTS_PROC_LOCK_ATOMIC_IMPL 0 #define ERTS_PROC_LOCK_SPINLOCK_IMPL 0 #define ERTS_PROC_LOCK_MUTEX_IMPL 0 +#if !ERTS_PROC_LOCK_OWN_IMPL +#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 1 +#else +#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 0 + #if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) # undef ERTS_PROC_LOCK_ATOMIC_IMPL # define ERTS_PROC_LOCK_ATOMIC_IMPL 1 @@ -52,27 +63,38 @@ # define ERTS_PROC_LOCK_MUTEX_IMPL 1 #endif +#endif + #define ERTS_PROC_LOCK_MAX_BIT 3 typedef erts_aint32_t ErtsProcLocks; typedef struct erts_proc_lock_t_ { +#if ERTS_PROC_LOCK_OWN_IMPL #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_smp_atomic32_t flags; #else ErtsProcLocks flags; #endif erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; - Sint32 refc; -#ifdef ERTS_PROC_LOCK_DEBUG - erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; -#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_t lcnt_main; erts_lcnt_lock_t lcnt_link; erts_lcnt_lock_t lcnt_msgq; erts_lcnt_lock_t lcnt_status; #endif +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + erts_mtx_t main; + erts_mtx_t link; + erts_mtx_t msgq; + erts_mtx_t status; +#else +# error "no implementation" +#endif + erts_atomic32_t refc; +#ifdef ERTS_PROC_LOCK_DEBUG + erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; +#endif } erts_proc_lock_t; /* Process lock flags */ @@ -105,11 +127,9 @@ typedef struct erts_proc_lock_t_ { /* * Status lock: * Protects the following fields in the process structure: - * * status - * * rstatus - * * status_flags * * pending_suspenders * * suspendee + * * ... */ #define ERTS_PROC_LOCK_STATUS (((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT) @@ -141,14 +161,11 @@ typedef struct erts_proc_lock_t_ { * Other rules regarding process locking: * * Exiting processes: - * When changing status to P_EXITING on a process, you are required - * to take all process locks (ERTS_PROC_LOCKS_ALL). Thus, by holding - * at least one process lock (whichever one doesn't matter) you - * are guaranteed that the process won't exit until the lock you are - * holding has been released. Appart from all process locks also - * the pix lock corresponding to the process has to be held. - * At the same time as status is changed to P_EXITING, also the - * field 'is_exiting' in the process structure is set to a value != 0. + * When changing state to exiting (ERTS_PSFLG_EXITING) on a process, + * you are required to take all process locks (ERTS_PROC_LOCKS_ALL). + * Thus, by holding at least one process lock (whichever one doesn't + * matter) you are guaranteed that the process won't exit until the + * lock you are holding has been released. * * Lock order: * Process locks with low numeric values has to be locked before @@ -159,8 +176,8 @@ typedef struct erts_proc_lock_t_ { * on multiple processes, locks on processes with low process ids * have to be locked before locks on processes with high process * ids. E.g., if the main and the message queue locks are to be - * locked on processes p1 and p2 and p1->id < p2->id, then locks - * should be locked in the following order: + * locked on processes p1 and p2 and p1->common.id < p2->common.id, + * then locks should be locked in the following order: * 1. main lock on p1 * 2. main lock on p2 * 3. message queue lock on p1 @@ -186,7 +203,7 @@ typedef struct erts_proc_lock_t_ { & ~ERTS_PROC_LOCK_MAIN) -#define ERTS_PIX_LOCKS_BITS 8 +#define ERTS_PIX_LOCKS_BITS 10 #define ERTS_NO_OF_PIX_LOCKS (1 << ERTS_PIX_LOCKS_BITS) @@ -260,12 +277,10 @@ typedef struct { } u; } erts_pix_lock_t; -#define ERTS_PIX2PIXLOCKIX(PIX) \ - ((PIX) & ((1 << ERTS_PIX_LOCKS_BITS) - 1)) -#define ERTS_PIX2PIXLOCK(PIX) \ - (&erts_pix_locks[ERTS_PIX2PIXLOCKIX((PIX))]) #define ERTS_PID2PIXLOCK(PID) \ - ERTS_PIX2PIXLOCK(internal_pid_data((PID))) + (&erts_pix_locks[(internal_pid_data((PID)) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))]) + +#if ERTS_PROC_LOCK_OWN_IMPL #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -335,11 +350,13 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags) #endif /* end no opt atomic ops */ +#endif /* ERTS_PROC_LOCK_OWN_IMPL */ extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; void erts_init_proc_lock(int cpus); void erts_proc_lock_prepare_proc_lock_waiter(void); +#if ERTS_PROC_LOCK_OWN_IMPL void erts_proc_lock_failed(Process *, erts_pix_lock_t *, ErtsProcLocks, @@ -347,6 +364,7 @@ void erts_proc_lock_failed(Process *, void erts_proc_unlock_failed(Process *, erts_pix_lock_t *, ErtsProcLocks); +#endif ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *); ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *); @@ -410,6 +428,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL ErtsProcLocks expct_lflgs = 0; while (1) { @@ -429,8 +448,38 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) /* cmpxchg failed, try again (should be rare). */ expct_lflgs = lflgs; } -} +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + + if (locks & ERTS_PROC_LOCK_MAIN) + if (erts_mtx_trylock(&p->lock.main) == EBUSY) + goto busy_main; + if (locks & ERTS_PROC_LOCK_LINK) + if (erts_mtx_trylock(&p->lock.link) == EBUSY) + goto busy_link; + if (locks & ERTS_PROC_LOCK_MSGQ) + if (erts_mtx_trylock(&p->lock.msgq) == EBUSY) + goto busy_msgq; + if (locks & ERTS_PROC_LOCK_STATUS) + if (erts_mtx_trylock(&p->lock.status) == EBUSY) + goto busy_status; + + return 0; + +busy_status: + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_mtx_unlock(&p->lock.msgq); +busy_msgq: + if (locks & ERTS_PROC_LOCK_LINK) + erts_mtx_unlock(&p->lock.link); +busy_link: + if (locks & ERTS_PROC_LOCK_MAIN) + erts_mtx_unlock(&p->lock.main); +busy_main: + + return EBUSY; +#endif +} ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_COUNT @@ -444,10 +493,13 @@ erts_smp_proc_lock__(Process *p, ErtsProcLocks locks) #endif { +#if ERTS_PROC_LOCK_OWN_IMPL + ErtsProcLocks old_lflgs; #if !ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lck); #endif + #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock(&(p->lock), locks); #endif @@ -471,12 +523,14 @@ erts_smp_proc_lock__(Process *p, erts_pix_unlock(pix_lck); } #endif + #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line); #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_proc_lc_lock(p, locks); #endif + #ifdef ERTS_PROC_LOCK_DEBUG erts_proc_lock_op_debug(p, locks, 1); #endif @@ -484,6 +538,22 @@ erts_smp_proc_lock__(Process *p, #if ERTS_PROC_LOCK_ATOMIC_IMPL ETHR_COMPILER_BARRIER; #endif + +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (locks & ERTS_PROC_LOCK_MAIN) + erts_mtx_lock(&p->lock.main); + if (locks & ERTS_PROC_LOCK_LINK) + erts_mtx_lock(&p->lock.link); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_mtx_lock(&p->lock.msgq); + if (locks & ERTS_PROC_LOCK_STATUS) + erts_mtx_lock(&p->lock.status); + +#ifdef ERTS_PROC_LOCK_DEBUG + erts_proc_lock_op_debug(p, locks, 1); +#endif + +#endif } ERTS_GLB_INLINE void @@ -491,6 +561,7 @@ erts_smp_proc_unlock__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL ErtsProcLocks old_lflgs; #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -555,6 +626,23 @@ erts_smp_proc_unlock__(Process *p, break; } + +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + +#ifdef ERTS_PROC_LOCK_DEBUG + erts_proc_lock_op_debug(p, locks, 0); +#endif + + if (locks & ERTS_PROC_LOCK_STATUS) + erts_mtx_unlock(&p->lock.status); + if (locks & ERTS_PROC_LOCK_MSGQ) + erts_mtx_unlock(&p->lock.msgq); + if (locks & ERTS_PROC_LOCK_LINK) + erts_mtx_unlock(&p->lock.link); + if (locks & ERTS_PROC_LOCK_MAIN) + erts_mtx_unlock(&p->lock.main); +#endif + } ERTS_GLB_INLINE int @@ -562,6 +650,7 @@ erts_smp_proc_trylock__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks) { +#if ERTS_PROC_LOCK_OWN_IMPL int res; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -573,6 +662,7 @@ erts_smp_proc_trylock__(Process *p, else #endif { + #if !ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lck); #endif @@ -611,8 +701,18 @@ erts_smp_proc_trylock__(Process *p, #if ERTS_PROC_LOCK_ATOMIC_IMPL ETHR_COMPILER_BARRIER; #endif - return res; + +#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL + if (erts_smp_proc_raw_trylock__(p, locks) != 0) + return EBUSY; + else { +#ifdef ERTS_PROC_LOCK_DEBUG + erts_proc_lock_op_debug(p, locks, 1); +#endif + return 0; + } +#endif } #ifdef ERTS_PROC_LOCK_DEBUG @@ -667,7 +767,7 @@ erts_smp_proc_lock(Process *p, ErtsProcLocks locks) #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else - ERTS_PID2PIXLOCK(p->id), + ERTS_PID2PIXLOCK(p->common.id), #endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/ locks, file, line); #elif defined(ERTS_SMP) @@ -675,7 +775,7 @@ erts_smp_proc_lock(Process *p, ErtsProcLocks locks) #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else - ERTS_PID2PIXLOCK(p->id), + ERTS_PID2PIXLOCK(p->common.id), #endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/ locks); #endif /*ERTS_SMP*/ @@ -689,7 +789,7 @@ erts_smp_proc_unlock(Process *p, ErtsProcLocks locks) #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else - ERTS_PID2PIXLOCK(p->id), + ERTS_PID2PIXLOCK(p->common.id), #endif locks); #endif @@ -705,50 +805,34 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks) #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else - ERTS_PID2PIXLOCK(p->id), + ERTS_PID2PIXLOCK(p->common.id), #endif locks); #endif } - ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p) { #ifdef ERTS_SMP - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - p->lock.refc++; - erts_pix_unlock(pixlck); + erts_ptab_inc_refc(&p->common); #endif } ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p) { #ifdef ERTS_SMP - Process *fp; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - fp = --p->lock.refc == 0 ? p : NULL; - erts_pix_unlock(pixlck); - if (fp) - erts_free_proc(fp); + int referred = erts_ptab_dec_test_refc(&p->common); + if (!referred) + erts_free_proc(p); #endif } -ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc) +ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc) { #ifdef ERTS_SMP - Process *fp; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - ERTS_LC_ASSERT(p->lock.refc > 0); - p->lock.refc += refc; - fp = p->lock.refc == 0 ? p : NULL; - erts_pix_unlock(pixlck); - if (fp) - erts_free_proc(fp); + int referred = erts_ptab_add_test_refc(&p->common, add_refc); + if (!referred) + erts_free_proc(p); #endif } @@ -756,6 +840,7 @@ ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc) #ifdef ERTS_SMP void erts_proc_lock_init(Process *); +void erts_proc_lock_fin(Process *); void erts_proc_safelock(Process *a_proc, ErtsProcLocks a_have_locks, ErtsProcLocks a_need_locks, @@ -782,217 +867,71 @@ void erts_proc_safelock(Process *a_proc, #define ERTS_P2P_FLG_TRY_LOCK (1 << 1) #define ERTS_P2P_FLG_SMP_INC_REFC (1 << 2) -#define ERTS_PROC_LOCK_BUSY ((Process *) &erts_proc_lock_busy) -extern const Process erts_proc_lock_busy; +#define ERTS_PROC_LOCK_BUSY ((Process *) &erts_invalid_process) #define erts_pid2proc(PROC, HL, PID, NL) \ erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0) -ERTS_GLB_INLINE Process * -erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); -#ifdef ERTS_SMP -void -erts_pid2proc_safelock(Process *c_p, - ErtsProcLocks c_p_have_locks, - Process **proc, - ErtsProcLocks need_locks, - erts_pix_lock_t *pix_lock, - int flags); -ERTS_GLB_INLINE Process *erts_pid2proc_unlocked_opt(Eterm pid, int flags); -#define erts_pid2proc_unlocked(PID) erts_pid2proc_unlocked_opt((PID), 0) -#else -#define erts_pid2proc_unlocked_opt(PID, FLGS) \ - erts_pid2proc_opt(NULL, 0, (PID), 0, FLGS) -#define erts_pid2proc_unlocked(PID) erts_pid2proc_opt(NULL, 0, (PID), 0, 0) +ERTS_GLB_INLINE Process *erts_pix2proc(int ix); +ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid); +ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid); + +#ifndef ERTS_SMP +ERTS_GLB_INLINE #endif +Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Process * -#ifdef ERTS_SMP -erts_pid2proc_unlocked_opt(Eterm pid, int flags) -#else -erts_pid2proc_opt(Process *c_p_unused, - ErtsProcLocks c_p_have_locks_unused, - Eterm pid, - ErtsProcLocks pid_need_locks_unused, - int flags) -#endif +ERTS_GLB_INLINE Process *erts_pix2proc(int ix) { - Uint pix; Process *proc; + ASSERT(0 <= ix && ix < erts_ptab_max(&erts_proc)); + proc = (Process *) erts_ptab_pix2intptr_nob(&erts_proc, ix); + return proc == ERTS_PROC_LOCK_BUSY ? NULL : proc; +} + +ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid) +{ + Process *proc; + + ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); if (is_not_internal_pid(pid)) return NULL; - pix = internal_pid_index(pid); - if(pix >= erts_max_processes) + + proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + if (proc && proc->common.id != pid) return NULL; - proc = process_tab[pix]; - if (proc) { - if (proc->id != pid - || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && proc->status == P_EXITING)) - proc = NULL; - } return proc; } -#ifdef ERTS_SMP +ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid) +{ + Process *proc = erts_proc_lookup_raw(pid); + if (proc && ERTS_PROC_IS_EXITING(proc)) + return NULL; + return proc; +} +#ifndef ERTS_SMP ERTS_GLB_INLINE Process * -erts_pid2proc_opt(Process *c_p, - ErtsProcLocks c_p_have_locks, +erts_pid2proc_opt(Process *c_p_unused, + ErtsProcLocks c_p_have_locks_unused, Eterm pid, - ErtsProcLocks pid_need_locks, + ErtsProcLocks pid_need_locks_unused, int flags) { - erts_pix_lock_t *pix_lock; - ErtsProcLocks need_locks; - Uint pix; - Process *proc; -#ifdef ERTS_ENABLE_LOCK_COUNT - ErtsProcLocks lcnt_locks; -#endif - -#ifdef ERTS_ENABLE_LOCK_CHECK - if (c_p) { - ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks; - if (might_unlock) - erts_proc_lc_might_unlock(c_p, might_unlock); - } -#endif - if (is_not_internal_pid(pid)) { - proc = NULL; - goto done; - } - pix = internal_pid_index(pid); - if(pix >= erts_max_processes) { - proc = NULL; - goto done; - } - - ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks); - need_locks = pid_need_locks; - - pix_lock = ERTS_PIX2PIXLOCK(pix); - - if (c_p && c_p->id == pid) { - ASSERT(c_p->id != ERTS_INVALID_PID); - ASSERT(c_p == process_tab[pix]); - if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) && c_p->is_exiting) { - proc = NULL; - goto done; - } - need_locks &= ~c_p_have_locks; - if (!need_locks) { - proc = c_p; - erts_pix_lock(pix_lock); - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; - erts_pix_unlock(pix_lock); - goto done; - } - } - - erts_pix_lock(pix_lock); - - proc = process_tab[pix]; - if (proc) { - if (proc->id != pid || (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && ERTS_PROC_IS_EXITING(proc))) { - proc = NULL; - } - else if (!need_locks) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; - } - else { - int busy; - -#ifdef ERTS_ENABLE_LOCK_COUNT - lcnt_locks = need_locks; - if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { - erts_lcnt_proc_lock(&proc->lock, need_locks); - } -#endif - -#ifdef ERTS_ENABLE_LOCK_CHECK - /* Make sure erts_pid2proc_safelock() is enough to handle - a potential lock order violation situation... */ - busy = erts_proc_lc_trylock_force_busy(proc, need_locks); - if (!busy) -#endif - { - /* Try a quick trylock to grab all the locks we need. */ - busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_trylock(proc, need_locks, !busy); -#endif -#ifdef ERTS_PROC_LOCK_DEBUG - if (!busy) - erts_proc_lock_op_debug(proc, need_locks, 1); -#endif - } - -#ifdef ERTS_ENABLE_LOCK_COUNT - if (flags & ERTS_P2P_FLG_TRY_LOCK) { - if (busy) { - erts_lcnt_proc_trylock(&proc->lock, need_locks, EBUSY); - } else { - erts_lcnt_proc_trylock(&proc->lock, need_locks, 0); - } - } -#endif - if (!busy) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - proc->lock.refc++; -#ifdef ERTS_ENABLE_LOCK_COUNT - /* all is great */ - if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) { - erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks, __FILE__, __LINE__); - } -#endif - } - else { - if (flags & ERTS_P2P_FLG_TRY_LOCK) - proc = ERTS_PROC_LOCK_BUSY; - else { -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); -#endif - erts_pid2proc_safelock(c_p, - c_p_have_locks, - &proc, - pid_need_locks, - pix_lock, - flags); - if (proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC)) - proc->lock.refc++; - } - } - } - } - - erts_pix_unlock(pix_lock); -#ifdef ERTS_PROC_LOCK_DEBUG - ERTS_LC_ASSERT(!proc - || proc == ERTS_PROC_LOCK_BUSY - || (pid_need_locks == - (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock) - & pid_need_locks))); -#endif - - - done: - -#if ERTS_PROC_LOCK_ATOMIC_IMPL - ETHR_COMPILER_BARRIER; -#endif - - return proc; + Process *proc = erts_proc_lookup_raw(pid); + return ((!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && proc + && ERTS_PROC_IS_EXITING(proc)) + ? NULL + : proc); } -#endif /* ERTS_SMP */ +#endif /* !ERTS_SMP */ #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c new file mode 100644 index 0000000000..fa5482b841 --- /dev/null +++ b/erts/emulator/beam/erl_ptab.c @@ -0,0 +1,1772 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Process/Port table implementation. + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#define ERTS_PTAB_WANT_BIF_IMPL__ +#define ERTS_PTAB_WANT_DEBUG_FUNCS__ +#include "erl_ptab.h" +#include "global.h" +#include "erl_binary.h" + +typedef struct ErtsPTabListBifData_ ErtsPTabListBifData; + +#define ERTS_PTAB_NEW_MAX_RESERVE_FAIL 1000 + +#define ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED 25 +#define ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE 1000 +#define ERTS_PTAB_LIST_BIF_MIN_START_REDS \ + (ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE \ + / ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED) + +#define ERTS_PTAB_LIST_BIF_TAB_FREE_DELETED_REDS 1 + +#define ERTS_PTAB_LIST_BIF_INSPECT_DELETED_PER_RED 10 + +#define ERTS_PTAB_LIST_INSPECT_DELETED_MAX_REDS \ + (ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE \ + / ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED) + + +#define ERTS_PTAB_LIST_BIF_BUILD_RESULT_CONSES_PER_RED 75 + +#define ERTS_PTAB_LIST_DBG_DO_TRACE 0 + +#ifdef DEBUG +# define ERTS_PTAB_LIST_BIF_DEBUGLEVEL 100 +#else +# define ERTS_PTAB_LIST_BIF_DEBUGLEVEL 0 +#endif + +#define ERTS_PTAB_LIST_DBGLVL_CHK_HALLOC 1 +#define ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS 5 +#define ERTS_PTAB_LIST_DBGLVL_CHK_PIDS 10 +#define ERTS_PTAB_LIST_DBGLVL_CHK_DEL_LIST 20 +#define ERTS_PTAB_LIST_DBGLVL_CHK_RESLIST 20 + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL == 0 +# define ERTS_PTAB_LIST_ASSERT(EXP) +#else +# define ERTS_PTAB_LIST_ASSERT(EXP) \ + ((void) ((EXP) \ + ? 1 \ + : (debug_ptab_list_assert_error(#EXP, \ + __FILE__, \ + __LINE__, \ + __func__), \ + 0))) +#endif + + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_HALLOC +# define ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(PTLBDP, HP, SZ) \ +do { \ + ERTS_PTAB_LIST_ASSERT(!(PTLBDP)->debug.heap); \ + ERTS_PTAB_LIST_ASSERT(!(PTLBDP)->debug.heap_size); \ + (PTLBDP)->debug.heap = (HP); \ + (PTLBDP)->debug.heap_size = (SZ); \ +} while (0) +# define ERTS_PTAB_LIST_DBG_VERIFY_HEAP_ALLOC_USED(PTLBDP, HP) \ +do { \ + ERTS_PTAB_LIST_ASSERT((PTLBDP)->debug.heap); \ + ERTS_PTAB_LIST_ASSERT((PTLBDP)->debug.heap_size); \ + ERTS_PTAB_LIST_ASSERT(((PTLBDP)->debug.heap \ + + (PTLBDP)->debug.heap_size) \ + == (HP)); \ + (PTLBDP)->debug.heap = NULL; \ + (PTLBDP)->debug.heap_size = 0; \ +} while (0) +# define ERTS_PTAB_LIST_DBG_HEAP_ALLOC_INIT(PTLBDP) \ +do { \ + (PTLBDP)->debug.heap = NULL; \ + (PTLBDP)->debug.heap_size = 0; \ +} while (0) +#else +# define ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(PTLBDP, HP, SZ) +# define ERTS_PTAB_LIST_DBG_VERIFY_HEAP_ALLOC_USED(PTLBDP, HP) +# define ERTS_PTAB_LIST_DBG_HEAP_ALLOC_INIT(PTLBDP) +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_RESLIST +# define ERTS_PTAB_LIST_DBG_CHK_RESLIST(R) \ + debug_ptab_list_check_res_list((R)) +#else +# define ERTS_PTAB_LIST_DBG_CHK_RESLIST(R) +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_PIDS +# define ERTS_PTAB_LIST_DBG_SAVE_PIDS(PTLBDP) \ + debug_ptab_list_save_all_pids((PTLBDP)) +# define ERTS_PTAB_LIST_DBG_VERIFY_PIDS(PTLBDP) \ +do { \ + if (!(PTLBDP)->debug.correct_pids_verified) \ + debug_ptab_list_verify_all_pids((PTLBDP)); \ +} while (0) +# define ERTS_PTAB_LIST_DBG_CLEANUP_CHK_PIDS(PTLBDP) \ +do { \ + if ((PTLBDP)->debug.correct_pids) { \ + erts_free(ERTS_ALC_T_PTAB_LIST_PIDS, \ + (PTLBDP)->debug.correct_pids); \ + (PTLBDP)->debug.correct_pids = NULL; \ + } \ +} while(0) +# define ERTS_PTAB_LIST_DBG_CHK_PIDS_INIT(PTLBDP) \ +do { \ + (PTLBDP)->debug.correct_pids_verified = 0; \ + (PTLBDP)->debug.correct_pids = NULL; \ +} while (0) +#else +# define ERTS_PTAB_LIST_DBG_SAVE_PIDS(PTLBDP) +# define ERTS_PTAB_LIST_DBG_VERIFY_PIDS(PTLBDP) +# define ERTS_PTAB_LIST_DBG_CLEANUP_CHK_PIDS(PTLBDP) +# define ERTS_PTAB_LIST_DBG_CHK_PIDS_INIT(PTLBDP) +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS +# define ERTS_PTAB_LIST_DBG_CHK_PID_FOUND(PTLBDP, PID, IC) \ + debug_ptab_list_check_found_pid((PTLBDP), (PID), (IC), 1) +# define ERTS_PTAB_LIST_DBG_CHK_PID_NOT_FOUND(PTLBDP, PID, IC) \ + debug_ptab_list_check_found_pid((PTLBDP), (PID), (IC), 0) +#else +# define ERTS_PTAB_LIST_DBG_CHK_PID_FOUND(PTLBDP, PID, IC) +# define ERTS_PTAB_LIST_DBG_CHK_PID_NOT_FOUND(PTLBDP, PID, IC) +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_DEL_LIST +# define ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(PTab) \ + debug_ptab_list_check_del_list((PTab)) +# define ERTS_PTAB_LIST_DBG_CHK_FREELIST(PTab, FL) \ + debug_ptab_list_check_del_free_list((PTab), (FL)) +#else +# define ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(PTab) +# define ERTS_PTAB_LIST_DBG_CHK_FREELIST(PTab, FL) +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL == 0 +#if ERTS_PTAB_LIST_DBG_DO_TRACE +# define ERTS_PTAB_LIST_DBG_INIT(P, PTLBDP) \ + (PTLBDP)->debug.caller = (P)->common.id +# else +# define ERTS_PTAB_LIST_DBG_INIT(P, PTLBDP) +# endif +# define ERTS_PTAB_LIST_DBG_CLEANUP(PTLBDP) +#else +# define ERTS_PTAB_LIST_DBG_INIT(P, PTLBDP) \ +do { \ + (PTLBDP)->debug.caller = (P)->common.id; \ + ERTS_PTAB_LIST_DBG_HEAP_ALLOC_INIT((PTLBDP)); \ + ERTS_PTAB_LIST_DBG_CHK_PIDS_INIT((PTLBDP)); \ +} while (0) +# define ERTS_PTAB_LIST_DBG_CLEANUP(PTLBDP) \ +do { \ + ERTS_PTAB_LIST_DBG_CLEANUP_CHK_PIDS((PTLBDP)); \ +} while (0) +#endif + +#if ERTS_PTAB_LIST_DBG_DO_TRACE +# define ERTS_PTAB_LIST_DBG_TRACE(PID, WHAT) \ + erts_fprintf(stderr, "%T %s:%d:%s(): %s\n", \ + (PID), __FILE__, __LINE__, __func__, #WHAT) +#else +# define ERTS_PTAB_LIST_DBG_TRACE(PID, WHAT) +#endif + + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL != 0 +static void debug_ptab_list_assert_error(char* expr, + const char* file, + int line, + const char *func); +#endif +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_RESLIST +static void debug_ptab_list_check_res_list(Eterm list); +#endif +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_PIDS +static void debug_ptab_list_save_all_pids(ErtsPTabListBifData *ptlbdp); +static void debug_ptab_list_verify_all_pids(ErtsPTabListBifData *ptlbdp); +#endif +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS +static void debug_ptab_list_check_found_pid(ErtsPTabListBifData *ptlbdp, + Eterm pid, + Uint64 ic, + int pid_should_be_found); +#endif +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_DEL_LIST +static void debug_ptab_list_check_del_list(ErtsPTab *ptab); +static void debug_ptab_list_check_del_free_list(ErtsPTab *ptab, + ErtsPTabDeletedElement *ptdep); +#endif + +struct ErtsPTabDeletedElement_ { + ErtsPTabDeletedElement *next; + ErtsPTabDeletedElement *prev; + int ix; + union { + struct { + Eterm id; + Uint64 inserted; + Uint64 deleted; + } element; + struct { + Uint64 interval; + } bif_invocation; + } u; +}; + +static Export ptab_list_continue_export; + +typedef struct { + Uint64 interval; +} ErtsPTabListBifChunkInfo; + +typedef enum { + INITIALIZING, + INSPECTING_TABLE, + INSPECTING_DELETED, + BUILDING_RESULT, + RETURN_RESULT +} ErtsPTabListBifState; + +struct ErtsPTabListBifData_ { + ErtsPTab *ptab; + ErtsPTabListBifState state; + Eterm caller; + ErtsPTabListBifChunkInfo *chunk; + int tix; + int pid_ix; + int pid_sz; + Eterm *pid; + ErtsPTabDeletedElement *bif_invocation; /* Only used when > 1 chunk */ + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL != 0 || ERTS_PTAB_LIST_DBG_DO_TRACE + struct { + Eterm caller; +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS + Uint64 *pid_started; +#endif +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_HALLOC + Eterm *heap; + Uint heap_size; +#endif +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_PIDS + int correct_pids_verified; + Eterm *correct_pids; +#endif + } debug; +#endif + +}; + +#ifdef ARCH_32 + +static ERTS_INLINE Uint64 +dw_aint_to_uint64(erts_dw_aint_t *dw) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (Uint64) dw->dw_sint; +#else + Uint64 res; + res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); + res <<= 32; + res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); + return res; +#endif +} + +static void +unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val; +#else + dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); + dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); +#endif +} + +static ERTS_INLINE void +last_data_init_nob(ErtsPTab *ptab, Uint64 val) +{ + erts_dw_aint_t dw; + unint64_to_dw_aint(&dw, val); + erts_smp_dw_atomic_init_nob(&ptab->vola.tile.last_data, &dw); +} + +static ERTS_INLINE void +last_data_set_relb(ErtsPTab *ptab, Uint64 val) +{ + erts_dw_aint_t dw; + unint64_to_dw_aint(&dw, val); + erts_smp_dw_atomic_set_relb(&ptab->vola.tile.last_data, &dw); +} + +static ERTS_INLINE Uint64 +last_data_read_nob(ErtsPTab *ptab) +{ + erts_dw_aint_t dw; + erts_smp_dw_atomic_read_nob(&ptab->vola.tile.last_data, &dw); + return dw_aint_to_uint64(&dw); +} + +static ERTS_INLINE Uint64 +last_data_read_acqb(ErtsPTab *ptab) +{ + erts_dw_aint_t dw; + erts_smp_dw_atomic_read_acqb(&ptab->vola.tile.last_data, &dw); + return dw_aint_to_uint64(&dw); +} + +static ERTS_INLINE Uint64 +last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) +{ + erts_dw_aint_t dw_new, dw_xchg; + + unint64_to_dw_aint(&dw_new, new); + unint64_to_dw_aint(&dw_xchg, exp); + + if (erts_smp_dw_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, + &dw_new, + &dw_xchg)) + return exp; + else + return dw_aint_to_uint64(&dw_xchg); +} + +#elif defined(ARCH_64) + +union { + erts_smp_atomic_t pid_data; + char align[ERTS_CACHE_LINE_SIZE]; +} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static ERTS_INLINE void +last_data_init_nob(ErtsPTab *ptab, Uint64 val) +{ + erts_smp_atomic_init_nob(&ptab->vola.tile.last_data, (erts_aint_t) val); +} + +static ERTS_INLINE void +last_data_set_relb(ErtsPTab *ptab, Uint64 val) +{ + erts_smp_atomic_set_relb(&ptab->vola.tile.last_data, (erts_aint_t) val); +} + +static ERTS_INLINE Uint64 +last_data_read_nob(ErtsPTab *ptab) +{ + return (Uint64) erts_smp_atomic_read_nob(&ptab->vola.tile.last_data); +} + +static ERTS_INLINE Uint64 +last_data_read_acqb(ErtsPTab *ptab) +{ + return (Uint64) erts_smp_atomic_read_acqb(&ptab->vola.tile.last_data); +} + +static ERTS_INLINE Uint64 +last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) +{ + return (Uint64) erts_smp_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, + (erts_aint_t) new, + (erts_aint_t) exp); +} + +#else +# error "Not 64-bit, nor 32-bit architecture..." +#endif + +static ERTS_INLINE int +last_data_cmp(Uint64 ld1, Uint64 ld2) +{ + Uint64 ld1_wrap; + + if (ld1 == ld2) + return 0; + + ld1_wrap = ld1 + (((Uint64) 1) << 63); + + if (ld1 < ld1_wrap) + return (ld1 < ld2 && ld2 < ld1_wrap) ? -1 : 1; + else + return (ld1_wrap <= ld2 && ld2 < ld1) ? 1 : -1; +} + +#define ERTS_PTAB_LastData2EtermData(LD) \ + ((Eterm) ((LD) & ~(~((Uint64) 0) << ERTS_PTAB_ID_DATA_SIZE))) + +static ERTS_INLINE Uint32 +ix_to_free_id_data_ix(ErtsPTab *ptab, Uint32 ix) +{ + Uint32 dix; + + dix = ((ix & ptab->r.o.dix_cl_mask) << ptab->r.o.dix_cl_shift); + dix += ((ix >> ptab->r.o.dix_cli_shift) & ptab->r.o.dix_cli_mask); + ASSERT(0 <= dix && dix < ptab->r.o.max); + return dix; +} + +UWord +erts_ptab_mem_size(ErtsPTab *ptab) +{ + UWord size = ptab->r.o.max*sizeof(erts_smp_atomic_t); + if (ptab->r.o.free_id_data) + size += ptab->r.o.max*sizeof(erts_smp_atomic32_t); + return size; +} + + +void +erts_ptab_init_table(ErtsPTab *ptab, + ErtsAlcType_t atype, + void (*release_element)(void *), + ErtsPTabElementCommon *invalid_element, + int size, + UWord element_size, + char *name, + int legacy) +{ + size_t tab_sz, alloc_sz; + Uint32 bits, cl, cli, ix, ix_per_cache_line, tab_cache_lines; + char *tab_end; + erts_smp_atomic_t *tab_entry; + erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&ptab->list.data.rwmtx, &rwmtx_opts, name); + erts_smp_atomic32_init_nob(&ptab->vola.tile.count, 0); + last_data_init_nob(ptab, ~((Uint64) 0)); + + /* A size that is a power of 2 is to prefer performance wise */ + bits = erts_fit_in_bits_int32(size-1); + size = 1 << bits; + if (size > ERTS_PTAB_MAX_SIZE) { + size = ERTS_PTAB_MAX_SIZE; + bits = erts_fit_in_bits_int32((Sint32) size - 1); + } + + ptab->r.o.element_size = element_size; + ptab->r.o.max = size; + + tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic_t)); + alloc_sz = tab_sz; + if (!legacy) + alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); + ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, alloc_sz); + tab_end = ((char *) ptab->r.o.tab) + tab_sz; + tab_entry = ptab->r.o.tab; + while (tab_end > ((char *) tab_entry)) { + erts_smp_atomic_init_nob(tab_entry, ERTS_AINT_NULL); + tab_entry++; + } + + tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t)); + ASSERT((ptab->r.o.max & (ptab->r.o.max - 1)) == 0); /* power of 2 */ + ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */ + ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */ + + ptab->r.o.pix_mask = (1 << bits) - 1; + ptab->r.o.pix_cl_mask = tab_cache_lines-1; + ptab->r.o.pix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1); + ptab->r.o.pix_cli_shift = erts_fit_in_bits_int32(ptab->r.o.pix_cl_mask); + ptab->r.o.pix_cli_mask = (1 << (bits - ptab->r.o.pix_cli_shift)) - 1; + + ASSERT(ptab->r.o.pix_cl_shift + ptab->r.o.pix_cli_shift == bits); + + ptab->r.o.invalid_element = invalid_element; + ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id); + ptab->r.o.release_element = release_element; + + if (legacy) { + ptab->r.o.free_id_data = NULL; + ptab->r.o.dix_cl_mask = 0; + ptab->r.o.dix_cl_shift = 0; + ptab->r.o.dix_cli_shift = 0; + ptab->r.o.dix_cli_mask = 0; + } + else { + + tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); + ptab->r.o.free_id_data = (erts_smp_atomic32_t *) tab_end; + + tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic32_t)); + + ptab->r.o.dix_cl_mask = tab_cache_lines-1; + ptab->r.o.dix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1); + ptab->r.o.dix_cli_shift = erts_fit_in_bits_int32(ptab->r.o.dix_cl_mask); + ptab->r.o.dix_cli_mask = (1 << (bits - ptab->r.o.dix_cli_shift)) - 1; + + ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */ + ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */ + + ASSERT(ptab->r.o.dix_cl_shift + ptab->r.o.dix_cli_shift == bits); + + ix = 0; + for (cl = 0; cl < tab_cache_lines; cl++) { + for (cli = 0; cli < ix_per_cache_line; cli++) { + erts_smp_atomic32_init_nob(&ptab->r.o.free_id_data[ix], + cli*tab_cache_lines+cl); + ASSERT(erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data); + ix++; + } + } + + erts_smp_atomic32_init_nob(&ptab->vola.tile.aid_ix, -1); + erts_smp_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1); + + } + + erts_smp_interval_init(&ptab->list.data.interval); + ptab->list.data.deleted.start = NULL; + ptab->list.data.deleted.end = NULL; + ptab->list.data.chunks = (((ptab->r.o.max - 1) + / ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE) + + 1); + + if (size == ERTS_PTAB_MAX_SIZE) { + int pix; + /* + * We want a table size of a power of 2 which ERTS_PTAB_MAX_SIZE + * is. We only have ERTS_PTAB_MAX_SIZE-1 unique identifiers and + * we don't want to shrink the size to ERTS_PTAB_MAX_SIZE/2. + * + * In order to fix this, we insert a pointer from the table + * to the invalid_element, wich will be interpreted as a + * slot currently being modified. This way we will be able to + * have ERTS_PTAB_MAX_SIZE-1 valid elements in the table while + * still having a table size of the power of 2. + */ + erts_smp_atomic32_inc_nob(&ptab->vola.tile.count); + pix = erts_ptab_data2pix(ptab, ptab->r.o.invalid_data); + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], + (erts_aint_t) ptab->r.o.invalid_element); + } + +} + +int +erts_ptab_initialized(ErtsPTab *ptab) +{ + return ptab->r.o.tab != NULL; +} + +int +erts_ptab_new_element(ErtsPTab *ptab, + ErtsPTabElementCommon *ptab_el, + void *init_arg, + void (*init_ptab_el)(void *, Eterm)) +{ + Uint32 pix, ix, data; + erts_aint32_t count; + erts_aint_t invalid = (erts_aint_t) ptab->r.o.invalid_element; + + erts_ptab_rlock(ptab); + + count = erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.count); + if (count > ptab->r.o.max) { + while (1) { + erts_aint32_t act_count; + + act_count = erts_smp_atomic32_cmpxchg_relb(&ptab->vola.tile.count, + count-1, + count); + if (act_count == count) { + erts_ptab_runlock(ptab); + return 0; + } + count = act_count; + if (count <= ptab->r.o.max) + break; + } + } + + ptab_el->u.alive.started_interval + = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + + if (ptab->r.o.free_id_data) { + do { + ix = (Uint32) erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix); + ix = ix_to_free_id_data_ix(ptab, ix); + + data = erts_smp_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix], + (erts_aint32_t)ptab->r.o.invalid_data); + }while ((Eterm)data == ptab->r.o.invalid_data); + + init_ptab_el(init_arg, (Eterm) data); + +#ifdef ERTS_SMP + erts_smp_atomic32_init_nob(&ptab_el->refc, 1); +#endif + + pix = erts_ptab_data2pix(ptab, (Eterm) data); + +#ifdef DEBUG + ASSERT(ERTS_AINT_NULL == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + (erts_aint_t) ptab_el)); +#else + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); +#endif + + erts_ptab_runlock(ptab); + + } + else { + int rlocked = ERTS_PTAB_NEW_MAX_RESERVE_FAIL; + Uint64 ld, exp_ld; + /* Deprecated legacy algorithm... */ + + restart: + + ptab_el->u.alive.started_interval + = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + + ld = last_data_read_acqb(ptab); + + /* Reserve slot */ + while (1) { + ld++; + pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld)); + if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix]) + == ERTS_AINT_NULL) { + erts_aint_t val; + val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], + invalid, + ERTS_AINT_NULL); + + if (ERTS_AINT_NULL == val) + break; + } + if (rlocked && --rlocked == 0) { + erts_ptab_runlock(ptab); + erts_ptab_rwlock(ptab); + goto restart; + } + } + + data = ERTS_PTAB_LastData2EtermData(ld); + + if (data == ptab->r.o.invalid_data) { + /* Do not use invalid data; fix it... */ + ld += ptab->r.o.max; + ASSERT(pix == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); + data = ERTS_PTAB_LastData2EtermData(ld); + ASSERT(data != ptab->r.o.invalid_data); + } + + exp_ld = last_data_read_nob(ptab); + + /* Move last data forward */ + while (1) { + Uint64 act_ld; + if (last_data_cmp(ld, exp_ld) < 0) + break; + act_ld = last_data_cmpxchg_relb(ptab, ld, exp_ld); + if (act_ld == exp_ld) + break; + exp_ld = act_ld; + } + + init_ptab_el(init_arg, data); + +#ifdef ERTS_SMP + erts_smp_atomic32_init_nob(&ptab_el->refc, 1); +#endif + + /* Move into slot reserved */ +#ifdef DEBUG + ASSERT(invalid == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + (erts_aint_t) ptab_el)); +#else + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); +#endif + + if (rlocked) + erts_ptab_runlock(ptab); + else + erts_ptab_rwunlock(ptab); + + } + + return 1; +} + +static void +save_deleted_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el) +{ + ErtsPTabDeletedElement *ptdep = erts_alloc(ERTS_ALC_T_PTAB_LIST_DEL, + sizeof(ErtsPTabDeletedElement)); + ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start + && ptab->list.data.deleted.end); + ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab)); + + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + + ptdep->prev = ptab->list.data.deleted.end; + ptdep->next = NULL; + ptdep->ix = erts_ptab_id2pix(ptab, ptab_el->id); + ptdep->u.element.id = ptab_el->id; + ptdep->u.element.inserted = ptab_el->u.alive.started_interval; + ptdep->u.element.deleted = + erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + + ptab->list.data.deleted.end->next = ptdep; + ptab->list.data.deleted.end = ptdep; + + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + + ERTS_PTAB_LIST_ASSERT(ptdep->prev->ix >= 0 + ? (ptdep->u.element.deleted + >= ptdep->prev->u.element.deleted) + : (ptdep->u.element.deleted + >= ptdep->prev->u.bif_invocation.interval)); +} + +void +erts_ptab_delete_element(ErtsPTab *ptab, + ErtsPTabElementCommon *ptab_el) +{ + int maybe_save; + Uint32 pix, ix, data; + + pix = erts_ptab_id2pix(ptab, ptab_el->id); + + ASSERT(erts_get_scheduler_id()); /* *Need* to be a scheduler */ + + erts_ptab_rlock(ptab); + maybe_save = ptab->list.data.deleted.end != NULL; + if (maybe_save) { + erts_ptab_runlock(ptab); + erts_ptab_rwlock(ptab); + } + + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL); + + if (ptab->r.o.free_id_data) { + Uint32 prev_data; + /* Next data for this slot... */ + data = (Uint32) erts_ptab_id2data(ptab, ptab_el->id); + data += ptab->r.o.max; + data &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + if (data == ptab->r.o.invalid_data) { /* make sure not invalid */ + data += ptab->r.o.max; + data &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + } + ASSERT(data != ptab->r.o.invalid_data); + ASSERT(pix == erts_ptab_data2pix(ptab, data)); + + do { + ix = (Uint32) erts_smp_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix); + ix = ix_to_free_id_data_ix(ptab, ix); + + prev_data = erts_smp_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix], + data, + ptab->r.o.invalid_data); + }while ((Eterm)prev_data != ptab->r.o.invalid_data); + } + + ASSERT(erts_smp_atomic32_read_nob(&ptab->vola.tile.count) > 0); + erts_smp_atomic32_dec_relb(&ptab->vola.tile.count); + + if (!maybe_save) + erts_ptab_runlock(ptab); + else { + if (ptab->list.data.deleted.end) + save_deleted_element(ptab, ptab_el); + erts_ptab_rwunlock(ptab); + } + + if (ptab->r.o.release_element) + erts_schedule_thr_prgr_later_cleanup_op(ptab->r.o.release_element, + (void *) ptab_el, + &ptab_el->u.release, + ptab->r.o.element_size); +} + +/* + * erts_ptab_list() implements BIFs listing the content of the table, + * e.g. erlang:processes/0. + */ +static void cleanup_ptab_list_bif_data(Binary *bp); +static int ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp); + + +BIF_RETTYPE +erts_ptab_list(Process *c_p, ErtsPTab *ptab) +{ + /* + * A requirement: The list of identifiers returned should be a + * consistent snapshot of all elements existing + * in the table at some point in time during the + * execution of the BIF calling this function. + * Since elements might be deleted while the BIF + * is executing, we have to keep track of all + * deleted elements and add them to the result. + * We also ignore elements created after the BIF + * has begun executing. + */ + BIF_RETTYPE ret_val; + Eterm res_acc = NIL; + Binary *mbp = erts_create_magic_binary(sizeof(ErtsPTabListBifData), + cleanup_ptab_list_bif_data); + ErtsPTabListBifData *ptlbdp = ERTS_MAGIC_BIN_DATA(mbp); + + ERTS_PTAB_LIST_DBG_TRACE(c_p->common.id, call); + ptlbdp->ptab = ptab; + ptlbdp->state = INITIALIZING; + ERTS_PTAB_LIST_DBG_INIT(c_p, ptlbdp); + + if (ERTS_BIF_REDS_LEFT(c_p) >= ERTS_PTAB_LIST_BIF_MIN_START_REDS + && ptab_list_bif_engine(c_p, &res_acc, mbp)) { + erts_bin_free(mbp); + ERTS_PTAB_LIST_DBG_CHK_RESLIST(res_acc); + ERTS_PTAB_LIST_DBG_TRACE(c_p->common.id, return); + ERTS_BIF_PREP_RET(ret_val, res_acc); + } + else { + Eterm *hp; + Eterm magic_bin; + ERTS_PTAB_LIST_DBG_CHK_RESLIST(res_acc); + hp = HAlloc(c_p, PROC_BIN_SIZE); + ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(ptlbdp, hp, PROC_BIN_SIZE); + magic_bin = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + ERTS_PTAB_LIST_DBG_VERIFY_HEAP_ALLOC_USED(ptlbdp, hp); + ERTS_PTAB_LIST_DBG_TRACE(c_p->common.id, trap); + ERTS_BIF_PREP_YIELD2(ret_val, + &ptab_list_continue_export, + c_p, + res_acc, + magic_bin); + } + return ret_val; +} + +static void +cleanup_ptab_list_bif_data(Binary *bp) +{ + ErtsPTabListBifData *ptlbdp = ERTS_MAGIC_BIN_DATA(bp); + ErtsPTab *ptab = ptlbdp->ptab; + + ERTS_PTAB_LIST_DBG_TRACE(ptlbdp->debug.caller, call); + + if (ptlbdp->state != INITIALIZING) { + + if (ptlbdp->chunk) { + erts_free(ERTS_ALC_T_PTAB_LIST_CNKI, ptlbdp->chunk); + ptlbdp->chunk = NULL; + } + if (ptlbdp->pid) { + erts_free(ERTS_ALC_T_PTAB_LIST_PIDS, ptlbdp->pid); + ptlbdp->pid = NULL; + } + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS + if (ptlbdp->debug.pid_started) { + erts_free(ERTS_ALC_T_PTAB_LIST_PIDS, ptlbdp->debug.pid_started); + ptlbdp->debug.pid_started = NULL; + } +#endif + + if (ptlbdp->bif_invocation) { + ErtsPTabDeletedElement *ptdep; + + erts_ptab_rwlock(ptab); + + ERTS_PTAB_LIST_DBG_TRACE(ptlbdp->debug.caller, deleted_cleanup); + + ptdep = ptlbdp->bif_invocation; + ptlbdp->bif_invocation = NULL; + + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + + if (ptdep->prev) { + /* + * Only remove this bif invokation when we + * have preceding invokations. + */ + ptdep->prev->next = ptdep->next; + if (ptdep->next) + ptdep->next->prev = ptdep->prev; + else { + /* + * At the time of writing this branch cannot be + * reached. I don't want to remove this code though + * since it may be possible to reach this line + * in the future if the cleanup order in + * erts_do_exit_process() is changed. The ASSERT(0) + * is only here to make us aware that the reorder + * has happened. /rickard + */ + ASSERT(0); + ptab->list.data.deleted.end = ptdep->prev; + } + erts_free(ERTS_ALC_T_PTAB_LIST_DEL, ptdep); + } + else { + /* + * Free all elements until next bif invokation + * is found. + */ + ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start == ptdep); + do { + ErtsPTabDeletedElement *fptdep = ptdep; + ptdep = ptdep->next; + erts_free(ERTS_ALC_T_PTAB_LIST_DEL, fptdep); + } while (ptdep && ptdep->ix >= 0); + ptab->list.data.deleted.start = ptdep; + if (ptdep) + ptdep->prev = NULL; + else + ptab->list.data.deleted.end = NULL; + } + + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + + erts_ptab_rwunlock(ptab); + + } + } + + ERTS_PTAB_LIST_DBG_TRACE(ptlbdp->debug.caller, return); + ERTS_PTAB_LIST_DBG_CLEANUP(ptlbdp); +} + +static int +ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp) +{ + ErtsPTabListBifData *ptlbdp = ERTS_MAGIC_BIN_DATA(mbp); + ErtsPTab *ptab = ptlbdp->ptab; + int have_reds; + int reds; + int locked = 0; + + do { + switch (ptlbdp->state) { + case INITIALIZING: + ptlbdp->chunk = erts_alloc(ERTS_ALC_T_PTAB_LIST_CNKI, + (sizeof(ErtsPTabListBifChunkInfo) + * ptab->list.data.chunks)); + ptlbdp->tix = 0; + ptlbdp->pid_ix = 0; + + erts_ptab_rwlock(ptab); + locked = 1; + + ERTS_PTAB_LIST_DBG_TRACE(c_p->common.id, init); + + ptlbdp->pid_sz = erts_ptab_count(ptab); + ptlbdp->pid = erts_alloc(ERTS_ALC_T_PTAB_LIST_PIDS, + sizeof(Eterm)*ptlbdp->pid_sz); + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS + ptlbdp->debug.pid_started + = erts_alloc(ERTS_ALC_T_PTAB_LIST_PIDS, + sizeof(Uint64)*ptlbdp->pid_sz); +#endif + + ERTS_PTAB_LIST_DBG_SAVE_PIDS(ptlbdp); + + if (ptab->list.data.chunks == 1) + ptlbdp->bif_invocation = NULL; + else { + /* + * We will have to access the table multiple times + * releasing the table lock in between chunks. + */ + ptlbdp->bif_invocation + = erts_alloc(ERTS_ALC_T_PTAB_LIST_DEL, + sizeof(ErtsPTabDeletedElement)); + ptlbdp->bif_invocation->ix = -1; + ptlbdp->bif_invocation->u.bif_invocation.interval + = erts_smp_step_interval_nob(erts_ptab_interval(ptab)); + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + + ptlbdp->bif_invocation->next = NULL; + if (ptab->list.data.deleted.end) { + ptlbdp->bif_invocation->prev = ptab->list.data.deleted.end; + ptab->list.data.deleted.end->next = ptlbdp->bif_invocation; + ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start); + } + else { + ptlbdp->bif_invocation->prev = NULL; + ptab->list.data.deleted.start = ptlbdp->bif_invocation; + } + ptab->list.data.deleted.end = ptlbdp->bif_invocation; + + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + + } + + ptlbdp->state = INSPECTING_TABLE; + /* Fall through */ + + case INSPECTING_TABLE: { + int ix = ptlbdp->tix; + int indices = ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE; + int cix = ix / ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE; + int end_ix = ix + indices; + Uint64 *invocation_interval_p; + ErtsPTabElementCommon *invalid_element; + + invocation_interval_p + = (ptlbdp->bif_invocation + ? &ptlbdp->bif_invocation->u.bif_invocation.interval + : NULL); + + ERTS_PTAB_LIST_ASSERT(is_nil(*res_accp)); + if (!locked) { + erts_ptab_rwlock(ptab); + locked = 1; + } + + ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab)); + ERTS_PTAB_LIST_DBG_TRACE(p->common.id, insp_table); + + if (cix != 0) + ptlbdp->chunk[cix].interval + = erts_smp_step_interval_nob(erts_ptab_interval(ptab)); + else if (ptlbdp->bif_invocation) + ptlbdp->chunk[0].interval = *invocation_interval_p; + /* else: interval is irrelevant */ + + if (end_ix >= ptab->r.o.max) { + ERTS_PTAB_LIST_ASSERT(cix+1 == ptab->list.data.chunks); + end_ix = ptab->r.o.max; + indices = end_ix - ix; + /* What to do when done with this chunk */ + ptlbdp->state = (ptab->list.data.chunks == 1 + ? BUILDING_RESULT + : INSPECTING_DELETED); + } + + invalid_element = ptab->r.o.invalid_element; + for (; ix < end_ix; ix++) { + ErtsPTabElementCommon *el; + el = (ErtsPTabElementCommon *) erts_ptab_pix2intptr_nob(ptab, + ix); + if (el + && el != invalid_element + && (!invocation_interval_p + || el->u.alive.started_interval < *invocation_interval_p)) { + ERTS_PTAB_LIST_ASSERT(erts_ptab_is_valid_id(el->id)); + ptlbdp->pid[ptlbdp->pid_ix] = el->id; + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS + ptlbdp->debug.pid_started[ptlbdp->pid_ix] + = el->u.alive.started_interval; +#endif + + ptlbdp->pid_ix++; + ERTS_PTAB_LIST_ASSERT(ptlbdp->pid_ix <= ptlbdp->pid_sz); + } + } + + ptlbdp->tix = end_ix; + + erts_ptab_rwunlock(ptab); + locked = 0; + + reds = indices/ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED; + BUMP_REDS(c_p, reds); + + have_reds = ERTS_BIF_REDS_LEFT(c_p); + + if (have_reds && ptlbdp->state == INSPECTING_TABLE) { + ix = ptlbdp->tix; + indices = ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE; + end_ix = ix + indices; + if (end_ix > ptab->r.o.max) { + end_ix = ptab->r.o.max; + indices = end_ix - ix; + } + + reds = indices/ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED; + + /* Pretend we have no reds left if we haven't got enough + reductions to complete next chunk */ + if (reds > have_reds) + have_reds = 0; + } + + break; + } + + case INSPECTING_DELETED: { + int i; + int max_reds; + int free_deleted = 0; + Uint64 invocation_interval; + ErtsPTabDeletedElement *ptdep; + ErtsPTabDeletedElement *free_list = NULL; + + ptdep = ptlbdp->bif_invocation; + ERTS_PTAB_LIST_ASSERT(ptdep); + invocation_interval = ptdep->u.bif_invocation.interval; + + max_reds = have_reds = ERTS_BIF_REDS_LEFT(c_p); + if (max_reds > ERTS_PTAB_LIST_INSPECT_DELETED_MAX_REDS) + max_reds = ERTS_PTAB_LIST_INSPECT_DELETED_MAX_REDS; + + reds = 0; + erts_ptab_rwlock(ptab); + ERTS_PTAB_LIST_DBG_TRACE(p->common.id, insp_term_procs); + + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + + if (ptdep->prev) + ptdep->prev->next = ptdep->next; + else { + ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start == ptdep); + ptab->list.data.deleted.start = ptdep->next; + + if (ptab->list.data.deleted.start + && ptab->list.data.deleted.start->ix >= 0) { + free_list = ptab->list.data.deleted.start; + free_deleted = 1; + } + } + + if (ptdep->next) + ptdep->next->prev = ptdep->prev; + else + ptab->list.data.deleted.end = ptdep->prev; + + ptdep = ptdep->next; + + i = 0; + while (reds < max_reds && ptdep) { + if (ptdep->ix < 0) { + if (free_deleted) { + ERTS_PTAB_LIST_ASSERT(free_list); + ERTS_PTAB_LIST_ASSERT(ptdep->prev); + + ptdep->prev->next = NULL; /* end of free_list */ + ptab->list.data.deleted.start = ptdep; + ptdep->prev = NULL; + free_deleted = 0; + } + } + else { + int cix = ptdep->ix/ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE; + Uint64 chunk_interval = ptlbdp->chunk[cix].interval; + Eterm pid = ptdep->u.element.id; + ERTS_PTAB_LIST_ASSERT(erts_ptab_is_valid_id(pid)); + + if (ptdep->u.element.inserted < invocation_interval) { + if (ptdep->u.element.deleted < chunk_interval) { + ERTS_PTAB_LIST_DBG_CHK_PID_NOT_FOUND( + ptlbdp, + pid, + ptdep->u.element.inserted); + ptlbdp->pid[ptlbdp->pid_ix] = pid; +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS + ptlbdp->debug.pid_started[ptlbdp->pid_ix] + = ptdep->u.element.inserted; +#endif + ptlbdp->pid_ix++; + ERTS_PTAB_LIST_ASSERT(ptlbdp->pid_ix + <= ptlbdp->pid_sz); + } + else { + ERTS_PTAB_LIST_DBG_CHK_PID_FOUND( + ptlbdp, + pid, + ptdep->u.element.inserted); + } + } + else { + ERTS_PTAB_LIST_DBG_CHK_PID_NOT_FOUND( + ptlbdp, + pid, + ptdep->u.element.inserted); + } + + i++; + if (i == ERTS_PTAB_LIST_BIF_INSPECT_DELETED_PER_RED) { + reds++; + i = 0; + } + if (free_deleted) + reds += ERTS_PTAB_LIST_BIF_TAB_FREE_DELETED_REDS; + } + ptdep = ptdep->next; + } + + if (free_deleted) { + ERTS_PTAB_LIST_ASSERT(free_list); + ptab->list.data.deleted.start = ptdep; + if (!ptdep) + ptab->list.data.deleted.end = NULL; + else { + ERTS_PTAB_LIST_ASSERT(ptdep->prev); + ptdep->prev->next = NULL; /* end of free_list */ + ptdep->prev = NULL; + } + } + + if (!ptdep) { + /* Done */ + ERTS_PTAB_LIST_ASSERT(ptlbdp->pid_ix == ptlbdp->pid_sz); + ptlbdp->state = BUILDING_RESULT; + ptlbdp->bif_invocation->next = free_list; + free_list = ptlbdp->bif_invocation; + ptlbdp->bif_invocation = NULL; + } + else { + /* Link in bif_invocation again where we left off */ + ptlbdp->bif_invocation->prev = ptdep->prev; + ptlbdp->bif_invocation->next = ptdep; + ptdep->prev = ptlbdp->bif_invocation; + if (ptlbdp->bif_invocation->prev) + ptlbdp->bif_invocation->prev->next = ptlbdp->bif_invocation; + else { + ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start + == ptdep); + ptab->list.data.deleted.start = ptlbdp->bif_invocation; + } + } + + ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); + ERTS_PTAB_LIST_DBG_CHK_FREELIST(ptab, free_list); + erts_ptab_rwunlock(ptab); + + /* + * We do the actual free of deleted structures now when we + * have released the table lock instead of when we encountered + * them. This since free() isn't for free and we don't want to + * unnecessarily block other schedulers. + */ + while (free_list) { + ptdep = free_list; + free_list = ptdep->next; + erts_free(ERTS_ALC_T_PTAB_LIST_DEL, ptdep); + } + + have_reds -= reds; + if (have_reds < 0) + have_reds = 0; + BUMP_REDS(c_p, reds); + break; + } + + case BUILDING_RESULT: { + int conses, ix, min_ix; + Eterm *hp; + Eterm res = *res_accp; + + ERTS_PTAB_LIST_DBG_VERIFY_PIDS(ptlbdp); + ERTS_PTAB_LIST_DBG_CHK_RESLIST(res); + + ERTS_PTAB_LIST_DBG_TRACE(p->common.id, begin_build_res); + + have_reds = ERTS_BIF_REDS_LEFT(c_p); + conses = ERTS_PTAB_LIST_BIF_BUILD_RESULT_CONSES_PER_RED*have_reds; + min_ix = ptlbdp->pid_ix - conses; + if (min_ix < 0) { + min_ix = 0; + conses = ptlbdp->pid_ix; + } + + if (conses) { + hp = HAlloc(c_p, conses*2); + ERTS_PTAB_LIST_DBG_SAVE_HEAP_ALLOC(ptlbdp, hp, conses*2); + + for (ix = ptlbdp->pid_ix - 1; ix >= min_ix; ix--) { + ERTS_PTAB_LIST_ASSERT(erts_ptab_is_valid_id(ptlbdp->pid[ix])); + res = CONS(hp, ptlbdp->pid[ix], res); + hp += 2; + } + + ERTS_PTAB_LIST_DBG_VERIFY_HEAP_ALLOC_USED(ptlbdp, hp); + } + + ptlbdp->pid_ix = min_ix; + if (min_ix == 0) + ptlbdp->state = RETURN_RESULT; + else { + ptlbdp->pid_sz = min_ix; + ptlbdp->pid = erts_realloc(ERTS_ALC_T_PTAB_LIST_PIDS, + ptlbdp->pid, + sizeof(Eterm)*ptlbdp->pid_sz); +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS + ptlbdp->debug.pid_started + = erts_realloc(ERTS_ALC_T_PTAB_LIST_PIDS, + ptlbdp->debug.pid_started, + sizeof(Uint64) * ptlbdp->pid_sz); +#endif + } + reds = conses/ERTS_PTAB_LIST_BIF_BUILD_RESULT_CONSES_PER_RED; + BUMP_REDS(c_p, reds); + have_reds -= reds; + + ERTS_PTAB_LIST_DBG_CHK_RESLIST(res); + ERTS_PTAB_LIST_DBG_TRACE(c_p->common.id, end_build_res); + *res_accp = res; + break; + } + case RETURN_RESULT: + cleanup_ptab_list_bif_data(mbp); + return 1; + + default: + erl_exit(ERTS_ABORT_EXIT, + "%s:%d:ptab_list_bif_engine(): Invalid state: %d\n", + __FILE__, __LINE__, (int) ptlbdp->state); + } + + + } while (have_reds || ptlbdp->state == RETURN_RESULT); + + return 0; +} + +/* + * ptab_list_continue/2 is a hidden BIF that the original BIF traps to + * if there are too much work to do in one go. + */ + +static BIF_RETTYPE ptab_list_continue(BIF_ALIST_2) +{ + Eterm res_acc; + Binary *mbp; + + /* + * This bif cannot be called from erlang code. It can only be + * trapped to from other BIFs; therefore, a bad argument + * is an internal error and should never occur... + */ + + ERTS_PTAB_LIST_DBG_TRACE(BIF_P->common.id, call); + ERTS_PTAB_LIST_ASSERT(is_nil(BIF_ARG_1) || is_list(BIF_ARG_1)); + + res_acc = BIF_ARG_1; + + ERTS_PTAB_LIST_ASSERT(ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_2)); + + mbp = ((ProcBin *) binary_val(BIF_ARG_2))->val; + + ERTS_PTAB_LIST_ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) + == cleanup_ptab_list_bif_data); + ERTS_PTAB_LIST_ASSERT( + ((ErtsPTabListBifData *) ERTS_MAGIC_BIN_DATA(mbp))->debug.caller + == BIF_P->common.id); + + if (ptab_list_bif_engine(BIF_P, &res_acc, mbp)) { + ERTS_PTAB_LIST_DBG_TRACE(BIF_P->common.id, return); + BIF_RET(res_acc); + } + else { + ERTS_PTAB_LIST_DBG_TRACE(BIF_P->common.id, trap); + ERTS_BIF_YIELD2(&ptab_list_continue_export, BIF_P, res_acc, BIF_ARG_2); + } +} + +void +erts_ptab_init(void) +{ + /* ptab_list_continue/2 is a hidden BIF that the original BIF traps to. */ + erts_init_trap_export(&ptab_list_continue_export, + am_erlang, am_ptab_list_continue, 2, + &ptab_list_continue); + +} + +/* + * Debug stuff + */ + +static void assert_ptab_consistency(ErtsPTab *ptab) +{ +#ifdef DEBUG + if (ptab->r.o.free_id_data) { + Uint32 ix, pix, data; + int free_pids = 0; + int null_slots = 0; + + for (ix=0; ix < ptab->r.o.max; ix++) { + if (erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) { + ++free_pids; + data = erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]); + pix = erts_ptab_data2pix(ptab, (Eterm) data); + ASSERT(erts_ptab_pix2intptr_nob(ptab, pix) == ERTS_AINT_NULL); + } + if (erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) { + ++null_slots; + } + } + ASSERT(free_pids == null_slots); + ASSERT(free_pids == ptab->r.o.max - erts_smp_atomic32_read_nob(&ptab->vola.tile.count)); + } +#endif +} + +Sint +erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) +{ + Uint64 ld; + Sint res; + Eterm data; + int first_pix = -1; + + erts_ptab_rwlock(ptab); + + assert_ptab_consistency(ptab); + + if (ptab->r.o.free_id_data) { + Uint32 id_ix, dix; + + if (set) { + Uint32 i, max_ix, num, stop_id_ix; + max_ix = ptab->r.o.max - 1; + num = next; + id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix); + + for (i=0; i <= max_ix; ++i) { + Uint32 pix; + ++num; + num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + if (num == ptab->r.o.invalid_data) { + num += ptab->r.o.max; + num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + } + pix = erts_ptab_data2pix(ptab, num); + if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { + ++id_ix; + dix = ix_to_free_id_data_ix(ptab, id_ix); + erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num); + ASSERT(pix == erts_ptab_data2pix(ptab, num)); + } + } + erts_smp_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix); + + /* Write invalid_data in rest of free_id_data[]: */ + stop_id_ix = (1 + erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix; + while (1) { + id_ix = (id_ix+1) & max_ix; + if (id_ix == stop_id_ix) + break; + dix = ix_to_free_id_data_ix(ptab, id_ix); + erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], + ptab->r.o.invalid_data); + } + } + id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1; + dix = ix_to_free_id_data_ix(ptab, id_ix); + res = (Sint) erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[dix]); + } + else { + /* Deprecated legacy algorithm... */ + if (!set) + ld = last_data_read_nob(ptab); + else { + + ld = (Uint64) next; + data = ERTS_PTAB_LastData2EtermData(ld); + if (ptab->r.o.invalid_data == data) { + ld += ptab->r.o.max; + ASSERT(erts_ptab_data2pix(ptab, data) + == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); + } + last_data_set_relb(ptab, ld); + } + + while (1) { + int pix; + ld++; + pix = (int) (ld % ptab->r.o.max); + if (first_pix < 0) + first_pix = pix; + else if (pix == first_pix) { + res = -1; + break; + } + if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { + data = ERTS_PTAB_LastData2EtermData(ld); + if (ptab->r.o.invalid_data == data) { + ld += ptab->r.o.max; + ASSERT(erts_ptab_data2pix(ptab, data) + == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); + data = ERTS_PTAB_LastData2EtermData(ld); + } + res = data; + break; + } + } + } + + assert_ptab_consistency(ptab); + erts_ptab_rwunlock(ptab); + + return res; +} + +static ERTS_INLINE ErtsPTabElementCommon * +ptab_pix2el(ErtsPTab *ptab, int ix) +{ + ErtsPTabElementCommon *ptab_el; + ASSERT(0 <= ix && ix < ptab->r.o.max); + ptab_el = (ErtsPTabElementCommon *) erts_ptab_pix2intptr_nob(ptab, ix); + if (ptab_el == ptab->r.o.invalid_element) + return NULL; + else + return ptab_el; +} + +Eterm +erts_debug_ptab_list(Process *c_p, ErtsPTab *ptab) +{ + int i; + Uint need; + Eterm res; + Eterm* hp; + Eterm *hp_end; + + erts_ptab_rwlock(ptab); + + res = NIL; + need = erts_ptab_count(ptab) * 2; + hp = HAlloc(c_p, need); /* we need two heap words for each id */ + hp_end = hp + need; + + /* make the list by scanning bakward */ + + + for (i = ptab->r.o.max-1; i >= 0; i--) { + ErtsPTabElementCommon *el = ptab_pix2el(ptab, i); + if (el) { + res = CONS(hp, el->id, res); + hp += 2; + } + } + + erts_ptab_rwunlock(ptab); + + HRelease(c_p, hp_end, hp); + + return res; +} + +Eterm +erts_debug_ptab_list_bif_info(Process *c_p, ErtsPTab *ptab) +{ + ERTS_DECL_AM(ptab_list_bif_info); + Eterm elements[] = { + AM_ptab_list_bif_info, + make_small((Uint) ERTS_PTAB_LIST_BIF_MIN_START_REDS), + make_small((Uint) ptab->list.data.chunks), + make_small((Uint) ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE), + make_small((Uint) ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED), + make_small((Uint) ERTS_PTAB_LIST_BIF_TAB_FREE_DELETED_REDS), + make_small((Uint) ERTS_PTAB_LIST_BIF_INSPECT_DELETED_PER_RED), + make_small((Uint) ERTS_PTAB_LIST_INSPECT_DELETED_MAX_REDS), + make_small((Uint) ERTS_PTAB_LIST_BIF_BUILD_RESULT_CONSES_PER_RED), + make_small((Uint) ERTS_PTAB_LIST_BIF_DEBUGLEVEL) + }; + Uint sz = 0; + Eterm *hp; + (void) erts_bld_tuplev(NULL, &sz, sizeof(elements)/sizeof(Eterm), elements); + hp = HAlloc(c_p, sz); + return erts_bld_tuplev(&hp, NULL, sizeof(elements)/sizeof(Eterm), elements); +} + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_FOUND_PIDS +static void +debug_ptab_list_check_found_pid(ErtsPTabListBifData *ptlbdp, + Eterm pid, + Uint64 ic, + int pid_should_be_found) +{ + int i; + for (i = 0; i < ptlbdp->pid_ix; i++) { + if (ptlbdp->pid[i] == pid && ptlbdp->debug.pid_started[i] == ic) { + ERTS_PTAB_LIST_ASSERT(pid_should_be_found); + return; + } + } + ERTS_PTAB_LIST_ASSERT(!pid_should_be_found); +} +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_RESLIST +static void +debug_ptab_list_check_res_list(Eterm list) +{ + while (is_list(list)) { + Eterm* consp = list_val(list); + Eterm hd = CAR(consp); + ERTS_PTAB_LIST_ASSERT(erts_ptab_is_valid_id(hd)); + list = CDR(consp); + } + + ERTS_PTAB_LIST_ASSERT(is_nil(list)); +} +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_PIDS + +static void +debug_ptab_list_save_all_pids(ErtsPTabListBifData *ptlbdp) +{ + int ix, tix, cpix; + ErtsPTab *ptab = ptlbdp->ptab; + ptlbdp->debug.correct_pids_verified = 0; + ptlbdp->debug.correct_pids = erts_alloc(ERTS_ALC_T_PTAB_LIST_PIDS, + sizeof(Eterm)*ptlbdp->pid_sz); + + for (tix = 0, cpix = 0; tix < ptab->r.o.max; tix++) { + ErtsPTabElementCommon *el = ptab_pix2el(ptab, tix); + if (el) { + ERTS_PTAB_LIST_ASSERT(erts_ptab_is_valid_id(el->id)); + ptlbdp->debug.correct_pids[cpix++] = el->id; + ERTS_PTAB_LIST_ASSERT(cpix <= ptlbdp->pid_sz); + } + } + ERTS_PTAB_LIST_ASSERT(cpix == ptlbdp->pid_sz); + + for (ix = 0; ix < ptlbdp->pid_sz; ix++) + ptlbdp->pid[ix] = make_small(ix); +} + +static void +debug_ptab_list_verify_all_pids(ErtsPTabListBifData *ptlbdp) +{ + int ix, cpix; + + ERTS_PTAB_LIST_ASSERT(ptlbdp->pid_ix == ptlbdp->pid_sz); + + for (ix = 0; ix < ptlbdp->pid_sz; ix++) { + int found = 0; + Eterm pid = ptlbdp->pid[ix]; + ERTS_PTAB_LIST_ASSERT(erts_ptab_is_valid_id(pid)); + for (cpix = ix; cpix < ptlbdp->pid_sz; cpix++) { + if (ptlbdp->debug.correct_pids[cpix] == pid) { + ptlbdp->debug.correct_pids[cpix] = NIL; + found = 1; + break; + } + } + if (!found) { + for (cpix = 0; cpix < ix; cpix++) { + if (ptlbdp->debug.correct_pids[cpix] == pid) { + ptlbdp->debug.correct_pids[cpix] = NIL; + found = 1; + break; + } + } + } + ERTS_PTAB_LIST_ASSERT(found); + } + ptlbdp->debug.correct_pids_verified = 1; + + erts_free(ERTS_ALC_T_PTAB_LIST_PIDS, ptlbdp->debug.correct_pids); + ptlbdp->debug.correct_pids = NULL; +} +#endif /* ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_PIDS */ + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL >= ERTS_PTAB_LIST_DBGLVL_CHK_DEL_LIST +static void +debug_ptab_list_check_del_list(ErtsPTab *ptab) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab)); + if (!ptab->list.data.deleted.start) + ERTS_PTAB_LIST_ASSERT(!ptab->list.data.deleted.end); + else { + Uint64 curr_interval = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + Uint64 *prev_x_interval_p = NULL; + ErtsPTabDeletedElement *ptdep; + + for (ptdep = ptab->list.data.deleted.start; + ptdep; + ptdep = ptdep->next) { + if (!ptdep->prev) + ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start == ptdep); + else + ERTS_PTAB_LIST_ASSERT(ptdep->prev->next == ptdep); + if (!ptdep->next) + ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.end == ptdep); + else + ERTS_PTAB_LIST_ASSERT(ptdep->next->prev == ptdep); + if (ptdep->ix < 0) { + Uint64 interval = ptdep->u.bif_invocation.interval; + ERTS_PTAB_LIST_ASSERT(interval <= curr_interval); + } + else { + Uint64 s_interval = ptdep->u.element.inserted; + Uint64 x_interval = ptdep->u.element.deleted; + + ERTS_PTAB_LIST_ASSERT(s_interval <= x_interval); + if (prev_x_interval_p) + ERTS_PTAB_LIST_ASSERT(*prev_x_interval_p <= x_interval); + prev_x_interval_p = &ptdep->u.element.deleted; + ERTS_PTAB_LIST_ASSERT( + erts_ptab_is_valid_id(ptdep->u.element.id)); + ERTS_PTAB_LIST_ASSERT(erts_ptab_id2pix(ptab, + ptdep->u.element.id) + == ptdep->ix); + + } + } + + } +} + +static void +debug_ptab_list_check_del_free_list(ErtsPTab *ptab, + ErtsPTabDeletedElement *free_list) +{ + if (ptab->list.data.deleted.start) { + ErtsPTabDeletedElement *fptdep; + ErtsPTabDeletedElement *ptdep; + + for (fptdep = free_list; fptdep; fptdep = fptdep->next) { + for (ptdep = ptab->list.data.deleted.start; + ptdep; + ptdep = ptdep->next) { + ERTS_PTAB_LIST_ASSERT(fptdep != ptdep); + } + } + } +} + +#endif + +#if ERTS_PTAB_LIST_BIF_DEBUGLEVEL != 0 + +static void +debug_ptab_list_assert_error(char* expr, const char* file, int line, const char *func) +{ + fflush(stdout); + erts_fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n", + (char *) file, line, (char *) func, expr); + fflush(stderr); + abort(); +} + +#endif diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h new file mode 100644 index 0000000000..e3e05f14af --- /dev/null +++ b/erts/emulator/beam/erl_ptab.h @@ -0,0 +1,481 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Process/Port table implementation. + * + * Author: Rickard Green + */ + +#ifndef ERL_PTAB_H__ +#define ERL_PTAB_H__ + +#include "sys.h" +#include "erl_term.h" +#include "erl_time.h" +#include "erl_utils.h" +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_alloc.h" +#include "erl_monitors.h" + +#define ERTS_TRACER_PROC(P) ((P)->common.tracer_proc) +#define ERTS_TRACE_FLAGS(P) ((P)->common.trace_flags) + +#define ERTS_P_LINKS(P) ((P)->common.u.alive.links) +#define ERTS_P_MONITORS(P) ((P)->common.u.alive.monitors) + +#define IS_TRACED(p) \ + (ERTS_TRACER_PROC((p)) != NIL) +#define ARE_TRACE_FLAGS_ON(p,tf) \ + ((ERTS_TRACE_FLAGS((p)) & (tf|F_SENSITIVE)) == (tf)) +#define IS_TRACED_FL(p,tf) \ + ( IS_TRACED(p) && ARE_TRACE_FLAGS_ON(p,tf) ) + +typedef struct { + Eterm id; +#ifdef ERTS_SMP + erts_atomic32_t refc; +#endif + Eterm tracer_proc; + Uint trace_flags; + union { + /* --- While being alive --- */ + struct { + Uint64 started_interval; + struct reg_proc *reg; + ErtsLink *links; + ErtsMonitor *monitors; +#ifdef ERTS_SMP + ErtsSmpPTimer *ptimer; +#else + ErlTimer tm; +#endif + } alive; + + /* --- While being released --- */ + ErtsThrPrgrLaterOp release; + } u; +} ErtsPTabElementCommon; + +typedef struct ErtsPTabDeletedElement_ ErtsPTabDeletedElement; + +typedef struct { + erts_smp_rwmtx_t rwmtx; + erts_interval_t interval; + struct { + ErtsPTabDeletedElement *start; + ErtsPTabDeletedElement *end; + } deleted; + int chunks; +} ErtsPTabListData; + +typedef struct { +#ifdef ARCH_32 + erts_smp_dw_atomic_t last_data; +#else + erts_smp_atomic_t last_data; +#endif + erts_smp_atomic32_t count; + erts_smp_atomic32_t aid_ix; + erts_smp_atomic32_t fid_ix; +} ErtsPTabVolatileData; + +typedef struct { + erts_smp_atomic_t *tab; + erts_smp_atomic32_t *free_id_data; + Uint32 max; + Uint32 pix_mask; + Uint32 pix_cl_mask; + Uint32 pix_cl_shift; + Uint32 pix_cli_mask; + Uint32 pix_cli_shift; + Uint32 dix_cl_mask; + Uint32 dix_cl_shift; + Uint32 dix_cli_mask; + Uint32 dix_cli_shift; + ErtsPTabElementCommon *invalid_element; + Eterm invalid_data; + void (*release_element)(void *); + UWord element_size; +} ErtsPTabReadOnlyData; + +typedef struct { + /* + * Data mainly modified when someone is listing + * the content of the table. + */ + union { + ErtsPTabListData data; + char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabListData))]; + } list; + + /* + * Frequently modified data. + */ + union { + ErtsPTabVolatileData tile; + char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabVolatileData))]; + } vola; + + /* + * Read only data. + */ + union { + ErtsPTabReadOnlyData o; + char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabReadOnlyData))]; + } r; +} ErtsPTab; + +#define ERTS_PTAB_ID_DATA_SIZE 28 +#define ERTS_PTAB_ID_DATA_SHIFT (_TAG_IMMED1_SIZE) +/* ERTS_PTAB_MAX_SIZE must be a power of 2 */ +#define ERTS_PTAB_MAX_SIZE (SWORD_CONSTANT(1) << 27) +#if (ERTS_PTAB_MAX_SIZE-1) > MAX_SMALL +# error "The maximum number of processes/ports must fit in a SMALL." +#endif + + +/* + * Currently pids and ports are allowed. + */ +#if _PID_DATA_SIZE != ERTS_PTAB_ID_DATA_SIZE +# error "Unexpected pid data size" +#endif +#if _PID_DATA_SHIFT != ERTS_PTAB_ID_DATA_SHIFT +# error "Unexpected pid tag size" +#endif +#if _PORT_DATA_SIZE != ERTS_PTAB_ID_DATA_SIZE +# error "Unexpected port data size" +#endif +#if _PORT_DATA_SHIFT != ERTS_PTAB_ID_DATA_SHIFT +# error "Unexpected port tag size" +#endif + +#define ERTS_PTAB_INVALID_ID(TAG) \ + ((Eterm) \ + ((((1 << ERTS_PTAB_ID_DATA_SIZE) - 1) << ERTS_PTAB_ID_DATA_SHIFT) \ + | (TAG))) + +#define erts_ptab_is_valid_id(ID) \ + (is_internal_pid((ID)) || is_internal_port((ID))) + +void erts_ptab_init(void); +void erts_ptab_init_table(ErtsPTab *ptab, + ErtsAlcType_t atype, + void (*release_element)(void *), + ErtsPTabElementCommon *invalid_element, + int size, + UWord element_size, + char *name, + int legacy); +int erts_ptab_new_element(ErtsPTab *ptab, + ErtsPTabElementCommon *ptab_el, + void *init_arg, + void (*init_ptab_el)(void *, Eterm)); +void erts_ptab_delete_element(ErtsPTab *ptab, + ErtsPTabElementCommon *ptab_el); +int erts_ptab_initialized(ErtsPTab *ptab); +UWord erts_ptab_mem_size(ErtsPTab *ptab); + +ERTS_GLB_INLINE erts_interval_t *erts_ptab_interval(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_ptab_max(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_ptab_count(ErtsPTab *ptab); +ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata); +ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata); +ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data); +ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data); +ERTS_GLB_INLINE Eterm erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag); +ERTS_GLB_INLINE int erts_ptab_id2pix(ErtsPTab *ptab, Eterm id); +ERTS_GLB_INLINE Uint erts_ptab_id2data(ErtsPTab *ptab, Eterm id); +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix); +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix); +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix); +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix); +ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint32 add_refc); +ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab); +ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab); +ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab); +ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_interval_t * +erts_ptab_interval(ErtsPTab *ptab) +{ + return &ptab->list.data.interval; +} + +ERTS_GLB_INLINE int +erts_ptab_max(ErtsPTab *ptab) +{ + int max = ptab->r.o.max; + return max == ERTS_PTAB_MAX_SIZE ? max - 1 : max; +} + +ERTS_GLB_INLINE int +erts_ptab_count(ErtsPTab *ptab) +{ + int max = ptab->r.o.max; + erts_aint32_t res = erts_smp_atomic32_read_nob(&ptab->vola.tile.count); + if (max == ERTS_PTAB_MAX_SIZE) { + max--; + res--; + } + if (res > max) + return max; + ASSERT(res >= 0); + return (int) res; + +} + +ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata) +{ + Uint32 data = ((Uint32) pixdata) & ~ptab->r.o.pix_mask; + data |= (pixdata >> ptab->r.o.pix_cl_shift) & ptab->r.o.pix_cl_mask; + data |= (pixdata & ptab->r.o.pix_cli_mask) << ptab->r.o.pix_cli_shift; + return data; +} + +ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata) +{ + return ((Uint32) pixdata) & ptab->r.o.pix_mask; +} + +ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data) +{ + Uint32 n, pix; + n = (Uint32) data; + pix = ((n & ptab->r.o.pix_cl_mask) << ptab->r.o.pix_cl_shift); + pix += ((n >> ptab->r.o.pix_cli_shift) & ptab->r.o.pix_cli_mask); + ASSERT(0 <= pix && pix < ptab->r.o.max); + return pix; +} + +ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data) +{ + Uint pixdata = data & ~((Uint) ptab->r.o.pix_mask); + pixdata |= (Uint) erts_ptab_data2pix(ptab, data); + ASSERT(data == erts_ptab_pixdata2data(ptab, pixdata)); + return pixdata; +} + +#if ERTS_SIZEOF_TERM == 8 + +ERTS_GLB_INLINE Eterm +erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag) +{ + HUint huint; + Uint32 low_data = (Uint32) data; + low_data &= (1 << ERTS_PTAB_ID_DATA_SIZE) - 1; + low_data <<= ERTS_PTAB_ID_DATA_SHIFT; + huint.hval[ERTS_HUINT_HVAL_HIGH] = erts_ptab_data2pix(ptab, data); + huint.hval[ERTS_HUINT_HVAL_LOW] = low_data | ((Uint32) tag); + return (Eterm) huint.val; +} + +ERTS_GLB_INLINE int +erts_ptab_id2pix(ErtsPTab *ptab, Eterm id) +{ + HUint huint; + huint.val = id; + return (int) huint.hval[ERTS_HUINT_HVAL_HIGH]; +} + +ERTS_GLB_INLINE Uint +erts_ptab_id2data(ErtsPTab *ptab, Eterm id) +{ + HUint huint; + huint.val = id; + return (Uint) (huint.hval[ERTS_HUINT_HVAL_LOW] >> ERTS_PTAB_ID_DATA_SHIFT); +} + +#elif ERTS_SIZEOF_TERM == 4 + +ERTS_GLB_INLINE Eterm +erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag) +{ + Eterm id; + data &= ((1 << ERTS_PTAB_ID_DATA_SIZE) - 1); + id = (Eterm) erts_ptab_data2pixdata(ptab, data); + return (id << ERTS_PTAB_ID_DATA_SHIFT) | tag; +} + +ERTS_GLB_INLINE int +erts_ptab_id2pix(ErtsPTab *ptab, Eterm id) +{ + Uint pixdata = (Uint) id; + pixdata >>= ERTS_PTAB_ID_DATA_SHIFT; + return (int) erts_ptab_pixdata2pix(ptab, pixdata); +} + +ERTS_GLB_INLINE Uint +erts_ptab_id2data(ErtsPTab *ptab, Eterm id) +{ + Uint pixdata = (Uint) id; + pixdata >>= ERTS_PTAB_ID_DATA_SHIFT; + return erts_ptab_pixdata2data(ptab, pixdata); +} + +#else +#error "Unsupported size of term" +#endif + +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix) +{ + ASSERT(0 <= ix && ix < ptab->r.o.max); + return erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]); +} + +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix) +{ + ASSERT(0 <= ix && ix < ptab->r.o.max); + return erts_smp_atomic_read_ddrb(&ptab->r.o.tab[ix]); +} + +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix) +{ + ASSERT(0 <= ix && ix < ptab->r.o.max); + return erts_smp_atomic_read_rb(&ptab->r.o.tab[ix]); +} + +ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix) +{ + ASSERT(0 <= ix && ix < ptab->r.o.max); + return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]); +} + +ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el) +{ +#ifdef ERTS_SMP +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_aint32_t refc = erts_atomic32_inc_read_nob(&ptab_el->refc); + ERTS_SMP_LC_ASSERT(refc > 1); +#else + erts_atomic32_inc_nob(&ptab_el->refc); +#endif +#endif +} + +ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el) +{ +#ifdef ERTS_SMP + erts_aint32_t refc = erts_atomic32_dec_read_nob(&ptab_el->refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + return (int) refc; +#else + return 0; +#endif +} + +ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint32 add_refc) +{ +#ifdef ERTS_SMP + erts_aint32_t refc; + +#ifndef ERTS_ENABLE_LOCK_CHECK + if (add_refc >= 0) { + erts_atomic32_add_nob(&ptab_el->refc, + (erts_aint32_t) add_refc); + return 1; + } +#endif + + refc = erts_atomic32_add_read_nob(&ptab_el->refc, + (erts_aint32_t) add_refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + return (int) refc; +#else + return 0; +#endif +} + +ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab) +{ + erts_smp_rwmtx_rlock(&ptab->list.data.rwmtx); +} + +ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab) +{ + return erts_smp_rwmtx_tryrlock(&ptab->list.data.rwmtx); +} + +ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab) +{ + erts_smp_rwmtx_runlock(&ptab->list.data.rwmtx); +} + +ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab) +{ + erts_smp_rwmtx_rwlock(&ptab->list.data.rwmtx); +} + +ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab) +{ + return erts_smp_rwmtx_tryrwlock(&ptab->list.data.rwmtx); +} + +ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab) +{ + erts_smp_rwmtx_rwunlock(&ptab->list.data.rwmtx); +} + +ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab) +{ + return erts_smp_lc_rwmtx_is_rlocked(&ptab->list.data.rwmtx); +} + +ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab) +{ + return erts_smp_lc_rwmtx_is_rwlocked(&ptab->list.data.rwmtx); +} + +#endif + +#endif + +#if defined(ERTS_PTAB_WANT_BIF_IMPL__) && !defined(ERTS_PTAB_LIST__) +#define ERTS_PTAB_LIST__ + +#include "erl_process.h" +#include "bif.h" + +BIF_RETTYPE erts_ptab_list(struct process *c_p, ErtsPTab *ptab); + +#endif + +#if defined(ERTS_PTAB_WANT_DEBUG_FUNCS__) && !defined(ERTS_PTAB_DEBUG_FUNCS__) +#define ERTS_PTAB_DEBUG_FUNCS__ +#include "erl_process.h" + +/* Debug functions */ +Sint erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next); +Eterm erts_debug_ptab_list(Process *c_p, ErtsPTab *ptab); +Eterm erts_debug_ptab_list_bif_info(Process *c_p, ErtsPTab *ptab); + +#endif diff --git a/erts/emulator/beam/erl_resolv_dns.c b/erts/emulator/beam/erl_resolv_dns.c deleted file mode 100644 index 9d76fa89f8..0000000000 --- a/erts/emulator/beam/erl_resolv_dns.c +++ /dev/null @@ -1,23 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* - * Set this to non-zero value if DNS should be used. - */ -int erl_use_resolver = 1; diff --git a/erts/emulator/beam/erl_resolv_nodns.c b/erts/emulator/beam/erl_resolv_nodns.c deleted file mode 100644 index f14ab68e27..0000000000 --- a/erts/emulator/beam/erl_resolv_nodns.c +++ /dev/null @@ -1,23 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* - * Set this to non-zero value if DNS should be used. - */ -int erl_use_resolver = 0; diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index a32e9d9d7c..ecb5525022 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2012. All Rights Reserved. + * Copyright Ericsson AB 2005-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -259,6 +259,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb #define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb +#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty +#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty + /* Word size atomics */ #define erts_smp_atomic_init_nob erts_atomic_init_nob @@ -274,6 +277,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob #define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob #define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob +#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob #define erts_smp_atomic_init_mb erts_atomic_init_mb #define erts_smp_atomic_set_mb erts_atomic_set_mb @@ -288,6 +292,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb #define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb #define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb +#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb #define erts_smp_atomic_init_acqb erts_atomic_init_acqb #define erts_smp_atomic_set_acqb erts_atomic_set_acqb @@ -302,6 +307,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb #define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb #define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb +#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb #define erts_smp_atomic_init_relb erts_atomic_init_relb #define erts_smp_atomic_set_relb erts_atomic_set_relb @@ -316,6 +322,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb #define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb #define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb +#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb #define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb #define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb @@ -330,6 +337,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb #define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb #define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb +#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb #define erts_smp_atomic_init_rb erts_atomic_init_rb #define erts_smp_atomic_set_rb erts_atomic_set_rb @@ -344,6 +352,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb #define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb #define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb +#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb #define erts_smp_atomic_init_wb erts_atomic_init_wb #define erts_smp_atomic_set_wb erts_atomic_set_wb @@ -358,6 +367,10 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb #define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb #define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb +#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb + +#define erts_smp_atomic_set_dirty erts_atomic_set_dirty +#define erts_smp_atomic_read_dirty erts_atomic_read_dirty /* 32-bit atomics */ @@ -374,6 +387,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob #define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob #define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob +#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob #define erts_smp_atomic32_init_mb erts_atomic32_init_mb #define erts_smp_atomic32_set_mb erts_atomic32_set_mb @@ -388,6 +402,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb #define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb #define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb +#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb #define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb #define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb @@ -402,6 +417,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb #define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb #define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb +#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb #define erts_smp_atomic32_init_relb erts_atomic32_init_relb #define erts_smp_atomic32_set_relb erts_atomic32_set_relb @@ -416,6 +432,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb #define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb #define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb +#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb #define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb #define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb @@ -430,6 +447,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb #define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb #define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb +#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb #define erts_smp_atomic32_init_rb erts_atomic32_init_rb #define erts_smp_atomic32_set_rb erts_atomic32_set_rb @@ -444,6 +462,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb #define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb #define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb +#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb #define erts_smp_atomic32_init_wb erts_atomic32_init_wb #define erts_smp_atomic32_set_wb erts_atomic32_set_wb @@ -458,6 +477,10 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb #define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb #define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb +#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb + +#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty +#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty #else /* !ERTS_SMP */ @@ -498,6 +521,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read #define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg +#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read + /* Word size atomics */ #define erts_smp_atomic_init_nob erts_no_atomic_set @@ -513,6 +539,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_nob erts_no_atomic_read_band #define erts_smp_atomic_xchg_nob erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset #define erts_smp_atomic_init_mb erts_no_atomic_set #define erts_smp_atomic_set_mb erts_no_atomic_set @@ -527,6 +554,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_mb erts_no_atomic_read_band #define erts_smp_atomic_xchg_mb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset #define erts_smp_atomic_init_acqb erts_no_atomic_set #define erts_smp_atomic_set_acqb erts_no_atomic_set @@ -541,6 +569,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band #define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset #define erts_smp_atomic_init_relb erts_no_atomic_set #define erts_smp_atomic_set_relb erts_no_atomic_set @@ -555,6 +584,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_relb erts_no_atomic_read_band #define erts_smp_atomic_xchg_relb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset #define erts_smp_atomic_init_ddrb erts_no_atomic_set #define erts_smp_atomic_set_ddrb erts_no_atomic_set @@ -569,6 +599,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band #define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset #define erts_smp_atomic_init_rb erts_no_atomic_set #define erts_smp_atomic_set_rb erts_no_atomic_set @@ -583,6 +614,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_rb erts_no_atomic_read_band #define erts_smp_atomic_xchg_rb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset #define erts_smp_atomic_init_wb erts_no_atomic_set #define erts_smp_atomic_set_wb erts_no_atomic_set @@ -597,6 +629,10 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_wb erts_no_atomic_read_band #define erts_smp_atomic_xchg_wb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset + +#define erts_smp_atomic_set_dirty erts_no_atomic_set +#define erts_smp_atomic_read_dirty erts_no_atomic_read /* 32-bit atomics */ @@ -613,6 +649,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset #define erts_smp_atomic32_init_mb erts_no_atomic32_set #define erts_smp_atomic32_set_mb erts_no_atomic32_set @@ -627,6 +664,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_acqb erts_no_atomic32_set #define erts_smp_atomic32_set_acqb erts_no_atomic32_set @@ -641,6 +679,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_relb erts_no_atomic32_set #define erts_smp_atomic32_set_relb erts_no_atomic32_set @@ -655,6 +694,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_ddrb erts_no_atomic32_set #define erts_smp_atomic32_set_ddrb erts_no_atomic32_set @@ -669,6 +709,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_rb erts_no_atomic32_set #define erts_smp_atomic32_set_rb erts_no_atomic32_set @@ -683,6 +724,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_wb erts_no_atomic32_set #define erts_smp_atomic32_set_wb erts_no_atomic32_set @@ -697,6 +739,10 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset + +#define erts_smp_atomic32_set_dirty erts_no_atomic32_set +#define erts_smp_atomic32_read_dirty erts_no_atomic32_read #endif /* !ERTS_SMP */ diff --git a/erts/emulator/beam/erl_sys_driver.h b/erts/emulator/beam/erl_sys_driver.h index d429d0ce96..dab4a94a9b 100644 --- a/erts/emulator/beam/erl_sys_driver.h +++ b/erts/emulator/beam/erl_sys_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,7 +31,6 @@ #define ERL_SYS_DRV typedef long ErlDrvEvent; /* An event to be selected on. */ -typedef long ErlDrvPort; /* A port descriptor. */ /* typedef struct _SysDriverOpts SysDriverOpts; defined in sys.h */ diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index f77e8b798f..2f206ffbec 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -105,7 +105,7 @@ unsigned tag_val_def(Wterm x) break; } } - sprintf(msg, "tag_val_def: %#lx", (unsigned long) x); + erts_snprintf(msg, sizeof(msg), "tag_val_def: %#lx", (unsigned long) x); et_abort(msg, file, line); #undef file #undef line @@ -133,7 +133,7 @@ ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small); ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom); ET_DEFINE_CHECKED(Uint,header_arity,Eterm,is_header); -ET_DEFINE_CHECKED(Uint,arityval,Eterm,is_arity_value); +ET_DEFINE_CHECKED(Uint,arityval,Eterm,is_sane_arity_value); ET_DEFINE_CHECKED(Uint,thing_arityval,Eterm,is_thing); ET_DEFINE_CHECKED(Uint,thing_subtag,Eterm,is_thing); ET_DEFINE_CHECKED(Eterm*,binary_val,Wterm,is_binary); @@ -144,9 +144,7 @@ ET_DEFINE_CHECKED(Uint,bignum_header_arity,Eterm,_is_bignum_header); ET_DEFINE_CHECKED(Eterm*,big_val,Wterm,is_big); ET_DEFINE_CHECKED(Eterm*,float_val,Wterm,is_float); ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple); -ET_DEFINE_CHECKED(Uint,internal_pid_data,Eterm,is_internal_pid); ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid); -ET_DEFINE_CHECKED(Uint,internal_port_data,Eterm,is_internal_port); ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port); ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref); ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index c270d13365..953edf79ea 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -300,8 +300,17 @@ _ET_DECLARE_CHECKED(Uint,header_arity,Eterm) #define header_arity(x) _ET_APPLY(header_arity,(x)) /* arityval access methods */ +/* Erlang Spec. 4.7.3 defines max arity to 65535 + * we will however enforce max arity of 16777215 (24 bits) + * (checked in bifs and asserted in debug) + */ +#define MAX_ARITYVAL ((((Uint)1) << 24) - 1) +#define ERTS_MAX_TUPLE_SIZE MAX_ARITYVAL + #define make_arityval(sz) _make_header((sz),_TAG_HEADER_ARITYVAL) #define is_arity_value(x) (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) +#define is_sane_arity_value(x) ((((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) && \ + (((x) >> _HEADER_ARITY_OFFS) <= MAX_ARITYVAL)) #define is_not_arity_value(x) (!is_arity_value((x))) #define _unchecked_arityval(x) _unchecked_header_arity((x)) _ET_DECLARE_CHECKED(Uint,arityval,Eterm) @@ -542,12 +551,6 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) #define _GETBITS(X,Pos,Size) (((X) >> (Pos)) & ~(~((Uint) 0) << (Size))) /* - * Observe! New layout for pids, ports and references in R9 (see also note - * in erl_node_container_utils.h). - */ - - -/* * Creation in node specific data (pids, ports, refs) */ @@ -584,7 +587,6 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) * */ -#define _PID_R9_SER_SIZE 3 #define _PID_SER_SIZE (_PID_DATA_SIZE - _PID_NUM_SIZE) #define _PID_NUM_SIZE 15 @@ -598,23 +600,13 @@ _ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) #define make_pid_data(Ser, Num) \ ((Uint) ((Ser) << _PID_NUM_SIZE | (Num))) -#define make_internal_pid(X) \ - ((Eterm) (((X) << _PID_DATA_SHIFT) | _TAG_IMMED1_PID)) - #define is_internal_pid(x) (((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_PID) #define is_not_internal_pid(x) (!is_internal_pid((x))) -#define _unchecked_internal_pid_data(x) _GET_PID_DATA((x)) -_ET_DECLARE_CHECKED(Uint,internal_pid_data,Eterm) -#define internal_pid_data(x) _ET_APPLY(internal_pid_data,(x)) - #define _unchecked_internal_pid_node(x) erts_this_node _ET_DECLARE_CHECKED(struct erl_node_*,internal_pid_node,Eterm) #define internal_pid_node(x) _ET_APPLY(internal_pid_node,(x)) -#define internal_pid_number(x) _GET_PID_NUM(internal_pid_data((x))) -#define internal_pid_serial(x) _GET_PID_SER(internal_pid_data((x))) - #define internal_pid_data_words(x) (1) /* @@ -644,7 +636,6 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_pid_node,Eterm) * N : node number * */ -#define _PORT_R9_NUM_SIZE 18 #define _PORT_NUM_SIZE _PORT_DATA_SIZE #define _PORT_DATA_SIZE 28 @@ -654,18 +645,9 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_pid_node,Eterm) #define _GET_PORT_NUM(X) _GETBITS((X), 0, _PORT_NUM_SIZE) -#define make_internal_port(X) \ - ((Eterm) (((X) << _PORT_DATA_SHIFT) | _TAG_IMMED1_PORT)) - #define is_internal_port(x) (((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_PORT) #define is_not_internal_port(x) (!is_internal_port(x)) -#define _unchecked_internal_port_data(x) _GET_PORT_DATA((x)) -_ET_DECLARE_CHECKED(Uint,internal_port_data,Eterm) -#define internal_port_data(x) _ET_APPLY(internal_port_data,(x)) - -#define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x))) - #define _unchecked_internal_port_node(x) erts_this_node _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define internal_port_node(x) _ET_APPLY(internal_port_node,(x)) diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 88524bdd4c..cf5e3dc012 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2012. All Rights Reserved. + * Copyright Ericsson AB 2011-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -96,17 +96,14 @@ #define ERTS_THR_PRGR_LFLG_BLOCK (((erts_aint32_t) 1) << 31) #define ERTS_THR_PRGR_LFLG_NO_LEADER (((erts_aint32_t) 1) << 30) -#define ERTS_THR_PRGR_LFLG_ACTIVE_MASK (~(ERTS_THR_PRGR_LFLG_NO_LEADER \ - | ERTS_THR_PRGR_LFLG_BLOCK)) +#define ERTS_THR_PRGR_LFLG_WAITING_UM (((erts_aint32_t) 1) << 29) +#define ERTS_THR_PRGR_LFLG_ACTIVE_MASK (~(ERTS_THR_PRGR_LFLG_NO_LEADER \ + | ERTS_THR_PRGR_LFLG_BLOCK \ + | ERTS_THR_PRGR_LFLG_WAITING_UM)) -#define ERTS_THR_PRGR_LFLGS_ACTIVE(LFLGS) \ +#define ERTS_THR_PRGR_LFLGS_ACTIVE(LFLGS) \ ((LFLGS) & ERTS_THR_PRGR_LFLG_ACTIVE_MASK) -#define ERTS_THR_PRGR_LFLGS_ALL_WAITING(LFLGS) \ - (((LFLGS) & (ERTS_THR_PRGR_LFLG_NO_LEADER \ - |ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) \ - == ERTS_THR_PRGR_LFLG_NO_LEADER) - /* * We use a 64-bit value for thread progress. By this wrapping of * the thread progress will more or less never occur. @@ -262,6 +259,11 @@ typedef struct { erts_atomic32_t managed_count; erts_atomic32_t managed_id; erts_atomic32_t unmanaged_id; + int chk_next_ix; + struct { + int waiting; + erts_atomic32_t current; + } umrefc_ix; } ErtsThrPrgrMiscData; typedef struct { @@ -276,12 +278,18 @@ typedef union { char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrPrgrElement))]; } ErtsThrPrgrArray; +typedef union { + erts_atomic_t refc; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_atomic_t))]; +} ErtsThrPrgrUnmanagedRefc; + typedef struct { union { ErtsThrPrgrMiscData data; char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( sizeof(ErtsThrPrgrMiscData))]; } misc; + ErtsThrPrgrUnmanagedRefc umrefc[2]; ErtsThrPrgrArray *thr; struct { int no; @@ -346,7 +354,9 @@ init_tmp_thr_prgr_data(ErtsThrPrgrData *tpd) tpd->is_managed = 0; tpd->is_blocking = 0; tpd->is_temporary = 1; - +#ifdef ERTS_ENABLE_LOCK_CHECK + tpd->is_delaying = 0; +#endif erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); } @@ -408,7 +418,7 @@ erts_thr_progress_pre_init(void) { intrnl = NULL; erts_tsd_key_create(&erts_thr_prgr_data_key__); - init_nob(&erts_thr_prgr__.current, 0); + init_nob(&erts_thr_prgr__.current, ERTS_THR_PRGR_VAL_FIRST); } void @@ -461,6 +471,12 @@ erts_thr_progress_init(int no_schedulers, int managed, int unmanaged) erts_atomic32_init_nob(&intrnl->misc.data.managed_count, 0); erts_atomic32_init_nob(&intrnl->misc.data.managed_id, no_schedulers); erts_atomic32_init_nob(&intrnl->misc.data.unmanaged_id, -1); + intrnl->misc.data.chk_next_ix = 0; + intrnl->misc.data.umrefc_ix.waiting = -1; + erts_atomic32_init_nob(&intrnl->misc.data.umrefc_ix.current, 0); + + erts_atomic_init_nob(&intrnl->umrefc[0].refc, (erts_aint_t) 0); + erts_atomic_init_nob(&intrnl->umrefc[1].refc, (erts_aint_t) 0); intrnl->thr = (ErtsThrPrgrArray *) ptr; ptr += thr_arr_sz; @@ -547,6 +563,9 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) tpd->is_managed = 0; tpd->is_blocking = is_blocking; tpd->is_temporary = 0; +#ifdef ERTS_ENABLE_LOCK_CHECK + tpd->is_delaying = 0; +#endif ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->unmanaged.no) erl_exit(ERTS_ABORT_EXIT, @@ -600,6 +619,9 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, tpd->is_managed = 1; tpd->is_blocking = is_blocking; tpd->is_temporary = 0; +#ifdef ERTS_ENABLE_LOCK_CHECK + tpd->is_delaying = 1; +#endif init_wakeup_request_array(&tpd->wakeup_request[0]); @@ -607,8 +629,8 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, tpd->leader = 0; tpd->active = 1; - tpd->previous.local = 0; - tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; + tpd->confirmed = 0; + tpd->leader_state.current = ERTS_THR_PRGR_VAL_WAITING; erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd); erts_atomic32_inc_nob(&intrnl->misc.data.lflgs); @@ -651,60 +673,113 @@ leader_update(ErtsThrPrgrData *tpd) block_thread(tpd); } else { + ErtsThrPrgrVal current; + int ix, chk_next_ix, umrefc_ix, my_ix, no_managed, waiting_unmanaged; erts_aint32_t lflgs; ErtsThrPrgrVal next; - int ix, sz, make_progress; + erts_aint_t refc; - if (tpd->previous.current == ERTS_THR_PRGR_VAL_WAITING) { - /* Took over as leader from another thread */ - tpd->previous.current = read_acqb(&erts_thr_prgr__.current); - tpd->previous.next = tpd->previous.current; - tpd->previous.next++; - if (tpd->previous.next == ERTS_THR_PRGR_VAL_WAITING) - tpd->previous.next = 0; - } + my_ix = tpd->id; - if (tpd->previous.local == tpd->previous.current) { - ErtsThrPrgrVal val = tpd->previous.current + 1; - if (val == ERTS_THR_PRGR_VAL_WAITING) - val = 0; - tpd->previous.local = val; - set_mb(&intrnl->thr[tpd->id].data.current, val); + if (tpd->leader_state.current == ERTS_THR_PRGR_VAL_WAITING) { + /* Took over as leader from another thread */ + tpd->leader_state.current = read_nob(&erts_thr_prgr__.current); + tpd->leader_state.next = tpd->leader_state.current; + tpd->leader_state.next++; + if (tpd->leader_state.next == ERTS_THR_PRGR_VAL_WAITING) + tpd->leader_state.next = 0; + tpd->leader_state.chk_next_ix = intrnl->misc.data.chk_next_ix; + tpd->leader_state.umrefc_ix.waiting = intrnl->misc.data.umrefc_ix.waiting; + tpd->leader_state.umrefc_ix.current = + (int) erts_atomic32_read_nob(&intrnl->misc.data.umrefc_ix.current); + + if (tpd->confirmed == tpd->leader_state.current) { + ErtsThrPrgrVal val = tpd->leader_state.current + 1; + if (val == ERTS_THR_PRGR_VAL_WAITING) + val = 0; + tpd->confirmed = val; + set_mb(&intrnl->thr[my_ix].data.current, val); + } } - next = tpd->previous.next; - make_progress = 1; - sz = intrnl->managed.no; - for (ix = 0; ix < sz; ix++) { - ErtsThrPrgrVal tmp; - tmp = read_nob(&intrnl->thr[ix].data.current); - if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) { - make_progress = 0; - ASSERT(erts_thr_progress_has_passed__(next, tmp)); - break; + next = tpd->leader_state.next; + + waiting_unmanaged = 0; + umrefc_ix = -1; /* Shut up annoying warning */ + + chk_next_ix = tpd->leader_state.chk_next_ix; + no_managed = intrnl->managed.no; + ASSERT(0 <= chk_next_ix && chk_next_ix <= no_managed); + /* Check manged threads */ + if (chk_next_ix < no_managed) { + for (ix = chk_next_ix; ix < no_managed; ix++) { + ErtsThrPrgrVal tmp; + if (ix == my_ix) + continue; + tmp = read_nob(&intrnl->thr[ix].data.current); + if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) { + tpd->leader_state.chk_next_ix = ix; + ASSERT(erts_thr_progress_has_passed__(next, tmp)); + goto done; + } } } - if (make_progress) { - ErtsThrPrgrVal current = next; + /* Check unmanged threads */ + waiting_unmanaged = tpd->leader_state.umrefc_ix.waiting != -1; + umrefc_ix = (waiting_unmanaged + ? tpd->leader_state.umrefc_ix.waiting + : tpd->leader_state.umrefc_ix.current); + refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); + ASSERT(refc >= 0); + if (refc != 0) { + int new_umrefc_ix; + + if (waiting_unmanaged) + goto done; + + new_umrefc_ix = (umrefc_ix + 1) & 0x1; + tpd->leader_state.umrefc_ix.waiting = umrefc_ix; + tpd->leader_state.chk_next_ix = no_managed; + erts_atomic32_set_nob(&intrnl->misc.data.umrefc_ix.current, + (erts_aint32_t) new_umrefc_ix); + ETHR_MEMBAR(ETHR_StoreLoad); + refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); + ASSERT(refc >= 0); + waiting_unmanaged = 1; + if (refc != 0) + goto done; + } - next++; - if (next == ERTS_THR_PRGR_VAL_WAITING) - next = 0; + /* Make progress */ + current = next; - set_nob(&intrnl->thr[tpd->id].data.current, next); - set_mb(&erts_thr_prgr__.current, current); - tpd->previous.local = next; - tpd->previous.next = next; - tpd->previous.current = current; + next++; + if (next == ERTS_THR_PRGR_VAL_WAITING) + next = 0; + + set_nob(&intrnl->thr[my_ix].data.current, next); + set_mb(&erts_thr_prgr__.current, current); + tpd->confirmed = next; + tpd->leader_state.next = next; + tpd->leader_state.current = current; #if ERTS_THR_PRGR_PRINT_VAL - if (current % 1000 == 0) - erts_fprintf(stderr, "%b64u\n", current); + if (current % 1000 == 0) + erts_fprintf(stderr, "%b64u\n", current); #endif - handle_wakeup_requests(current); + handle_wakeup_requests(current); + + if (waiting_unmanaged) { + waiting_unmanaged = 0; + tpd->leader_state.umrefc_ix.waiting = -1; + erts_atomic32_read_band_nob(&intrnl->misc.data.lflgs, + ~ERTS_THR_PRGR_LFLG_WAITING_UM); } + tpd->leader_state.chk_next_ix = 0; + + done: if (tpd->active) { lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); @@ -712,20 +787,46 @@ leader_update(ErtsThrPrgrData *tpd) (void) block_thread(tpd); } else { + int force_wakeup_check = 0; + erts_aint32_t set_flags = ERTS_THR_PRGR_LFLG_NO_LEADER; tpd->leader = 0; - tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING; + tpd->leader_state.current = ERTS_THR_PRGR_VAL_WAITING; #if ERTS_THR_PRGR_PRINT_LEADER erts_fprintf(stderr, "L <- %d\n", tpd->id); #endif ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0); + intrnl->misc.data.umrefc_ix.waiting + = tpd->leader_state.umrefc_ix.waiting; + if (waiting_unmanaged) + set_flags |= ERTS_THR_PRGR_LFLG_WAITING_UM; + lflgs = erts_atomic32_read_bor_relb(&intrnl->misc.data.lflgs, - ERTS_THR_PRGR_LFLG_NO_LEADER); + set_flags); + lflgs |= set_flags; if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK) lflgs = block_thread(tpd); - if (ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0 && got_sched_wakeups()) + + if (waiting_unmanaged) { + /* Need to check umrefc again */ + ETHR_MEMBAR(ETHR_StoreLoad); + refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); + if (refc == 0) { + /* Need to force wakeup check */ + force_wakeup_check = 1; + } + } + + if ((force_wakeup_check + || ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER + | ERTS_THR_PRGR_LFLG_WAITING_UM + | ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) + == ERTS_THR_PRGR_LFLG_NO_LEADER)) + && got_sched_wakeups()) { + /* Someone need to make progress */ wakeup_managed(0); + } } } @@ -744,11 +845,11 @@ update(ErtsThrPrgrData *tpd) erts_aint32_t lflgs; res = 0; val = read_acqb(&erts_thr_prgr__.current); - if (tpd->previous.local == val) { + if (tpd->confirmed == val) { val++; if (val == ERTS_THR_PRGR_VAL_WAITING) val = 0; - tpd->previous.local = val; + tpd->confirmed = val; set_mb(&intrnl->thr[tpd->id].data.current, val); } @@ -801,12 +902,19 @@ erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp) block_count_dec(); - tpd->previous.local = ERTS_THR_PRGR_VAL_WAITING; + tpd->confirmed = ERTS_THR_PRGR_VAL_WAITING; set_mb(&intrnl->thr[tpd->id].data.current, ERTS_THR_PRGR_VAL_WAITING); lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); - if (ERTS_THR_PRGR_LFLGS_ALL_WAITING(lflgs) && got_sched_wakeups()) - wakeup_managed(0); /* Someone need to make progress */ + + if ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER + | ERTS_THR_PRGR_LFLG_WAITING_UM + | ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) + == ERTS_THR_PRGR_LFLG_NO_LEADER + && got_sched_wakeups()) { + /* Someone need to make progress */ + wakeup_managed(0); + } } void @@ -828,7 +936,7 @@ erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp) val++; if (val == ERTS_THR_PRGR_VAL_WAITING) val = 0; - tpd->previous.local = val; + tpd->confirmed = val; set_mb(&intrnl->thr[tpd->id].data.current, val); val = read_acqb(&erts_thr_prgr__.current); if (current == val) @@ -875,6 +983,68 @@ erts_thr_progress_active(ErtsSchedulerData *esdp, int on) } +static ERTS_INLINE void +unmanaged_continue(ErtsThrPrgrDelayHandle handle) +{ + int umrefc_ix = (int) handle; + erts_aint_t refc; + + ASSERT(umrefc_ix == 0 || umrefc_ix == 1); + refc = erts_atomic_dec_read_relb(&intrnl->umrefc[umrefc_ix].refc); + ASSERT(refc >= 0); + if (refc == 0) { + erts_aint_t lflgs; + ERTS_THR_READ_MEMORY_BARRIER; + lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs); + if ((lflgs & (ERTS_THR_PRGR_LFLG_NO_LEADER + | ERTS_THR_PRGR_LFLG_WAITING_UM + | ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) + == (ERTS_THR_PRGR_LFLG_NO_LEADER|ERTS_THR_PRGR_LFLG_WAITING_UM) + && got_sched_wakeups()) { + /* Others waiting for us... */ + wakeup_managed(0); + } + } +} + +void +erts_thr_progress_unmanaged_continue__(ErtsThrPrgrDelayHandle handle) +{ +#ifdef ERTS_ENABLE_LOCK_CHECK + ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); + ERTS_LC_ASSERT(tpd && tpd->is_delaying); + tpd->is_delaying = 0; + return_tmp_thr_prgr_data(tpd); +#endif + ASSERT(!erts_thr_progress_is_managed_thread()); + + unmanaged_continue(handle); +} + +ErtsThrPrgrDelayHandle +erts_thr_progress_unmanaged_delay__(void) +{ + int umrefc_ix; + ASSERT(!erts_thr_progress_is_managed_thread()); + umrefc_ix = (int) erts_atomic32_read_acqb(&intrnl->misc.data.umrefc_ix.current); + while (1) { + int tmp_ix; + erts_atomic_inc_acqb(&intrnl->umrefc[umrefc_ix].refc); + tmp_ix = (int) erts_atomic32_read_acqb(&intrnl->misc.data.umrefc_ix.current); + if (tmp_ix == umrefc_ix) + break; + unmanaged_continue(umrefc_ix); + umrefc_ix = tmp_ix; + } +#ifdef ERTS_ENABLE_LOCK_CHECK + { + ErtsThrPrgrData *tpd = tmp_thr_prgr_data(NULL); + tpd->is_delaying = 1; + } +#endif + return (ErtsThrPrgrDelayHandle) umrefc_ix; +} + static ERTS_INLINE int has_reached_wakeup(ErtsThrPrgrVal wakeup) { @@ -931,7 +1101,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) */ ASSERT(tpd->is_managed); - ASSERT(tpd->previous.local != ERTS_THR_PRGR_VAL_WAITING); + ASSERT(tpd->confirmed != ERTS_THR_PRGR_VAL_WAITING); if (has_reached_wakeup(value)) { wakeup_managed(tpd->id); @@ -946,7 +1116,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) tpd->wakeup_request[wix])); - if (tpd->previous.local == value) { + if (tpd->confirmed == value) { /* * We have already confirmed this value. We need to request * wakeup for a value later than our latest confirmed value in diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 89486b065b..5f392944c2 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2012. All Rights Reserved. + * Copyright Ericsson AB 2011-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -53,9 +53,22 @@ typedef Uint64 ErtsThrPrgrVal; #define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */ typedef struct { + ErtsThrPrgrVal next; + ErtsThrPrgrVal current; + int chk_next_ix; + struct { + int current; + int waiting; + } umrefc_ix; +} ErtsThrPrgrLeaderState; + +typedef struct { int id; int is_managed; int is_blocking; +#ifdef ERTS_ENABLE_LOCK_CHECK + int is_delaying; /* managed is always delaying */ +#endif int is_temporary; /* --- Part below only for registered threads --- */ @@ -66,11 +79,8 @@ typedef struct { int leader; /* Needs to be first in the managed threads part */ int active; - struct { - ErtsThrPrgrVal local; - ErtsThrPrgrVal next; - ErtsThrPrgrVal current; - } previous; + ErtsThrPrgrVal confirmed; + ErtsThrPrgrLeaderState leader_state; } ErtsThrPrgrData; void erts_thr_progress_fatal_error_block(SWord timeout, @@ -78,6 +88,16 @@ void erts_thr_progress_fatal_error_block(SWord timeout, #endif /* ERTS_SMP */ +typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp; +struct ErtsThrPrgrLaterOp_ { +#ifdef ERTS_SMP + ErtsThrPrgrVal later; +#endif + void (*func)(void *); + void *data; + ErtsThrPrgrLaterOp *next; +}; + #endif #if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY) @@ -88,6 +108,8 @@ void erts_thr_progress_fatal_error_block(SWord timeout, #ifdef ERTS_SMP +/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */ +#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0) #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0)) #define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0)) @@ -111,6 +133,11 @@ typedef struct { ERTS_THR_PRGR_ATOMIC current; } ErtsThrPrgr; +typedef int ErtsThrPrgrDelayHandle; +#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1) +/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */ +#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2) + extern ErtsThrPrgr erts_thr_prgr__; void erts_thr_progress_pre_init(void); @@ -126,6 +153,8 @@ int erts_thr_progress_update(ErtsSchedulerData *esdp); int erts_thr_progress_leader_update(ErtsSchedulerData *esdp); void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp); void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp); +ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void); +void erts_thr_progress_unmanaged_continue__(int umrefc_ix); void erts_thr_progress_dbg_print_state(void); @@ -138,6 +167,11 @@ ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *a ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void); +ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void); +ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle); +#ifdef ERTS_ENABLE_LOCK_CHECK +ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void); +#endif ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void); @@ -219,6 +253,35 @@ erts_thr_progress_is_managed_thread(void) return tpd && tpd->is_managed; } +ERTS_GLB_INLINE ErtsThrPrgrDelayHandle +erts_thr_progress_unmanaged_delay(void) +{ + if (erts_thr_progress_is_managed_thread()) + return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */ + else + return erts_thr_progress_unmanaged_delay__(); +} + +ERTS_GLB_INLINE void +erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle) +{ + ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED + || erts_thr_progress_is_managed_thread()); + if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_thr_progress_unmanaged_continue__(handle); +} + +#ifdef ERTS_ENABLE_LOCK_CHECK + +ERTS_GLB_INLINE int +erts_thr_progress_lc_is_delaying(void) +{ + ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__); + return tpd && tpd->is_delaying; +} + +#endif + ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val) { @@ -238,7 +301,7 @@ erts_thr_progress_later(ErtsSchedulerData *esdp) if (esdp) { tpd = &esdp->thr_progress_data; managed_thread: - val = tpd->previous.local; + val = tpd->confirmed; ERTS_THR_MEMORY_BARRIER; } else { diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index f07964a265..f8ca87ddcc 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2012. All Rights Reserved. + * Copyright Ericsson AB 2011-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -113,6 +113,11 @@ sl_element_free(ErtsThrQElement_t *p) #endif +#define ErtsThrQDirtyReadEl(A) \ + ((ErtsThrQElement_t *) erts_atomic_read_dirty((A))) +#define ErtsThrQDirtySetEl(A, V) \ + erts_atomic_set_dirty((A), (erts_aint_t) (V)) + typedef union { ErtsThrQ_t q; char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))]; @@ -137,7 +142,7 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) q->last = NULL; q->q.blk = NULL; #else - erts_atomic_init_nob(&q->tail.data.marker.next.atmc, ERTS_AINT_NULL); + erts_atomic_init_nob(&q->tail.data.marker.next, ERTS_AINT_NULL); q->tail.data.marker.data.ptr = NULL; erts_atomic_init_nob(&q->tail.data.last, (erts_aint_t) &q->tail.data.marker); @@ -150,7 +155,7 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) if (!q->tail.data.notify) q->tail.data.notify = noop_callback; - q->head.head.ptr = &q->tail.data.marker; + erts_atomic_init_nob(&q->head.head, (erts_aint_t) &q->tail.data.marker); q->head.live = qi->live.objects; q->head.first = &q->tail.data.marker; q->head.unref_end = &q->tail.data.marker; @@ -296,17 +301,17 @@ element_free(ErtsThrQ_t *q, ErtsThrQElement_t *el) #ifdef USE_THREADS static ERTS_INLINE ErtsThrQElement_t * -enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this, int want_last) +enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this) { erts_aint_t ilast, itmp; - erts_atomic_init_nob(&this->next.atmc, ERTS_AINT_NULL); + erts_atomic_init_nob(&this->next, ERTS_AINT_NULL); /* Enqueue at end of list... */ ilast = erts_atomic_read_nob(&q->tail.data.last); while (1) { ErtsThrQElement_t *last = (ErtsThrQElement_t *) ilast; - itmp = erts_atomic_cmpxchg_mb(&last->next.atmc, + itmp = erts_atomic_cmpxchg_mb(&last->next, (erts_aint_t) this, ERTS_AINT_NULL); if (itmp == ERTS_AINT_NULL) @@ -316,31 +321,57 @@ enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this, int want_last) /* Move last pointer forward... */ while (1) { - if (want_last) { - if (erts_atomic_read_rb(&this->next.atmc) != ERTS_AINT_NULL) { - /* Someone else will move it forward */ - ilast = erts_atomic_read_rb(&q->tail.data.last); - return (ErtsThrQElement_t *) ilast; - } - } - else { - if (erts_atomic_read_nob(&this->next.atmc) != ERTS_AINT_NULL) { - /* Someone else will move it forward */ - return NULL; - } + if (erts_atomic_read_rb(&this->next) != ERTS_AINT_NULL) { + /* Someone else will move it forward */ + ilast = erts_atomic_read_rb(&q->tail.data.last); + return (ErtsThrQElement_t *) ilast; } itmp = erts_atomic_cmpxchg_mb(&q->tail.data.last, (erts_aint_t) this, ilast); if (ilast == itmp) - return want_last ? this : NULL; + return this; ilast = itmp; } } +static ERTS_INLINE ErtsThrQElement_t * +enqueue_marker(ErtsThrQ_t *q, ErtsThrQElement_t **headp) +{ + int maybe_notify; + erts_aint_t inext; + ErtsThrQElement_t *last, *head; + + if (headp) + head = *headp; + else + head = ErtsThrQDirtyReadEl(&q->head.head); + + ASSERT(!q->head.used_marker); + q->head.used_marker = 1; + last = enqueue_managed(q, &q->tail.data.marker); + maybe_notify = &q->tail.data.marker == last; + inext = erts_atomic_read_acqb(&head->next); + if (inext == (erts_aint_t) &q->tail.data.marker) { + ErtsThrQDirtySetEl(&q->head.head, &q->tail.data.marker); + if (headp) + *headp = &q->tail.data.marker; + } + else if (maybe_notify) { + /* + * We need to notify; otherwise, we might loose a notification + * for a concurrently inserted element. + */ + q->head.notify(q->head.arg); + } + return last; +} + + static ErtsThrQCleanState_t clean(ErtsThrQ_t *q, int max_ops, int do_notify) { + ErtsThrQElement_t *head; erts_aint_t ilast; int um_refc_ix; int ops; @@ -349,7 +380,8 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) ErtsThrQElement_t *tmp; restart: ASSERT(q->head.first); - if (q->head.first == q->head.head.ptr) { + head = ErtsThrQDirtyReadEl(&q->head.head); + if (q->head.first == head) { q->head.clean_reached_head_count++; if (q->head.clean_reached_head_count >= ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT) { @@ -362,19 +394,20 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) break; if (q->head.first == &q->tail.data.marker) { q->head.used_marker = 0; - q->head.first = q->head.first->next.ptr; + q->head.first = ErtsThrQDirtyReadEl(&q->head.first->next); goto restart; } tmp = q->head.first; - q->head.first = q->head.first->next.ptr; + q->head.first = ErtsThrQDirtyReadEl(&q->head.first->next); if (q->head.deq_fini.automatic) element_free(q, tmp); else { tmp->data.ptr = (void *) (UWord) q->head.live; if (!q->head.deq_fini.start) q->head.deq_fini.start = tmp; - else if (q->head.deq_fini.end->next.ptr == &q->tail.data.marker) - q->head.deq_fini.end->next.ptr = tmp; + else if (ErtsThrQDirtyReadEl(&q->head.deq_fini.end->next) + == &q->tail.data.marker) + ErtsThrQDirtySetEl(&q->head.deq_fini.end->next, tmp); q->head.deq_fini.end = tmp; } } @@ -401,21 +434,8 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) q->head.unref_end = q->head.next.unref_end; if (!q->head.used_marker - && q->head.unref_end == (ErtsThrQElement_t *) ilast) { - q->head.used_marker = 1; - ilast = (erts_aint_t) enqueue_managed(q, - &q->tail.data.marker, - 1); - if (q->head.head.ptr == q->head.unref_end) { - ErtsThrQElement_t *next; - next = ((ErtsThrQElement_t *) - erts_atomic_read_acqb(&q->head.head.ptr->next.atmc)); - if (next == &q->tail.data.marker) { - q->head.head.ptr->next.ptr = &q->tail.data.marker; - q->head.head.ptr = &q->tail.data.marker; - } - } - } + && q->head.unref_end == (ErtsThrQElement_t *) ilast) + ilast = (erts_aint_t) enqueue_marker(q, NULL); if (q->head.unref_end == (ErtsThrQElement_t *) ilast) ERTS_SMP_MEMORY_BARRIER; @@ -436,20 +456,16 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) } #endif - if (q->head.first == q->head.head.ptr) { + head = ErtsThrQDirtyReadEl(&q->head.head); + if (q->head.first == head) { inspect_head: if (!q->head.used_marker) { erts_aint_t inext; - inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + inext = erts_atomic_read_acqb(&head->next); if (inext == ERTS_AINT_NULL) { - q->head.used_marker = 1; - (void) enqueue_managed(q, &q->tail.data.marker, 0); - inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); - if (inext == (erts_aint_t) &q->tail.data.marker) { - q->head.head.ptr->next.ptr = &q->tail.data.marker; - q->head.head.ptr = &q->tail.data.marker; + enqueue_marker(q, &head); + if (head == &q->tail.data.marker) goto check_thr_progress; - } } } @@ -506,26 +522,27 @@ erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty) #ifndef USE_THREADS return ERTS_THR_Q_CLEAN; #else + ErtsThrQElement_t *head = ErtsThrQDirtyReadEl(&q->head.head); if (ensure_empty) { erts_aint_t inext; - inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + inext = erts_atomic_read_acqb(&head->next); if (inext != ERTS_AINT_NULL) { if (&q->tail.data.marker != (ErtsThrQElement_t *) inext) return ERTS_THR_Q_DIRTY; else { - q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; - q->head.head.ptr = (ErtsThrQElement_t *) inext; - inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + head = (ErtsThrQElement_t *) inext; + ErtsThrQDirtySetEl(&q->head.head, head); + inext = erts_atomic_read_acqb(&head->next); if (inext != ERTS_AINT_NULL) return ERTS_THR_Q_DIRTY; } } } - if (q->head.first == q->head.head.ptr) { + if (q->head.first == head) { if (!q->head.used_marker) { erts_aint_t inext; - inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + inext = erts_atomic_read_acqb(&head->next); if (inext == ERTS_AINT_NULL) return ERTS_THR_Q_DIRTY; } @@ -553,11 +570,11 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) #ifndef USE_THREADS ASSERT(data); - this->next.ptr = NULL; + this->next = NULL; this->data.ptr = data; if (q->last) - q->last->next.ptr = this; + q->last->next = this; else { q->first = q->last = this; q->init.notify(q->init.arg); @@ -595,7 +612,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) } } - notify = this == enqueue_managed(q, this, 1); + notify = this == enqueue_managed(q, this); #ifdef ERTS_SMP @@ -638,17 +655,17 @@ erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp) ErtsThrQElement_t *e = q->head.deq_fini.start; ErtsThrQElement_t *end = q->head.deq_fini.end; while (e != end) { - ASSERT(q->head.head.ptr != e); + ASSERT(ErtsThrQDirtyReadEl(&q->head.head) != e); ASSERT(q->head.first != e); ASSERT(q->head.unref_end != e); - e = e->next.ptr; + e = ErtsThrQDirtyReadEl(&e->next); } } #endif fdp->start = q->head.deq_fini.start; fdp->end = q->head.deq_fini.end; if (fdp->end) - fdp->end->next.ptr = NULL; + ErtsThrQDirtySetEl(&fdp->end->next, NULL); q->head.deq_fini.start = NULL; q->head.deq_fini.end = NULL; return fdp->start != NULL; @@ -662,7 +679,7 @@ erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0, #ifdef USE_THREADS if (fdp1->start) { if (fdp0->end) - fdp0->end->next.ptr = fdp1->start; + ErtsThrQDirtySetEl(&fdp0->end->next, fdp1->start); else fdp0->start = fdp1->start; fdp0->end = fdp1->end; @@ -683,7 +700,7 @@ int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state) if (!start) break; tmp = start; - start = start->next.ptr; + start = ErtsThrQDirtyReadEl(&start->next); live = (ErtsThrQLive_t) (UWord) tmp->data.ptr; element_live_free(live, tmp); } @@ -724,7 +741,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) return NULL; tmp = q->first; res = tmp->data.ptr; - q->first = tmp->next.ptr; + q->first = tmp->next; if (!q->first) q->last = NULL; @@ -732,24 +749,26 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) return res; #else + ErtsThrQElement_t *head; erts_aint_t inext; void *res; - inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + head = ErtsThrQDirtyReadEl(&q->head.head); + inext = erts_atomic_read_acqb(&head->next); if (inext == ERTS_AINT_NULL) return NULL; - q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; - q->head.head.ptr = (ErtsThrQElement_t *) inext; - if (q->head.head.ptr == &q->tail.data.marker) { - inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc); + head = (ErtsThrQElement_t *) inext; + ErtsThrQDirtySetEl(&q->head.head, head); + if (head == &q->tail.data.marker) { + inext = erts_atomic_read_acqb(&head->next); if (inext == ERTS_AINT_NULL) return NULL; - q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext; - q->head.head.ptr = (ErtsThrQElement_t *) inext; + head = (ErtsThrQElement_t *) inext; + ErtsThrQDirtySetEl(&q->head.head, head); } - res = q->head.head.ptr->data.ptr; + res = head->data.ptr; #if ERTS_THR_Q_DBG_CHK_DATA - q->head.head.ptr->data.ptr = NULL; + head->data.ptr = NULL; if (!res) erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); #endif diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h index edcf2c3823..13af758b3f 100644 --- a/erts/emulator/beam/erl_thr_queue.h +++ b/erts/emulator/beam/erl_thr_queue.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -76,13 +76,12 @@ typedef struct { typedef struct ErtsThrQElement_t_ ErtsThrQElement_t; typedef struct ErtsThrQElement_t ErtsThrQPrepEnQ_t; -typedef union { - erts_atomic_t atmc; - ErtsThrQElement_t *ptr; -} ErtsThrQPtr_t; - struct ErtsThrQElement_t_ { - ErtsThrQPtr_t next; +#ifdef USE_THREADS + erts_atomic_t next; +#else + ErtsThrQElement_t *next; +#endif union { erts_atomic_t atmc; void *ptr; @@ -130,7 +129,7 @@ struct ErtsThrQ_t_ { * thread dequeuing. */ struct { - ErtsThrQPtr_t head; + erts_atomic_t head; ErtsThrQLive_t live; ErtsThrQElement_t *first; ErtsThrQElement_t *unref_end; diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index ee47c98009..759c8f4c33 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2012. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -258,10 +258,6 @@ #include "sys.h" -typedef struct { SWord sint[2]; } erts_no_dw_atomic_t; -typedef SWord erts_no_atomic_t; -typedef Sint32 erts_no_atomic32_t; - #ifdef USE_THREADS #define ETHR_TRY_INLINE_FUNCS @@ -411,12 +407,15 @@ typedef struct { typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; typedef int erts_tse_t; -#define erts_dw_aint_t erts_no_dw_atomic_t -#define erts_dw_atomic_t erts_no_dw_atomic_t -#define erts_aint_t SWord -#define erts_atomic_t erts_no_atomic_t -#define erts_aint32_t Sint32 -#define erts_atomic32_t erts_no_atomic32_t + +typedef struct { SWord sint[2]; } erts_dw_aint_t; +typedef SWord erts_aint_t; +typedef Sint32 erts_aint32_t; + +#define erts_dw_atomic_t erts_dw_aint_t +#define erts_atomic_t erts_aint_t +#define erts_atomic32_t erts_aint32_t + #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -425,6 +424,14 @@ typedef struct { int gcc_is_buggy; } erts_spinlock_t; typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif +#ifdef WORDS_BIGENDIAN +#define ERTS_DW_AINT_LOW_WORD 1 +#define ERTS_DW_AINT_HIGH_WORD 0 +#else +#define ERTS_DW_AINT_LOW_WORD 0 +#define ERTS_DW_AINT_HIGH_WORD 1 +#endif + #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -433,6 +440,10 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif /* #ifdef USE_THREADS */ +#define erts_no_dw_atomic_t erts_dw_aint_t +#define erts_no_atomic_t erts_aint_t +#define erts_no_atomic32_t erts_aint32_t + #define ERTS_AINT_NULL ((erts_aint_t) NULL) #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) @@ -522,6 +533,9 @@ ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp, ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i); ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var); @@ -542,6 +556,9 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, erts_aint32_t new, erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, @@ -601,6 +618,91 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #ifdef USE_THREADS +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_nob(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_ddrb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_rb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_wb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_acqb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_relb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_mb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_nob(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_ddrb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_rb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_wb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_acqb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_relb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_mb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +#define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \ +do { \ + Type act = ReadOp((VarP)); \ + while (1) { \ + Type exp = act; \ + Type new = exp & ~(Mask); \ + new |= ((Mask) & (Set)); \ + act = CmpxchgOp((VarP), new, exp); \ + if (act == exp) \ + return act; \ + } \ +} while (0) +#endif + +ERTS_GLB_INLINE void +erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val); +ERTS_GLB_INLINE void +erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val); +ERTS_GLB_INLINE void +erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_dirty(erts_atomic_t *var); +ERTS_GLB_INLINE void +erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_dirty(erts_atomic32_t *var); + /* * See "Documentation of atomics and memory barriers" at the top * of this file for info on atomics. @@ -643,6 +745,26 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb #define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val) +{ + ethr_sint_t *sint = ethr_dw_atomic_addr(var); + sint[0] = val->sint[0]; + sint[1] = val->sint[1]; +} + +ERTS_GLB_INLINE void +erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val) +{ + ethr_sint_t *sint = ethr_dw_atomic_addr(var); + val->sint[0] = sint[0]; + val->sint[1] = sint[1]; +} + +#endif + /* Word size atomics */ #define erts_atomic_init_nob ethr_atomic_init @@ -659,6 +781,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_nob ethr_atomic_xchg #define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_nob(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg, + var, mask, set); +} +#endif + #define erts_atomic_init_mb ethr_atomic_init_mb #define erts_atomic_set_mb ethr_atomic_set_mb #define erts_atomic_read_mb ethr_atomic_read_mb @@ -673,6 +808,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_mb ethr_atomic_xchg_mb #define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_mb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_mb, + var, mask, set); +} +#endif + #define erts_atomic_init_acqb ethr_atomic_init_acqb #define erts_atomic_set_acqb ethr_atomic_set_acqb #define erts_atomic_read_acqb ethr_atomic_read_acqb @@ -687,6 +835,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb #define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_acqb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_acqb, + var, mask, set); +} +#endif + #define erts_atomic_init_relb ethr_atomic_init_relb #define erts_atomic_set_relb ethr_atomic_set_relb #define erts_atomic_read_relb ethr_atomic_read_relb @@ -701,6 +862,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb ethr_atomic_xchg_relb #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_relb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_relb, + var, mask, set); +} +#endif + #define erts_atomic_init_ddrb ethr_atomic_init_ddrb #define erts_atomic_set_ddrb ethr_atomic_set_ddrb #define erts_atomic_read_ddrb ethr_atomic_read_ddrb @@ -715,6 +889,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb #define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_ddrb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_ddrb, + var, mask, set); +} +#endif + #define erts_atomic_init_rb ethr_atomic_init_rb #define erts_atomic_set_rb ethr_atomic_set_rb #define erts_atomic_read_rb ethr_atomic_read_rb @@ -729,6 +916,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_rb ethr_atomic_xchg_rb #define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_rb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_rb, + var, mask, set); +} +#endif + #define erts_atomic_init_wb ethr_atomic_init_wb #define erts_atomic_set_wb ethr_atomic_set_wb #define erts_atomic_read_wb ethr_atomic_read_wb @@ -743,6 +943,39 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_wb ethr_atomic_xchg_wb #define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_wb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_wb, + var, mask, set); +} + +#endif + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val) +{ + ethr_sint_t *sint = ethr_atomic_addr(var); + *sint = val; +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_dirty(erts_atomic_t *var) +{ + ethr_sint_t *sint = ethr_atomic_addr(var); + return *sint; +} + +#endif + /* 32-bit atomics */ #define erts_atomic32_init_nob ethr_atomic32_init @@ -759,6 +992,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_nob ethr_atomic32_xchg #define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_nob(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg, + var, mask, set); +} +#endif + #define erts_atomic32_init_mb ethr_atomic32_init_mb #define erts_atomic32_set_mb ethr_atomic32_set_mb #define erts_atomic32_read_mb ethr_atomic32_read_mb @@ -773,6 +1019,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb #define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_mb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_mb, + var, mask, set); +} +#endif + #define erts_atomic32_init_acqb ethr_atomic32_init_acqb #define erts_atomic32_set_acqb ethr_atomic32_set_acqb #define erts_atomic32_read_acqb ethr_atomic32_read_acqb @@ -787,6 +1046,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb #define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_acqb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_acqb, + var, mask, set); +} +#endif + #define erts_atomic32_init_relb ethr_atomic32_init_relb #define erts_atomic32_set_relb ethr_atomic32_set_relb #define erts_atomic32_read_relb ethr_atomic32_read_relb @@ -801,6 +1073,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_relb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_relb, + var, mask, set); +} +#endif + #define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb #define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb #define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb @@ -815,6 +1100,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb #define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_ddrb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_ddrb, + var, mask, set); +} +#endif + #define erts_atomic32_init_rb ethr_atomic32_init_rb #define erts_atomic32_set_rb ethr_atomic32_set_rb #define erts_atomic32_read_rb ethr_atomic32_read_rb @@ -829,6 +1127,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb #define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_rb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_rb, + var, mask, set); +} +#endif + #define erts_atomic32_init_wb ethr_atomic32_init_wb #define erts_atomic32_set_wb ethr_atomic32_set_wb #define erts_atomic32_read_wb ethr_atomic32_read_wb @@ -843,6 +1154,41 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb #define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_wb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_wb, + var, mask, set); +} + +#endif + +#undef ERTS_ATOMIC_BSET_IMPL__ + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val) +{ + ethr_sint32_t *sint = ethr_atomic32_addr(var); + *sint = val; +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_dirty(erts_atomic32_t *var) +{ + ethr_sint32_t *sint = ethr_atomic32_addr(var); + return *sint; +} + +#endif + #else /* !USE_THREADS */ /* Double word size atomics */ @@ -882,6 +1228,9 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_wb erts_no_dw_atomic_read #define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg +#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set +#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read + /* Word size atomics */ #define erts_atomic_init_nob erts_no_atomic_set @@ -897,6 +1246,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_nob erts_no_atomic_read_band #define erts_atomic_xchg_nob erts_no_atomic_xchg #define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_nob erts_no_atomic_read_bset #define erts_atomic_init_mb erts_no_atomic_set #define erts_atomic_set_mb erts_no_atomic_set @@ -911,6 +1261,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_mb erts_no_atomic_read_band #define erts_atomic_xchg_mb erts_no_atomic_xchg #define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_mb erts_no_atomic_read_bset #define erts_atomic_init_acqb erts_no_atomic_set #define erts_atomic_set_acqb erts_no_atomic_set @@ -925,6 +1276,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_acqb erts_no_atomic_read_band #define erts_atomic_xchg_acqb erts_no_atomic_xchg #define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset #define erts_atomic_init_relb erts_no_atomic_set #define erts_atomic_set_relb erts_no_atomic_set @@ -939,6 +1291,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_relb erts_no_atomic_read_band #define erts_atomic_xchg_relb erts_no_atomic_xchg #define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_relb erts_no_atomic_read_bset #define erts_atomic_init_ddrb erts_no_atomic_set #define erts_atomic_set_ddrb erts_no_atomic_set @@ -953,6 +1306,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_ddrb erts_no_atomic_read_band #define erts_atomic_xchg_ddrb erts_no_atomic_xchg #define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset #define erts_atomic_init_rb erts_no_atomic_set #define erts_atomic_set_rb erts_no_atomic_set @@ -967,6 +1321,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_rb erts_no_atomic_read_band #define erts_atomic_xchg_rb erts_no_atomic_xchg #define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_rb erts_no_atomic_read_bset #define erts_atomic_init_wb erts_no_atomic_set #define erts_atomic_set_wb erts_no_atomic_set @@ -981,6 +1336,10 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_wb erts_no_atomic_read_band #define erts_atomic_xchg_wb erts_no_atomic_xchg #define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_wb erts_no_atomic_read_bset + +#define erts_atomic_set_dirty erts_no_atomic_set +#define erts_atomic_read_dirty erts_no_atomic_read /* 32-bit atomics */ @@ -997,6 +1356,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_nob erts_no_atomic32_read_band #define erts_atomic32_xchg_nob erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset #define erts_atomic32_init_mb erts_no_atomic32_set #define erts_atomic32_set_mb erts_no_atomic32_set @@ -1011,6 +1371,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_mb erts_no_atomic32_read_band #define erts_atomic32_xchg_mb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset #define erts_atomic32_init_acqb erts_no_atomic32_set #define erts_atomic32_set_acqb erts_no_atomic32_set @@ -1025,6 +1386,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_acqb erts_no_atomic32_read_band #define erts_atomic32_xchg_acqb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset #define erts_atomic32_init_relb erts_no_atomic32_set #define erts_atomic32_set_relb erts_no_atomic32_set @@ -1039,6 +1401,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_relb erts_no_atomic32_read_band #define erts_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset #define erts_atomic32_init_ddrb erts_no_atomic32_set #define erts_atomic32_set_ddrb erts_no_atomic32_set @@ -1053,6 +1416,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band #define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset #define erts_atomic32_init_rb erts_no_atomic32_set #define erts_atomic32_set_rb erts_no_atomic32_set @@ -1067,6 +1431,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_rb erts_no_atomic32_read_band #define erts_atomic32_xchg_rb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset #define erts_atomic32_init_wb erts_no_atomic32_set #define erts_atomic32_set_wb erts_no_atomic32_set @@ -1081,6 +1446,10 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_wb erts_no_atomic32_read_band #define erts_atomic32_xchg_wb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset + +#define erts_atomic32_set_dirty erts_no_atomic32_set +#define erts_atomic32_read_dirty erts_no_atomic32_read #endif /* !USE_THREADS */ @@ -1845,6 +2214,17 @@ erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, return old; } +ERTS_GLB_INLINE erts_aint_t +erts_no_atomic_read_bset(erts_no_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + erts_aint_t old = *var; + *var &= ~mask; + *var |= (mask & set); + return old; +} + /* atomic32 */ ERTS_GLB_INLINE void @@ -1932,6 +2312,17 @@ erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, return old; } +ERTS_GLB_INLINE erts_aint32_t +erts_no_atomic32_read_bset(erts_no_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + erts_aint32_t old = *var; + *var &= ~mask; + *var |= (mask & set); + return old; +} + /* spinlock */ ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 6c6e193818..4bbdcaa3e3 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -118,9 +118,11 @@ ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif +typedef UWord erts_approx_time_t; +erts_approx_time_t erts_get_approx_time(void); + void erts_get_timeval(SysTimeval *tv); erts_time_t erts_get_time(void); -void erts_get_emu_time(SysTimeval *); ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 04147408d5..3272a5326d 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -91,6 +91,41 @@ static SysTimeval then; /* Used in get_now */ static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */ SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */ +union { + erts_smp_atomic_t time; + char align[ERTS_CACHE_LINE_SIZE]; +} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static void +init_approx_time(void) +{ + erts_smp_atomic_init_nob(&approx.time, 0); +} + +static ERTS_INLINE erts_approx_time_t +get_approx_time(void) +{ + return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time); +} + +static ERTS_INLINE void +update_approx_time(SysTimeval *tv) +{ + erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec; + erts_approx_time_t old_secs = get_approx_time(); + if (old_secs != new_secs) + erts_smp_atomic_set_nob(&approx.time, new_secs); +} + +/* + * erts_get_approx_time() returns an *approximate* time + * in seconds. NOTE that this time may jump backwards!!! + */ +erts_approx_time_t +erts_get_approx_time(void) +{ + return get_approx_time(); +} #ifdef HAVE_GETHRTIME @@ -398,6 +433,8 @@ erts_init_time_sup(void) { erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + init_approx_time(); + last_emu_time.tv_sec = 0; last_emu_time.tv_usec = 0; @@ -417,7 +454,7 @@ erts_init_time_sup(void) gtv = inittv; then.tv_sec = then.tv_usec = 0; - erts_get_emu_time(&erts_first_emu_time); + erts_deliver_time(); return CLOCK_RESOLUTION; } @@ -883,6 +920,8 @@ get_now(Uint* megasec, Uint* sec, Uint* microsec) *megasec = (Uint) (now.tv_sec / 1000000); *sec = (Uint) (now.tv_sec % 1000000); *microsec = (Uint) (now.tv_usec); + + update_approx_time(&now); } void @@ -895,6 +934,8 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) *megasec = (Uint) (now.tv_sec / 1000000); *sec = (Uint) (now.tv_sec % 1000000); *microsec = (Uint) (now.tv_usec); + + update_approx_time(&now); } @@ -911,6 +952,8 @@ void erts_deliver_time(void) { do_erts_deliver_time(&now); erts_smp_mtx_unlock(&erts_timeofday_mtx); + + update_approx_time(&now); } /* get *real* time (not ticks) remaining until next timeout - if there @@ -959,6 +1002,7 @@ void erts_get_timeval(SysTimeval *tv) erts_smp_mtx_lock(&erts_timeofday_mtx); get_tolerant_timeofday(tv); erts_smp_mtx_unlock(&erts_timeofday_mtx); + update_approx_time(tv); } erts_time_t @@ -971,7 +1015,9 @@ erts_get_time(void) get_tolerant_timeofday(&sys_tv); erts_smp_mtx_unlock(&erts_timeofday_mtx); - + + update_approx_time(&sys_tv); + return sys_tv.tv_sec; } @@ -987,38 +1033,3 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { *sec = (Uint)(tp.tv_sec % 1000000); } #endif - - -/* - * erts_get_emu_time() is similar to get_now(). You will - * always get different times from erts_get_emu_time(), but they - * may equal a time from get_now(). - * - * erts_get_emu_time() is only used internally in the emulator in - * order to order emulator internal events. - */ - -void -erts_get_emu_time(SysTimeval *this_emu_time_p) -{ - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(this_emu_time_p); - - /* Make sure time is later than last */ - if (last_emu_time.tv_sec > this_emu_time_p->tv_sec || - (last_emu_time.tv_sec == this_emu_time_p->tv_sec - && last_emu_time.tv_usec >= this_emu_time_p->tv_usec)) { - *this_emu_time_p = last_emu_time; - this_emu_time_p->tv_usec++; - } - /* Check for carry from above + general reasonability */ - if (this_emu_time_p->tv_usec >= 1000000) { - this_emu_time_p->tv_usec = 0; - this_emu_time_p->tv_sec++; - } - - last_emu_time = *this_emu_time_p; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); -} diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index dc8b4bbe6e..fa015ee4b9 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,9 +44,9 @@ #undef DEBUG_PRINTOUTS #endif -extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ -extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ +extern BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +extern BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* Pseudo export entries. Never filled in with data, only used to yield unique pointers of the correct type. */ @@ -64,7 +64,7 @@ int erts_cpu_timestamp; #endif static erts_smp_mtx_t smq_mtx; -static erts_smp_mtx_t sys_trace_mtx; +static erts_smp_rwmtx_t sys_trace_rwmtx; enum ErtsSysMsgType { SYS_MSG_TYPE_UNDEFINED, @@ -91,7 +91,12 @@ static void init_sys_msg_dispatcher(void); #endif void erts_init_trace(void) { - erts_smp_mtx_init(&sys_trace_mtx, "sys_tracers"); + erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers"); + #ifdef HAVE_ERTS_NOW_CPU erts_cpu_timestamp = 0; #endif @@ -151,8 +156,8 @@ do { (RES) = (TPID); } while(0) #define ERTS_TRACER_REF_TYPE Process * #define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \ do { \ - (RES) = process_tab[internal_pid_index((TPID))]; \ - if (INVALID_PID((RES), (TPID)) || !((RES)->trace_flags & F_TRACER)) { \ + (RES) = erts_proc_lookup((TPID)); \ + if (!(RES) || !(ERTS_TRACE_FLAGS((RES)) & F_TRACER)) { \ (TPID) = NIL; \ (TRACEE_FLGS) &= ~TRACEE_FLAGS; \ return; \ @@ -169,10 +174,10 @@ erts_system_profile_setup_active_schedulers(void) active_sched = erts_active_schedulers(); } -void -erts_trace_check_exiting(Eterm exiting) +static void +exiting_reset(Eterm exiting) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); if (exiting == default_tracer) { default_tracer = NIL; default_trace_flags &= TRACEE_FLAGS; @@ -202,29 +207,49 @@ erts_trace_check_exiting(Eterm exiting) erts_system_profile_clear(NULL); #endif } - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); +} + +void +erts_trace_check_exiting(Eterm exiting) +{ + int reset = 0; + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + if (exiting == default_tracer) + reset = 1; + else if (exiting == system_seq_tracer) + reset = 1; + else if (exiting == system_monitor) + reset = 1; + else if (exiting == system_profile) + reset = 1; + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + if (reset) + exiting_reset(exiting); +} + +static ERTS_INLINE int +is_valid_tracer(Eterm tracer) +{ + return erts_proc_lookup(tracer) || erts_is_valid_tracer_port(tracer); } Eterm erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, Eterm new) { - Eterm old = THE_NON_VALUE; + Eterm old; - if (new != am_false) { - if (!erts_pid2proc(c_p, c_p_locks, new, 0) - && !erts_is_valid_tracer_port(new)) { - return old; - } - } + if (new != am_false && !is_valid_tracer(new)) + return THE_NON_VALUE; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); old = system_seq_tracer; system_seq_tracer = new; #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old); #endif - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); return old; } @@ -232,12 +257,12 @@ Eterm erts_get_system_seq_tracer(void) { Eterm st; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); st = system_seq_tracer; #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "get seq tracer %T\n", st); #endif - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); return st; } @@ -250,7 +275,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp) if (is_nil(default_tracer)) { default_trace_flags &= ~TRACEE_FLAGS; } else if (is_internal_pid(default_tracer)) { - if (!erts_pid2proc(NULL, 0, default_tracer, 0)) { + if (!erts_proc_lookup(default_tracer)) { reset_tracer: default_trace_flags &= ~TRACEE_FLAGS; default_tracer = NIL; @@ -270,7 +295,7 @@ get_default_tracing(Uint *flagsp, Eterm *tracerp) void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); if (flagsp) { if (setflags) default_trace_flags |= *flagsp; @@ -280,48 +305,48 @@ erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp) if (tracerp) default_tracer = *tracerp; get_default_tracing(flagsp, tracerp); - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); } void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); get_default_tracing(flagsp, tracerp); - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); } void erts_set_system_monitor(Eterm monitor) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); system_monitor = monitor; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); } Eterm erts_get_system_monitor(void) { Eterm monitor; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); monitor = system_monitor; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); return monitor; } /* Performance monitoring */ void erts_set_system_profile(Eterm profile) { - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); system_profile = profile; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); } Eterm erts_get_system_profile(void) { Eterm profile; - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rlock(&sys_trace_rwmtx); profile = system_profile; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_runlock(&sys_trace_rwmtx); return profile; } @@ -384,13 +409,9 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, } #ifndef ERTS_SMP - if (!INVALID_TRACER_PORT(trace_port, trace_port->id)) { + if (!INVALID_TRACER_PORT(trace_port, trace_port->common.id)) #endif erts_raw_port_command(trace_port, buffer, ptr-buffer); -#ifndef ERTS_SMP - erts_port_release(trace_port); - } -#endif erts_free(ERTS_ALC_T_TMP, (void *) buffer); } @@ -420,7 +441,7 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { message = TUPLE5(hp, am_trace_ts, pid, am_out, mfarity, timestamp); /* Note, hp is deliberately NOT incremented since it will be reused */ - do_send_to_port(trace_port->id, + do_send_to_port(trace_port->common.id, trace_port, pid, SYS_MSG_TYPE_UNDEFINED, @@ -430,7 +451,7 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { hp += 5; hp = patch_ts(message, hp); - do_send_to_port(trace_port->id, + do_send_to_port(trace_port->common.id, trace_port, pid, SYS_MSG_TYPE_UNDEFINED, @@ -465,13 +486,13 @@ send_to_port(Process *c_p, Eterm message, trace_port = NULL; #else - if (is_not_internal_port(*tracer_pid)) - goto invalid_tracer_port; - trace_port = &erts_port[internal_port_index(*tracer_pid)]; + trace_port = erts_id2port_sflgs(*tracer_pid, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - if (INVALID_TRACER_PORT(trace_port, *tracer_pid)) { - invalid_tracer_port: + if (!trace_port) { *tracee_flags &= ~TRACEE_FLAGS; *tracer_pid = NIL; return; @@ -487,10 +508,11 @@ send_to_port(Process *c_p, Eterm message, #endif do_send_to_port(*tracer_pid, trace_port, - c_p ? c_p->id : NIL, + c_p ? c_p->common.id : NIL, SYS_MSG_TYPE_TRACE, message); #ifndef ERTS_SMP + erts_port_release(trace_port); return; } @@ -521,7 +543,7 @@ send_to_port(Process *c_p, Eterm message, trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(*tracer_pid, trace_port, - c_p ? c_p->id : NIL, + c_p ? c_p->common.id : NIL, SYS_MSG_TYPE_TRACE, message); @@ -535,8 +557,11 @@ send_to_port(Process *c_p, Eterm message, * just after writning the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts); } + + erts_port_release(trace_port); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #endif @@ -566,23 +591,27 @@ profile_send(Eterm from, Eterm message) { Port *profiler_port = NULL; /* not smp */ - - - profiler_port = &erts_port[internal_port_index(profiler)]; - - do_send_to_port(profiler, - profiler_port, - NIL, /* or current process->id */ - SYS_MSG_TYPE_SYSPROF, - message); + + profiler_port = erts_id2port_sflgs(profiler, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); + if (profiler_port) { + do_send_to_port(profiler, + profiler_port, + NIL, /* or current process->common.id */ + SYS_MSG_TYPE_SYSPROF, + message); + erts_port_release(profiler_port); + } } else { - ASSERT(is_internal_pid(profiler) - && internal_pid_index(profiler) < erts_max_processes); + ASSERT(is_internal_pid(profiler)); - profile_p = process_tab[internal_pid_index(profiler)]; + profile_p = erts_proc_lookup(profiler); - if (INVALID_PID(profile_p, profiler)) return; + if (!profile_p) + return; sz = size_object(message); hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); @@ -626,13 +655,11 @@ seq_trace_send_to_port(Process *c_p, trace_port = NULL; #else - if (is_not_internal_port(seq_tracer)) - goto invalid_tracer_port; - - trace_port = &erts_port[internal_port_index(seq_tracer)]; - - if (INVALID_TRACER_PORT(trace_port, seq_tracer)) { - invalid_tracer_port: + trace_port = erts_id2port_sflgs(seq_tracer, + NULL, + 0, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); + if (!trace_port) { system_seq_tracer = am_false; #ifndef ERTS_SMP UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -645,11 +672,12 @@ seq_trace_send_to_port(Process *c_p, #endif do_send_to_port(seq_tracer, trace_port, - c_p ? c_p->id : NIL, + c_p ? c_p->common.id : NIL, SYS_MSG_TYPE_SEQTRACE, message); #ifndef ERTS_SMP + erts_port_release(trace_port); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); return; } @@ -675,7 +703,7 @@ seq_trace_send_to_port(Process *c_p, trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(seq_tracer, trace_port, - c_p ? c_p->id : NIL, + c_p ? c_p->common.id : NIL, SYS_MSG_TYPE_SEQTRACE, message); @@ -689,15 +717,20 @@ seq_trace_send_to_port(Process *c_p, * just after writing the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts); } + + erts_port_release(trace_port); + UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #endif } #define TS_HEAP_WORDS 5 -#define TS_SIZE(p) (((p)->trace_flags & F_TIMESTAMP) ? TS_HEAP_WORDS : 0) +#define TS_SIZE(p) ((ERTS_TRACE_FLAGS((p)) & F_TIMESTAMP) \ + ? TS_HEAP_WORDS \ + : 0) /* * Patch a timestamp into a tuple. The tuple must be the last thing @@ -732,17 +765,17 @@ send_to_tracer(Process *tracee, erts_smp_mtx_lock(&smq_mtx); - if (tracee->trace_flags & F_TIMESTAMP) + if (ERTS_TRACE_FLAGS(tracee) & F_TIMESTAMP) *hpp = patch_ts(msg, *hpp); - if (is_internal_pid(tracee->tracer_proc)) - ERTS_ENQ_TRACE_MSG(tracee->id, tracer_ref, msg, bp); + if (is_internal_pid(ERTS_TRACER_PROC(tracee))) + ERTS_ENQ_TRACE_MSG(tracee->common.id, tracer_ref, msg, bp); else { - ASSERT(is_internal_port(tracee->tracer_proc)); + ASSERT(is_internal_port(ERTS_TRACER_PROC(tracee))); send_to_port(no_fake_sched ? NULL : tracee, msg, - &tracee->tracer_proc, - &tracee->trace_flags); + &ERTS_TRACER_PROC(tracee), + &ERTS_TRACE_FLAGS(tracee)); } erts_smp_mtx_unlock(&smq_mtx); @@ -760,7 +793,7 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF; int sched_no, curr_func, to_port, no_fake_sched; - if (is_nil(p->tracer_proc)) + if (is_nil(ERTS_TRACER_PROC(p))) return; no_fake_sched = never_fake_sched; @@ -780,22 +813,18 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) } sched_no = IS_TRACED_FL(p, F_TRACE_SCHED_NO); - to_port = is_internal_port(p->tracer_proc); + to_port = is_internal_port(ERTS_TRACER_PROC(p)); if (!to_port) { - ASSERT(is_internal_pid(p->tracer_proc) - && internal_pid_index(p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(p), + ERTS_TRACE_FLAGS(p)); } - if (ERTS_PROC_IS_EXITING(p) -#ifndef ERTS_SMP - || p->status == P_FREE -#endif - ) { + if (ERTS_PROC_IS_EXITING(p)) curr_func = 0; - } else { if (!p->current) p->current = find_function_from_pc(p->i); @@ -824,7 +853,7 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) } if (!sched_no) { - mess = TUPLE4(hp, am_trace, p->id, what, tmp); + mess = TUPLE4(hp, am_trace, p->common.id, what, tmp); hp += 5; } else { @@ -833,7 +862,7 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) #else Eterm sched_id = make_small(1); #endif - mess = TUPLE5(hp, am_trace, p->id, what, sched_id, tmp); + mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, tmp); hp += 6; } @@ -874,7 +903,7 @@ trace_send(Process *p, Eterm to, Eterm msg) operation = am_send; if (is_internal_pid(to)) { - if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to, 0)) + if (!erts_proc_lookup(to)) goto send_to_non_existing_process; } else if(is_external_pid(to) @@ -885,19 +914,19 @@ trace_send(Process *p, Eterm to, Eterm msg) operation = am_atom_put(s, sys_strlen(s)); } - if (is_internal_port(p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(p))) { #define LOCAL_HEAP_SIZE (11) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; - mess = TUPLE5(hp, am_trace, p->id, operation, msg, to); + mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - send_to_port(p, mess, &p->tracer_proc, &p->trace_flags); + send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); @@ -907,10 +936,11 @@ trace_send(Process *p, Eterm to, Eterm msg) ErlOffHeap *off_heap; ERTS_TRACER_REF_TYPE tracer_ref; - ASSERT(is_internal_pid(p->tracer_proc) - && internal_pid_index(p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(p), + ERTS_TRACE_FLAGS(p)); sz_msg = size_object(msg); sz_to = size_object(to); @@ -926,16 +956,16 @@ trace_send(Process *p, Eterm to, Eterm msg) sz_msg, &hp, off_heap); - mess = TUPLE5(hp, am_trace, p->id/* Local pid */, operation, msg, to); + mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { patch_ts(mess, hp); } - ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } } @@ -950,19 +980,19 @@ trace_receive(Process *rp, Eterm msg) size_t sz_msg; Eterm* hp; - if (is_internal_port(rp->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(rp))) { #define LOCAL_HEAP_SIZE (10) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; - mess = TUPLE4(hp, am_trace, rp->id, am_receive, msg); + mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (rp->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - send_to_port(rp, mess, &rp->tracer_proc, &rp->trace_flags); + send_to_port(rp, mess, &ERTS_TRACER_PROC(rp), &ERTS_TRACE_FLAGS(rp)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); @@ -972,10 +1002,11 @@ trace_receive(Process *rp, Eterm msg) ErlOffHeap *off_heap; ERTS_TRACER_REF_TYPE tracer_ref; - ASSERT(is_internal_pid(rp->tracer_proc) - && internal_pid_index(rp->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(rp))); - ERTS_GET_TRACER_REF(tracer_ref, rp->tracer_proc, rp->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(rp), + ERTS_TRACE_FLAGS(rp)); sz_msg = size_object(msg); @@ -984,16 +1015,16 @@ trace_receive(Process *rp, Eterm msg) hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref); msg = copy_struct(msg, sz_msg, &hp, off_heap); - mess = TUPLE4(hp, am_trace, rp->id/* Local pid */, am_receive, msg); + mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (rp->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { patch_ts(mess, hp); } - ERTS_ENQ_TRACE_MSG(rp->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(rp->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } } @@ -1003,14 +1034,14 @@ seq_trace_update_send(Process *p) { Eterm seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); - if ( (p->id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL) + if ( (p->common.id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL) #ifdef USE_VM_PROBES || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) #endif ) { return 0; } - SEQ_TRACE_TOKEN_SENDER(p) = p->id; /* Internal pid */ + SEQ_TRACE_TOKEN_SENDER(p) = p->common.id; SEQ_TRACE_TOKEN_SERIAL(p) = make_small(++(p -> seq_trace_clock)); SEQ_TRACE_TOKEN_LASTCNT(p) = @@ -1047,7 +1078,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, ASSERT(is_tuple(token) || is_nil(token)); if (SEQ_TRACE_T_SENDER(token) == seq_tracer || token == NIL || - (process && process->trace_flags & F_SENSITIVE)) { + (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE)) { return; } @@ -1111,13 +1142,12 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, Uint sz_label, sz_lastcnt_serial, sz_msg, sz_ts, sz_sender, sz_exitfrom, sz_receiver; - ASSERT(is_internal_pid(seq_tracer) - && internal_pid_index(seq_tracer) < erts_max_processes); + ASSERT(is_internal_pid(seq_tracer)); #ifndef ERTS_SMP - tracer = process_tab[internal_pid_index(seq_tracer)]; - if (INVALID_PID(tracer, tracer->id)) { + tracer = erts_proc_lookup(seq_tracer); + if (!tracer) { system_seq_tracer = am_false; return; /* no need to send anything */ } @@ -1226,17 +1256,17 @@ erts_trace_return_to(Process *p, BeamInstr *pc) hp += 4; } - mess = TUPLE4(hp, am_trace, p->id, am_return_to, mfa); + mess = TUPLE4(hp, am_trace, p->common.id, am_return_to, mfa); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - if (is_internal_port(p->tracer_proc)) { - send_to_port(p, mess, &p->tracer_proc, &p->trace_flags); + if (is_internal_port(ERTS_TRACER_PROC(p))) { + send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); } else { ErlHeapFragment *bp; ErlOffHeap *off_heap; @@ -1246,10 +1276,11 @@ erts_trace_return_to(Process *p, BeamInstr *pc) /* * Find the tracer. */ - ASSERT(is_internal_pid(p->tracer_proc) - && internal_pid_index(p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(p), + ERTS_TRACE_FLAGS(p)); size = size_object(mess); @@ -1259,7 +1290,7 @@ erts_trace_return_to(Process *p, BeamInstr *pc) * Copy the trace message into the buffer and enqueue it. */ mess = copy_struct(mess, size, &hp, off_heap); - ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); } UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1288,25 +1319,25 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) /* Breakpoint trace enabled without specifying tracer => * use process tracer and flags */ - tracer_pid = &p->tracer_proc; + tracer_pid = &ERTS_TRACER_PROC(p); } if (is_nil(*tracer_pid)) { /* Trace disabled */ return; } ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid)); - if (*tracer_pid == p->id) { + if (*tracer_pid == p->common.id) { /* Do not generate trace messages to oneself */ return; } - if (tracer_pid == &p->tracer_proc) { + if (tracer_pid == &ERTS_TRACER_PROC(p)) { /* Tracer specified in process structure => * non-breakpoint trace => * use process flags */ - tracee_flags = &p->trace_flags; + tracee_flags = &ERTS_TRACE_FLAGS(p); #ifdef ERTS_SMP - tracee = p->id; + tracee = p->common.id; #endif } else { /* Tracer not specified in process structure => @@ -1335,7 +1366,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) hp = local_heap; mfa = TUPLE3(hp, mod, name, make_small(arity)); hp += 4; - mess = TUPLE5(hp, am_trace, p->id, am_return_from, mfa, retval); + mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval); hp += 6; erts_smp_mtx_lock(&smq_mtx); if (*tracee_flags & F_TIMESTAMP) { @@ -1355,8 +1386,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) Eterm* limit; #endif - ASSERT(is_internal_pid(*tracer_pid) - && internal_pid_index(*tracer_pid) < erts_max_processes); + ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); @@ -1378,7 +1408,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) mfa = TUPLE3(hp, mod, name, make_small(arity)); hp += 4; retval = copy_struct(retval, retval_size, &hp, off_heap); - mess = TUPLE5(hp, am_trace, p->id/* Local pid */, am_return_from, mfa, retval); + mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval); hp += 6; erts_smp_mtx_lock(&smq_mtx); @@ -1419,25 +1449,25 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, /* Breakpoint trace enabled without specifying tracer => * use process tracer and flags */ - tracer_pid = &p->tracer_proc; + tracer_pid = &ERTS_TRACER_PROC(p); } if (is_nil(*tracer_pid)) { /* Trace disabled */ return; } ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid)); - if (*tracer_pid == p->id) { + if (*tracer_pid == p->common.id) { /* Do not generate trace messages to oneself */ return; } - if (tracer_pid == &p->tracer_proc) { + if (tracer_pid == &ERTS_TRACER_PROC(p)) { /* Tracer specified in process structure => * non-breakpoint trace => * use process flags */ - tracee_flags = &p->trace_flags; + tracee_flags = &ERTS_TRACE_FLAGS(p); #ifdef ERTS_SMP - tracee = p->id; + tracee = p->common.id; #endif if (! (*tracee_flags & F_TRACE_CALLS)) { return; @@ -1465,7 +1495,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, hp += 4; cv = TUPLE2(hp, class, value); hp += 3; - mess = TUPLE5(hp, am_trace, p->id, am_exception_from, mfa_tuple, cv); + mess = TUPLE5(hp, am_trace, p->common.id, am_exception_from, mfa_tuple, cv); hp += 6; ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE); erts_smp_mtx_lock(&smq_mtx); @@ -1487,8 +1517,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, Eterm* limit; #endif - ASSERT(is_internal_pid(*tracer_pid) - && internal_pid_index(*tracer_pid) < erts_max_processes); + ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); @@ -1512,7 +1541,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, value = copy_struct(value, value_size, &hp, off_heap); cv = TUPLE2(hp, class, value); hp += 3; - mess = TUPLE5(hp, am_trace, p->id/* Local pid */, + mess = TUPLE5(hp, am_trace, p->common.id, am_exception_from, mfa_tuple, cv); hp += 6; @@ -1566,25 +1595,25 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* Breakpoint trace enabled without specifying tracer => * use process tracer and flags */ - tracer_pid = &p->tracer_proc; + tracer_pid = &ERTS_TRACER_PROC(p); } if (is_nil(*tracer_pid)) { /* Trace disabled */ return 0; } ASSERT(is_internal_pid(*tracer_pid) || is_internal_port(*tracer_pid)); - if (*tracer_pid == p->id) { + if (*tracer_pid == p->common.id) { /* Do not generate trace messages to oneself */ return 0; } - if (tracer_pid == &p->tracer_proc) { + if (tracer_pid == &ERTS_TRACER_PROC(p)) { /* Tracer specified in process structure => * non-breakpoint trace => * use process flags */ - tracee_flags = &p->trace_flags; + tracee_flags = &ERTS_TRACE_FLAGS(p); #ifdef ERTS_SMP - tracee = p->id; + tracee = p->common.id; #endif } else { /* Tracer not specified in process structure => @@ -1592,7 +1621,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * meta trace => * use fixed flag set instead of process flags */ - if (p->trace_flags & F_SENSITIVE) { + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) { /* No trace messages for sensitive processes. */ return 0; } @@ -1650,7 +1679,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, if (!erts_is_valid_tracer_port(*tracer_pid)) { #ifdef ERTS_SMP - ASSERT(is_nil(tracee) || tracer_pid == &p->tracer_proc); + ASSERT(is_nil(tracee) || tracer_pid == &ERTS_TRACER_PROC(p)); if (is_not_nil(tracee)) erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif @@ -1752,7 +1781,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * Build the trace tuple and send it to the port. */ - mess = TUPLE4(hp, am_trace, p->id, am_call, mfa_tuple); + mess = TUPLE4(hp, am_trace, p->common.id, am_call, mfa_tuple); hp += 5; if (pam_result != am_true) { hp[-5] = make_arityval(5); @@ -1787,21 +1816,21 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm* limit; #endif - ASSERT(is_internal_pid(*tracer_pid) - && internal_pid_index(*tracer_pid) < erts_max_processes); + ASSERT(is_internal_pid(*tracer_pid)); tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, *tracer_pid, ERTS_PROC_LOCK_STATUS); if (!tracer) invalid_tracer = 1; else { - invalid_tracer = (tracer->trace_flags & F_TRACER) == 0; + invalid_tracer = !(ERTS_TRACE_FLAGS(tracer) & F_TRACER); erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS); } if (invalid_tracer) { #ifdef ERTS_SMP - ASSERT(is_nil(tracee) || tracer_pid == &p->tracer_proc); + ASSERT(is_nil(tracee) + || tracer_pid == &ERTS_TRACER_PROC(p)); if (is_not_nil(tracee)) erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif @@ -1925,7 +1954,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * Build the trace tuple and enqueue it. */ - mess = TUPLE4(hp, am_trace, p->id/* Local pid */, am_call, mfa_tuple); + mess = TUPLE4(hp, am_trace, p->common.id, am_call, mfa_tuple); hp += 5; if (pam_result != am_true) { hp[-5] = make_arityval(5); @@ -1963,17 +1992,17 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_thr_progress_is_blocking()); - if (is_internal_port(t_p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(t_p))) { #define LOCAL_HEAP_SIZE (5+5) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; - mess = TUPLE4(hp, am_trace, t_p->id, what, data); + mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (t_p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } send_to_port( @@ -1984,7 +2013,9 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) /* Fake schedule out and in are never sent when smp enabled */ c_p, #endif - mess, &t_p->tracer_proc, &t_p->trace_flags); + mess, + &ERTS_TRACER_PROC(t_p), + &ERTS_TRACE_FLAGS(t_p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); @@ -1995,10 +2026,11 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) ERTS_TRACER_REF_TYPE tracer_ref; size_t sz_data; - ASSERT(is_internal_pid(t_p->tracer_proc) - && internal_pid_index(t_p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p))); - ERTS_GET_TRACER_REF(tracer_ref, t_p->tracer_proc, t_p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(t_p), + ERTS_TRACE_FLAGS(t_p)); sz_data = size_object(data); @@ -2007,16 +2039,16 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); tmp = copy_struct(data, sz_data, &hp, off_heap); - mess = TUPLE4(hp, am_trace, t_p->id/* Local pid */, what, tmp); + mess = TUPLE4(hp, am_trace, t_p->common.id, what, tmp); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (t_p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - ERTS_ENQ_TRACE_MSG(t_p->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } } @@ -2037,7 +2069,7 @@ trace_proc_spawn(Process *p, Eterm pid, Eterm mess; Eterm* hp; - if (is_internal_port(p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(p))) { #define LOCAL_HEAP_SIZE (4+6+5) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2045,13 +2077,13 @@ trace_proc_spawn(Process *p, Eterm pid, hp = local_heap; mfa = TUPLE3(hp, mod, func, args); hp += 4; - mess = TUPLE5(hp, am_trace, p->id, am_spawn, pid, mfa); + mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, pid, mfa); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - send_to_port(p, mess, &p->tracer_proc, &p->trace_flags); + send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); @@ -2063,10 +2095,11 @@ trace_proc_spawn(Process *p, Eterm pid, size_t sz_args, sz_pid; Uint need; - ASSERT(is_internal_pid(p->tracer_proc) - && internal_pid_index(p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(p), + ERTS_TRACE_FLAGS(p)); sz_args = size_object(args); sz_pid = size_object(pid); @@ -2078,16 +2111,16 @@ trace_proc_spawn(Process *p, Eterm pid, mfa = TUPLE3(hp, mod, func, tmp); hp += 4; tmp = copy_struct(pid, sz_pid, &hp, off_heap); - mess = TUPLE5(hp, am_trace, p->id, am_spawn, tmp, mfa); + mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, tmp, mfa); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } } @@ -2107,187 +2140,6 @@ void save_calls(Process *p, Export *e) } } -/* - * Entry point called by the trace wrap functions in erl_bif_wrap.c - * - * The trace wrap functions are themselves called through the export - * entries instead of the original BIF functions. - */ -Eterm -erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) -{ - Eterm result; - int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META); - - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - - if (!ARE_TRACE_FLAGS_ON(p, F_TRACE_CALLS) && (! meta)) { - /* Warning! This is an Optimization. - * - * If neither meta trace is active nor process trace flags then - * no tracing will occur. Doing the whole else branch will - * also do nothing, only slower. - */ - Eterm (*func)(Process*, Eterm*, BeamInstr*) = bif_table[bif_index].f; - result = func(p, args, I); - } else { - Eterm (*func)(Process*, Eterm*, BeamInstr*); - Export* ep = bif_export[bif_index]; - Uint32 flags = 0, flags_meta = 0; - int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL); - int local = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_LOCAL); - int time = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_CALL_TIME); - Eterm meta_tracer_pid = NIL; - int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif - * is actually in the - * export entry */ - BeamInstr *cp = p->cp; - - /* - * Make continuation pointer OK, it is not during direct BIF calls, - * but it is correct during apply of bif. - */ - if (!applying) { - p->cp = I; - } - if (global || local) { - flags = erts_call_trace(p, ep->code, ep->match_prog_set, args, - local, &p->tracer_proc); - } - if (meta) { - flags_meta = erts_bif_mtrace(p, ep->code+3, args, local, - &meta_tracer_pid); - } - if (time) { - BpDataTime *bdt = NULL; - BeamInstr *pc = (BeamInstr *)ep->code+3; - - bdt = (BpDataTime *) erts_get_time_break(p, pc); - ASSERT(bdt); - - if (!bdt->pause) { - erts_trace_time_break(p, pc, bdt, ERTS_BP_CALL_TIME_CALL); - } - } - /* Restore original continuation pointer (if changed). */ - p->cp = cp; - - func = bif_table[bif_index].f; - - result = func(p, args, I); - - if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { - BeamInstr i_return_trace = beam_return_trace[0]; - BeamInstr i_return_to_trace = beam_return_to_trace[0]; - BeamInstr i_return_time_trace = beam_return_time_trace[0]; - Eterm *cpp; - /* Maybe advance cp to skip trace stack frames */ - for (cpp = p->stop; ; cp = cp_val(*cpp++)) { - if (*cp == i_return_trace) { - /* Skip stack frame variables */ - while (is_not_CP(*cpp)) cpp++; - cpp += 2; /* Skip return_trace parameters */ - } else if (*cp == i_return_time_trace) { - /* Skip stack frame variables */ - while (is_not_CP(*cpp)) cpp++; - cpp += 1; /* Skip return_time_trace parameters */ - } else if (*cp == i_return_to_trace) { - /* A return_to trace message is going to be generated - * by normal means, so we do not have to. - */ - cp = NULL; - break; - } else break; - } - } - - /* Try to get these in the order - * they usually appear in normal code... */ - if (is_non_value(result)) { - Uint reason = p->freason; - if (reason != TRAP) { - Eterm class; - Eterm value = p->fvalue; - DeclareTmpHeapNoproc(nocatch,3); - UseTmpHeapNoproc(3); - /* Expand error value like in handle_error() */ - if (reason & EXF_ARGLIST) { - Eterm *tp; - ASSERT(is_tuple(value)); - tp = tuple_val(value); - value = tp[1]; - } - if ((reason & EXF_THROWN) && (p->catches <= 0)) { - value = TUPLE2(nocatch, am_nocatch, value); - reason = EXC_ERROR; - } - /* Note: expand_error_value() could theoretically - * allocate on the heap, but not for any error - * returned by a BIF, and it would do no harm, - * just be annoying. - */ - value = expand_error_value(p, reason, value); - class = exception_tag[GET_EXC_CLASS(reason)]; - - if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, - &meta_tracer_pid); - } - if (flags & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, - &p->tracer_proc); - } - if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { - /* can only happen if(local)*/ - Eterm *ptr = p->stop; - ASSERT(is_CP(*ptr)); - ASSERT(ptr <= STACK_START(p)); - /* Search the nearest stack frame for a catch */ - while (++ptr < STACK_START(p)) { - if (is_CP(*ptr)) break; - if (is_catch(*ptr)) { - if (applying) { - /* Apply of BIF, cp is in calling function */ - if (cp) erts_trace_return_to(p, cp); - } else { - /* Direct bif call, I points into - * calling function */ - erts_trace_return_to(p, I); - } - } - } - } - UnUseTmpHeapNoproc(3); - if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - p->trace_flags |= F_EXCEPTION_TRACE; - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - } else { - if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &meta_tracer_pid); - } - /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ - if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &p->tracer_proc); - } - if (flags & MATCH_SET_RETURN_TO_TRACE) { - /* can only happen if(local)*/ - if (applying) { - /* Apply of BIF, cp is in calling function */ - if (cp) erts_trace_return_to(p, cp); - } else { - /* Direct bif call, I points into calling function */ - erts_trace_return_to(p, I); - } - } - } - } - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - return result; -} - /* Sends trace message: * {trace_ts, Pid, What, Msg, Timestamp} * or {trace, Pid, What, Msg} @@ -2358,7 +2210,7 @@ trace_gc(Process *p, Eterm what) UseTmpHeap(LOCAL_HEAP_SIZE,p); - if (is_internal_port(p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(p))) { hp = local_heap; #ifdef DEBUG size = 0; @@ -2370,10 +2222,11 @@ trace_gc(Process *p, Eterm what) size += 5/*4-tuple*/ + TS_SIZE(p); #endif } else { - ASSERT(is_internal_pid(p->tracer_proc) - && internal_pid_index(p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(p), + ERTS_TRACE_FLAGS(p)); size = 0; (void) erts_bld_atom_uint_2tup_list(NULL, @@ -2397,19 +2250,19 @@ trace_gc(Process *p, Eterm what) tags, values); - msg = TUPLE4(hp, am_trace, p->id/* Local pid */, what, msg); + msg = TUPLE4(hp, am_trace, p->common.id, what, msg); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(msg, hp); } ASSERT(hp == limit); - if (is_internal_port(p->tracer_proc)) - send_to_port(p, msg, &p->tracer_proc, &p->trace_flags); + if (is_internal_port(ERTS_TRACER_PROC(p))) + send_to_port(p, msg, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); else - ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, msg, bp); + ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, msg, bp); erts_smp_mtx_unlock(&smq_mtx); UnUseTmpHeap(LOCAL_HEAP_SIZE,p); #undef LOCAL_HEAP_SIZE @@ -2429,10 +2282,9 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint #ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor) - && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) { return; } #endif @@ -2467,10 +2319,10 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint hp += 2; list = CONS(hp,tmo_tpl,list); hp += 2; - msg = TUPLE4(hp, am_monitor, p->id, am_long_schedule, list); + msg = TUPLE4(hp, am_monitor, p->common.id, am_long_schedule, list); hp += 5; #ifdef ERTS_SMP - enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp); + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else erts_queue_message(monitor_p, NULL, bp, msg, NIL #ifdef USE_VM_PROBES @@ -2493,10 +2345,9 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) #ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor) - && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor)) { + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p) { return; } #endif @@ -2512,7 +2363,7 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); switch (type) { - case ERTS_PORT_TASK_FREE: op = am_free; break; + case ERTS_PORT_TASK_PROC_SIG: op = am_proc_sig; break; case ERTS_PORT_TASK_TIMEOUT: op = am_timeout; break; case ERTS_PORT_TASK_INPUT: op = am_input; break; case ERTS_PORT_TASK_OUTPUT: op = am_output; break; @@ -2533,10 +2384,10 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) hp += 2; list = CONS(hp,tmo_tpl,list); hp += 2; - msg = TUPLE4(hp, am_monitor, pp->id, am_long_schedule, list); + msg = TUPLE4(hp, am_monitor, pp->common.id, am_long_schedule, list); hp += 5; #ifdef ERTS_SMP - enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->id, NIL, msg, bp); + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp); #else erts_queue_message(monitor_p, NULL, bp, msg, NIL #ifdef USE_VM_PROBES @@ -2578,12 +2429,10 @@ monitor_long_gc(Process *p, Uint time) { #endif #ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor) - && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) return; - } #endif hsz = 0; @@ -2605,7 +2454,7 @@ monitor_long_gc(Process *p, Uint time) { sizeof(values)/sizeof(Uint), tags, values); - msg = TUPLE4(hp, am_monitor, p->id/* Local pid */, am_long_gc, list); + msg = TUPLE4(hp, am_monitor, p->common.id, am_long_gc, list); #ifdef DEBUG hp += 5 /* 4-tuple */; @@ -2613,7 +2462,7 @@ monitor_long_gc(Process *p, Uint time) { #endif #ifdef ERTS_SMP - enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp); + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else erts_queue_message(monitor_p, NULL, bp, msg, NIL #ifdef USE_VM_PROBES @@ -2654,10 +2503,9 @@ monitor_large_heap(Process *p) { #ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor) - && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (monitor_p || p == monitor_p) { return; } #endif @@ -2681,7 +2529,7 @@ monitor_large_heap(Process *p) { sizeof(values)/sizeof(Uint), tags, values); - msg = TUPLE4(hp, am_monitor, p->id/* Local pid */, am_large_heap, list); + msg = TUPLE4(hp, am_monitor, p->common.id, am_large_heap, list); #ifdef DEBUG hp += 5 /* 4-tuple */; @@ -2689,7 +2537,7 @@ monitor_large_heap(Process *p) { #endif #ifdef ERTS_SMP - enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp); + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else erts_queue_message(monitor_p, NULL, bp, msg, NIL #ifdef USE_VM_PROBES @@ -2709,21 +2557,19 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { Eterm *hp, msg; #ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor) - && internal_pid_index(system_monitor) < erts_max_processes); - monitor_p = process_tab[internal_pid_index(system_monitor)]; - if (INVALID_PID(monitor_p, system_monitor) || p == monitor_p) { + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) return; - } #endif hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p); - msg = TUPLE4(hp, am_monitor, p->id/* Local pid */, type, spec); + msg = TUPLE4(hp, am_monitor, p->common.id, type, spec); hp += 5; #ifdef ERTS_SMP - enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp); + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else erts_queue_message(monitor_p, NULL, bp, msg, NIL #ifdef USE_VM_PROBES @@ -2845,21 +2691,21 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { Eterm mess; Eterm* hp; - if (is_internal_port(p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(p))) { #define LOCAL_HEAP_SIZE (5+6) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; - mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name); + mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } /* No fake schedule */ - send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags); + send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); @@ -2869,25 +2715,26 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { size_t sz_data; ERTS_TRACER_REF_TYPE tracer_ref; - ASSERT(is_internal_pid(p->tracer_proc) - && internal_pid_index(p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); sz_data = 6 + TS_SIZE(p); - ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(p), + ERTS_TRACE_FLAGS(p)); hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref); - mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name); + mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -2908,20 +2755,20 @@ trace_port(Port *t_p, Eterm what, Eterm data) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); - if (is_internal_port(t_p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(t_p))) { #define LOCAL_HEAP_SIZE (5+5) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; - mess = TUPLE4(hp, am_trace, t_p->id, what, data); + mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (t_p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } /* No fake schedule */ - send_to_port(NULL, mess, &t_p->tracer_proc, &t_p->trace_flags); + send_to_port(NULL,mess,&ERTS_TRACER_PROC(t_p),&ERTS_TRACE_FLAGS(t_p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); @@ -2931,25 +2778,26 @@ trace_port(Port *t_p, Eterm what, Eterm data) { size_t sz_data; ERTS_TRACER_REF_TYPE tracer_ref; - ASSERT(is_internal_pid(t_p->tracer_proc) - && internal_pid_index(t_p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p))); sz_data = 5 + TS_SIZE(t_p); - ERTS_GET_TRACER_REF(tracer_ref, t_p->tracer_proc, t_p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(t_p), + ERTS_TRACE_FLAGS(t_p)); hp = ERTS_ALLOC_SYSMSG_HEAP(sz_data, &bp, &off_heap, tracer_ref); - mess = TUPLE4(hp, am_trace, t_p->id, what, data); + mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (t_p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - ERTS_ENQ_TRACE_MSG(t_p->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } } @@ -2974,7 +2822,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { int ws = 5; Eterm sched_id = am_undefined; - if (is_internal_port(p->tracer_proc)) { + if (is_internal_port(ERTS_TRACER_PROC(p))) { #define LOCAL_HEAP_SIZE (5+6) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2989,21 +2837,21 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { #else sched_id = make_small(1); #endif - mess = TUPLE5(hp, am_trace, p->id, what, sched_id, where); + mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, where); ws = 6; } else { - mess = TUPLE4(hp, am_trace, p->id, what, where); + mess = TUPLE4(hp, am_trace, p->common.id, what, where); ws = 5; } hp += ws; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } /* No fake scheduling */ - send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags); + send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE erts_smp_mtx_unlock(&smq_mtx); @@ -3012,12 +2860,13 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { ErlOffHeap *off_heap; ERTS_TRACER_REF_TYPE tracer_ref; - ASSERT(is_internal_pid(p->tracer_proc) - && internal_pid_index(p->tracer_proc) < erts_max_processes); + ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) ws = 6; /* Make place for scheduler id */ - ERTS_GET_TRACER_REF(tracer_ref, p->tracer_proc, p->trace_flags); + ERTS_GET_TRACER_REF(tracer_ref, + ERTS_TRACER_PROC(p), + ERTS_TRACE_FLAGS(p)); hp = ERTS_ALLOC_SYSMSG_HEAP(ws+TS_SIZE(p), &bp, &off_heap, tracer_ref); @@ -3029,19 +2878,19 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { #else sched_id = make_small(1); #endif - mess = TUPLE5(hp, am_trace, p->id, what, sched_id, where); + mess = TUPLE5(hp, am_trace, p->common.id, what, sched_id, where); } else { - mess = TUPLE4(hp, am_trace, p->id, what, where); + mess = TUPLE4(hp, am_trace, p->common.id, what, where); } hp += ws; erts_smp_mtx_lock(&smq_mtx); - if (p->trace_flags & F_TIMESTAMP) { + if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { hp = patch_ts(mess, hp); } - ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp); + ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } } @@ -3077,14 +2926,14 @@ profile_runnable_port(Port *p, Eterm status) { GET_NOW(&Ms, &s, &us); timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->id, status, count, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, count, timestamp); hp += 6; #ifndef ERTS_SMP - profile_send(p->id, msg); + profile_send(p->common.id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp); + enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp); #endif erts_smp_mtx_unlock(&smq_mtx); } @@ -3131,13 +2980,13 @@ profile_runnable_proc(Process *p, Eterm status){ GET_NOW(&Ms, &s, &us); timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, where, timestamp); hp += 6; #ifndef ERTS_SMP - profile_send(p->id, msg); + profile_send(p->common.id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE #else - enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp); + enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp); #endif erts_smp_mtx_unlock(&smq_mtx); } @@ -3150,16 +2999,19 @@ profile_runnable_proc(Process *p, Eterm status){ void erts_check_my_tracer_proc(Process *p) { - if (is_internal_pid(p->tracer_proc)) { - Process *tracer = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, - p->tracer_proc, ERTS_PROC_LOCK_STATUS); - int invalid_tracer = !tracer || !(tracer->trace_flags & F_TRACER); + if (is_internal_pid(ERTS_TRACER_PROC(p))) { + Process *tracer = erts_pid2proc(p, + ERTS_PROC_LOCK_MAIN, + ERTS_TRACER_PROC(p), + ERTS_PROC_LOCK_STATUS); + int invalid_tracer = (!tracer + || !(ERTS_TRACE_FLAGS(tracer) & F_TRACER)); if (tracer) erts_smp_proc_unlock(tracer, ERTS_PROC_LOCK_STATUS); if (invalid_tracer) { erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - p->trace_flags &= ~TRACEE_FLAGS; - p->tracer_proc = NIL; + ERTS_TRACE_FLAGS(p) &= ~TRACEE_FLAGS; + ERTS_TRACER_PROC(p) = NIL; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); } } @@ -3278,10 +3130,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) break; case SYS_MSG_TYPE_SEQTRACE: /* Reset seq_tracer if it hasn't changed */ - erts_smp_mtx_lock(&sys_trace_mtx); + erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); if (system_seq_tracer == receiver) system_seq_tracer = am_false; - erts_smp_mtx_unlock(&sys_trace_mtx); + erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); break; case SYS_MSG_TYPE_SYSMON: if (receiver == NIL @@ -3504,7 +3356,7 @@ sys_msg_dispatcher_func(void *unused) proc = erts_pid2proc(NULL, 0, receiver, proc_locks); if (!proc || (smqp->type == SYS_MSG_TYPE_TRACE - && !(proc->trace_flags & F_TRACER))) { + && !(ERTS_TRACE_FLAGS(proc) & F_TRACER))) { /* Bad tracer */ #ifdef DEBUG_PRINTOUTS if (smqp->type == SYS_MSG_TYPE_TRACE && proc) @@ -3531,18 +3383,16 @@ sys_msg_dispatcher_func(void *unused) proc = erts_whereis_process(NULL,0,receiver,proc_locks,0); if (!proc) goto failure; - else if (smqp->from == proc->id) + else if (smqp->from == proc->common.id) goto drop_sys_msg; else goto queue_proc_msg; } else if (is_internal_port(receiver)) { - port = erts_id2port(receiver, NULL, 0); - if (INVALID_TRACER_PORT(port, receiver)) { - if (port) - erts_port_release(port); + port = erts_thr_id2port_sflgs(receiver, + ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); + if (!port) goto failure; - } else { write_sys_msg_to_port(receiver, port, @@ -3554,7 +3404,7 @@ sys_msg_dispatcher_func(void *unused) #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif - erts_port_release(port); + erts_thr_port_release(port); if (smqp->bp) free_message_buffer(smqp->bp); } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h new file mode 100644 index 0000000000..853c6cb0d8 --- /dev/null +++ b/erts/emulator/beam/erl_trace.h @@ -0,0 +1,143 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +#ifndef ERL_TRACE_H__ +#define ERL_TRACE_H__ + +struct binary; + +/* erl_bif_trace.c */ +Eterm erl_seq_trace_info(Process *p, Eterm arg1); +void erts_system_monitor_clear(Process *c_p); +void erts_system_profile_clear(Process *c_p); + +/* erl_trace.c */ +void erts_init_trace(void); +void erts_trace_check_exiting(Eterm exiting); +Eterm erts_set_system_seq_tracer(Process *c_p, + ErtsProcLocks c_p_locks, + Eterm new); +Eterm erts_get_system_seq_tracer(void); +void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp); +void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp); +void erts_set_system_monitor(Eterm monitor); +Eterm erts_get_system_monitor(void); + +#ifdef ERTS_SMP +void erts_check_my_tracer_proc(Process *); +void erts_block_sys_msg_dispatcher(void); +void erts_release_sys_msg_dispatcher(void); +void erts_foreach_sys_msg_in_q(void (*func)(Eterm, + Eterm, + Eterm, + ErlHeapFragment *)); +void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *); +#endif + +void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); +void trace_send(Process*, Eterm, Eterm); +void trace_receive(Process*, Eterm); +Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, Eterm* args, + int local, Eterm *tracer_pid); +void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid); +void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, + Eterm *tracer); +void erts_trace_return_to(Process *p, BeamInstr *pc); +void trace_sched(Process*, Eterm); +void trace_proc(Process*, Process*, Eterm, Eterm); +void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args); +void save_calls(Process *p, Export *); +void trace_gc(Process *p, Eterm what); +/* port tracing */ +void trace_virtual_sched(Process*, Eterm); +void trace_sched_ports(Port *pp, Eterm); +void trace_sched_ports_where(Port *pp, Eterm, Eterm); +void trace_port(Port *, Eterm what, Eterm data); +void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name); + +/* system_profile */ +void erts_set_system_profile(Eterm profile); +Eterm erts_get_system_profile(void); +void profile_scheduler(Eterm scheduler_id, Eterm); +void profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us); +void profile_runnable_proc(Process* p, Eterm status); +void profile_runnable_port(Port* p, Eterm status); +void erts_system_profile_setup_active_schedulers(void); + +/* system_monitor */ +void monitor_long_gc(Process *p, Uint time); +void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); +void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); +void monitor_large_heap(Process *p); +void monitor_generic(Process *p, Eterm type, Eterm spec); +Uint erts_trace_flag2bit(Eterm flag); +int erts_trace_flags(Eterm List, + Uint *pMask, Eterm *pTracer, int *pCpuTimestamp); +Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); + +#ifdef ERTS_SMP +void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); +#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) \ +do { \ + if ((ESDP)->pending_trace_msgs) \ + erts_send_pending_trace_msgs((ESDP)); \ +} while (0) +#else +#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) +#endif + +#define seq_trace_output(token, msg, type, receiver, process) \ +seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) +#define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \ +seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom)) +void seq_trace_output_generic(Eterm token, Eterm msg, Uint type, + Eterm receiver, Process *process, Eterm exitfrom); + +int seq_trace_update_send(Process *process); + +Eterm erts_seq_trace(Process *process, + Eterm atom_type, Eterm atom_true_or_false, + int build_result); + +struct trace_pattern_flags { + unsigned int breakpoint : 1; /* Set if any other is set */ + unsigned int local : 1; /* Local call trace breakpoint */ + unsigned int meta : 1; /* Metadata trace breakpoint */ + unsigned int call_count : 1; /* Fast call count breakpoint */ + unsigned int call_time : 1; /* Fast call time breakpoint */ +}; +extern const struct trace_pattern_flags erts_trace_pattern_flags_off; +extern int erts_call_time_breakpoint_tracing; +int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, + struct binary* match_prog_set, + struct binary *meta_match_prog_set, + int on, struct trace_pattern_flags, + Eterm meta_tracer_pid, int is_blocking); +void +erts_get_default_trace_pattern(int *trace_pattern_is_on, + struct binary **match_spec, + struct binary **meta_match_spec, + struct trace_pattern_flags *trace_pattern_flags, + Eterm *meta_tracer_pid); +int erts_is_default_trace_enabled(void); +void erts_bif_trace_init(void); +int erts_finish_breakpointing(void); + +#endif /* ERL_TRACE_H__ */ diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 26ac231ddb..e00440b905 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2012. All Rights Reserved. + * Copyright Ericsson AB 2008-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -77,66 +77,25 @@ void erts_init_unicode(void) { max_loop_limit = CONTEXT_REDS * LOOP_FACTOR; /* Non visual BIFs to trap to. */ - memset(&characters_to_utf8_trap_exp, 0, sizeof(Export)); - characters_to_utf8_trap_exp.address = - &characters_to_utf8_trap_exp.code[3]; - characters_to_utf8_trap_exp.code[0] = am_erlang; - characters_to_utf8_trap_exp.code[1] = - am_atom_put("characters_to_utf8_trap",23); - characters_to_utf8_trap_exp.code[2] = 3; - characters_to_utf8_trap_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_utf8_trap_exp.code[4] = - (BeamInstr) &characters_to_utf8_trap; - - memset(&characters_to_list_trap_1_exp, 0, sizeof(Export)); - characters_to_list_trap_1_exp.address = - &characters_to_list_trap_1_exp.code[3]; - characters_to_list_trap_1_exp.code[0] = am_erlang; - characters_to_list_trap_1_exp.code[1] = - am_atom_put("characters_to_list_trap_1",25); - characters_to_list_trap_1_exp.code[2] = 3; - characters_to_list_trap_1_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_1_exp.code[4] = - (BeamInstr) &characters_to_list_trap_1; - - memset(&characters_to_list_trap_2_exp, 0, sizeof(Export)); - characters_to_list_trap_2_exp.address = - &characters_to_list_trap_2_exp.code[3]; - characters_to_list_trap_2_exp.code[0] = am_erlang; - characters_to_list_trap_2_exp.code[1] = - am_atom_put("characters_to_list_trap_2",25); - characters_to_list_trap_2_exp.code[2] = 3; - characters_to_list_trap_2_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_2_exp.code[4] = - (BeamInstr) &characters_to_list_trap_2; - - - memset(&characters_to_list_trap_3_exp, 0, sizeof(Export)); - characters_to_list_trap_3_exp.address = - &characters_to_list_trap_3_exp.code[3]; - characters_to_list_trap_3_exp.code[0] = am_erlang; - characters_to_list_trap_3_exp.code[1] = - am_atom_put("characters_to_list_trap_3",25); - characters_to_list_trap_3_exp.code[2] = 3; - characters_to_list_trap_3_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_3_exp.code[4] = - (BeamInstr) &characters_to_list_trap_3; - - memset(&characters_to_list_trap_4_exp, 0, sizeof(Export)); - characters_to_list_trap_4_exp.address = - &characters_to_list_trap_4_exp.code[3]; - characters_to_list_trap_4_exp.code[0] = am_erlang; - characters_to_list_trap_4_exp.code[1] = - am_atom_put("characters_to_list_trap_4",25); - characters_to_list_trap_4_exp.code[2] = 1; - characters_to_list_trap_4_exp.code[3] = - (BeamInstr) em_apply_bif; - characters_to_list_trap_4_exp.code[4] = - (BeamInstr) &characters_to_list_trap_4; + erts_init_trap_export(&characters_to_utf8_trap_exp, + am_erlang, am_atom_put("characters_to_utf8_trap",23), 3, + &characters_to_utf8_trap); + + erts_init_trap_export(&characters_to_list_trap_1_exp, + am_erlang, am_atom_put("characters_to_list_trap_1",25), 3, + &characters_to_list_trap_1); + + erts_init_trap_export(&characters_to_list_trap_2_exp, + am_erlang, am_atom_put("characters_to_list_trap_2",25), 3, + &characters_to_list_trap_2); + + erts_init_trap_export(&characters_to_list_trap_3_exp, + am_erlang, am_atom_put("characters_to_list_trap_3",25), 3, + &characters_to_list_trap_3); + + erts_init_trap_export(&characters_to_list_trap_4_exp, + am_erlang, am_atom_put("characters_to_list_trap_4",25), 1, + &characters_to_list_trap_4); c_to_b_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_binary_int,2); c_to_l_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_list_int,2); @@ -765,7 +724,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ hp = HAlloc(p, 2); obj = CDR(objp); ioterm = CONS(hp, rest_term, obj); - //(*left) = 0; + /* (*left) = 0; */ goto done; } if (rest_term != NIL) { @@ -1195,15 +1154,24 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2) * When input to characters_to_list is a plain binary and the format is 'unicode', we do * a faster analyze and size count with this function. */ -int erts_analyze_utf8(byte *source, Uint size, - byte **err_pos, Uint *num_chars, int *left) +static ERTS_INLINE int +analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left, + Sint *num_latin1_chars, Uint max_chars) { + Uint latin1_count; + int is_latin1; *err_pos = source; + if (num_latin1_chars) { + is_latin1 = 1; + latin1_count = 0; + } *num_chars = 0; while (size) { if (((*source) & ((byte) 0x80)) == 0) { source++; - --size; + --size; + if (num_latin1_chars) + latin1_count++; } else if (((*source) & ((byte) 0xE0)) == 0xC0) { if (size < 2) { return ERTS_UTF8_INCOMPLETE; @@ -1212,6 +1180,11 @@ int erts_analyze_utf8(byte *source, Uint size, ((*source) < 0xC2) /* overlong */) { return ERTS_UTF8_ERROR; } + if (num_latin1_chars) { + latin1_count++; + if ((source[0] & ((byte) 0xFC)) != ((byte) 0xC0)) + is_latin1 = 0; + } source += 2; size -= 2; } else if (((*source) & ((byte) 0xF0)) == 0xE0) { @@ -1229,6 +1202,8 @@ int erts_analyze_utf8(byte *source, Uint size, } source += 3; size -= 3; + if (num_latin1_chars) + is_latin1 = 0; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { if (size < 4) { return ERTS_UTF8_INCOMPLETE; @@ -1246,21 +1221,40 @@ int erts_analyze_utf8(byte *source, Uint size, } source += 4; size -= 4; + if (num_latin1_chars) + is_latin1 = 0; } else { return ERTS_UTF8_ERROR; } ++(*num_chars); *err_pos = source; - if (left && --(*left) <= 0) { + if (max_chars && size > 0 && *num_chars == max_chars) + return ERTS_UTF8_OK_MAX_CHARS; + if (left && --(*left) <= 0 && size) { return ERTS_UTF8_ANALYZE_MORE; } } + if (num_latin1_chars) + *num_latin1_chars = is_latin1 ? latin1_count : -1; return ERTS_UTF8_OK; } +int erts_analyze_utf8(byte *source, Uint size, + byte **err_pos, Uint *num_chars, int *left) +{ + return analyze_utf8(source, size, err_pos, num_chars, left, NULL, 0); +} + +int erts_analyze_utf8_x(byte *source, Uint size, + byte **err_pos, Uint *num_chars, int *left, + Sint *num_latin1_chars, Uint max_chars) +{ + return analyze_utf8(source, size, err_pos, num_chars, left, num_latin1_chars, max_chars); +} + /* * No errors should be able to occur - no overlongs, no malformed, no nothing - */ + */ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left, Uint *num_built, Uint *num_eaten, Eterm tail) @@ -1316,6 +1310,12 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, return ret; } +Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left, + Uint *num_built, Uint *num_eaten, Eterm tail) +{ + return do_utf8_to_list(p, num, bytes, sz, left, num_built, num_eaten, tail); +} + static int is_candidate(Uint cp) { int index,pos; @@ -1723,14 +1723,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, if (b_sz) { ErlSubBin *sb; Eterm orig; - ERTS_DECLARE_DUMMY(Uint offset); + Uint offset; ASSERT(state != ERTS_UTF8_OK); hp = HAlloc(p, ERL_SUB_BIN_SIZE); sb = (ErlSubBin *) hp; ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize); sb->thing_word = HEADER_SUB_BIN; sb->size = b_sz; - sb->offs = num_bytes_to_process + num_processed_bytes; + sb->offs = offset + num_bytes_to_process + num_processed_bytes; sb->orig = orig; sb->bitoffs = bitoffs; sb->bitsize = bitsize; @@ -1853,31 +1853,25 @@ BIF_RETTYPE atom_to_binary_2(BIF_ALIST_2) ap = atom_tab(atom_val(BIF_ARG_1)); if (BIF_ARG_2 == am_latin1) { - BIF_RET(new_binary(BIF_P, ap->name, ap->len)); - } else if (BIF_ARG_2 == am_utf8 || BIF_ARG_2 == am_unicode) { - int bin_size = 0; - int i; Eterm bin_term; - byte* bin_p; - for (i = 0; i < ap->len; i++) { - bin_size += (ap->name[i] >= 0x80) ? 2 : 1; + if (ap->latin1_chars < 0) { + goto error; } - if (bin_size == ap->len) { - BIF_RET(new_binary(BIF_P, ap->name, ap->len)); + if (ap->latin1_chars == ap->len) { + bin_term = new_binary(BIF_P, ap->name, ap->len); } - bin_term = new_binary(BIF_P, 0, bin_size); - bin_p = binary_bytes(bin_term); - for (i = 0; i < ap->len; i++) { - byte b = ap->name[i]; - if (b < 0x80) { - *bin_p++ = b; - } else { - *bin_p++ = 0xC0 | (b >> 6); - *bin_p++ = 0x80 | (b & 0x3F); - } + else { + byte* bin_p; + int dbg_sz; + bin_term = new_binary(BIF_P, 0, ap->latin1_chars); + bin_p = binary_bytes(bin_term); + dbg_sz = erts_utf8_to_latin1(bin_p, ap->name, ap->len); + ASSERT(dbg_sz == ap->latin1_chars); (void)dbg_sz; } BIF_RET(bin_term); + } else if (BIF_ARG_2 == am_utf8 || BIF_ARG_2 == am_unicode) { + BIF_RET(new_binary(BIF_P, ap->name, ap->len)); } else { error: BIF_ERROR(BIF_P, BADARG); @@ -1885,118 +1879,78 @@ BIF_RETTYPE atom_to_binary_2(BIF_ALIST_2) } static BIF_RETTYPE -binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist) +binary_to_atom(Process* proc, Eterm bin, Eterm enc, int must_exist) { byte* bytes; byte *temp_alloc = NULL; Uint bin_size; if ((bytes = erts_get_aligned_binary_bytes(bin, &temp_alloc)) == 0) { - BIF_ERROR(p, BADARG); + BIF_ERROR(proc, BADARG); } bin_size = binary_size(bin); if (enc == am_latin1) { Eterm a; - if (bin_size > MAX_ATOM_LENGTH) { + if (bin_size > MAX_ATOM_CHARACTERS) { system_limit: erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(p, SYSTEM_LIMIT); + BIF_ERROR(proc, SYSTEM_LIMIT); } if (!must_exist) { - a = am_atom_put((char *)bytes, bin_size); - erts_free_aligned_binary_bytes(temp_alloc); + a = erts_atom_put((byte *) bytes, + bin_size, + ERTS_ATOM_ENC_LATIN1, + 0); + erts_free_aligned_binary_bytes(temp_alloc); + if (is_non_value(a)) + goto badarg; BIF_RET(a); - } else if (erts_atom_get((char *)bytes, bin_size, &a)) { + } else if (erts_atom_get((char *)bytes, bin_size, &a, ERTS_ATOM_ENC_LATIN1)) { erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(a); } else { goto badarg; } } else if (enc == am_utf8 || enc == am_unicode) { - char *buf; - char *dst; - int i; - int num_chars; Eterm res; + Uint num_chars = 0; + const byte* p = bytes; + Uint left = bin_size; - if (bin_size > 2*MAX_ATOM_LENGTH) { - byte* err_pos; - Uint n; - int reds_left = bin_size+1; /* Number of reductions left. */ - - if (erts_analyze_utf8(bytes, bin_size, &err_pos, - &n, &reds_left) == ERTS_UTF8_OK) { - /* - * Correct UTF-8 encoding, but too many characters to - * fit in an atom. - */ + while (left) { + if (++num_chars > MAX_ATOM_CHARACTERS) { goto system_limit; - } else { - /* - * Something wrong in the UTF-8 encoding or Unicode code - * points > 255. - */ - goto badarg; } - } - - /* - * Allocate a temporary buffer the same size as the binary, - * so that we don't need an extra overflow test. - */ - buf = (char *) erts_alloc(ERTS_ALC_T_TMP, bin_size); - dst = buf; - for (i = 0; i < bin_size; i++) { - int c = bytes[i]; - if (c < 0x80) { - *dst++ = c; - } else if (i < bin_size-1) { - int c2; - if ((c & 0xE0) != 0xC0) { - goto free_badarg; - } - i++; - c = (c & 0x3F) << 6; - c2 = bytes[i]; - if ((c2 & 0xC0) != 0x80) { - goto free_badarg; - } - c = c | (c2 & 0x3F); - if (0x80 <= c && c < 256) { - *dst++ = c; - } else { - goto free_badarg; - } - } else { - free_badarg: - erts_free(ERTS_ALC_T_TMP, (void *) buf); - goto badarg; + if ((p[0] & 0x80) == 0) { + ++p; + --left; } + else if (left >= 2 + && (p[0] & 0xFE) == 0xC2 /* only allow latin1 subset */ + && (p[1] & 0xC0) == 0x80) { + p += 2; + left -= 2; + } + else goto badarg; } - num_chars = dst - buf; - if (num_chars > MAX_ATOM_LENGTH) { - erts_free(ERTS_ALC_T_TMP, (void *) buf); - goto system_limit; - } + if (!must_exist) { - res = am_atom_put(buf, num_chars); - erts_free(ERTS_ALC_T_TMP, (void *) buf); - erts_free_aligned_binary_bytes(temp_alloc); - BIF_RET(res); - } else { - int exists = erts_atom_get(buf, num_chars, &res); - erts_free(ERTS_ALC_T_TMP, (void *) buf); - if (exists) { - erts_free_aligned_binary_bytes(temp_alloc); - BIF_RET(res); - } else { - goto badarg; - } + res = erts_atom_put((byte *) bytes, + bin_size, + ERTS_ATOM_ENC_UTF8, + 0); + } + else if (!erts_atom_get((char*)bytes, bin_size, &res, ERTS_ATOM_ENC_UTF8)) { + goto badarg; } + erts_free_aligned_binary_bytes(temp_alloc); + if (is_non_value(res)) + goto badarg; + BIF_RET(res); } else { badarg: erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(p, BADARG); + BIF_ERROR(proc, BADARG); } } @@ -2619,8 +2573,20 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1) case ERL_FILENAME_UTF8: bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { + Eterm *hp = HAlloc(BIF_P,3); + Eterm warn_type = NIL; erts_free_aligned_binary_bytes(temp_alloc); - goto noconvert; + switch (erts_get_filename_warning_type()) { + case ERL_FILENAME_WARNING_IGNORE: + warn_type = am_ignore; + break; + case ERL_FILENAME_WARNING_ERROR: + warn_type = am_error; + break; + default: + warn_type = am_warning; + } + BIF_RET(TUPLE2(hp,am_error,warn_type)); } num_built = 0; num_eaten = 0; @@ -2653,9 +2619,8 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1) erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(ret); default: - goto noconvert; + break; } - noconvert: BIF_RET(BIF_ARG_1); } @@ -2692,6 +2657,52 @@ BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1) BIF_RET(ret); } +BIF_RETTYPE prim_file_is_translatable_1(BIF_ALIST_1) +{ + ERTS_DECLARE_DUMMY(Eterm real_bin); + ERTS_DECLARE_DUMMY(Uint offset); + Uint size; + Uint num_chars; + Uint bitsize; + ERTS_DECLARE_DUMMY(Uint bitoffs); + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + int status; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + size = binary_size(BIF_ARG_1); + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (size == 0) { + BIF_RET(am_true); + } + + /* + * If the encoding is latin1, the pathname is always translatable. + */ + switch (erts_get_native_filename_encoding()) { + case ERL_FILENAME_LATIN1: + BIF_RET(am_true); + case ERL_FILENAME_WIN_WCHAR: + if (erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + BIF_RET(am_true); + } + } + + /* + * Check whether the binary contains legal UTF-8 sequences. + */ + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + status = erts_analyze_utf8(bytes, size, &err_pos, &num_chars, NULL); + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(status == ERTS_UTF8_OK ? am_true : am_false); +} + BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0) { switch (erts_get_native_filename_encoding()) { @@ -2711,3 +2722,36 @@ BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0) } } +int erts_utf8_to_latin1(byte* dest, const byte* source, int slen) +{ + /* + * Assumes source contains valid utf8 that can be encoded as latin1, + * and that dest has enough room. + */ + byte* dp = dest; + + while (slen > 0) { + if ((source[0] & 0x80) == 0) { + *dp++ = *source++; + --slen; + } + else { + ASSERT(slen > 1); + ASSERT((source[0] & 0xFE) == 0xC2); + ASSERT((source[1] & 0xC0) == 0x80); + *dp++ = (char) ((source[0] << 6) | (source[1] & 0x3F)); + source += 2; + slen -= 2; + } + } + return dp - dest; +} + +BIF_RETTYPE io_printable_range_0(BIF_ALIST_0) +{ + if (erts_get_printable_characters() == ERL_PRINTABLE_CHARACTERS_UNICODE) { + BIF_RET(am_unicode); + } else { + BIF_RET(am_latin1); + } +} diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h new file mode 100644 index 0000000000..80d29d554a --- /dev/null +++ b/erts/emulator/beam/erl_utils.h @@ -0,0 +1,220 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2012. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_UTILS_H__ +#define ERL_UTILS_H__ + +#include "sys.h" +#include "erl_smp.h" +#include "erl_printf.h" + +struct process; + +typedef struct { +#ifdef DEBUG + int smp_api; +#endif + union { + Uint64 not_atomic; +#ifdef ARCH_64 + erts_atomic_t atomic; +#else + erts_dw_atomic_t atomic; +#endif + } counter; +} erts_interval_t; + +void erts_interval_init(erts_interval_t *); +void erts_smp_interval_init(erts_interval_t *); +Uint64 erts_step_interval_nob(erts_interval_t *); +Uint64 erts_step_interval_relb(erts_interval_t *); +Uint64 erts_smp_step_interval_nob(erts_interval_t *); +Uint64 erts_smp_step_interval_relb(erts_interval_t *); +Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64); +Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64); +Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64); +Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64); +#ifdef ARCH_32 +ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *); +#endif +ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *); +ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +#ifdef ARCH_32 + +ERTS_GLB_INLINE Uint64 +erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw) +{ +#ifdef ETHR_SU_DW_NAINT_T__ + return (Uint64) dw->dw_sint; +#else + Uint64 res; + res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); + res <<= 32; + res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); + return res; +#endif +} + +#endif + +ERTS_GLB_INLINE Uint64 +erts_current_interval_nob__(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t dw; + erts_dw_atomic_read_nob(&icp->counter.atomic, &dw); + return erts_interval_dw_aint_to_val__(&dw); +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_acqb__(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); +#else + erts_dw_aint_t dw; + erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw); + return erts_interval_dw_aint_to_val__(&dw); +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_nob(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return erts_current_interval_nob__(icp); +} + +ERTS_GLB_INLINE Uint64 +erts_current_interval_acqb(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return erts_current_interval_acqb__(icp); +} + +ERTS_GLB_INLINE Uint64 +erts_smp_current_interval_nob(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return erts_current_interval_nob__(icp); +#else + return icp->counter.not_atomic; +#endif +} + +ERTS_GLB_INLINE Uint64 +erts_smp_current_interval_acqb(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return erts_current_interval_acqb__(icp); +#else + return icp->counter.not_atomic; +#endif +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +/* + * To be used to silence unused result warnings, but do not abuse it. + */ +void erts_silence_warn_unused_result(long unused); + + +int erts_fit_in_bits_int64(Sint64); +int erts_fit_in_bits_int32(Sint32); +int list_length(Eterm); +int erts_is_builtin(Eterm, Eterm, int); +Uint32 make_broken_hash(Eterm); +Uint32 block_hash(byte *, unsigned, Uint32); +Uint32 make_hash2(Eterm); +Uint32 make_hash(Eterm); + +void erts_save_emu_args(int argc, char **argv); +Eterm erts_get_emu_args(struct process *c_p); +Eterm erts_get_ethread_info(struct process * c_p); + +Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str); +Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); +Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw); +Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64); +Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64); +Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr); +Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...); +Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]); +Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len); +#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str)) +Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]); +Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp, + Sint length, Eterm terms1[], Uint terms2[]); +Eterm +erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp, + Sint length, Eterm atoms[], Uint uints[]); +Eterm +erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, + Eterm atoms[], Uint uints1[], Uint uints2[]); + +void erts_init_utils(void); +void erts_init_utils_mem(void); + +erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint); +void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *); + +#if HALFWORD_HEAP +int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); +# define eq(A,B) eq_rel(A,NULL,B,NULL) +#else +int eq(Eterm, Eterm); +# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B) +#endif + +#define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) + +#if HALFWORD_HEAP +Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*); +#define CMP(A,B) cmp_rel(A,NULL,B,NULL) +#else +Sint cmp(Eterm, Eterm); +#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B) +#define CMP(A,B) cmp(A,B) +#endif +#define cmp_lt(a,b) (CMP((a),(b)) < 0) +#define cmp_le(a,b) (CMP((a),(b)) <= 0) +#define cmp_eq(a,b) (CMP((a),(b)) == 0) +#define cmp_ne(a,b) (CMP((a),(b)) != 0) +#define cmp_ge(a,b) (CMP((a),(b)) >= 0) +#define cmp_gt(a,b) (CMP((a),(b)) > 0) + +#define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) +#define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) +#define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) +#define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) + +#endif diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c index f73d48b6c2..47fd92988e 100644 --- a/erts/emulator/beam/erl_zlib.c +++ b/erts/emulator/beam/erl_zlib.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,6 +44,48 @@ void erl_zlib_zfree_callback (voidpf opaque, voidpf ptr) erts_free(ERTS_ALC_T_ZLIB, ptr); } +/* + * Initialize a z_stream with a source, to later *chunk* data into a destination + * Returns Z_OK or Error. + */ +int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen, int level) +{ + streamp->next_in = (Bytef*)source; + streamp->avail_in = (uInt)sourceLen; + streamp->total_out = streamp->avail_out = 0; + streamp->next_out = NULL; + erl_zlib_alloc_init(streamp); + return deflateInit(streamp, level); +} +/* + * Deflate a chunk, The destination length is the limit. + * Returns Z_OK if more to process, Z_STREAM_END if we are done. + */ +int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen) +{ + int err; + uLongf last_tot = streamp->total_out; + + streamp->next_out = dest; + streamp->avail_out = (uInt)*destLen; + + if ((uLong)streamp->avail_out != *destLen) return Z_BUF_ERROR; + + err = deflate(streamp, Z_FINISH); + *destLen = streamp->total_out - last_tot; + return err; +} + + +/* + * When we are done, free up the deflate structure + * Retyurns Z_OK or Error + */ +int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp) +{ + return deflateEnd(streamp); +} int ZEXPORT erl_zlib_compress2 (Bytef* dest, uLongf* destLen, const Bytef* source, uLong sourceLen, diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h index 9054a5e428..5ac849d21c 100644 --- a/erts/emulator/beam/erl_zlib.h +++ b/erts/emulator/beam/erl_zlib.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,6 +31,14 @@ (s)->zfree = erl_zlib_zfree_callback; \ } while (0) +/* + * Chunked interface, used by term_to_binary among others. + */ +int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen, int level); +int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen); +int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp); + /* Use instead of compress */ #define erl_zlib_compress(dest,destLen,source,sourceLen) \ diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index bdfde58845..e3ebbb84f4 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -639,9 +639,9 @@ provider erlang { * Entry into the efile_drv.c file I/O driver * * For a list of command numbers used by this driver, see the section - * "Guide to probe arguments" in ../../../README.md. That section - * also contains explanation of the various integer and string - * arguments that may be present when any particular probe fires. + * "Guide to efile_drv.c probe arguments" in ../../../HOWTO/DTRACE.md. + * That section also contains explanation of the various integer and + * string arguments that may be present when any particular probe fires. * * NOTE: Not all Linux platforms (using SystemTap) can support * arguments beyond arg9. diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index fb0ee99119..b0f08d8245 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -32,98 +32,162 @@ #define EXPORT_HASH(m,f,a) ((m)*(f)+(a)) -static IndexTable export_table; /* Not locked. */ -static Hash secondary_export_table; /* Locked. */ +#ifdef DEBUG +# define IF_DEBUG(x) x +#else +# define IF_DEBUG(x) +#endif -#include "erl_smp.h" +static IndexTable export_tables[ERTS_NUM_CODE_IX]; /* Active not locked */ -static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. */ +static erts_smp_atomic_t total_entries_bytes; + +#include "erl_smp.h" -#define export_read_lock() erts_smp_rwmtx_rlock(&export_table_lock) -#define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock) -#define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock) -#define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock) +/* This lock protects the staging export table from concurrent access + * AND it protects the staging table from becoming active. + */ +erts_smp_mtx_t export_staging_lock; extern BeamInstr* em_call_error_handler; extern BeamInstr* em_call_traced_function; +struct export_entry +{ + IndexSlot slot; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ + Export* ep; +}; + +/* Helper struct that brings things together in one allocation +*/ +struct export_blob +{ + Export exp; + struct export_entry entryv[ERTS_NUM_CODE_IX]; + /* Note that entryv is not indexed by "code_ix". + */ +}; + +/* Helper struct only used as template +*/ +struct export_templ +{ + struct export_entry entry; + Export exp; +}; + +static struct export_blob* entry_to_blob(struct export_entry* ee) +{ + return (struct export_blob*) + ((char*)ee->ep - offsetof(struct export_blob,exp)); +} + void export_info(int to, void *to_arg) { #ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - export_read_lock(); + export_staging_lock(); #endif - index_info(to, to_arg, &export_table); - hash_info(to, to_arg, &secondary_export_table); + index_info(to, to_arg, &export_tables[erts_active_code_ix()]); + hash_info(to, to_arg, &export_tables[erts_staging_code_ix()].htable); #ifdef ERTS_SMP if (lock) - export_read_unlock(); + export_staging_unlock(); #endif } static HashValue -export_hash(Export* x) +export_hash(struct export_entry* ee) { + Export* x = ee->ep; return EXPORT_HASH(x->code[0], x->code[1], x->code[2]); } static int -export_cmp(Export* tmpl, Export* obj) +export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e) { + Export* tmpl = tmpl_e->ep; + Export* obj = obj_e->ep; return !(tmpl->code[0] == obj->code[0] && tmpl->code[1] == obj->code[1] && tmpl->code[2] == obj->code[2]); } -static Export* -export_alloc(Export* tmpl) +static struct export_entry* +export_alloc(struct export_entry* tmpl_e) { - Export* obj = (Export*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(Export)); - - obj->fake_op_func_info_for_hipe[0] = 0; - obj->fake_op_func_info_for_hipe[1] = 0; - obj->code[0] = tmpl->code[0]; - obj->code[1] = tmpl->code[1]; - obj->code[2] = tmpl->code[2]; - obj->slot.index = -1; - obj->address = obj->code+3; - obj->code[3] = (BeamInstr) em_call_error_handler; - obj->code[4] = 0; - obj->match_prog_set = NULL; - return obj; + struct export_blob* blob; + unsigned ix; + + if (tmpl_e->slot.index == -1) { /* Template, allocate blob */ + Export* tmpl = tmpl_e->ep; + Export* obj; + + blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob)); + erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); + obj = &blob->exp; + obj->fake_op_func_info_for_hipe[0] = 0; + obj->fake_op_func_info_for_hipe[1] = 0; + obj->code[0] = tmpl->code[0]; + obj->code[1] = tmpl->code[1]; + obj->code[2] = tmpl->code[2]; + obj->code[3] = (BeamInstr) em_call_error_handler; + obj->code[4] = 0; + + for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { + obj->addressv[ix] = obj->code+3; + + blob->entryv[ix].slot.index = -1; + blob->entryv[ix].ep = &blob->exp; + } + ix = 0; + } + else { /* Existing entry in another table, use free entry in blob */ + blob = entry_to_blob(tmpl_e); + for (ix = 0; blob->entryv[ix].slot.index >= 0; ix++) { + ASSERT(ix < ERTS_NUM_CODE_IX); + } + } + return &blob->entryv[ix]; } - -static void -export_free(Export* obj) +static void +export_free(struct export_entry* obj) { - erts_free(ERTS_ALC_T_EXPORT, (void*) obj); + struct export_blob* blob = entry_to_blob(obj); + int i; + obj->slot.index = -1; + for (i=0; i < ERTS_NUM_CODE_IX; i++) { + if (blob->entryv[i].slot.index >= 0) { + return; + } + } + erts_free(ERTS_ALC_T_EXPORT, blob); + erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } - void init_export_table(void) { HashFunctions f; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + int i; - erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab"); + erts_smp_mtx_init(&export_staging_lock, "export_tab"); + erts_smp_atomic_init_nob(&total_entries_bytes, 0); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; f.free = (HFREE_FUN) export_free; - erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_table, "export_list", - EXPORT_INITIAL_SIZE, EXPORT_LIMIT, f); - hash_init(ERTS_ALC_T_EXPORT_TABLE, &secondary_export_table, - "secondary_export_table", 50, f); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_tables[i], "export_list", + EXPORT_INITIAL_SIZE, EXPORT_LIMIT, f); + } } /* @@ -138,29 +202,42 @@ init_export_table(void) * called through such an export entry. * 3) This function is suitable for the implementation of erlang:apply/3. */ +extern Export* /* inline-helper */ +erts_find_export_entry(Eterm m, Eterm f, unsigned int a,ErtsCodeIndex code_ix); Export* -erts_find_export_entry(Eterm m, Eterm f, unsigned int a) +erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) { HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a); int ix; HashBucket* b; - ix = hval % export_table.htable.size; - b = export_table.htable.bucket[ix]; + ix = hval % export_tables[code_ix].htable.size; + b = export_tables[code_ix].htable.bucket[ix]; /* * Note: We have inlined the code from hash.c for speed. */ while (b != (HashBucket*) 0) { - Export* ep = (Export *) b; + Export* ep = ((struct export_entry*) b)->ep; if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) { - break; + return ep; } b = b->next; } - return (Export*)b; + return NULL; +} + +static struct export_entry* init_template(struct export_templ* templ, + Eterm m, Eterm f, unsigned a) +{ + templ->entry.ep = &templ->exp; + templ->entry.slot.index = -1; + templ->exp.code[0] = m; + templ->exp.code[1] = f; + templ->exp.code[2] = a; + return &templ->entry; } @@ -176,47 +253,42 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a) */ Export* -erts_find_function(Eterm m, Eterm f, unsigned int a) +erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) { - Export e; - Export* ep; - - e.code[0] = m; - e.code[1] = f; - e.code[2] = a; - - ep = hash_get(&export_table.htable, (void*) &e); - if (ep != NULL && ep->address == ep->code+3 && - ep->code[3] != (BeamInstr) em_call_traced_function) { - ep = NULL; + struct export_templ templ; + struct export_entry* ee; + + ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); + if (ee == NULL || + (ee->ep->addressv[code_ix] == ee->ep->code+3 && + ee->ep->code[3] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + return NULL; } - return ep; + return ee->ep; } /* * Returns a pointer to an existing export entry for a MFA, * or creates a new one and returns the pointer. * - * This function provides unlocked write access to the main export - * table. It should only be used during start up or when - * all other threads are blocked. + * This function acts on the staging export table. It should only be used + * to load new code. */ Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity) { - Export e; - int ix; + ErtsCodeIndex code_ix = erts_staging_code_ix(); + struct export_templ templ; + struct export_entry* ee; - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_smp_thr_progress_is_blocking()); ASSERT(is_atom(mod)); ASSERT(is_atom(func)); - e.code[0] = mod; - e.code[1] = func; - e.code[2] = arity; - ix = index_put(&export_table, (void*) &e); - return (Export*) erts_index_lookup(&export_table, ix); + export_staging_lock(); + ee = (struct export_entry*) index_put_entry(&export_tables[code_ix], + init_template(&templ, mod, func, arity)); + export_staging_unlock(); + return ee->ep; } /* @@ -224,77 +296,122 @@ erts_export_put(Eterm mod, Eterm func, unsigned int arity) * export entry (making a call through it will cause the error_handler to * be called). * - * Stub export entries will be placed in the secondary export table. - * erts_export_consolidate() will move all stub export entries into the - * main export table (will be done the next time code is loaded). + * Stub export entries will be placed in the loader export table. */ Export* erts_export_get_or_make_stub(Eterm mod, Eterm func, unsigned int arity) { - Export e; + ErtsCodeIndex code_ix; Export* ep; + IF_DEBUG(int retrying = 0;) ASSERT(is_atom(mod)); ASSERT(is_atom(func)); - - e.code[0] = mod; - e.code[1] = func; - e.code[2] = arity; - ep = erts_find_export_entry(mod, func, arity); - if (ep == 0) { - /* - * The code is not loaded (yet). Put the export in the secondary - * export table, to avoid having to lock the main export table. - */ - export_write_lock(); - ep = (Export *) hash_put(&secondary_export_table, (void*) &e); - export_write_unlock(); - } + + do { + code_ix = erts_active_code_ix(); + ep = erts_find_export_entry(mod, func, arity, code_ix); + if (ep == 0) { + /* + * The code is not loaded (yet). Put the export in the staging + * export table, to avoid having to lock the active export table. + */ + export_staging_lock(); + if (erts_active_code_ix() == code_ix) { + struct export_templ templ; + struct export_entry* entry; + + IndexTable* tab = &export_tables[erts_staging_code_ix()]; + init_template(&templ, mod, func, arity); + entry = (struct export_entry *) index_put_entry(tab, &templ.entry); + ep = entry->ep; + ASSERT(ep); + } + else { /* race */ + ASSERT(!retrying); + IF_DEBUG(retrying = 1); + } + export_staging_unlock(); + } + } while (!ep); return ep; } -/* - * To be called before loading code (with other threads blocked). - * This function will move all export entries from the secondary - * export table into the primary. - */ -void -erts_export_consolidate(void) +Export *export_list(int i, ErtsCodeIndex code_ix) { -#ifdef DEBUG - HashInfo hi; -#endif - - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_smp_thr_progress_is_blocking()); - - export_write_lock(); - erts_index_merge(&secondary_export_table, &export_table); - erts_hash_merge(&secondary_export_table, &export_table.htable); - export_write_unlock(); -#ifdef DEBUG - hash_get_info(&hi, &export_table.htable); - ASSERT(export_table.entries == hi.objs); -#endif + return ((struct export_entry*) erts_index_lookup(&export_tables[code_ix], i))->ep; } -Export *export_list(int i) +int export_list_size(ErtsCodeIndex code_ix) { - return (Export*) erts_index_lookup(&export_table, i); + return export_tables[code_ix].entries; } -int export_list_size(void) +int export_table_sz(void) { - return export_table.entries; + int i, bytes = 0; + + export_staging_lock(); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + bytes += index_table_sz(&export_tables[i]); + } + export_staging_unlock(); + return bytes; +} +int export_entries_sz(void) +{ + return erts_smp_atomic_read_nob(&total_entries_bytes); } +Export *export_get(Export *e) +{ + struct export_entry ee; + struct export_entry* entry; -int export_table_sz(void) + ee.ep = e; + entry = (struct export_entry*)hash_get(&export_tables[erts_active_code_ix()].htable, &ee); + return entry ? entry->ep : NULL; +} + +IF_DEBUG(static ErtsCodeIndex debug_start_load_ix = 0;) + + +void export_start_staging(void) { - return index_table_sz(&export_table); + ErtsCodeIndex dst_ix = erts_staging_code_ix(); + ErtsCodeIndex src_ix = erts_active_code_ix(); + IndexTable* dst = &export_tables[dst_ix]; + IndexTable* src = &export_tables[src_ix]; + struct export_entry* src_entry; +#ifdef DEBUG + struct export_entry* dst_entry; +#endif + int i; + + ASSERT(dst_ix != src_ix); + ASSERT(debug_start_load_ix == -1); + + export_staging_lock(); + /* + * Insert all entries in src into dst table + */ + for (i = 0; i < src->entries; i++) { + src_entry = (struct export_entry*) erts_index_lookup(src, i); + src_entry->ep->addressv[dst_ix] = src_entry->ep->addressv[src_ix]; +#ifdef DEBUG + dst_entry = (struct export_entry*) +#endif + index_put_entry(dst, src_entry); + ASSERT(entry_to_blob(src_entry) == entry_to_blob(dst_entry)); + } + export_staging_unlock(); + + IF_DEBUG(debug_start_load_ix = dst_ix); } -Export *export_get(Export *e) +void export_end_staging(int commit) { - return hash_get(&export_table.htable, e); + ASSERT(debug_start_load_ix == erts_staging_code_ix()); + IF_DEBUG(debug_start_load_ix = -1); } + diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index c604fdf7c3..61a54de59f 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,14 +28,15 @@ #include "index.h" #endif +#include "code_ix.h" + /* ** Export entry */ + typedef struct export { - IndexSlot slot; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ - void* address; /* Pointer to code for function. */ - struct binary* match_prog_set; /* Match program for tracing. */ + void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ /* @@ -44,12 +45,12 @@ typedef struct export * code[2]: Arity (untagged integer). * code[3]: This entry is 0 unless the 'address' field points to it. * Threaded code instruction to load function - * (em_call_error_handler), execute BIF (em_apply_bif, - * em_apply_apply), or call a traced function - * (em_call_traced_function). - * code[4]: Function pointer to BIF function (for BIFs only) + * (em_call_error_handler), execute BIF (em_apply_bif), + * or a breakpoint instruction (op_i_generic_breakpoint). + * code[4]: Function pointer to BIF function (for BIFs only), * or pointer to threaded code if the module has an - * on_load function that has not been run yet. + * on_load function that has not been run yet, or pointer + * to code for function code[3] is a breakpont instruction. * Otherwise: 0. */ BeamInstr code[5]; @@ -59,21 +60,38 @@ typedef struct export void init_export_table(void); void export_info(int, void *); -Export* erts_find_export_entry(Eterm m, Eterm f, unsigned int a); +ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a); Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity); - Export* erts_export_get_or_make_stub(Eterm, Eterm, unsigned); -void erts_export_consolidate(void); -Export *export_list(int); -int export_list_size(void); +Export *export_list(int,ErtsCodeIndex); +int export_list_size(ErtsCodeIndex); int export_table_sz(void); +int export_entries_sz(void); Export *export_get(Export*); +void export_start_staging(void); +void export_end_staging(int commit); + +extern erts_smp_mtx_t export_staging_lock; +#define export_staging_lock() erts_smp_mtx_lock(&export_staging_lock) +#define export_staging_unlock() erts_smp_mtx_unlock(&export_staging_lock) #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->address == (EntryPtr)->code + 3) && \ +(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \ ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) -#endif +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Export* +erts_active_export_entry(Eterm m, Eterm f, unsigned int a) +{ + extern Export* erts_find_export_entry(Eterm m, Eterm f, unsigned a, ErtsCodeIndex); + return erts_find_export_entry(m, f, a, erts_active_code_ix()); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* __EXPORT_H__ */ + diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 52f45b924f..1c88765381 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -81,7 +81,11 @@ * */ +static Export term_to_binary_trap_export; + static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); +static int enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap, Sint *reds, byte **res); static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); @@ -89,9 +93,31 @@ static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static Sint decoded_size(byte *ep, byte* endp, int internal_tags); +static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); +static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, + Binary *context_b); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); +static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, + unsigned dflags, Sint *reds, Uint *res); + +void erts_init_external(void) { +#if 1 /* In R16 */ + erts_init_trap_export(&term_to_binary_trap_export, + am_erlang, am_term_to_binary_trap, 1, + &term_to_binary_trap_1); +#else + sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export)); + term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3]; + term_to_binary_trap_export.code[0] = am_erlang; + term_to_binary_trap_export.code[1] = am_term_to_binary_trap; + term_to_binary_trap_export.code[2] = 1; + term_to_binary_trap_export.code[3] = (BeamInstr) em_apply_bif; + term_to_binary_trap_export.code[4] = (BeamInstr) &term_to_binary_trap_1; +#endif + return; +} #define ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES 255 @@ -142,6 +168,7 @@ erts_init_atom_cache_map(ErtsAtomCacheMap *acmp) { if (acmp) { int ix; + acmp->long_atoms = 0; for (ix = 0; ix < ERTS_ATOM_CACHE_SIZE; ix++) acmp->cache[ix].iix = -1; acmp->sz = 0; @@ -154,6 +181,7 @@ erts_reset_atom_cache_map(ErtsAtomCacheMap *acmp) { if (acmp) { int i; + acmp->long_atoms = 0; for (i = 0; i < acmp->sz; i++) { ASSERT(0 <= acmp->cix[i] && acmp->cix[i] < ERTS_ATOM_CACHE_SIZE); acmp->cache[acmp->cix[i]].iix = -1; @@ -175,9 +203,23 @@ erts_destroy_atom_cache_map(ErtsAtomCacheMap *acmp) } static ERTS_INLINE void -insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom) +insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) { - if (acmp && acmp->sz < ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES) { + /* + * If the receiver do not understand utf8 atoms + * and this atom cannot be represented in latin1, + * we are not allowed to cache it. + * + * In this case all atoms are assumed to have + * latin1 encoding in the cache. By refusing it + * in the cache we will instead encode it using + * ATOM_UTF8_EXT/SMALL_ATOM_UTF8_EXT which the + * receiver do not recognize and tear down the + * connection. + */ + if (acmp && acmp->sz < ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES + && ((dflags & DFLAG_UTF8_ATOMS) + || atom_tab(atom_val(atom))->latin1_chars >= 0)) { int ix; ASSERT(acmp->hdr_sz < 0); ix = atom2cix(atom); @@ -190,7 +232,7 @@ insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom) } static ERTS_INLINE int -get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom) +get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) { if (!acmp) return -1; @@ -199,7 +241,9 @@ get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom) ASSERT(is_atom(atom)); ix = atom2cix(atom); if (acmp->cache[ix].iix < 0) { - ASSERT(acmp->sz == ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES); + ASSERT(acmp->sz == ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES + || (!(dflags & DFLAG_UTF8_ATOMS) + && atom_tab(atom_val(atom))->latin1_chars < 0)); return -1; } else { @@ -210,18 +254,17 @@ get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom) } void -erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp) +erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) { if (acmp) { -#if MAX_ATOM_LENGTH > 255 -#error "This code is not complete; long_atoms info need to be passed to the following stages." - int long_atoms = 0; /* !0 if one or more atoms are long than 255. */ -#endif + int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); + int long_atoms = 0; /* !0 if one or more atoms are longer than 255. */ int i; int sz; int fix_sz = 1 /* VERSION_MAGIC */ + 1 /* DIST_HEADER */ + + 1 /* dist header flags */ + 1 /* number of internal cache entries */ ; int min_sz; @@ -230,22 +273,23 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp) min_sz = fix_sz+(2+4)*acmp->sz; sz = fix_sz; for (i = 0; i < acmp->sz; i++) { + Atom *a; Eterm atom; int len; atom = acmp->cache[acmp->cix[i]].atom; ASSERT(is_atom(atom)); - len = atom_tab(atom_val(atom))->len; -#if MAX_ATOM_LENGTH > 255 + a = atom_tab(atom_val(atom)); + len = (int) (utf8_atoms ? a->len : a->latin1_chars); + ASSERT(len >= 0); if (!long_atoms && len > 255) long_atoms = 1; -#endif /* Enough for a new atom cache value */ sz += 1 /* cix */ + 1 /* length */ + len /* text */; } -#if MAX_ATOM_LENGTH > 255 - if (long_atoms) + if (long_atoms) { + acmp->long_atoms = 1; sz += acmp->sz; /* we need 2 bytes per atom for length */ -#endif + } /* Dynamically sized flag field */ sz += ERTS_DIST_HDR_ATOM_CACHE_FLAG_BYTES(acmp->sz); if (sz < min_sz) @@ -274,6 +318,7 @@ byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp) else { int i; byte *ep = ctl_ext; + byte dist_hdr_flags = acmp->long_atoms ? ERTS_DIST_HDR_LONG_ATOMS_FLG : 0; ASSERT(acmp->hdr_sz >= 0); /* * Write cache update instructions. Note that this is a purely @@ -296,28 +341,36 @@ byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp) } --ep; put_int8(acmp->sz, ep); + --ep; + put_int8(dist_hdr_flags, ep); *--ep = DIST_HEADER; *--ep = VERSION_MAGIC; return ep; } } -byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache) +byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint32 dflags) { byte *ip; byte instr_buf[(2+4)*ERTS_ATOM_CACHE_SIZE]; int ci, sz; + byte dist_hdr_flags; + int long_atoms; + int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); register byte *ep = ext; ASSERT(ep[0] == VERSION_MAGIC); if (ep[1] != DIST_HEADER) return ext; + dist_hdr_flags = ep[2]; + long_atoms = ERTS_DIST_HDR_LONG_ATOMS_FLG & ((int) dist_hdr_flags); + /* * Update output atom cache and write the external version of * the dist header. We write the header backwards just * before the actual term(s). */ - ep += 2; + ep += 3; ci = (int) get_int8(ep); ASSERT(0 <= ci && ci < ERTS_ATOM_CACHE_SIZE); ep += 1; @@ -342,12 +395,7 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache) flgs_bytes = ERTS_DIST_HDR_ATOM_CACHE_FLAG_BYTES(ci); ASSERT(flgs_bytes <= sizeof(flgs_buf)); -#if MAX_ATOM_LENGTH > 255 - /* long_atoms info needs to be passed from previous stages */ - if (long_atoms) - flgs |= ERTS_DIST_HDR_LONG_ATOMS_FLG; -#endif - flgs = 0; + flgs = (Uint32) dist_hdr_flags; flgs_buf_ix = 0; if ((ci & 1) == 0) used_half_bytes = 2; @@ -382,17 +430,22 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache) Atom *a; cache->out_arr[cix] = atom; a = atom_tab(atom_val(atom)); - sz = a->len; - ep -= sz; - sys_memcpy((void *) ep, (void *) a->name, sz); -#if MAX_ATOM_LENGTH > 255 + if (utf8_atoms) { + sz = a->len; + ep -= sz; + sys_memcpy((void *) ep, (void *) a->name, sz); + } + else { + ASSERT(0 <= a->latin1_chars && a->latin1_chars <= MAX_ATOM_CHARACTERS); + ep -= a->latin1_chars; + sz = erts_utf8_to_latin1(ep, a->name, a->len); + ASSERT(a->latin1_chars == sz); + } if (long_atoms) { ep -= 2; put_int16(sz, ep); } - else -#endif - { + else { ASSERT(0 <= sz && sz <= 255); --ep; put_int8(sz, ep); @@ -467,7 +520,7 @@ Uint erts_encode_ext_size_2(Eterm term, unsigned dflags) Uint erts_encode_ext_size_ets(Eterm term) { - return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS); + return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAG_INTERNAL_TAGS); } @@ -500,7 +553,7 @@ void erts_encode_ext(Eterm term, byte **ext) byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off_heap) { - return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS, + return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAG_INTERNAL_TAGS, off_heap); } @@ -553,6 +606,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, #endif register byte *ep = ext; + int utf8_atoms = (int) (dep->flags & DFLAG_UTF8_ATOMS); edep->heap_size = -1; edep->ext_endp = ext+size; @@ -611,9 +665,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, ERTS_EXT_HDR_FAIL; ep++; if (no_atoms) { -#if MAX_ATOM_LENGTH > 255 int long_atoms = 0; -#endif #ifdef DEBUG byte *flgs_buf = ep; #endif @@ -632,14 +684,8 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, */ byte_ix = ERTS_DIST_HDR_ATOM_CACHE_FLAG_BYTE_IX(no_atoms); bit_ix = ERTS_DIST_HDR_ATOM_CACHE_FLAG_BIT_IX(no_atoms); - if (flgsp[byte_ix] & (((byte) ERTS_DIST_HDR_LONG_ATOMS_FLG) - << bit_ix)) { -#if MAX_ATOM_LENGTH > 255 + if (flgsp[byte_ix] & (((byte) ERTS_DIST_HDR_LONG_ATOMS_FLG) << bit_ix)) long_atoms = 1; -#else - ERTS_EXT_HDR_FAIL; /* Long atoms not supported yet */ -#endif - } #ifdef DEBUG byte_ix = 0; @@ -707,23 +753,25 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, if (cix >= ERTS_ATOM_CACHE_SIZE) ERTS_EXT_HDR_FAIL; ep++; -#if MAX_ATOM_LENGTH > 255 if (long_atoms) { CHKSIZE(2); len = get_int16(ep); ep += 2; } - else -#endif - { + else { CHKSIZE(1); len = get_int8(ep); ep++; } - if (len > MAX_ATOM_LENGTH) - ERTS_EXT_HDR_FAIL; /* Too long atom */ CHKSIZE(len); - atom = am_atom_put((char *) ep, len); + atom = erts_atom_put((byte *) ep, + len, + (utf8_atoms + ? ERTS_ATOM_ENC_UTF8 + : ERTS_ATOM_ENC_LATIN1), + 0); + if (is_non_value(atom)) + ERTS_EXT_HDR_FAIL; ep += len; cache->in_arr[cix] = atom; edep->attab.atom[tix] = atom; @@ -926,7 +974,7 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) Eterm res; Eterm *hp; Eterm *hendp; - Uint hsz; + Sint hsz; ErtsDistExternal ede; Eterm *tp; Eterm real_bin; @@ -972,7 +1020,7 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) if (hsz < 0) goto badarg; - hp = HAlloc(BIF_P, hsz); + hp = HAlloc(BIF_P, (Uint) hsz); hendp = hp + hsz; res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede); @@ -987,10 +1035,28 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } - +static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) +{ + Eterm *tp = tuple_val(BIF_ARG_1); + Eterm Term = tp[1]; + Eterm bt = tp[2]; + Binary *bin = ((ProcBin *) binary_val(bt))->val; + Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin); + if (is_tuple(res)) { + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + BIF_RET(res); + } +} + BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { - return erts_term_to_binary(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS); + Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL); + if (is_tuple(res)) { + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + BIF_RET(res); + } } BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) @@ -1000,6 +1066,8 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) Eterm Flags = BIF_ARG_2; int level = 0; Uint flags = TERM_TO_BINARY_DFLAGS; + Eterm res; + Binary *bin = NULL; while (is_list(Flags)) { Eterm arg = CAR(list_val(Flags)); @@ -1036,7 +1104,12 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) goto error; } - return erts_term_to_binary(p, Term, level, flags); + res = erts_term_to_binary_int(p, Term, level, flags, bin); + if (is_tuple(res)) { + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + BIF_RET(res); + } } static uLongf binary2term_uncomp_size(byte* data, Sint size) @@ -1313,16 +1386,13 @@ external_size_2(BIF_ALIST_2) } } -Eterm -erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) +static Eterm +erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint flags) { - Uint size; Eterm bin; size_t real_size; byte* endp; - size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */; - if (level != 0) { byte buf[256]; byte* bytes = buf; @@ -1392,6 +1462,327 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) } } +Eterm +erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { + Uint size; + size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */; + return erts_term_to_binary_simple(p, Term, size, level, flags); +} + +/* Define for testing */ +/* #define EXTREME_TTB_TRAPPING 1 */ + +#ifndef EXTREME_TTB_TRAPPING +#define TERM_TO_BINARY_LOOP_FACTOR 500 +#define TERM_TO_BINARY_SIZE_FACTOR 500000 +#define TERM_TO_BINARY_COMPRESS_CHUNK 500000 +#else +#define TERM_TO_BINARY_LOOP_FACTOR 1 +#define TERM_TO_BINARY_SIZE_FACTOR 10 +#define TERM_TO_BINARY_COMPRESS_CHUNK 10 +#endif + + +typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; +typedef struct { + Uint flags; + int level; +} TTBSizeContext; + +typedef struct { + Uint flags; + int level; + Binary *result_bin; +} TTBEncodeContext; + +typedef struct { + Uint real_size; + Uint dest_len; + byte *dbytes; + Binary *result_bin; + Binary *destination_bin; + z_stream stream; +} TTBCompressContext; + +typedef struct { + int alive; + TTBState state; + union { + TTBSizeContext sc; + TTBEncodeContext ec; + TTBCompressContext cc; + } s; +} TTBContext; + +static void context_destructor(Binary *context_bin) +{ + TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin); + if (context->alive) { + context->alive = 0; + switch (context->state) { + case TTBSize: + break; + case TTBEncode: + if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0); + erts_bin_free(context->s.ec.result_bin); + context->s.ec.result_bin = NULL; + } + break; + case TTBCompress: + erl_zlib_deflate_finish(&(context->s.cc.stream)); + + if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),0) == 0); + erts_bin_free(context->s.cc.destination_bin); + context->s.cc.destination_bin = NULL; + } + + if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),0) == 0); + erts_bin_free(context->s.cc.result_bin); + context->s.cc.result_bin = NULL; + } + break; + } + } +} + +static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, + Binary *context_b) +{ + Eterm *hp; + Eterm res; + Eterm c_term; +#ifndef EXTREME_TTB_TRAPPING + Sint reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); +#else + Sint reds = 20; /* For testing */ +#endif + Sint initial_reds = reds; + TTBContext c_buff; + TTBContext *context = &c_buff; + +#define EXPORT_CONTEXT() \ + do { \ + if (context_b == NULL) { \ + context_b = erts_create_magic_binary(sizeof(TTBContext), \ + context_destructor); \ + context = ERTS_MAGIC_BIN_DATA(context_b); \ + memcpy(context,&c_buff,sizeof(TTBContext)); \ + } \ + } while (0) + +#define RETURN_STATE() \ + do { \ + hp = HAlloc(p, PROC_BIN_SIZE+3); \ + c_term = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); \ + res = TUPLE2(hp, Term, c_term); \ + BUMP_ALL_REDS(p); \ + return res; \ + } while (0); + + + if (context_b == NULL) { + /* Setup enough to get started */ + context->state = TTBSize; + context->alive = 1; + context->s.sc.flags = flags; + context->s.sc.level = level; + } else { + context = ERTS_MAGIC_BIN_DATA(context_b); + } + /* Initialization done, now we will go through the states */ + for (;;) { + switch (context->state) { + case TTBSize: + { + Uint size; + Binary *result_bin; + int level; + Uint flags; + /* Try for fast path */ + if (encode_size_struct_int(p, NULL, Term, context->s.sc.flags, &reds, &size) < 0) { + EXPORT_CONTEXT(); + /* Same state */ + RETURN_STATE(); + } + ++size; /* VERSION_MAGIC */ + /* Move these to next state */ + flags = context->s.sc.flags; + level = context->s.sc.level; + if (size <= ERL_ONHEAP_BIN_LIMIT) { + /* Finish in one go */ + res = erts_term_to_binary_simple(p, Term, size, + level, flags); + BUMP_REDS(p, size / TERM_TO_BINARY_SIZE_FACTOR); + return res; + } + + result_bin = erts_bin_nrml_alloc(size); + result_bin->flags = 0; + result_bin->orig_size = size; + erts_refc_init(&result_bin->refc, 0); + result_bin->orig_bytes[0] = VERSION_MAGIC; + /* Next state immediately, no need to export context */ + context->state = TTBEncode; + context->s.ec.flags = flags; + context->s.ec.level = level; + context->s.ec.result_bin = result_bin; + break; + } + case TTBEncode: + { + byte *endp; + byte *bytes = (byte *) context->s.ec.result_bin->orig_bytes; + size_t real_size; + Binary *result_bin; + + flags = context->s.ec.flags; + if (enc_term_int(p,NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) { + EXPORT_CONTEXT(); + RETURN_STATE(); + } + real_size = endp - bytes; + result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size); + level = context->s.ec.level; + BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); + if (level == 0 || real_size < 6) { /* We are done */ + ProcBin* pb; + return_normal: + context->s.ec.result_bin = NULL; + context->alive = 0; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = real_size; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + /* Continue with compression... */ + /* To make absolutely sure that zlib does not barf on a reallocated context, + we make sure it's "exported" before doing anything compession-like */ + EXPORT_CONTEXT(); + bytes = (byte *) result_bin->orig_bytes; /* result_bin is reallocated */ + if (erl_zlib_deflate_start(&(context->s.cc.stream),bytes+1,real_size-1,level) + != Z_OK) { + goto return_normal; + } + context->state = TTBCompress; + context->s.cc.real_size = real_size; + context->s.cc.result_bin = result_bin; + + result_bin = erts_bin_nrml_alloc(real_size); + result_bin->flags = 0; + result_bin->orig_size = real_size; + erts_refc_init(&result_bin->refc, 0); + result_bin->orig_bytes[0] = VERSION_MAGIC; + + context->s.cc.destination_bin = result_bin; + context->s.cc.dest_len = 0; + context->s.cc.dbytes = (byte *) result_bin->orig_bytes+6; + break; + } + case TTBCompress: + { + uLongf tot_dest_len = context->s.cc.real_size - 6; + uLongf left = (tot_dest_len - context->s.cc.dest_len); + uLongf this_time = (left > TERM_TO_BINARY_COMPRESS_CHUNK) ? + TERM_TO_BINARY_COMPRESS_CHUNK : + left; + Binary *result_bin; + ProcBin *pb; + Uint max = (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_COMPRESS_CHUNK) / CONTEXT_REDS; + + if (max < this_time) { + this_time = max + 1; /* do not set this_time to 0 */ + } + + res = erl_zlib_deflate_chunk(&(context->s.cc.stream), context->s.cc.dbytes, &this_time); + context->s.cc.dbytes += this_time; + context->s.cc.dest_len += this_time; + switch (res) { + case Z_OK: + if (context->s.cc.dest_len >= tot_dest_len) { + goto no_use_compressing; + } + RETURN_STATE(); + case Z_STREAM_END: + { + byte *dbytes = (byte *) context->s.cc.destination_bin->orig_bytes + 1; + + dbytes[0] = COMPRESSED; + put_int32(context->s.cc.real_size-1,dbytes+1); + erl_zlib_deflate_finish(&(context->s.cc.stream)); + result_bin = erts_bin_realloc(context->s.cc.destination_bin, + context->s.cc.dest_len+6); + context->s.cc.destination_bin = NULL; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = context->s.cc.dest_len+6; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + erts_bin_free(context->s.cc.result_bin); + context->s.cc.result_bin = NULL; + context->alive = 0; + BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + default: /* Compression error, revert to uncompressed binary (still in + context) */ + no_use_compressing: + result_bin = context->s.cc.result_bin; + context->s.cc.result_bin = NULL; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = context->s.cc.real_size; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + erl_zlib_deflate_finish(&(context->s.cc.stream)); + erts_bin_free(context->s.cc.destination_bin); + context->s.cc.destination_bin = NULL; + context->alive = 0; + BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + } + } + } +#undef EXPORT_CONTEXT +#undef RETURN_STATE +} + + + + + + + + /* * This function fills ext with the external format of atom. * If it's an old atom we just supply an index, otherwise @@ -1404,11 +1795,12 @@ static byte* enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) { int iix; - int i, j; + int len; + int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); ASSERT(is_atom(atom)); - if (dflags & DFLAGS_INTERNAL_TAGS) { + if (dflags & DFLAG_INTERNAL_TAGS) { Uint aval = atom_val(atom); ASSERT(aval < (1<<24)); if (aval >= (1 << 16)) { @@ -1423,27 +1815,56 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) } return ep; } + /* * term_to_binary/1,2 and the initial distribution message * don't use the cache. */ - iix = get_iix_acache_map(acmp, atom); - if (iix < 0) { - i = atom_val(atom); - j = atom_tab(i)->len; - if ((MAX_ATOM_LENGTH <= 255 || j <= 255) - && (dflags & DFLAG_SMALL_ATOM_TAGS)) { - *ep++ = SMALL_ATOM_EXT; - put_int8(j, ep); - ep++; + + iix = get_iix_acache_map(acmp, atom, dflags); + if (iix < 0) { + Atom *a = atom_tab(atom_val(atom)); + len = a->len; + if (utf8_atoms || a->latin1_chars < 0) { + if (len > 255) { + *ep++ = ATOM_UTF8_EXT; + put_int16(len, ep); + ep += 2; + } + else { + *ep++ = SMALL_ATOM_UTF8_EXT; + put_int8(len, ep); + ep += 1; + } + sys_memcpy((char *) ep, (char *) a->name, len); } else { - *ep++ = ATOM_EXT; - put_int16(j, ep); - ep += 2; + if (a->latin1_chars <= 255 && (dflags & DFLAG_SMALL_ATOM_TAGS)) { + *ep++ = SMALL_ATOM_EXT; + if (len == a->latin1_chars) { + sys_memcpy(ep+1, a->name, len); + } + else { + len = erts_utf8_to_latin1(ep+1, a->name, len); + ASSERT(len == a->latin1_chars); + } + put_int8(len, ep); + ep++; + } + else { + *ep++ = ATOM_EXT; + if (len == a->latin1_chars) { + sys_memcpy(ep+2, a->name, len); + } + else { + len = erts_utf8_to_latin1(ep+2, a->name, len); + ASSERT(len == a->latin1_chars); + } + put_int16(len, ep); + ep += 2; + } } - sys_memcpy((char *) ep, (char*)atom_tab(i)->name, (int) j); - ep += j; + ep += len; return ep; } @@ -1472,7 +1893,7 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - *ep++ = (is_internal_pid(pid) && (dflags & DFLAGS_INTERNAL_TAGS)) ? + *ep++ = (is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) ? INTERNAL_CREATION : pid_creation(pid); return ep; } @@ -1483,6 +1904,7 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) { Uint len; int n; + ErtsAtomEncoding char_enc; switch (*ep++) { case ATOM_CACHE_REF: @@ -1498,17 +1920,32 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) case ATOM_EXT: len = get_int16(ep), ep += 2; + char_enc = ERTS_ATOM_ENC_LATIN1; goto dec_atom_common; case SMALL_ATOM_EXT: len = get_int8(ep); ep++; + char_enc = ERTS_ATOM_ENC_LATIN1; + goto dec_atom_common; + case ATOM_UTF8_EXT: + len = get_int16(ep), + ep += 2; + char_enc = ERTS_ATOM_ENC_UTF8; + goto dec_atom_common; + case SMALL_ATOM_UTF8_EXT: + len = get_int8(ep), + ep++; + char_enc = ERTS_ATOM_ENC_UTF8; dec_atom_common: if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) { - if (!erts_atom_get((char*)ep, len, objp)) { + if (!erts_atom_get((char*)ep, len, objp, char_enc)) { goto error; } } else { - *objp = am_atom_put((char*)ep, len); + Eterm atom = erts_atom_put(ep, len, char_enc, 0); + if (is_non_value(atom)) + goto error; + *objp = atom; } ep += len; break; @@ -1610,32 +2047,71 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_PATCH_FUN_SIZE ((Eterm) 2) #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) +/* Free extra rootset (used when trapping) */ +static void cleanup_ttb_extra_root(ErlExtraRootSet *rs) +{ + if (rs->objv != NULL) { + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv); + } + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs); +} + +/* Same as above, but we have an extra "stack" beyond GC reach, i.e. an array of two extra roots */ +static void cleanup_ttb_extra_root_2(ErlExtraRootSet *rs) +{ + if (rs->objv != NULL) { + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv); + } + if (rs[1].objv != NULL) { + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs[1].objv); + } + + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs); +} + static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, struct erl_off_heap_header** off_heap) { - DECLARE_WSTACK(s); + byte *res; + (void) enc_term_int(NULL, acmp, obj, ep, dflags, off_heap, NULL, &res); + return res; +} + +static int +enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap, Sint *reds, byte **res) +{ + DECLARE_ESTACK(s); + DECLARE_WSTACK(com); Uint n; Uint i; Uint j; Uint* ptr; Eterm val; FloatDef f; -#if HALFWORD_HEAP - UWord wobj; -#endif + int count_reds = (p != NULL && reds != NULL); + Sint r = 0; + if (count_reds) { + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT); + WSTACK_CHANGE_ALLOCATOR(com, ERTS_ALC_T_EXTRA_ROOT); + r = *reds; + } + + if (p && p->extra_root) { /* restore saved stacks and byte pointer */ + ESTACK_RESTORE(s,p->extra_root[0].objv, p->extra_root[0].sz); + obj = ESTACK_POP(s); + WSTACK_RESTORE(com, p->extra_root[1].objv, p->extra_root[1].sz); + ep = (byte *) WSTACK_POP(com); + } goto L_jump_start; outer_loop: - while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else - obj = WSTACK_POP(s); -#endif - switch (val = WSTACK_POP(s)) { + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + switch (val = WSTACK_POP(com)) { case ENC_TERM: break; case ENC_ONE_CONS: @@ -1646,45 +2122,57 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, obj = CAR(cons); tl = CDR(cons); - WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - WSTACK_PUSH(s, tl); + WSTACK_PUSH(com, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); + ESTACK_PUSH(s, tl); } break; case ENC_PATCH_FUN_SIZE: + /* obj will be discarded, it was NIL */ { -#if HALFWORD_HEAP - byte* size_p = (byte *) wobj; -#else - byte* size_p = (byte *) obj; -#endif + byte* size_p = (byte *) WSTACK_POP(com); put_int32(ep - size_p, size_p); } goto outer_loop; case ENC_LAST_ARRAY_ELEMENT: + /* obj is the tuple */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr; + Eterm* ptr = tuple_val(obj); + i = arityval(*ptr); + obj = ptr[i]; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr++; - WSTACK_PUSH(s, val-1); - WSTACK_PUSH(s, (UWord) ptr); + Eterm* ptr = tuple_val(obj); + i = arityval(*ptr); + ESTACK_PUSH(s, obj); /* put back tuple and next element index */ + WSTACK_PUSH(com, val-1); + obj = ptr[i - (val - ENC_LAST_ARRAY_ELEMENT)]; /* the index is counting down */ } break; } L_jump_start: + + if (count_reds && --r == 0) { + *reds = r; + ESTACK_PUSH(s,obj); /* push back current object, to be popped on restore */ + WSTACK_PUSH(com,((UWord) ep)); + if (p->extra_root == NULL) { + /* NB. Allocate an array of two "extra-roots", of which only the first element + is seen and handled by the GC. Index 1 holds the Wstack. */ + p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet)*2); + p->extra_root->objv = NULL; + p->extra_root->sz = 0; + p->extra_root->cleanup = cleanup_ttb_extra_root_2; + p->extra_root[1].objv = NULL; + p->extra_root[1].sz = 0; + p->extra_root[1].cleanup = NULL; /* Never used */ + } + ESTACK_SAVE(s, p->extra_root[0].objv, p->extra_root[0].sz); + WSTACK_SAVE(com, p->extra_root[1].objv, (p->extra_root[1].sz)); + return -1; + } switch(tag_val_def(obj)) { case NIL_DEF: *ep++ = NIL_EXT; @@ -1770,7 +2258,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, put_int16(i, ep); ep += 2; ep = enc_atom(acmp,ref_node_name(obj),ep,dflags); - *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_ref(obj)) ? + *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) ? INTERNAL_CREATION : ref_creation(obj); ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { @@ -1787,7 +2275,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, j = port_number(obj); put_int32(j, ep); ep += 4; - *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_port(obj)) ? + *ep++ = ((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) ? INTERNAL_CREATION : port_creation(obj); break; @@ -1828,8 +2316,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, ep += 4; } if (i > 0) { - WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - WSTACK_PUSH(s, (UWord) ptr); + WSTACK_PUSH(com, ENC_LAST_ARRAY_ELEMENT+i-1); + ESTACK_PUSH(s, obj); } break; @@ -1837,7 +2325,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { *ep++ = NEW_FLOAT_EXT; -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) put_int32(f.fw[0], ep); ep += 4; put_int32(f.fw[1], ep); @@ -1850,8 +2338,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, } else { *ep++ = FLOAT_EXT; - /* now the sprintf which does the work */ - i = sys_double_to_chars(f.fd, (char*) ep); + /* now the erts_snprintf which does the work */ + i = sys_double_to_chars(f.fd, (char*) ep, (size_t)31); /* Don't leave garbage after the float! (Bad practice in general, * and Purify complains.) @@ -1868,7 +2356,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, byte* bytes; ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); - if (dflags & DFLAGS_INTERNAL_TAGS) { + if (dflags & DFLAG_INTERNAL_TAGS) { ProcBin* pb = (ProcBin*) binary_val(obj); Uint bytesize = pb->size; if (pb->thing_word == HEADER_SUB_BIN) { @@ -1973,8 +2461,9 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, int ei; *ep++ = NEW_FUN_EXT; - WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ + WSTACK_PUSH(com, (UWord) ep); /* Position for patching in size */ + WSTACK_PUSH(com, ENC_PATCH_FUN_SIZE); + ESTACK_PUSH(s,NIL); /* Will be thrown away */ ep += 4; *ep = funp->arity; ep += 1; @@ -1991,8 +2480,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, fun_env: for (ei = funp->num_free-1; ei > 0; ei--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) funp->env[ei]); + WSTACK_PUSH(com, ENC_TERM); + ESTACK_PUSH(s, (UWord) funp->env[ei]); } if (funp->num_free != 0) { obj = funp->env[0]; @@ -2035,8 +2524,17 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, break; } } - DESTROY_WSTACK(s); - return ep; + DESTROY_ESTACK(s); + DESTROY_WSTACK(com); + if (p && p->extra_root) { + cleanup_ttb_extra_root_2(p->extra_root); + p->extra_root = NULL; + } + if (count_reds) { + *reds = r; + } + *res = ep; + return 0; } static @@ -2114,6 +2612,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et { Eterm* hp_saved = *hpp; int n; + ErtsAtomEncoding char_enc; register Eterm* hp = *hpp; /* Please don't take the address of hp */ Eterm* next = objp; @@ -2199,17 +2698,32 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et case ATOM_EXT: n = get_int16(ep); ep += 2; - goto dec_term_atom_common; + char_enc = ERTS_ATOM_ENC_LATIN1; + goto dec_term_atom_common; case SMALL_ATOM_EXT: n = get_int8(ep); ep++; + char_enc = ERTS_ATOM_ENC_LATIN1; + goto dec_term_atom_common; + case ATOM_UTF8_EXT: + n = get_int16(ep); + ep += 2; + char_enc = ERTS_ATOM_ENC_UTF8; + goto dec_term_atom_common; + case SMALL_ATOM_UTF8_EXT: + n = get_int8(ep); + ep++; + char_enc = ERTS_ATOM_ENC_UTF8; dec_term_atom_common: if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) { - if (!erts_atom_get((char*)ep, n, objp)) { + if (!erts_atom_get((char*)ep, n, objp, char_enc)) { goto error; } } else { - *objp = am_atom_put((char*)ep, n); + Eterm atom = erts_atom_put(ep, n, char_enc, 0); + if (is_non_value(atom)) + goto error; + *objp = atom; } ep += n; break; @@ -2290,7 +2804,7 @@ dec_term_atom_common: volatile unsigned long *fpexnp = erts_get_current_fp_exception(); #endif -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) ff.fw[0] = get_int32(ep); ep += 4; ff.fw[1] = get_int32(ep); @@ -2564,7 +3078,7 @@ dec_term_atom_common: goto error; } if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) { - if (!erts_find_export_entry(mod, name, arity)) + if (!erts_active_export_entry(mod, name, arity)) goto error; } *objp = make_export(hp); @@ -2808,51 +3322,47 @@ dec_term_atom_common: to a sequence of bytes N.B. That this must agree with to_external2() above!!! (except for cached atoms) */ +static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) { + Uint res; + (void) encode_size_struct_int(NULL, acmp, obj, dflags, NULL, &res); + return res; +} -static Uint -encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) +static int +encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, + unsigned dflags, Sint *reds, Uint *res) { - DECLARE_WSTACK(s); + DECLARE_ESTACK(s); Uint m, i, arity; Uint result = 0; -#if HALFWORD_HEAP - UWord wobj = 0; -#endif + int count_reds = (p != NULL && reds != 0); + Sint r = 0; + + if (count_reds) { + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT); + r = *reds; + } + + if (p && p->extra_root) { /* restore saved stack */ + ESTACK_RESTORE(s,p->extra_root->objv, p->extra_root->sz + 1); + result = ESTACK_POP(s); /*Untagged, beyond p->extra_root->sz */ + obj = ESTACK_POP(s); + + } goto L_jump_start; outer_loop: - while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else - obj = WSTACK_POP(s); -#endif + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); handle_popped_obj: - if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */ -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - /* - * Pointer into a tuple. - */ - obj = *ptr--; - if (!is_header(obj)) { - WSTACK_PUSH(s, (UWord)ptr); - } else { - /* Reached tuple header */ - ASSERT(header_is_arityval(obj)); - goto outer_loop; - } - } else if (is_list(obj)) { + if (is_list(obj)) { Eterm* cons = list_val(obj); Eterm tl; tl = CDR(cons); obj = CAR(cons); - WSTACK_PUSH(s, tl); + ESTACK_PUSH(s, tl); } else if (is_nil(obj)) { result++; goto outer_loop; @@ -2864,12 +3374,26 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } L_jump_start: + if (count_reds && --r == 0) { + *reds = r; + ESTACK_PUSH(s,obj); /* push back current object */ + ESTACK_PUSH(s,result); /* Untagged, will be out of GC reach */ + if (p->extra_root == NULL) { + p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet)); + p->extra_root->objv = NULL; + p->extra_root->sz = 0; + p->extra_root->cleanup = cleanup_ttb_extra_root; + } + ESTACK_SAVE(s, p->extra_root->objv, p->extra_root->sz); + --p->extra_root->sz; /* Hide result from GC */ + return -1; + } switch (tag_val_def(obj)) { case NIL_DEF: result++; break; case ATOM_DEF: - if (dflags & DFLAGS_INTERNAL_TAGS) { + if (dflags & DFLAG_INTERNAL_TAGS) { if (atom_val(obj) >= (1<<16)) { result += 1 + 3; } @@ -2878,17 +3402,22 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } } else { - int alen = atom_tab(atom_val(obj))->len; - if ((MAX_ATOM_LENGTH <= 255 || alen <= 255) - && (dflags & DFLAG_SMALL_ATOM_TAGS)) { - /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */ - result += 1 + 1 + alen; + Atom *a = atom_tab(atom_val(obj)); + int alen; + if ((dflags & DFLAG_UTF8_ATOMS) || a->latin1_chars < 0) { + alen = a->len; + result += 1 + 1 + alen; + if (alen > 255) { + result++; /* ATOM_UTF8_EXT (not small) */ + } } else { - /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */ - result += 1 + 2 + alen; + alen = a->latin1_chars; + result += 1 + 1 + alen; + if (alen > 255 || !(dflags & DFLAG_SMALL_ATOM_TAGS)) + result++; /* ATOM_EXT (not small) */ } - insert_acache_map(acmp, obj); + insert_acache_map(acmp, obj, dflags); } break; case SMALL_DEF: @@ -2945,20 +3474,24 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case TUPLE_DEF: { Eterm* ptr = tuple_val(obj); - + Uint i; arity = arityval(*ptr); if (arity <= 0xff) { result += 1 + 1; } else { result += 1 + 4; } - ptr += arity; -#if HALFWORD_HEAP - obj = (Eterm) (wobj = (UWord) ptr); -#else - obj = (Eterm) ptr; -#endif - goto handle_popped_obj; + for (i = 1; i <= arity; ++i) { + if (is_list(ptr[i])) { + if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + } + } + ESTACK_PUSH(s,ptr[i]); + } + goto outer_loop; } break; case FLOAT_DEF: @@ -2969,7 +3502,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } break; case BINARY_DEF: - if (dflags & DFLAGS_INTERNAL_TAGS) { + if (dflags & DFLAG_INTERNAL_TAGS) { ProcBin* pb = (ProcBin*) binary_val(obj); Uint sub_extra = 0; Uint tot_bytes = pb->size; @@ -3016,14 +3549,14 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) if (is_not_list(obj)) { /* Push any non-list terms on the stack */ - WSTACK_PUSH(s, obj); + ESTACK_PUSH(s, obj); } else { /* Lists must be handled specially. */ if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; } else { result += 5; - WSTACK_PUSH(s, obj); + ESTACK_PUSH(s, obj); } } } @@ -3054,8 +3587,16 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } } - DESTROY_WSTACK(s); - return result; + DESTROY_ESTACK(s); + if (p && p->extra_root) { + cleanup_ttb_extra_root(p->extra_root); + p->extra_root = NULL; + } + if (count_reds) { + *reds = r; + } + *res = result; + return 0; } static Sint @@ -3125,21 +3666,41 @@ decoded_size(byte *ep, byte* endp, int internal_tags) case ATOM_EXT: CHKSIZE(2); n = get_int16(ep); - if (n > MAX_ATOM_LENGTH) { + if (n > MAX_ATOM_CHARACTERS) { return -1; } SKIP(n+2+atom_extra_skip); atom_extra_skip = 0; break; + case ATOM_UTF8_EXT: + CHKSIZE(2); + n = get_int16(ep); + ep += 2; + if (n > MAX_ATOM_SZ_LIMIT) { + return -1; + } + SKIP(n+atom_extra_skip); + atom_extra_skip = 0; + break; case SMALL_ATOM_EXT: CHKSIZE(1); n = get_int8(ep); - if (n > MAX_ATOM_LENGTH) { + if (n > MAX_ATOM_CHARACTERS) { return -1; } SKIP(n+1+atom_extra_skip); atom_extra_skip = 0; break; + case SMALL_ATOM_UTF8_EXT: + CHKSIZE(1); + n = get_int8(ep); + ep++; + if (n > MAX_ATOM_SZ_LIMIT) { + return -1; + } + SKIP(n+atom_extra_skip); + atom_extra_skip = 0; + break; case ATOM_CACHE_REF: SKIP(1+atom_extra_skip); atom_extra_skip = 0; diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index eddd4571dd..e37d47919e 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -51,6 +51,8 @@ #define NEW_FUN_EXT 'p' #define EXPORT_EXT 'q' #define FUN_EXT 'u' +#define ATOM_UTF8_EXT 'v' +#define SMALL_ATOM_UTF8_EXT 'w' #define DIST_HEADER 'D' #define ATOM_CACHE_REF 'R' @@ -90,6 +92,7 @@ typedef struct cache { typedef struct { int hdr_sz; int sz; + int long_atoms; int cix[ERTS_ATOM_CACHE_SIZE]; struct { Eterm atom; @@ -150,12 +153,12 @@ typedef struct { void erts_init_atom_cache_map(ErtsAtomCacheMap *); void erts_reset_atom_cache_map(ErtsAtomCacheMap *); void erts_destroy_atom_cache_map(ErtsAtomCacheMap *); -void erts_finalize_atom_cache_map(ErtsAtomCacheMap *); +void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32); Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *); -byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *); +byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32); Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *); void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 5f41ed0dc4..bacd5a5752 100755 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,6 +28,7 @@ #include "hash.h" #include "index.h" #include "atom.h" +#include "code_ix.h" #include "export.h" #include "module.h" #include "register.h" @@ -38,38 +39,8 @@ #include "erl_sys_driver.h" #include "erl_debug.h" #include "error.h" - -typedef struct port Port; -#include "erl_port_task.h" - -typedef struct erts_driver_t_ erts_driver_t; - -#define SMALL_IO_QUEUE 5 /* Number of fixed elements */ - -typedef struct { - ErlDrvSizeT size; /* total size in bytes */ - - SysIOVec* v_start; - SysIOVec* v_end; - SysIOVec* v_head; - SysIOVec* v_tail; - SysIOVec v_small[SMALL_IO_QUEUE]; - - ErlDrvBinary** b_start; - ErlDrvBinary** b_end; - ErlDrvBinary** b_head; - ErlDrvBinary** b_tail; - ErlDrvBinary* b_small[SMALL_IO_QUEUE]; -} ErlIOQueue; - -typedef struct line_buf { /* Buffer used in line oriented I/O */ - ErlDrvSizeT bufsiz; /* Size of character buffer */ - ErlDrvSizeT ovlen; /* Length of overflow data */ - ErlDrvSizeT ovsiz; /* Actual size of overflow buffer */ - char data[1]; /* Starting point of buffer data, - data[0] is a flag indicating an unprocess CR, - The rest is the overflow buffer. */ -} LineBuf; +#include "erl_utils.h" +#include "erl_port.h" struct enif_environment_t /* ErlNifEnv */ { @@ -89,158 +60,6 @@ extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); -/* - * Port Specific Data. - * - * Only use PrtSD for very rarely used data. - */ - -#define ERTS_PRTSD_SCHED_ID 0 - -#define ERTS_PRTSD_SIZE 1 - -typedef struct { - void *data[ERTS_PRTSD_SIZE]; -} ErtsPrtSD; - -#ifdef ERTS_SMP -typedef struct ErtsXPortsList_ ErtsXPortsList; -#endif - -/* - * Port locking: - * - * Locking is done either driver specific or port specific. When - * driver specific locking is used, all instances of the driver, - * i.e. ports running the driver, share the same lock. When port - * specific locking is used each instance have its own lock. - * - * Most fields in the Port structure are protected by the lock - * referred to by the lock field. I'v called it the port lock. - * This lock is shared between all ports running the same driver - * when driver specific locking is used. - * - * The 'sched' field is protected by the port tasks lock - * (see erl_port_tasks.c) - * - * The 'status' field is protected by a combination of the port lock, - * the port tasks lock, and the state_lck. It may be read if - * the state_lck, or the port lock is held. It may only be - * modified if both the port lock and the state_lck is held - * (with one exception; see below). When changeing status from alive - * to dead or vice versa, also the port task lock has to be held. - * This in order to guarantee that tasks are scheduled only for - * ports that are alive. - * - * The status field may be modified with only the state_lck - * held when status is changed from dead to alive. This since no - * threads can have any references to the port other than via the - * port table. - * - * /rickard - */ - -struct port { - ErtsPortTaskSched sched; - ErtsPortTaskHandle timeout_task; -#ifdef ERTS_SMP - erts_smp_atomic_t refc; - erts_smp_mtx_t *lock; - ErtsXPortsList *xports; - erts_smp_atomic_t run_queue; - erts_smp_spinlock_t state_lck; /* protects: id, status, snapshot */ -#endif - Eterm id; /* The Port id of this port */ - Eterm connected; /* A connected process */ - Eterm caller; /* Current caller. */ - Eterm data; /* Data associated with port. */ - ErlHeapFragment* bp; /* Heap fragment holding data (NULL if imm data). */ - ErtsLink *nlinks; - ErtsMonitor *monitors; /* Only MON_ORIGIN monitors of pid's */ - Uint bytes_in; /* Number of bytes read */ - Uint bytes_out; /* Number of bytes written */ -#ifdef ERTS_SMP - ErtsSmpPTimer *ptimer; -#else - ErlTimer tm; /* Timer entry */ -#endif - - Eterm tracer_proc; /* If the port is traced, this is the tracer */ - Uint trace_flags; /* Trace flags */ - - ErlIOQueue ioq; /* driver accessible i/o queue */ - DistEntry *dist_entry; /* Dist entry used in DISTRIBUTION */ - char *name; /* String used in the open */ - erts_driver_t* drv_ptr; - UWord drv_data; - SWord os_pid; /* Child process ID */ - ErtsProcList *suspended; /* List of suspended processes. */ - LineBuf *linebuf; /* Buffer to hold data not ready for - process to get (line oriented I/O)*/ - Uint32 status; /* Status and type flags */ - int control_flags; /* Flags for port_control() */ - erts_aint32_t snapshot; /* Next snapshot that port should be part of */ - struct reg_proc *reg; - ErlDrvPDL port_data_lock; - - ErtsPrtSD *psd; /* Port specific data */ -}; - - -ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE ErtsRunQueue * -erts_port_runq(Port *prt) -{ -#ifdef ERTS_SMP - ErtsRunQueue *rq1, *rq2; - rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); - while (1) { - erts_smp_runq_lock(rq1); - rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); - if (rq1 == rq2) - return rq1; - erts_smp_runq_unlock(rq1); - rq1 = rq2; - } -#else - return ERTS_RUNQ_IX(0); -#endif -} - -#endif - - -ERTS_GLB_INLINE void *erts_prtsd_get(Port *p, int ix); -ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void * -erts_prtsd_get(Port *prt, int ix) -{ - return prt->psd ? prt->psd->data[ix] : NULL; -} - -ERTS_GLB_INLINE void * -erts_prtsd_set(Port *prt, int ix, void *data) -{ - if (prt->psd) { - void *old = prt->psd->data[ix]; - prt->psd->data[ix] = data; - return old; - } - else { - prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD)); - prt->psd->data[ix] = data; - return NULL; - } -} - -#endif - /* Driver handle (wrapper for old plain handle) */ #define ERL_DE_OK 0 #define ERL_DE_UNLOAD 1 @@ -292,7 +111,7 @@ typedef struct { or that wait for it to change state */ erts_refc_t refc; /* Number of ports/processes having references to the driver */ - Uint port_count; /* Number of ports using the driver */ + erts_smp_atomic32_t port_count; /* Number of ports using the driver */ Uint flags; /* ERL_DE_FL_KILL_PORTS */ int status; /* ERL_DE_xxx */ char *full_path; /* Full path of the driver */ @@ -344,7 +163,7 @@ struct erts_driver_t_ { }; extern erts_driver_t *driver_list; -extern erts_smp_mtx_t erts_driver_list_lock; +extern erts_smp_rwmtx_t erts_driver_list_lock; extern void erts_ddll_init(void); extern void erts_ddll_lock_driver(DE_Handle *dh, char *name); @@ -524,40 +343,9 @@ union erl_off_heap_ptr { void* voidp; }; -/* arrays that get malloced at startup */ -extern Port* erts_port; - -extern Uint erts_max_ports; -extern Uint erts_port_tab_index_mask; -extern erts_smp_atomic32_t erts_ports_snapshot; -extern erts_smp_atomic_t erts_dead_ports_ptr; - -ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) -{ - ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck)); - if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) { - /* Dead ports are added from the end of the snapshot buffer */ - Eterm* tombstone; - tombstone = (Eterm*) erts_smp_atomic_add_read_nob(&erts_dead_ports_ptr, - -(erts_aint_t)sizeof(Eterm)); - ASSERT(tombstone+1 != NULL); - ASSERT(prt->snapshot == erts_smp_atomic32_read_nob(&erts_ports_snapshot) - 1); - *tombstone = prt->id; - } - /*else no ongoing snapshot or port was already included or created after snapshot */ -} - -#endif - /* controls warning mapping in error_logger */ extern Eterm node_cookie; -extern erts_smp_atomic_t erts_bytes_out; /* no bytes written out */ -extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ extern Uint display_items; /* no of items to display in traces etc */ extern int erts_backtrace_depth; @@ -588,7 +376,7 @@ extern int stackdump_on_exit; */ -void erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end); +void erl_grow_stack(ErtsAlcType_t a_type, Eterm** start, Eterm** sp, Eterm** end); #define ESTK_CONCAT(a,b) a##b #define ESTK_SUBSCRIPT(s,i) *((Eterm *)((byte *)ESTK_CONCAT(s,_start) + (i))) #define DEF_ESTACK_SIZE (16) @@ -597,20 +385,79 @@ void erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end); Eterm ESTK_CONCAT(s,_default_stack)[DEF_ESTACK_SIZE]; \ Eterm* ESTK_CONCAT(s,_start) = ESTK_CONCAT(s,_default_stack); \ Eterm* ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start); \ - Eterm* ESTK_CONCAT(s,_end) = ESTK_CONCAT(s,_start) + DEF_ESTACK_SIZE + Eterm* ESTK_CONCAT(s,_end) = ESTK_CONCAT(s,_start) + DEF_ESTACK_SIZE;\ + ErtsAlcType_t ESTK_CONCAT(s,_alloc_type) = ERTS_ALC_T_ESTACK + +#define ESTACK_CHANGE_ALLOCATOR(s,t) \ +do { \ + if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) { \ + erl_exit(1, "Internal error - trying to change allocator " \ + "type of active estack\n"); \ + } \ + ESTK_CONCAT(s,_alloc_type) = (t); \ + } while (0) + +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'v'. 'v' and 'vsize' are changed by this macro. If + * 'v' points to anything, it should have been allocated by a previous + * call to this macro. Be careful to set a correct allocator prior to + * saving. + * 'v' can be any lvalue pointer, it will point to an array of UWord + * after calling this macro. + */ +#define ESTACK_SAVE(s,v,vsize) /* v and vsize are "name parameters" */ \ +do { \ + Uint _esz = ESTACK_COUNT(s); \ + if (ESTK_CONCAT(s,_start) == ESTK_CONCAT(s,_default_stack)) { \ + if ((v) == NULL) { \ + (v) = erts_alloc(ESTK_CONCAT(s,_alloc_type), \ + DEF_ESTACK_SIZE * sizeof(Eterm)); \ + } \ + memcpy((v),ESTK_CONCAT(s,_start),_esz*sizeof(Eterm)); \ + } else { \ + (v) = (void *) ESTK_CONCAT(s,_start); \ + } \ + (vsize) = _esz; \ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this + * The vector parameter is reset to NULL if the vector is moved to stack, + * otherwise it's kept for reuse, so a saved and restored vector might + * need freeing using the correct allocator parameter. + * 'v' can be any lvalue pointer, it's cast to an (Eterm *). + */ +#define ESTACK_RESTORE(s, v, vsize) /*v is a "name parameter"*/ \ +do { \ + if ((vsize) > DEF_ESTACK_SIZE) { \ + Uint _ca = DEF_ESTACK_SIZE; \ + while (_ca < (vsize)) \ + _ca = _ca * 2; \ + ESTK_CONCAT(s,_start) = (Eterm *) (v); \ + ESTK_CONCAT(s,_end) = ((Eterm *)(v)) + _ca; \ + ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start) + (vsize); \ + (v) = NULL; \ + } else { \ + memcpy(ESTK_CONCAT(s,_start),(v),(vsize)*sizeof(Eterm));\ + ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start) + (vsize); \ + } \ + } while (0) + +#define ESTACK_IS_STATIC(s) (ESTK_CONCAT(s,_start) == ESTK_CONCAT(s,_default_stack)) #define DESTROY_ESTACK(s) \ do { \ if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) { \ - erts_free(ERTS_ALC_T_ESTACK, ESTK_CONCAT(s,_start)); \ + erts_free(ESTK_CONCAT(s,_alloc_type), ESTK_CONCAT(s,_start)); \ } \ } while(0) #define ESTACK_PUSH(s, x) \ do { \ if (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_end)) { \ - erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \ - &ESTK_CONCAT(s,_end)); \ + erl_grow_stack(ESTK_CONCAT(s,_alloc_type),&ESTK_CONCAT(s,_start), \ + &ESTK_CONCAT(s,_sp), &ESTK_CONCAT(s,_end)); \ } \ *ESTK_CONCAT(s,_sp)++ = (x); \ } while(0) @@ -618,8 +465,8 @@ do { \ #define ESTACK_PUSH2(s, x, y) \ do { \ if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 2) { \ - erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \ - &ESTK_CONCAT(s,_end)); \ + erl_grow_stack(ESTK_CONCAT(s,_alloc_type),&ESTK_CONCAT(s,_start), \ + &ESTK_CONCAT(s,_sp), &ESTK_CONCAT(s,_end)); \ } \ *ESTK_CONCAT(s,_sp)++ = (x); \ *ESTK_CONCAT(s,_sp)++ = (y); \ @@ -642,7 +489,7 @@ do { \ #define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp))) -void erl_grow_wstack(UWord** start, UWord** sp, UWord** end); +void erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end); #define WSTK_CONCAT(a,b) a##b #define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i))) #define DEF_WSTACK_SIZE (16) @@ -651,20 +498,79 @@ void erl_grow_wstack(UWord** start, UWord** sp, UWord** end); UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE]; \ UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack); \ UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start); \ - UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE + UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE; \ + ErtsAlcType_t WSTK_CONCAT(s,_alloc_type) = ERTS_ALC_T_ESTACK + +#define WSTACK_CHANGE_ALLOCATOR(s,t) \ +do { \ + if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \ + erl_exit(1, "Internal error - trying to change allocator " \ + "type of active wstack\n"); \ + } \ + WSTK_CONCAT(s,_alloc_type) = (t); \ + } while (0) #define DESTROY_WSTACK(s) \ do { \ if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \ - erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start)); \ + erts_free(WSTK_CONCAT(s,_alloc_type), WSTK_CONCAT(s,_start)); \ } \ } while(0) +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'v'. 'v' and 'vsize' are changed by this macro. If + * 'v' points to anything, it should have been allocated by a previous + * call to this macro. Be careful to set a correct allocator prior to + * saving. + * 'v' can be any lvalue pointer, it will point to an array of UWord + * after calling this macro. + */ +#define WSTACK_SAVE(s,v,vsize) /* v and vsize are "name parameters" */ \ +do { \ + Uint _wsz = WSTACK_COUNT(s); \ + if (WSTK_CONCAT(s,_start) == WSTK_CONCAT(s,_default_stack)) { \ + if ((v) == NULL) { \ + (v) = erts_alloc(WSTK_CONCAT(s,_alloc_type), \ + DEF_WSTACK_SIZE * sizeof(UWord)); \ + } \ + memcpy((v),WSTK_CONCAT(s,_start),_wsz*sizeof(UWord)); \ + } else { \ + (v) = (void *) WSTK_CONCAT(s,_start); \ + } \ + (vsize) = _wsz; \ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this + * The vector parameter is reset to NULL if the vector is moved to stack, + * otherwise it's kept for reuse, so a saved and restored vector might + * need freeing using the correct allocator parameter. + * 'v' can be any lvalue pointer, it's cast to an (UWord *). + */ +#define WSTACK_RESTORE(s, v, vsize) /*v is a "name parameter"*/ \ +do { \ + if ((vsize) > DEF_WSTACK_SIZE) { \ + Uint _ca = DEF_WSTACK_SIZE; \ + while (_ca < (vsize)) \ + _ca = _ca * 2; \ + WSTK_CONCAT(s,_start) = (UWord *) (v); \ + WSTK_CONCAT(s,_end) = ((UWord *)(v)) + _ca; \ + WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start) + (vsize); \ + (v) = NULL; \ + } else { \ + memcpy(WSTK_CONCAT(s,_start),(v),(vsize)*sizeof(UWord));\ + WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start) + (vsize); \ + } \ + } while (0) + +#define WSTACK_IS_STATIC(s) (WSTK_CONCAT(s,_start) == WSTK_CONCAT(s,_default_stack)) + #define WSTACK_PUSH(s, x) \ do { \ if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ + erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \ + &WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end)); \ } \ *WSTK_CONCAT(s,_sp)++ = (x); \ } while(0) @@ -672,8 +578,8 @@ do { \ #define WSTACK_PUSH2(s, x, y) \ do { \ if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ + erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \ + &WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end)); \ } \ *WSTK_CONCAT(s,_sp)++ = (x); \ *WSTK_CONCAT(s,_sp)++ = (y); \ @@ -682,8 +588,8 @@ do { \ #define WSTACK_PUSH3(s, x, y, z) \ do { \ if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ + erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \ + &WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end)); \ } \ *WSTK_CONCAT(s,_sp)++ = (x); \ *WSTK_CONCAT(s,_sp)++ = (y); \ @@ -695,54 +601,6 @@ do { \ #define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start)) #define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp))) - -/* port status flags */ - -#define ERTS_PORT_SFLG_CONNECTED ((Uint32) (1 << 0)) -/* Port have begun exiting */ -#define ERTS_PORT_SFLG_EXITING ((Uint32) (1 << 1)) -/* Distribution port */ -#define ERTS_PORT_SFLG_DISTRIBUTION ((Uint32) (1 << 2)) -#define ERTS_PORT_SFLG_BINARY_IO ((Uint32) (1 << 3)) -#define ERTS_PORT_SFLG_SOFT_EOF ((Uint32) (1 << 4)) -/* Flow control */ -#define ERTS_PORT_SFLG_PORT_BUSY ((Uint32) (1 << 5)) -/* Port is closing (no i/o accepted) */ -#define ERTS_PORT_SFLG_CLOSING ((Uint32) (1 << 6)) -/* Send a closed message when terminating */ -#define ERTS_PORT_SFLG_SEND_CLOSED ((Uint32) (1 << 7)) -/* Line orinted io on port */ -#define ERTS_PORT_SFLG_LINEBUF_IO ((Uint32) (1 << 8)) -/* Immortal port (only certain system ports) */ -#define ERTS_PORT_SFLG_IMMORTAL ((Uint32) (1 << 9)) -#define ERTS_PORT_SFLG_FREE ((Uint32) (1 << 10)) -#define ERTS_PORT_SFLG_FREE_SCHEDULED ((Uint32) (1 << 11)) -#define ERTS_PORT_SFLG_INITIALIZING ((Uint32) (1 << 12)) -/* Port uses port specific locking (opposed to driver specific locking) */ -#define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 13)) -#define ERTS_PORT_SFLG_INVALID ((Uint32) (1 << 14)) -/* Last port to terminate halts the emulator */ -#define ERTS_PORT_SFLG_HALT ((Uint32) (1 << 15)) -#ifdef DEBUG -/* Only debug: make sure all flags aren't cleared unintentionally */ -#define ERTS_PORT_SFLG_PORT_DEBUG ((Uint32) (1 << 31)) -#endif - -/* Combinations of port status flags */ -#define ERTS_PORT_SFLGS_DEAD \ - (ERTS_PORT_SFLG_FREE \ - | ERTS_PORT_SFLG_FREE_SCHEDULED \ - | ERTS_PORT_SFLG_INITIALIZING) -#define ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP \ - (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID) -#define ERTS_PORT_SFLGS_INVALID_LOOKUP \ - (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP \ - | ERTS_PORT_SFLG_CLOSING) -#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP \ - (ERTS_PORT_SFLGS_INVALID_LOOKUP \ - | ERTS_PORT_SFLG_PORT_BUSY \ - | ERTS_PORT_SFLG_DISTRIBUTION) - /* binary.c */ void erts_emasculate_writable_binary(ProcBin* pb); @@ -753,17 +611,43 @@ Eterm erts_realloc_binary(Eterm bin, size_t size); /* erl_bif_info.c */ +Eterm +erts_bld_port_info(Eterm **hpp, + ErlOffHeap *ohp, + Uint *szp, + Port *prt, + Eterm item); + void erts_bif_info_init(void); /* bif.c */ Eterm erts_make_ref(Process *); Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); +void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); + +ERTS_GLB_INLINE Eterm +erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Eterm +erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Eterm *hp = HAlloc(c_p, REF_THING_SIZE); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +#endif + void erts_queue_monitor_message(Process *, ErtsProcLocks*, Eterm, Eterm, Eterm, Eterm); +void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, + Eterm (*bif)(Process*,Eterm*)); void erts_init_bif(void); Eterm erl_send(Process *p, Eterm to, Eterm msg); @@ -771,13 +655,6 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); -/* erl_bif_port.c */ - -/* erl_bif_trace.c */ -Eterm erl_seq_trace_info(Process *p, Eterm arg1); -void erts_system_monitor_clear(Process *c_p); -void erts_system_profile_clear(Process *c_p); - /* beam_load.c */ typedef struct { BeamInstr* current; /* Pointer to: Mod, Name, Arity */ @@ -786,24 +663,33 @@ typedef struct { Eterm* fname_ptr; /* Pointer to fname table */ } FunctionInfo; -struct LoaderState* erts_alloc_loader_state(void); -Eterm erts_prepare_loading(struct LoaderState*, Process *c_p, +Binary* erts_alloc_loader_state(void); +Eterm erts_module_for_prepared_code(Binary* magic); +Eterm erts_prepare_loading(Binary* loader_state, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint size); -Eterm erts_finish_loading(struct LoaderState* stp, Process* c_p, +Eterm erts_finish_loading(Binary* loader_state, Process* c_p, ErtsProcLocks c_p_locks, Eterm* modp); -Eterm erts_load_module(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm* mod, byte* code, Uint size); +Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, + Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); BeamInstr* find_function_from_pc(BeamInstr* pc); Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p); -void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +/* beam_ranges.c */ +void erts_init_ranges(void); +void erts_start_staging_ranges(void); +void erts_end_staging_ranges(int commit); +void erts_update_ranges(BeamInstr* code, Uint size); +void erts_remove_from_ranges(BeamInstr* code); +UWord erts_ranges_sz(void); +void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); + /* break.c */ void init_break_handler(void); void erts_set_ignore_break(void); @@ -920,12 +806,6 @@ void MD5Final(unsigned char [16], MD5_CTX *); /* ggc.c */ - -typedef struct { - Uint garbage_collections; - Uint reclaimed; -} ErtsGCInfo; - void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); int erts_garbage_collect(Process*, int, Eterm*, int); @@ -944,11 +824,6 @@ void erts_free_heap_frags(Process* p); /* io.c */ -struct erl_drv_port_data_lock { - erts_mtx_t mtx; - erts_atomic_t refc; -}; - typedef struct { char *name; char *driver_name; @@ -957,363 +832,35 @@ typedef struct { #define ERTS_SPAWN_DRIVER 1 #define ERTS_SPAWN_EXECUTABLE 2 #define ERTS_SPAWN_ANY (ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXECUTABLE) - int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_locked); void erts_destroy_driver(erts_driver_t *drv); -void erts_wake_process_later(Port*, Process*); -int erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *); -int erts_is_port_ioq_empty(Port *); -void erts_terminate_port(Port *); -void close_port(Eterm); -void init_io(void); -void cleanup_io(void); -void erts_do_exit_port(Port *, Eterm, Eterm); -void erts_port_command(Process *, Eterm, Port *, Eterm); -Eterm erts_port_control(Process*, Port*, Uint, Eterm); -int erts_write_to_port(Eterm caller_id, Port *p, Eterm list); -void print_port_info(int, void *, int); +int erts_save_suspend_process_on_port(Port*, Process*); +Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *); +void erts_init_io(int, int, int); void erts_raw_port_command(Port*, byte*, Uint); -void driver_report_exit(int, int); +void driver_report_exit(ErlDrvPort, int); LineBuf* allocate_linebuf(int); int async_ready(Port *, void*); -Sint erts_test_next_port(int, Uint); -ErtsPortNames *erts_get_port_names(Eterm); +ErtsPortNames *erts_get_port_names(Eterm, ErlDrvPort); void erts_free_port_names(ErtsPortNames *); Uint erts_port_ioq_size(Port *pp); -void erts_stale_drv_select(Eterm, ErlDrvEvent, int, int); -void erts_port_cleanup(Port *); -void erts_fire_port_monitor(Port *prt, Eterm ref); +void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int); Port *erts_get_heart_port(void); -#ifdef ERTS_SMP -void erts_smp_xports_unlock(Port *); -#endif - #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_enable_io_lock_count(int enable); #endif -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) -int erts_lc_is_port_locked(Port *); -#endif - -ERTS_GLB_INLINE void erts_smp_port_state_lock(Port*); -ERTS_GLB_INLINE void erts_smp_port_state_unlock(Port*); - -ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt); -ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt); -ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void -erts_smp_port_state_lock(Port* prt) -{ -#ifdef ERTS_SMP - erts_smp_spin_lock(&prt->state_lck); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_port_state_unlock(Port *prt) -{ -#ifdef ERTS_SMP - erts_smp_spin_unlock(&prt->state_lck); -#endif -} - - -ERTS_GLB_INLINE int -erts_smp_port_trylock(Port *prt) -{ -#ifdef ERTS_SMP - int res; - - ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); - erts_smp_atomic_inc_nob(&prt->refc); - res = erts_smp_mtx_trylock(prt->lock); - if (res == EBUSY) { - erts_smp_atomic_dec_nob(&prt->refc); - } - - return res; -#else /* !ERTS_SMP */ - return 0; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_port_lock(Port *prt) -{ -#ifdef ERTS_SMP - ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0); - erts_smp_atomic_inc_nob(&prt->refc); - erts_smp_mtx_lock(prt->lock); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_port_unlock(Port *prt) -{ -#ifdef ERTS_SMP - erts_aint_t refc; - erts_smp_mtx_unlock(prt->lock); - refc = erts_smp_atomic_dec_read_nob(&prt->refc); - ASSERT(refc >= 0); - if (refc == 0) - erts_port_cleanup(prt); -#endif -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - - -#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS) \ - (!(PP) || ((PP)->status & (FLGS)) || (PP)->id != (ID)) - -/* port lookup */ - -#define INVALID_PORT(PP, ID) \ - ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_LOOKUP) - -/* Invalidate trace port if anything suspicious, for instance - * that the port is a distribution port or it is busy. - */ -#define INVALID_TRACER_PORT(PP, ID) \ - ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP) - -#define ERTS_PORT_SCHED_ID(P, ID) \ - ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID))) - -#ifdef ERTS_SMP -Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks); -#endif - -#define erts_id2port(ID, P, PL) \ - erts_id2port_sflgs((ID), (P), (PL), ERTS_PORT_SFLGS_INVALID_LOOKUP) - -ERTS_GLB_INLINE Port*erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32); -ERTS_GLB_INLINE void erts_port_release(Port *); -ERTS_GLB_INLINE Port*erts_drvport2port(ErlDrvPort); -ERTS_GLB_INLINE Port*erts_drvportid2port(Eterm); -ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm id); -ERTS_GLB_INLINE int erts_is_port_alive(Eterm id); -ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm id); -ERTS_GLB_INLINE void erts_port_status_bandor_set(Port *, Uint32, Uint32); -ERTS_GLB_INLINE void erts_port_status_band_set(Port *, Uint32); -ERTS_GLB_INLINE void erts_port_status_bor_set(Port *, Uint32); -ERTS_GLB_INLINE void erts_port_status_set(Port *, Uint32); -ERTS_GLB_INLINE Uint32 erts_port_status_get(Port *); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE Port* -erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs) -{ -#ifdef ERTS_SMP - int no_proc_locks = !c_p || !c_p_locks; -#endif - Port *prt; - - if (is_not_internal_port(id)) - return NULL; - - prt = &erts_port[internal_port_index(id)]; - - erts_smp_port_state_lock(prt); - if (ERTS_INVALID_PORT_OPT(prt, id, sflgs)) { - erts_smp_port_state_unlock(prt); - prt = NULL; - } -#ifdef ERTS_SMP - else { - erts_smp_atomic_inc_nob(&prt->refc); - erts_smp_port_state_unlock(prt); - - if (no_proc_locks) - erts_smp_mtx_lock(prt->lock); - else if (erts_smp_mtx_trylock(prt->lock) == EBUSY) { - /* Unlock process locks, and acquire locks in lock order... */ - erts_smp_proc_unlock(c_p, c_p_locks); - erts_smp_mtx_lock(prt->lock); - erts_smp_proc_lock(c_p, c_p_locks); - } - - /* The id may not have changed... */ - ERTS_SMP_LC_ASSERT(prt->id == id); - /* ... but status may have... */ - if (prt->status & sflgs) { - erts_smp_port_unlock(prt); /* Also decrements refc... */ - prt = NULL; - } - } -#endif - - return prt; -} - -ERTS_GLB_INLINE void -erts_port_release(Port *prt) -{ -#ifdef ERTS_SMP - erts_smp_port_unlock(prt); -#else - if (prt->status & ERTS_PORT_SFLGS_DEAD) - erts_port_cleanup(prt); -#endif -} - -ERTS_GLB_INLINE Port* -erts_drvport2port(ErlDrvPort drvport) -{ - int ix = (int) drvport; - if (ix < 0 || erts_max_ports <= ix) - return NULL; - if (erts_port[ix].status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - return NULL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[ix])); - return &erts_port[ix]; -} - -ERTS_GLB_INLINE Port* -erts_drvportid2port(Eterm id) -{ - int ix; - if (is_not_internal_port(id)) - return NULL; - ix = (int) internal_port_index(id); - if (erts_max_ports <= ix) - return NULL; - if (erts_port[ix].status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - return NULL; - if (erts_port[ix].id != id) - return NULL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[ix])); - return &erts_port[ix]; -} - -ERTS_GLB_INLINE Uint32 -erts_portid2status(Eterm id) -{ - if (is_not_internal_port(id)) - return ERTS_PORT_SFLG_INVALID; - else { - Uint32 status; - int ix = internal_port_index(id); - if (erts_max_ports <= ix) - return ERTS_PORT_SFLG_INVALID; - erts_smp_port_state_lock(&erts_port[ix]); - if (erts_port[ix].id == id) - status = erts_port[ix].status; - else - status = ERTS_PORT_SFLG_INVALID; - erts_smp_port_state_unlock(&erts_port[ix]); - return status; - } -} - -ERTS_GLB_INLINE int -erts_is_port_alive(Eterm id) -{ - return !(erts_portid2status(id) & (ERTS_PORT_SFLG_INVALID - | ERTS_PORT_SFLGS_DEAD)); -} - -ERTS_GLB_INLINE int -erts_is_valid_tracer_port(Eterm id) -{ - return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); -} - -ERTS_GLB_INLINE void erts_port_status_bandor_set(Port *prt, - Uint32 band_status, - Uint32 bor_status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status &= band_status; - prt->status |= bor_status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE void erts_port_status_band_set(Port *prt, Uint32 status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status &= status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE void erts_port_status_bor_set(Port *prt, Uint32 status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status |= status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE void erts_port_status_set(Port *prt, Uint32 status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status = status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE Uint32 erts_port_status_get(Port *prt) -{ - Uint32 res; - erts_smp_port_state_lock(prt); - res = prt->status; - erts_smp_port_state_unlock(prt); - return res; -} -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - /* erl_drv_thread.c */ void erl_drv_thr_init(void); -/* time.c */ - /* utils.c */ - -/* - * To be used to silence unused result warnings, but do not abuse it. - */ -void erts_silence_warn_unused_result(long unused); - void erts_cleanup_offheap(ErlOffHeap *offheap); -Uint erts_fit_in_bits(Uint); -int list_length(Eterm); -Export* erts_find_function(Eterm, Eterm, unsigned int); -int erts_is_builtin(Eterm, Eterm, int); -Uint32 make_broken_hash(Eterm); -Uint32 block_hash(byte *, unsigned, Uint32); -Uint32 make_hash2(Eterm); -Uint32 make_hash(Eterm); - - -Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str); -Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); -Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw); -Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64); -Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64); -Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr); -Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...); -Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]); -Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len); -#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str)) -Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]); -Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp, - Sint length, Eterm terms1[], Uint terms2[]); -Eterm -erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp, - Sint length, Eterm atoms[], Uint uints[]); -Eterm -erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, - Eterm atoms[], Uint uints1[], Uint uints2[]); +Uint64 erts_timestamp_millis(void); + +Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex); Eterm store_external_or_ref_in_proc_(Process *, Eterm); Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); @@ -1328,42 +875,6 @@ Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); (ASSERT_EXPR(is_node_container((NC))), \ IS_CONST((NC)) ? (NC) : store_external_or_ref_in_proc_((Pp), (NC))) -void erts_init_utils(void); -void erts_init_utils_mem(void); - -erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint); -void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *); - -#if HALFWORD_HEAP -int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); -# define eq(A,B) eq_rel(A,NULL,B,NULL) -#else -int eq(Eterm, Eterm); -# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B) -#endif - -#define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) - -#if HALFWORD_HEAP -Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*); -#define CMP(A,B) cmp_rel(A,NULL,B,NULL) -#else -Sint cmp(Eterm, Eterm); -#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B) -#define CMP(A,B) cmp(A,B) -#endif -#define cmp_lt(a,b) (CMP((a),(b)) < 0) -#define cmp_le(a,b) (CMP((a),(b)) <= 0) -#define cmp_eq(a,b) (CMP((a),(b)) == 0) -#define cmp_ne(a,b) (CMP((a),(b)) != 0) -#define cmp_ge(a,b) (CMP((a),(b)) >= 0) -#define cmp_gt(a,b) (CMP((a),(b)) > 0) - -#define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) -#define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) -#define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) -#define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) - /* duplicates from big.h */ int term_to_Uint(Eterm term, Uint *up); int term_to_UWord(Eterm, UWord*); @@ -1380,6 +891,9 @@ Sint erts_re_set_loop_limit(Sint limit); void erts_init_bif_binary(void); Sint erts_binary_set_loop_limit(Sint limit); +/* external.c */ +void erts_init_external(void); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); @@ -1389,92 +903,23 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding); void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars); int erts_analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left); +int erts_analyze_utf8_x(byte *source, Uint size, + byte **err_pos, Uint *num_chars, int *left, + Sint *num_latin1_chars, Uint max_chars); char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_size, ErtsAlcType_t alloc_type, int allow_empty, int allow_atom, Sint *used /* out */); Eterm erts_convert_native_to_filename(Process *p, byte *bytes); +Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left, + Uint *num_built, Uint *num_eaten, Eterm tail); +int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); #define ERTS_UTF8_OK 0 #define ERTS_UTF8_INCOMPLETE 1 #define ERTS_UTF8_ERROR 2 #define ERTS_UTF8_ANALYZE_MORE 3 - -/* erl_trace.c */ -void erts_init_trace(void); -void erts_trace_check_exiting(Eterm exiting); -Eterm erts_set_system_seq_tracer(Process *c_p, - ErtsProcLocks c_p_locks, - Eterm new); -Eterm erts_get_system_seq_tracer(void); -void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp); -void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp); -void erts_set_system_monitor(Eterm monitor); -Eterm erts_get_system_monitor(void); - -#ifdef ERTS_SMP -void erts_check_my_tracer_proc(Process *); -void erts_block_sys_msg_dispatcher(void); -void erts_release_sys_msg_dispatcher(void); -void erts_foreach_sys_msg_in_q(void (*func)(Eterm, - Eterm, - Eterm, - ErlHeapFragment *)); -void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *); -#endif - -void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); -void trace_send(Process*, Eterm, Eterm); -void trace_receive(Process*, Eterm); -Uint32 erts_call_trace(Process *p, BeamInstr mfa[], Binary *match_spec, Eterm* args, - int local, Eterm *tracer_pid); -void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid); -void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, - Eterm *tracer); -void erts_trace_return_to(Process *p, BeamInstr *pc); -void trace_sched(Process*, Eterm); -void trace_proc(Process*, Process*, Eterm, Eterm); -void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args); -void save_calls(Process *p, Export *); -void trace_gc(Process *p, Eterm what); -/* port tracing */ -void trace_virtual_sched(Process*, Eterm); -void trace_sched_ports(Port *pp, Eterm); -void trace_sched_ports_where(Port *pp, Eterm, Eterm); -void trace_port(Port *, Eterm what, Eterm data); -void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name); - -/* system_profile */ -void erts_set_system_profile(Eterm profile); -Eterm erts_get_system_profile(void); -void profile_scheduler(Eterm scheduler_id, Eterm); -void profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us); -void profile_runnable_proc(Process* p, Eterm status); -void profile_runnable_port(Port* p, Eterm status); -void erts_system_profile_setup_active_schedulers(void); - -/* system_monitor */ -void monitor_long_gc(Process *p, Uint time); -void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); -void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); -void monitor_large_heap(Process *p); -void monitor_generic(Process *p, Eterm type, Eterm spec); -Uint erts_trace_flag2bit(Eterm flag); -int erts_trace_flags(Eterm List, - Uint *pMask, Eterm *pTracer, int *pCpuTimestamp); -Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); -Uint64 erts_timestamp_millis(void); - -#ifdef ERTS_SMP -void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); -#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) \ -do { \ - if ((ESDP)->pending_trace_msgs) \ - erts_send_pending_trace_msgs((ESDP)); \ -} while (0) -#else -#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) -#endif +#define ERTS_UTF8_OK_MAX_CHARS 4 void bin_write(int, void*, byte*, size_t); int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */ @@ -1492,10 +937,17 @@ char* Sint_to_buf(Sint, struct Sint_buf*); #define ERTS_IOLIST_OVERFLOW 1 #define ERTS_IOLIST_TYPE 2 -Eterm buf_to_intlist(Eterm**, char*, size_t, Eterm); /* most callers pass plain char*'s */ -int io_list_to_buf(Eterm, char*, int); -int io_list_to_buf2(Eterm, char*, int); -int erts_iolist_size(Eterm, Uint *); +Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */ + +#define ERTS_IOLIST_TO_BUF_OVERFLOW (~((ErlDrvSizeT) 0)) +#define ERTS_IOLIST_TO_BUF_TYPE_ERROR (~((ErlDrvSizeT) 1)) +#define ERTS_IOLIST_TO_BUF_FAILED(R) \ + (((R) & (~((ErlDrvSizeT) 1))) == (~((ErlDrvSizeT) 1))) +#define ERTS_IOLIST_TO_BUF_SUCCEEDED(R) \ + (!ERTS_IOLIST_TO_BUF_FAILED((R))) + +ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT); +int erts_iolist_size(Eterm, ErlDrvSizeT *); int is_string(Eterm); void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); @@ -1540,39 +992,6 @@ Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(int to, void *arg, Process *c_p); int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); -#define seq_trace_output(token, msg, type, receiver, process) \ -seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) -#define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \ -seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom)) -void seq_trace_output_generic(Eterm token, Eterm msg, Uint type, - Eterm receiver, Process *process, Eterm exitfrom); - -int seq_trace_update_send(Process *process); - -Eterm erts_seq_trace(Process *process, - Eterm atom_type, Eterm atom_true_or_false, - int build_result); - -struct trace_pattern_flags { - unsigned int breakpoint : 1; /* Set if any other is set */ - unsigned int local : 1; /* Local call trace breakpoint */ - unsigned int meta : 1; /* Metadata trace breakpoint */ - unsigned int call_count : 1; /* Fast call count breakpoint */ - unsigned int call_time : 1; /* Fast call time breakpoint */ -}; -extern const struct trace_pattern_flags erts_trace_pattern_flags_off; -extern int erts_call_time_breakpoint_tracing; -int erts_set_trace_pattern(Eterm* mfa, int specified, - Binary* match_prog_set, Binary *meta_match_prog_set, - int on, struct trace_pattern_flags, - Eterm meta_tracer_pid); -void -erts_get_default_trace_pattern(int *trace_pattern_is_on, - Binary **match_spec, - Binary **meta_match_spec, - struct trace_pattern_flags *trace_pattern_flags, - Eterm *meta_tracer_pid); -void erts_bif_trace_init(void); /* ** Call_trace uses this API for the parameter matching functions @@ -1599,9 +1018,10 @@ Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); enum erts_pam_run_flags { - ERTS_PAM_TMP_RESULT=0, - ERTS_PAM_COPY_RESULT=1, - ERTS_PAM_CONTIGUOUS_TUPLE=2 + ERTS_PAM_TMP_RESULT=1, + ERTS_PAM_COPY_RESULT=2, + ERTS_PAM_CONTIGUOUS_TUPLE=4, + ERTS_PAM_IGNORE_TRACE_SILENT=8 }; extern Eterm erts_match_set_run(Process *p, Binary *mpsp, Eterm *args, int num_args, @@ -1618,20 +1038,21 @@ extern void erts_match_prog_foreach_offheap(Binary *b, breakpoint functions */ #define MATCH_SET_EXCEPTION_TRACE (0x4) /* exception trace requested */ #define MATCH_SET_RX_TRACE (MATCH_SET_RETURN_TRACE|MATCH_SET_EXCEPTION_TRACE) -/* - * Flag values when tracing bif - * Future note: flag field is 8 bits - */ -#define BIF_TRACE_AS_LOCAL (0x1) -#define BIF_TRACE_AS_GLOBAL (0x2) -#define BIF_TRACE_AS_META (0x4) -#define BIF_TRACE_AS_CALL_TIME (0x8) extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; extern erts_driver_t fd_driver; +int erts_beam_jump_table(void); + /* Should maybe be placed in erl_message.h, but then we get an include mess. */ +ERTS_GLB_INLINE Eterm * +erts_alloc_message_heap_state(Uint size, + ErlHeapFragment **bpp, + ErlOffHeap **ohpp, + Process *receiver, + ErtsProcLocks *receiver_locks, + erts_aint32_t *statep); ERTS_GLB_INLINE Eterm * erts_alloc_message_heap(Uint size, @@ -1654,16 +1075,22 @@ erts_alloc_message_heap(Uint size, */ ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks) +erts_alloc_message_heap_state(Uint size, + ErlHeapFragment **bpp, + ErlOffHeap **ohpp, + Process *receiver, + ErtsProcLocks *receiver_locks, + erts_aint32_t *statep) { Eterm *hp; + erts_aint32_t state; #ifdef ERTS_SMP int locked_main = 0; - ErtsProcLocks ulocks = *receiver_locks & ERTS_PROC_LOCKS_MSG_SEND; + state = erts_smp_atomic32_read_acqb(&receiver->state); + if (statep) + *statep = state; + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + goto allocate_in_mbuf; #endif if (size > (Uint) INT_MAX) @@ -1679,20 +1106,19 @@ erts_alloc_message_heap(Uint size, #ifdef ERTS_SMP try_allocate_on_heap: #endif - if (ERTS_PROC_IS_EXITING(receiver) + state = erts_smp_atomic32_read_nob(&receiver->state); + if (statep) + *statep = state; + if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) { #ifdef ERTS_SMP - if (locked_main) - ulocks |= ERTS_PROC_LOCK_MAIN; + if (locked_main) { + *receiver_locks &= ~ERTS_PROC_LOCK_MAIN; + erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN); + } #endif goto allocate_in_mbuf; } -#ifdef ERTS_SMP - if (ulocks) { - erts_smp_proc_unlock(receiver, ulocks); - *receiver_locks &= ~ulocks; - } -#endif hp = HEAP_TOP(receiver); HEAP_TOP(receiver) = hp + size; *bpp = NULL; @@ -1708,12 +1134,6 @@ erts_alloc_message_heap(Uint size, else { ErlHeapFragment *bp; allocate_in_mbuf: -#ifdef ERTS_SMP - if (ulocks) { - *receiver_locks &= ~ulocks; - erts_smp_proc_unlock(receiver, ulocks); - } -#endif bp = new_message_buffer(size); hp = bp->mem; *bpp = bp; @@ -1723,6 +1143,17 @@ erts_alloc_message_heap(Uint size, return hp; } +ERTS_GLB_INLINE Eterm * +erts_alloc_message_heap(Uint size, + ErlHeapFragment **bpp, + ErlOffHeap **ohpp, + Process *receiver, + ErtsProcLocks *receiver_locks) +{ + return erts_alloc_message_heap_state(size, bpp, ohpp, receiver, + receiver_locks, NULL); +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #if !HEAP_ON_C_STACK @@ -1789,6 +1220,13 @@ erts_alloc_message_heap(Uint size, # define UnUseTmpHeapNoproc(Size) /* Nothing */ #endif /* HEAP_ON_C_STACK */ +ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); +ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); +ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf); +ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, + Eterm module, Eterm function, int arity, + char *process_buf, char *mfa_buf); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF #include "dtrace-wrapper.h" @@ -1805,15 +1243,15 @@ dtrace_pid_str(Eterm pid, char *process_buf) ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf) { - dtrace_pid_str(process->id, process_buf); + dtrace_pid_str(process->common.id, process_buf); } ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf) { erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>", - port_channel_no(port->id), - port_number(port->id)); + port_channel_no(port->common.id), + port_number(port->common.id)); } ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 75d0d598a2..79c3ecf1b3 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -68,14 +68,14 @@ erts_index_init(ErtsAlcType_t type, IndexTable* t, char* name, return t; } -int -index_put(IndexTable* t, void* tmpl) +IndexSlot* +index_put_entry(IndexTable* t, void* tmpl) { int ix; IndexSlot* p = (IndexSlot*) hash_put(&t->htable, tmpl); if (p->index >= 0) { - return p->index; + return p; } ix = t->entries; @@ -93,7 +93,7 @@ index_put(IndexTable* t, void* tmpl) t->entries++; p->index = ix; t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; - return ix; + return p; } int index_get(IndexTable* t, void* tmpl) @@ -136,3 +136,18 @@ void erts_index_merge(Hash* src, IndexTable* dst) } } } + +void index_erase_latest_from(IndexTable* t, Uint from_ix) +{ + if(from_ix < (Uint)t->entries) { + int ix; + for (ix = from_ix; ix < t->entries; ix++) { + IndexSlot* obj = t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK]; + hash_erase(&t->htable, obj); + } + t->entries = from_ix; + } + else { + ASSERT(from_ix == t->entries); + } +} diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 4eb9b1f992..537bc11056 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -55,12 +55,24 @@ void index_info(int, void *, IndexTable*); int index_table_sz(IndexTable *); int index_get(IndexTable*, void*); -int index_put(IndexTable*, void*); + +IndexSlot* index_put_entry(IndexTable*, void*); void erts_index_merge(Hash*, IndexTable*); +/* Erase all entries with index 'ix' and higher +*/ +void index_erase_latest_from(IndexTable*, Uint ix); + +ERTS_GLB_INLINE int index_put(IndexTable*, void*); ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint); #if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int index_put(IndexTable* t, void* tmpl) +{ + return index_put_entry(t, tmpl)->index; +} + ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable* t, Uint ix) { diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 609fe9f5fb..c1e66b59af 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -43,6 +43,8 @@ #include "erl_version.h" #include "error.h" #include "erl_async.h" +#define ERTS_WANT_EXTERNAL_TAGS +#include "external.h" #include "dtrace-wrapper.h" extern ErlDrvEntry fd_driver_entry; @@ -51,34 +53,40 @@ extern ErlDrvEntry spawn_driver_entry; extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */ erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */ -erts_smp_mtx_t erts_driver_list_lock; /* Mutex for driver list */ +erts_smp_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */ static erts_smp_tsd_key_t driver_list_lock_status_key; /*stop recursive locks when calling driver init */ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error on a per thread basis (for BC interfaces) */ -Port* erts_port; /* The port table */ +ErtsPTab erts_port erts_align_attribute(ERTS_CACHE_LINE_SIZE); /* The port table */ erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ -Uint erts_max_ports; -Uint erts_port_tab_index_mask; - const ErlDrvTermData driver_term_nil = (ErlDrvTermData)NIL; +const Port erts_invalid_port = {{ERTS_INVALID_PORT}}; + erts_driver_t vanilla_driver; erts_driver_t spawn_driver; erts_driver_t fd_driver; +int erts_port_synchronous_ops = 0; +int erts_port_schedule_all_ops = 0; +int erts_port_parallelism = 0; + +static void deliver_result(Eterm sender, Eterm pid, Eterm res); static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); static void pdl_init(void); #ifdef ERTS_SMP static void driver_monitor_lock_pdl(Port *p); static void driver_monitor_unlock_pdl(Port *p); +#define DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(Port) erts_thr_drvport2port((Port), 1) #define DRV_MONITOR_LOCK_PDL(Port) driver_monitor_lock_pdl(Port) #define DRV_MONITOR_UNLOCK_PDL(Port) driver_monitor_unlock_pdl(Port) #else +#define DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(Port) erts_thr_drvport2port((Port), 0) #define DRV_MONITOR_LOCK_PDL(Port) /* nothing */ #define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */ #endif @@ -89,36 +97,10 @@ static void driver_monitor_unlock_pdl(Port *p); static ERTS_INLINE ErlIOQueue* drvport2ioq(ErlDrvPort drvport) { - int ix = (int) drvport; - Uint32 status; - - if (ix < 0 || erts_max_ports <= ix) + Port *prt = erts_thr_drvport2port(drvport, 0); + if (prt == ERTS_INVALID_ERL_DRV_PORT) return NULL; - - if (erts_get_scheduler_data()) { - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[ix])); - ERTS_LC_ASSERT(!erts_port[ix].port_data_lock - || erts_lc_mtx_is_locked( - &erts_port[ix].port_data_lock->mtx)); - - status = erts_port[ix].status; - } - else { - erts_smp_port_state_lock(&erts_port[ix]); - status = erts_port[ix].status; - erts_smp_port_state_unlock(&erts_port[ix]); - - ERTS_LC_ASSERT((status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - || erts_port[ix].port_data_lock); - ERTS_LC_ASSERT(!erts_port[ix].port_data_lock - || erts_lc_mtx_is_locked( - &erts_port[ix].port_data_lock->mtx)); - - } - - return ((status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - ? NULL - : &erts_port[ix].ioq); + return &prt->ioq; } static ERTS_INLINE int @@ -194,29 +176,31 @@ typedef struct line_buf_context { \ dtrace_proc_str((PID), process_str); \ dtrace_port_str((PORT), port_str); -#endif -/* The 'number' field in a port now has two parts: the lowest bits - contain the index in the port table, and the higher bits are a counter - which is incremented each time we look for a free port and start from - the beginning of the table. erts_max_ports is the number of file descriptors, - rounded up to a power of 2. - To get the index from a port, use the macro 'internal_port_index'; - 'port_number' returns the whole number field. -*/ +void +dtrace_drvport_str(ErlDrvPort drvport, char *port_buf) +{ + Port *port = erts_drvport2port(drvport); -static erts_smp_spinlock_t get_free_port_lck; -static Uint last_port_num; -static Uint port_num_mask; -erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */ + if (port != ERTS_INVALID_ERL_DRV_PORT) + erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>", + port_channel_no(port->common.id), + port_number(port->common.id)); + else + erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<INVALID>", + port_channel_no(port->common.id), + port_number(port->common.id)); +} +#endif static ERTS_INLINE void kill_port(Port *pp) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + erts_ptab_delete_element(&erts_port, &pp->common); /* Time of death */ erts_port_task_free_port(pp); - ASSERT(pp->status & ERTS_PORT_SFLGS_DEAD); + /* In non-smp case the port structure may have been deallocated now */ } #ifdef ERTS_SMP @@ -227,148 +211,280 @@ erts_lc_is_port_locked(Port *prt) { if (!prt) return 0; + ERTS_SMP_LC_ASSERT(prt->lock); return erts_smp_lc_mtx_is_locked(prt->lock); } #endif #endif /* #ifdef ERTS_SMP */ -static int -get_free_port(void) -{ - Uint num; - Uint tries = erts_max_ports; - Port* port; +static void initq(Port* prt); - erts_smp_spin_lock(&get_free_port_lck); - num = last_port_num + 1; - for (;; ++num) { - port = &erts_port[num & erts_port_tab_index_mask]; +#if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT) +#define ERTS_PORT_INIT_INSTR_NEED_ID 1 +#else +#define ERTS_PORT_INIT_INSTR_NEED_ID 0 +#endif - erts_smp_port_state_lock(port); - if (port->status & ERTS_PORT_SFLG_FREE) { - last_port_num = num; - erts_smp_spin_unlock(&get_free_port_lck); - break; - } - erts_smp_port_state_unlock(port); +static ERTS_INLINE void port_init_instr(Port *prt +#if ERTS_PORT_INIT_INSTR_NEED_ID + , Eterm id +#endif + ) +{ +#if !ERTS_PORT_INIT_INSTR_NEED_ID + Eterm id = NIL; /* Not used */ +#endif - if (--tries == 0) { - erts_smp_spin_unlock(&get_free_port_lck); - return -1; - } + /* + * Stuff that need to be initialized with the port id + * in the instrumented case, but not in the normal case. + */ +#ifdef ERTS_SMP + ASSERT(prt->drv_ptr && prt->lock); + if (!prt->drv_ptr->lock) { + char *lock_str = "port_lock"; +#ifdef ERTS_ENABLE_LOCK_COUNT + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; +#endif + erts_mtx_init_locked_x(prt->lock, lock_str, id); } - port->status = ERTS_PORT_SFLG_INITIALIZING; +#endif + erts_port_task_init_sched(&prt->sched, id); +} + +#if !ERTS_PORT_INIT_INSTR_NEED_ID +static ERTS_INLINE void port_init_instr_abort(Port *prt) +{ #ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0); - erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */ -#endif - erts_smp_port_state_unlock(port); - return num & port_num_mask; + ASSERT(prt->drv_ptr && prt->lock); + if (!prt->drv_ptr->lock) { + erts_mtx_unlock(prt->lock); + erts_mtx_destroy(prt->lock); + } +#endif + erts_port_task_fini_sched(&prt->sched); } +#endif -/* - * erts_test_next_port() is only used for testing. - */ -Sint -erts_test_next_port(int set, Uint next) +static void insert_port_struct(void *vprt, Eterm data) { - Uint i, num; - Sint res = -1; + Port *prt = (Port *) vprt; + Eterm id = make_internal_port(data); +#if ERTS_PORT_INIT_INSTR_NEED_ID + /* + * This cannot be done earlier in the instrumented + * case since we don't now 'id' until now. + */ + port_init_instr(prt, id); +#endif + prt->common.id = id; + erts_atomic32_init_relb(&prt->state, ERTS_PORT_SFLG_INITIALIZING); +} - erts_smp_spin_lock(&get_free_port_lck); - if (set) { - last_port_num = (next - 1) & port_num_mask; +#define ERTS_CREATE_PORT_FLAG_PARALLELISM (1 << 0) + +static Port *create_port(char *name, + erts_driver_t *driver, + erts_mtx_t *driver_lock, + int create_flags, + Eterm pid, + int *enop) +{ + ErtsPortTaskBusyPortQ *busy_port_queue; + Port *prt; + char *p; + size_t port_size, busy_port_queue_size, size; + erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED; + erts_aint32_t x_pts_flgs = 0; +#ifdef DEBUG + /* Make sure the debug flags survives until port is freed */ + state |= ERTS_PORT_SFLG_PORT_DEBUG; +#endif + +#ifdef ERTS_SMP + if (!driver_lock) { + /* Align size for mutex following port struct */ + port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); + size += sizeof(erts_mtx_t); } - num = last_port_num + 1; + else +#endif + port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); - for (i=0; i < erts_max_ports && res<0; ++i, ++num) { - - Port* port = &erts_port[num & erts_port_tab_index_mask]; + busy_port_queue_size + = ((driver->flags & ERL_DRV_FLAG_NO_BUSY_MSGQ) + ? 0 + : ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ))); + size += busy_port_queue_size; - erts_smp_port_state_lock(port); + size += sys_strlen(name) + 1; - if (port->status & ERTS_PORT_SFLG_FREE) { - last_port_num = num - 1; - res = num & port_num_mask; - } - erts_smp_port_state_unlock(port); + p = erts_alloc_fnf(ERTS_ALC_T_PORT, size); + if (!p) { + if (enop) + *enop = ENOMEM; + return NULL; } - erts_smp_spin_unlock(&get_free_port_lck); - return res; -} + prt = (Port *) p; + p += port_size; -static void port_cleanup(Port *prt); + if (!busy_port_queue_size) + busy_port_queue = NULL; + else { + busy_port_queue = (ErtsPortTaskBusyPortQ *) p; + p += busy_port_queue_size; + } #ifdef ERTS_SMP - -static void -sched_port_cleanup(void *vprt) -{ - Port *prt = (Port *) vprt; - erts_smp_mtx_lock(prt->lock); - port_cleanup(prt); -} - + if (driver_lock) { + prt->lock = driver_lock; + erts_mtx_lock(driver_lock); + } + else { + prt->lock = (erts_mtx_t *) p; + p += sizeof(erts_mtx_t); + state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; + } + erts_smp_atomic_set_nob(&prt->run_queue, + (erts_aint_t) erts_get_runq_current(NULL)); + prt->xports = NULL; +#else + erts_atomic32_init_nob(&prt->refc, 1); + prt->cleanup = 0; #endif + + erts_port_task_pre_init_sched(&prt->sched, busy_port_queue); -void -erts_port_cleanup(Port *prt) -{ + prt->name = p; + sys_strcpy(p, name); + prt->drv_ptr = driver; + ERTS_P_LINKS(prt) = NULL; + ERTS_P_MONITORS(prt) = NULL; + prt->linebuf = NULL; + prt->suspended = NULL; + erts_init_port_data(prt); + prt->port_data_lock = NULL; + prt->control_flags = 0; + prt->bytes_in = 0; + prt->bytes_out = 0; + prt->dist_entry = NULL; + ERTS_PORT_INIT_CONNECTED(prt, pid); + prt->common.u.alive.reg = NULL; #ifdef ERTS_SMP - if (erts_smp_mtx_trylock(prt->lock) == EBUSY) - erts_schedule_misc_op(sched_port_cleanup, (void *) prt); - else + prt->common.u.alive.ptimer = NULL; +#else + sys_memset(&prt->common.u.alive.tm, 0, sizeof(ErlTimer)); #endif - port_cleanup(prt); -} + erts_port_task_handle_init(&prt->timeout_task); + prt->psd = NULL; + prt->drv_data = (SWord) 0; + prt->os_pid = -1; -void -port_cleanup(Port *prt) -{ + /* Set default tracing */ + erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt)); + + ASSERT(((char *) prt) == ((char *) &prt->common)); + +#if !ERTS_PORT_INIT_INSTR_NEED_ID + /* + * When 'id' isn't needed (the normal case), it is better to + * do the initialization here avoiding unnecessary contention + * on table... + */ + port_init_instr(prt); +#endif + + if (!erts_ptab_new_element(&erts_port, + &prt->common, + (void *) prt, + insert_port_struct)) { + +#if !ERTS_PORT_INIT_INSTR_NEED_ID + port_init_instr_abort(prt); +#endif #ifdef ERTS_SMP - Uint32 port_specific; - erts_smp_mtx_t *mtx; + if (driver_lock) + erts_mtx_unlock(driver_lock); #endif - erts_driver_t *driver; + if (enop) + *enop = 0; + erts_free(ERTS_ALC_T_PORT, prt); + return NULL; + } - erts_smp_port_state_lock(prt); + ASSERT(prt == (Port *) (erts_ptab_pix2intptr_nob( + &erts_port, + internal_port_index(prt->common.id)))); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - driver = prt->drv_ptr; - prt->drv_ptr = NULL; - ASSERT(driver); + initq(prt); -#ifdef ERTS_SMP + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED); - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0); + if (erts_port_schedule_all_ops) + x_pts_flgs |= ERTS_PTS_FLG_FORCE_SCHED; - port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK); + if (create_flags & ERTS_CREATE_PORT_FLAG_PARALLELISM) + x_pts_flgs |= ERTS_PTS_FLG_PARALLELISM; - mtx = prt->lock; - ASSERT(mtx); + if (x_pts_flgs) + erts_smp_atomic32_read_bor_nob(&prt->sched.flags, x_pts_flgs); - prt->lock = NULL; + erts_atomic32_set_relb(&prt->state, state); + return prt; +} - ASSERT(prt->status & ERTS_PORT_SFLG_PORT_DEBUG); - ASSERT(!(prt->status & ERTS_PORT_SFLG_FREE)); - prt->status = ERTS_PORT_SFLG_FREE; +#ifndef ERTS_SMP +void +erts_port_cleanup(Port *prt) +{ + if (prt->drv_ptr && prt->drv_ptr->handle) + erts_ddll_dereference_driver(prt->drv_ptr->handle); + prt->drv_ptr = NULL; + erts_port_dec_refc(prt); +} +#endif - erts_smp_port_state_unlock(prt); - erts_smp_mtx_unlock(mtx); +void +erts_port_free(Port *prt) +{ +#if defined(ERTS_SMP) || defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) + erts_aint32_t state = erts_atomic32_read_nob(&prt->state); +#endif + ERTS_LC_ASSERT(state & (ERTS_PORT_SFLG_INITIALIZING + | ERTS_PORT_SFLG_FREE)); + ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG); - if (port_specific) { - erts_smp_mtx_destroy(mtx); - erts_free(ERTS_ALC_T_PORT_LOCK, mtx); - } +#ifdef ERTS_SMP + ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->common.refc) == 0); +#else + ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->refc) == 0); #endif - if (driver->handle) - erts_ddll_dereference_driver(driver->handle); -} + erts_port_task_fini_sched(&prt->sched); + +#ifdef ERTS_SMP + ASSERT(prt->lock); + if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) + erts_mtx_destroy(prt->lock); + /* + * We cannot dereference a driver using driver + * locking until here in smp case. Otherwise, + * the driver lock may still be in use by others. + * + * In the non-smp case we cannot do it here since + * this function may be called by non-scheduler + * threads. This is done in erts_port_cleanup() + * in the non-smp case. + */ + if (prt->drv_ptr->handle) + erts_ddll_dereference_driver(prt->drv_ptr->handle); +#endif + erts_free(ERTS_ALC_T_PORT, prt); +} /* ** Initialize v_start to point to the small fixed vector. @@ -416,94 +532,21 @@ static void stopq(Port* prt) if (prt->port_data_lock) { driver_pdl_unlock(prt->port_data_lock); driver_pdl_dec_refc(prt->port_data_lock); - prt->port_data_lock = NULL; - } -} - - - -static void -setup_port(Port* prt, Eterm pid, erts_driver_t *driver, - ErlDrvData drv_data, char *name, Uint32 xstatus) -{ - ErtsRunQueue *runq = erts_get_runq_current(NULL); - char *new_name, *old_name; -#ifdef DEBUG - /* Make sure the debug flags survives until port is freed */ - xstatus |= ERTS_PORT_SFLG_PORT_DEBUG; -#endif - ASSERT(runq); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - - - new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1); - sys_strcpy(new_name, name); - erts_smp_runq_lock(runq); - erts_smp_port_state_lock(prt); - prt->os_pid = -1; - prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; - prt->snapshot = erts_smp_atomic32_read_nob(&erts_ports_snapshot); - old_name = prt->name; - prt->name = new_name; -#ifdef ERTS_SMP - erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); -#endif - ASSERT(!prt->drv_ptr); - prt->drv_ptr = driver; - erts_smp_port_state_unlock(prt); - erts_smp_runq_unlock(runq); -#ifdef ERTS_SMP - ASSERT(!prt->xports); -#endif - if (old_name) { - erts_free(ERTS_ALC_T_PORT_NAME, (void *) old_name); } - - prt->control_flags = 0; - prt->connected = pid; - prt->drv_data = (SWord) drv_data; - prt->bytes_in = 0; - prt->bytes_out = 0; - prt->dist_entry = NULL; - prt->reg = NULL; -#ifdef ERTS_SMP - prt->ptimer = NULL; -#else - sys_memset(&prt->tm, 0, sizeof(ErlTimer)); -#endif - erts_port_task_handle_init(&prt->timeout_task); - prt->suspended = NULL; - sys_strcpy(prt->name, name); - prt->nlinks = NULL; - prt->monitors = NULL; - prt->linebuf = NULL; - prt->bp = NULL; - prt->data = am_undefined; - /* Set default tracing */ - erts_get_default_tracing(&(prt->trace_flags), &(prt->tracer_proc)); - - prt->psd = NULL; - - initq(prt); } -void -erts_wake_process_later(Port *prt, Process *process) +int +erts_save_suspend_process_on_port(Port *prt, Process *process) { - ErtsProcList** p; - ErtsProcList* new_p; - - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - - if (prt->status & ERTS_PORT_SFLGS_DEAD) - return; - - for (p = &(prt->suspended); *p != NULL; p = &((*p)->next)) - /* Empty loop body */; - - new_p = erts_proclist_create(process); - new_p->next = NULL; - *p = new_p; + int saved; + erts_aint32_t flags; + erts_port_task_sched_lock(&prt->sched); + flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + saved = (flags & ERTS_PTS_FLGS_BUSY) && !(flags & ERTS_PTS_FLG_EXIT); + if (saved) + erts_proclist_store_last(&prt->suspended, erts_proclist_create(process)); + erts_port_task_sched_unlock(&prt->sched); + return saved; } /* @@ -515,47 +558,44 @@ erts_wake_process_later(Port *prt, Process *process) (*error_number_ptr must contain either BADARG or SYSTEM_LIMIT). The driver start function must obey the same conventions. */ -int +Port * erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ Eterm pid, /* Current process. */ char* name, /* Driver name. */ SysDriverOpts* opts, /* Options. */ - int *error_number_ptr) /* errno in case -2 is returned */ + int *error_type_ptr, /* error type */ + int *error_number_ptr) /* errno in case of error type -2 */ { - int port_num; - int port_ix; + +#undef ERTS_OPEN_DRIVER_RET +#define ERTS_OPEN_DRIVER_RET(Prt, EType, ENo) \ + do { \ + if (error_type_ptr) \ + *error_type_ptr = (EType); \ + if (error_number_ptr) \ + *error_number_ptr = (ENo); \ + return (Prt); \ + } while (0) + ErlDrvData drv_data = 0; - Uint32 xstatus = 0; Port *port; int fpe_was_unmasked; - - if (error_number_ptr) - *error_number_ptr = 0; + int error_type, error_number; + int port_errno = 0; + erts_mtx_t *driver_lock = NULL; + int cprt_flgs = 0; ERTS_SMP_CHK_NO_PROC_LOCKS; - if ((port_num = get_free_port()) < 0) { - if (error_number_ptr) { - *error_number_ptr = SYSTEM_LIMIT; - } - return -3; - } - - port_ix = port_num & erts_port_tab_index_mask; - port = &erts_port[port_ix]; - port->id = make_internal_port(port_num); - - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rlock(&erts_driver_list_lock); if (!driver) { for (driver = driver_list; driver; driver = driver->next) { if (sys_strcmp(driver->name, name) == 0) break; } if (!driver) { - erts_smp_mtx_unlock(&erts_driver_list_lock); - if (error_number_ptr) - *error_number_ptr = BADARG; - return -3; + erts_smp_rwmtx_runlock(&erts_driver_list_lock); + ERTS_OPEN_DRIVER_RET(NULL, -3, BADARG); } } if (driver == &spawn_driver) { @@ -599,61 +639,52 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ } if (driver == NULL || (driver != &spawn_driver && opts->exit_status)) { - erts_smp_mtx_unlock(&erts_driver_list_lock); - if (error_number_ptr) { - *error_number_ptr = BADARG; - } - /* Need to mark the port as free again */ - erts_smp_port_state_lock(port); - port->status = ERTS_PORT_SFLG_FREE; -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2); - erts_smp_atomic_set_nob(&port->refc, 0); -#endif - erts_smp_port_state_unlock(port); - return -3; + erts_smp_rwmtx_runlock(&erts_driver_list_lock); + ERTS_OPEN_DRIVER_RET(NULL, -3, BADARG); } - /* - * We'll set up the port before calling the start function, - * to allow message sending and setting timers in the start function. - */ - #ifdef ERTS_SMP - ASSERT(!port->lock); - port->lock = driver->lock; - if (!port->lock) { - port->lock = erts_alloc(ERTS_ALC_T_PORT_LOCK, - sizeof(erts_smp_mtx_t)); - erts_smp_mtx_init_x(port->lock, -#ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) ? "port_lock" : NULL, -#else - "port_lock", -#endif - port->id); - xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; - } + driver_lock = driver->lock; #endif if (driver->handle != NULL) { erts_ddll_increment_port_count(driver->handle); erts_ddll_reference_driver(driver->handle); } - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_runlock(&erts_driver_list_lock); -#ifdef ERTS_SMP - erts_smp_mtx_lock(port->lock); -#endif + /* + * We'll set up the port before calling the start function, + * to allow message sending and setting timers in the start function. + */ - setup_port(port, pid, driver, drv_data, name, xstatus); + if (opts->parallelism) + cprt_flgs |= ERTS_CREATE_PORT_FLAG_PARALLELISM; + + port = create_port(name, driver, driver_lock, cprt_flgs, pid, &port_errno); + if (!port) { + if (driver->handle) { + erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_ddll_decrement_port_count(driver->handle); + erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_ddll_dereference_driver(driver->handle); + } + if (port_errno) + ERTS_OPEN_DRIVER_RET(NULL, -2, port_errno); + else + ERTS_OPEN_DRIVER_RET(NULL, -3, SYSTEM_LIMIT); + } if (IS_TRACED_FL(port, F_TRACE_PORTS)) { trace_port_open(port, - pid, - am_atom_put(port->name, strlen(port->name))); + pid, + erts_atom_put((byte *) port->name, + strlen(port->name), + ERTS_ATOM_ENC_LATIN1, + 1)); } - + + error_number = error_type = 0; if (driver->start) { if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_in, am_start); @@ -666,56 +697,63 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ } #endif fpe_was_unmasked = erts_block_fpe(); - drv_data = (*driver->start)((ErlDrvPort)(port_ix), - name, opts); + drv_data = (*driver->start)(ERTS_Port2ErlDrvPort(port), name, opts); + if (((SWord) drv_data) == -1) + error_type = -1; + else if (((SWord) drv_data) == -2) { + /* + * We need to save errno quickly after the + * call to the 'start' callback before + * something else modify it. + */ + error_type = -2; + error_number = errno; + } + else if (((SWord) drv_data) == -3) { + error_type = -3; + error_number = BADARG; + } + erts_unblock_fpe(fpe_was_unmasked); port->caller = NIL; if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_out, am_start); } - if (error_number_ptr && ((SWord) drv_data) == (SWord) -2) - *error_number_ptr = errno; #ifdef ERTS_SMP if (port->xports) - erts_smp_xports_unlock(port); + erts_port_handle_xports(port); ASSERT(!port->xports); #endif } - if (((SWord)drv_data) == -1 || - ((SWord)drv_data) == -2 || - ((SWord)drv_data) == -3) { - int res = (int) ((SWord) drv_data); - - if (res == -3 && error_number_ptr) { - *error_number_ptr = BADARG; - } - + if (error_type) { /* * Must clean up the port. */ #ifdef ERTS_SMP - erts_cancel_smp_ptimer(port->ptimer); + erts_cancel_smp_ptimer(port->common.u.alive.ptimer); #else - erts_cancel_timer(&(port->tm)); + erts_cancel_timer(&(port->common.u.alive.tm)); #endif stopq(port); - kill_port(port); if (port->linebuf != NULL) { erts_free(ERTS_ALC_T_LINEBUF, (void *) port->linebuf); port->linebuf = NULL; } if (driver->handle != NULL) { - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(driver->handle); - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_runlock(&erts_driver_list_lock); } + kill_port(port); erts_port_release(port); - return res; + ERTS_OPEN_DRIVER_RET(NULL, error_type, error_number); } - port->drv_data = (SWord) drv_data; - return port_ix; + port->drv_data = (UWord) drv_data; + ERTS_OPEN_DRIVER_RET(port, 0, 0); + +#undef ERTS_OPEN_DRIVER_RET } #ifdef ERTS_SMP @@ -740,102 +778,122 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ char* name, /* Driver name */ ErlDrvData drv_data) /* Driver data */ { + int cprt_flgs = 0; Port *creator_port; Port* port; erts_driver_t *driver; Process *rp; - int port_num; - Eterm port_id; - Uint32 xstatus = 0; + erts_mtx_t *driver_lock = NULL; ERTS_SMP_CHK_NO_PROC_LOCKS; + /* Need to be called from a scheduler thread */ + if (!erts_get_scheduler_id()) + return ERTS_INVALID_ERL_DRV_PORT; + creator_port = erts_drvport2port(creator_port_ix); - if (!creator_port) - return (ErlDrvTermData) -1; + if (creator_port == ERTS_INVALID_ERL_DRV_PORT) + return ERTS_INVALID_ERL_DRV_PORT; + + rp = erts_proc_lookup(pid); + if (!rp) + return ERTS_INVALID_ERL_DRV_PORT; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(creator_port)); driver = creator_port->drv_ptr; - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rlock(&erts_driver_list_lock); if (!erts_ddll_driver_ok(driver->handle)) { - erts_smp_mtx_unlock(&erts_driver_list_lock); - return (ErlDrvTermData) -1; + erts_smp_rwmtx_runlock(&erts_driver_list_lock); + return ERTS_INVALID_ERL_DRV_PORT; } - rp = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_LINK); - if (!rp) { - erts_smp_mtx_unlock(&erts_driver_list_lock); - return (ErlDrvTermData) -1; /* pid does not exist */ + if (driver->handle != NULL) { + erts_ddll_increment_port_count(driver->handle); + erts_ddll_reference_referenced_driver(driver->handle); } - if ((port_num = get_free_port()) < 0) { - errno = SYSTEM_LIMIT; + +#ifdef ERTS_SMP + driver_lock = driver->lock; +#endif + + erts_smp_rwmtx_runlock(&erts_driver_list_lock); + + /* Inherit parallelism flag from parent */ + if (ERTS_PTS_FLG_PARALLELISM & + erts_smp_atomic32_read_nob(&creator_port->sched.flags)) + cprt_flgs |= ERTS_CREATE_PORT_FLAG_PARALLELISM; + port = create_port(name, driver, driver_lock, cprt_flgs, pid, NULL); + if (!port) { + if (driver->handle) { + erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_ddll_decrement_port_count(driver->handle); + erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_ddll_dereference_driver(driver->handle); + } + return ERTS_INVALID_ERL_DRV_PORT; + } + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); + + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (ERTS_PROC_IS_EXITING(rp)) { erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - erts_smp_mtx_unlock(&erts_driver_list_lock); - return (ErlDrvTermData) -1; + if (driver->handle) { + erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_ddll_decrement_port_count(driver->handle); + erts_smp_rwmtx_runlock(&erts_driver_list_lock); + } + kill_port(port); + erts_port_release(port); + return ERTS_INVALID_ERL_DRV_PORT; } - port_id = make_internal_port(port_num); - port = &erts_port[port_num & erts_port_tab_index_mask]; + erts_add_link(&ERTS_P_LINKS(port), LINK_PID, pid); + erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, port->common.id); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); #ifdef ERTS_SMP - ASSERT(!port->lock); - port->lock = driver->lock; - if (!port->lock) { + if (!driver_lock) { ErtsXPortsList *xplp = xports_list_alloc(); xplp->port = port; xplp->next = creator_port->xports; creator_port->xports = xplp; - port->lock = erts_alloc(ERTS_ALC_T_PORT_LOCK, - sizeof(erts_smp_mtx_t)); - erts_smp_mtx_init_locked_x(port->lock, -#ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) ? "port_lock" : NULL, -#else - "port_lock", -#endif - port_id); - xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } - #endif - if (driver->handle != NULL) { - erts_ddll_increment_port_count(driver->handle); - erts_ddll_reference_referenced_driver(driver->handle); - } - erts_smp_mtx_unlock(&erts_driver_list_lock); - - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); + port->drv_data = (UWord) drv_data; - setup_port(port, pid, driver, drv_data, name, xstatus); - port->id = port_id; - - erts_add_link(&(port->nlinks), LINK_PID, pid); - erts_add_link(&(rp->nlinks), LINK_PID, port_id); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - return port_num & erts_port_tab_index_mask; + return ERTS_Port2ErlDrvPort(port); } #ifdef ERTS_SMP -void -erts_smp_xports_unlock(Port *prt) +int erts_port_handle_xports(Port *prt) { + int reds = 0; ErtsXPortsList *xplp; ASSERT(prt); xplp = prt->xports; ASSERT(xplp); while (xplp) { + Port *rprt = xplp->port; ErtsXPortsList *free_xplp; - if (xplp->port->xports) - erts_smp_xports_unlock(xplp->port); - erts_port_release(xplp->port); + erts_aint32_t state; + if (rprt->xports) + reds += erts_port_handle_xports(rprt); + state = erts_atomic32_read_nob(&rprt->state); + if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(rprt)) { + terminate_port(rprt); + reds += ERTS_PORT_REDS_TERMINATE; + } + erts_port_release(rprt); free_xplp = xplp; xplp = xplp->next; xports_list_free(free_xplp); + reds++; } prt->xports = NULL; + return reds; } #endif @@ -870,8 +928,8 @@ io_list_to_vec(Eterm obj, /* io-list */ DECLARE_ESTACK(s); Eterm* objp; char *buf = cbin->orig_bytes; - ErlDrvSizeT len = cbin->orig_size; - ErlDrvSizeT csize = 0; + Uint len = cbin->orig_size; + Uint csize = 0; int vlen = 0; char* cptr = buf; @@ -986,7 +1044,7 @@ io_list_to_vec(Eterm obj, /* io-list */ #define IO_LIST_VEC_COUNT(obj) \ do { \ - ErlDrvSizeT _size = binary_size(obj); \ + Uint _size = binary_size(obj); \ Eterm _real; \ ERTS_DECLARE_DUMMY(Uint _offset); \ int _bitoffs; \ @@ -1037,8 +1095,9 @@ do { \ */ static int -io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, - Uint* pvsize, Uint* pcsize, Uint* total_size) +io_list_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + ErlDrvSizeT* total_size) { DECLARE_ESTACK(s); Eterm* objp; @@ -1049,7 +1108,7 @@ io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, Uint p_v_size = 0; Uint p_c_size = 0; Uint p_in_clist = 0; - Uint total; + Uint total; /* Uint due to halfword emulator */ goto L_jump_start; /* avoid a push */ @@ -1109,7 +1168,7 @@ io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, if (total < c_size) { goto L_overflow_error; } - *total_size = total; + *total_size = (ErlDrvSizeT) total; DESTROY_ESTACK(s); *vsize = v_size; @@ -1124,56 +1183,755 @@ io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize, return 1; } -/* write data to a port */ -int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) -{ - char *buf; - erts_driver_t *drv = p->drv_ptr; - Uint size; +typedef enum { + ERTS_TRY_IMM_DRV_CALL_OK, + ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK, + ERTS_TRY_IMM_DRV_CALL_INVALID_PORT, + ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS +} ErtsTryImmDrvCallResult; + +typedef struct { + Process *c_p; /* Currently executing process (unlocked) */ + Port *port; /* Port to operate on */ + Eterm port_op; /* port operation as an atom */ + erts_aint32_t state; /* in: invalid state; out: read state (if read) */ + erts_aint32_t sched_flags; /* in: invalid flags; out: read flags (if read) */ + int async; /* Asynchronous operation */ + int pre_chk_sched_flags; /* Check sched flags before lock? */ int fpe_was_unmasked; - - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p) || ERTS_IS_CRASH_DUMPING); + int reds_left_in; +} ErtsTryImmDrvCallState; + +#define ERTS_INIT_TRY_IMM_DRV_CALL_STATE(C_P, PRT, SFLGS, PTS_FLGS, A, PRT_OP) \ + {(C_P), (PRT), (PRT_OP), (SFLGS), (PTS_FLGS), (A), 1, 0} + +/* + * Try doing an immediate driver callback call from a process. If + * this fail, the operation should be scheduled in the normal case... + * + */ +static ERTS_INLINE ErtsTryImmDrvCallResult +try_imm_drv_call(ErtsTryImmDrvCallState *sp) +{ + ErtsTryImmDrvCallResult res; + int reds_left_in; + erts_aint32_t invalid_state, invalid_sched_flags; + Port *prt = sp->port; + Process *c_p = sp->c_p; + + ASSERT(is_atom(sp->port_op)); + + invalid_sched_flags = ERTS_PTS_FLGS_FORCE_SCHEDULE_OP; + invalid_sched_flags |= sp->sched_flags; + if (sp->async) + invalid_sched_flags |= ERTS_PTS_FLG_PARALLELISM; + + if (sp->pre_chk_sched_flags) { + sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + if (sp->sched_flags & invalid_sched_flags) + return ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS; + } + + if (erts_smp_port_trylock(prt) == EBUSY) + return ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK; + + invalid_state = sp->state; + sp->state = erts_atomic32_read_nob(&prt->state); + if (sp->state & invalid_state) { + res = ERTS_TRY_IMM_DRV_CALL_INVALID_PORT; + goto locked_fail; + } + + sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + if (sp->sched_flags & invalid_sched_flags) { + res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS; + goto locked_fail; + } + + + if (!c_p) + reds_left_in = CONTEXT_REDS/10; + else { + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) + trace_virtual_sched(c_p, am_out); + if (erts_system_profile_flags.runnable_procs + && erts_system_profile_flags.exclusive) + profile_runnable_proc(c_p, am_inactive); + + reds_left_in = ERTS_BIF_REDS_LEFT(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + } + + ASSERT(0 <= reds_left_in && reds_left_in <= CONTEXT_REDS); + sp->reds_left_in = reds_left_in; + prt->reds = CONTEXT_REDS - reds_left_in; + ERTS_SMP_CHK_NO_PROC_LOCKS; - p->caller = caller_id; - if (drv->outputv != NULL) { - Uint vsize; - Uint csize; - Uint pvsize; - Uint pcsize; - ErlDrvSizeT blimit; + if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) + trace_sched_ports_where(prt, am_in, sp->port_op); + if (erts_system_profile_flags.runnable_ports + && !erts_port_is_scheduled(prt)) + profile_runnable_port(prt, am_active); + + sp->fpe_was_unmasked = erts_block_fpe(); + + return ERTS_TRY_IMM_DRV_CALL_OK; + +locked_fail: + erts_port_release(prt); + return res; +} + +static ERTS_INLINE void +finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) +{ + int reds; + Port *prt = sp->port; + Process *c_p = sp->c_p; + + reds = prt->reds; + reds += erts_port_driver_callback_epilogue(prt, NULL); + + erts_unblock_fpe(sp->fpe_was_unmasked); + + if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) + trace_sched_ports_where(prt, am_out, sp->port_op); + if (erts_system_profile_flags.runnable_ports + && !erts_port_is_scheduled(prt)) + profile_runnable_port(prt, am_inactive); + + erts_port_release(prt); + + if (c_p) { + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + if (reds != (CONTEXT_REDS - sp->reds_left_in)) { + int bump_reds = reds - (CONTEXT_REDS - sp->reds_left_in); + ASSERT(bump_reds > 0); + BUMP_REDS(c_p, bump_reds); + } + + if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) + trace_virtual_sched(c_p, am_in); + if (erts_system_profile_flags.runnable_procs + && erts_system_profile_flags.exclusive) + profile_runnable_proc(c_p, am_active); + } +} + +/* + * force_imm_drv_call()/finalize_force_imm_drv_call() should *only* + * be used while crash dumping... + */ +static ErtsTryImmDrvCallResult +force_imm_drv_call(ErtsTryImmDrvCallState *sp) +{ + erts_aint32_t invalid_state; + Port *prt = sp->port; + + ASSERT(ERTS_IS_CRASH_DUMPING) + ASSERT(is_atom(sp->port_op)); + + invalid_state = sp->state; + sp->state = erts_atomic32_read_nob(&prt->state); + if (sp->state & invalid_state) + return ERTS_TRY_IMM_DRV_CALL_INVALID_PORT; + + sp->fpe_was_unmasked = erts_block_fpe(); + + return ERTS_TRY_IMM_DRV_CALL_OK; +} + +static void +finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) +{ + erts_unblock_fpe(sp->fpe_was_unmasked); +} + +#define ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE (REF_THING_SIZE + 3) + +static ERTS_INLINE void +queue_port_sched_op_reply(Process *rp, + ErtsProcLocks *rp_locksp, + Eterm *hp_start, + Eterm *hp, + Uint h_size, + ErlHeapFragment* bp, + Uint32 *ref_num, + Eterm msg) +{ + Eterm ref = make_internal_ref(hp); + write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]); + hp += REF_THING_SIZE; + + msg = TUPLE2(hp, ref, msg); + hp += 3; + + if (!bp) { + HRelease(rp, hp_start + h_size, hp); + } + else { + Uint used_h_size = hp - hp_start; + ASSERT(h_size >= used_h_size); + if (h_size > used_h_size) + bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1); + } + + erts_queue_message(rp, + rp_locksp, + bp, + msg, + NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); +} + +static void +port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) +{ + Process *rp = erts_proc_lookup_raw(to); + if (rp) { + ErlOffHeap *ohp; + ErlHeapFragment* bp; + Eterm msg_copy; + Uint hsz, msg_sz; + Eterm *hp, *hp_start; + ErtsProcLocks rp_locks = 0; + + hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; + if (is_immed(msg)) + msg_sz = 0; + else { + msg_sz = size_object(msg); + hsz += msg_sz; + } + + hp_start = hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + rp, + &rp_locks); + if (is_immed(msg)) + msg_copy = msg; + else + msg_copy = copy_struct(msg, msg_sz, &hp, ohp); + + queue_port_sched_op_reply(rp, + &rp_locks, + hp_start, + hp, + hsz, + bp, + ref_num, + msg_copy); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } +} + + +ErtsPortOpResult +erts_schedule_proc2port_signal(Process *c_p, + Port *prt, + Eterm caller, + Eterm *refp, + ErtsProc2PortSigData *sigdp, + int task_flags, + ErtsPortTaskHandle *pthp, + ErtsProc2PortSigCallback callback) +{ + int sched_res; + if (!refp) { + if (c_p) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + } + else { + ASSERT(c_p); + sigdp->flags |= ERTS_P2P_SIG_DATA_FLG_REPLY; + erts_make_ref_in_array(sigdp->ref); + *refp = erts_proc_store_ref(c_p, sigdp->ref); + + /* + * Caller needs to wait for a message containing + * the ref that we just created. No such message + * can exist in callers message queue at this time. + * We therefore move the save pointer of the + * callers message queue to the end of the queue. + * + * NOTE: It is of vital importance that the caller + * immediately do a receive unconditionaly + * waiting for the message with the reference; + * otherwise, next receive will *not* work + * as expected! + */ + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + + if (ERTS_PROC_PENDING_EXIT(c_p)) { + /* need to exit caller instead */ + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + KILL_CATCHES(c_p); + c_p->freason = EXC_EXIT; + return ERTS_PORT_OP_CALLER_EXIT; + } + + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + c_p->msg.save = c_p->msg.last; + + erts_smp_proc_unlock(c_p, + (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCKS_MSG_RECEIVE)); + } + + + sigdp->caller = caller; + + /* Schedule port close call for later execution... */ + sched_res = erts_port_task_schedule(prt->common.id, + pthp, + ERTS_PORT_TASK_PROC_SIG, + sigdp, + callback, + task_flags); + + if (c_p) + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + if (sched_res != 0) { + if (refp) + *refp = NIL; + return ERTS_PORT_OP_DROPPED; + } + return ERTS_PORT_OP_SCHEDULED; +} + +static ERTS_INLINE void +send_badsig(Port *prt) +{ + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + Process* rp; + Eterm connected = ERTS_PORT_GET_CONNECTED(prt); + + ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_get_scheduler_id()); + + ASSERT(is_internal_pid(connected)); + + rp = erts_proc_lookup_raw(connected); + if (rp) { + erts_smp_proc_lock(rp, rp_locks); + if (!ERTS_PROC_IS_EXITING(rp)) + (void) erts_send_exit_signal(NULL, + prt->common.id, + rp, + &rp_locks, + am_badsig, + NIL, + NULL, + 0); + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } +} + +static void +badsig_received(int bang_op, + Port *prt, + erts_aint32_t state, + int bad_output_value) +{ + /* + * if (bang_op) + * we are part of a "Prt ! Something" operation + * else + * we are part of a call to a port BIF + * behave accordingly... + */ + if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) { + if (bad_output_value) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Bad value on output port '%s'\n", prt->name); + erts_send_error_to_logger_nogl(dsbufp); + } + if (bang_op) + send_badsig(prt); + } +} + +static int +port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) +{ + if (op == ERTS_PROC2PORT_SIG_EXEC) + badsig_received(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP, + prt, + state, + sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT); + if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + return ERTS_PORT_REDS_BADSIG; +} + + +/* + * bad_port_signal() will + * - preserve signal order of signals. + * - send a 'badsig' exit signal to connected process if 'from' is an + * internal pid and the port is alive when the bad signal reaches + * it. + */ +static ErtsPortOpResult +bad_port_signal(Process *c_p, + int flags, + Port *prt, + Eterm from, + Eterm *refp, + Eterm port_op) +{ + ErtsProc2PortSigData *sigdp; + ErtsTryImmDrvCallResult try_call_res; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( + c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + 0, + !refp, + port_op); + + try_call_res = try_imm_drv_call(&try_call_state); + switch (try_call_res) { + case ERTS_TRY_IMM_DRV_CALL_OK: + badsig_received(flags & ERTS_PORT_SIG_FLG_BANG_OP, + prt, + try_call_state.state, + flags & ERTS_PORT_SIG_FLG_BAD_OUTPUT); + finalize_imm_drv_call(&try_call_state); + if (c_p) + BUMP_REDS(c_p, ERTS_PORT_REDS_BADSIG); + return ERTS_PORT_OP_BADARG; + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + return ERTS_PORT_OP_DROPPED; + case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: + case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: + /* Schedule badsig() call instead... */ + break; + } + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = (flags & ~ERTS_P2P_SIG_TYPE_MASK) | ERTS_P2P_SIG_TYPE_BAD; + + return erts_schedule_proc2port_signal(c_p, + prt, + c_p->common.id, + refp, + sigdp, + 0, + NULL, + port_badsig); +} + + +/* + * Driver outputv() callback + */ + +static ERTS_INLINE void +call_driver_outputv(int bang_op, + Eterm caller, + Eterm from, + Port *prt, + erts_driver_t *drv, + ErlIOVec *evp) +{ + /* + * if (bang_op) + * we are part of a "Prt ! {From, {command, Data}}" operation + * else + * we are part of a call to port_command/[2,3] + * behave accordingly... + */ + if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) + send_badsig(prt); + else { + ErlDrvSizeT size = evp->size; + + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) + || ERTS_IS_CRASH_DUMPING); + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_outputv)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt); + DTRACE4(driver_outputv, process_str, port_str, prt->name, size); + } +#endif + + prt->caller = caller; + (*drv->outputv)((ErlDrvData) prt->drv_data, evp); + prt->caller = NIL; + + prt->bytes_out += size; + erts_smp_atomic_add_nob(&erts_bytes_out, size); + } +} + +static ERTS_INLINE void +cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp) +{ + int i; + /* Need to free all binaries */ + for (i = 1; i < ev->vsize; i++) + if (ev->binv[i]) + driver_free_binary(ev->binv[i]); + if (cbinp) + driver_free_binary(cbinp); + erts_free(ERTS_ALC_T_DRV_CMD_DATA, ev); +} + +static int +port_sig_outputv(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) +{ + Eterm reply; + + switch (op) { + case ERTS_PROC2PORT_SIG_EXEC: + /* Execution of a scheduled outputv() call */ + + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + + if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) + reply = am_badarg; + else { + call_driver_outputv(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP, + sigdp->caller, + sigdp->u.outputv.from, + prt, + prt->drv_ptr, + sigdp->u.outputv.evp); + reply = am_true; + } + break; + case ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND: + reply = am_false; + break; + default: + reply = am_badarg; + break; + } + + if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) + port_sched_op_reply(sigdp->caller, sigdp->ref, reply); + + cleanup_scheduled_outputv(sigdp->u.outputv.evp, + sigdp->u.outputv.cbinp); + + return ERTS_PORT_REDS_CMD_OUTPUTV; +} + +/* + * Driver output() callback + */ + +static ERTS_INLINE void +call_driver_output(int bang_op, + Eterm caller, + Eterm from, + Port *prt, + erts_driver_t *drv, + char *bufp, + ErlDrvSizeT size) +{ + /* + * if (bang_op) + * we are part of a "Prt ! {From, {command, Data}}" operation + * else + * we are part of a call to port_command/[2,3] + * behave accordingly... + */ + if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) + send_badsig(prt); + else { + + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) + || ERTS_IS_CRASH_DUMPING); + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(driver_output)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt); + DTRACE4(driver_output, process_str, port_str, prt->name, size); + } +#endif + + prt->caller = caller; + (*drv->output)((ErlDrvData) prt->drv_data, bufp, size); + prt->caller = NIL; + + prt->bytes_out += size; + erts_smp_atomic_add_nob(&erts_bytes_out, size); + } +} + +static ERTS_INLINE void +cleanup_scheduled_output(char *bufp) +{ + erts_free(ERTS_ALC_T_DRV_CMD_DATA, bufp); +} + +static int +port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) +{ + Eterm reply; + + switch (op) { + case ERTS_PROC2PORT_SIG_EXEC: + /* Execution of a scheduled output() call */ + + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + + if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) + reply = am_badarg; + else { + call_driver_output(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP, + sigdp->caller, + sigdp->u.output.from, + prt, + prt->drv_ptr, + sigdp->u.output.bufp, + sigdp->u.output.size); + reply = am_true; + } + break; + case ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND: + reply = am_false; + break; + default: + reply = am_badarg; + break; + } + + if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) + port_sched_op_reply(sigdp->caller, sigdp->ref, reply); + + cleanup_scheduled_output(sigdp->u.output.bufp); + + return ERTS_PORT_REDS_CMD_OUTPUT; +} + +ErtsPortOpResult +erts_port_output(Process *c_p, + int flags, + Port *prt, + Eterm from, + Eterm list, + Eterm *refp) +{ + ErtsPortOpResult res; + ErtsProc2PortSigData *sigdp; + erts_driver_t *drv = prt->drv_ptr; + size_t size; + int try_call; + erts_aint32_t sched_flags, busy_flgs, invalid_flags; + int task_flags; + ErtsProc2PortSigCallback port_sig_callback; + ErlDrvBinary *cbin = NULL; + ErlIOVec *evp = NULL; + char *buf = NULL; + int force_immediate_call = (flags & ERTS_PORT_SIG_FLG_FORCE_IMM_CALL); + int async_nosuspend; + ErtsPortTaskHandle *ns_pthp; + + ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_NOSUSPEND + | ERTS_PORT_SIG_FLG_FORCE + | ERTS_PORT_SIG_FLG_FORCE_IMM_CALL)) == 0); + + busy_flgs = ((flags & ERTS_PORT_SIG_FLG_FORCE) + ? ((erts_aint32_t) 0) + : ERTS_PTS_FLGS_BUSY); + invalid_flags = busy_flgs; + if (!refp) + invalid_flags |= ERTS_PTS_FLG_PARALLELISM; + + /* + * Assumes caller have checked that port is valid... + */ + + sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + if (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT)) + return ((sched_flags & ERTS_PTS_FLG_EXIT) + ? ERTS_PORT_OP_DROPPED + : ERTS_PORT_OP_BUSY); + + async_nosuspend = ((flags & (ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_NOSUSPEND + | ERTS_PORT_SIG_FLG_FORCE)) + == (ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_NOSUSPEND)); + + try_call = (force_immediate_call /* crash dumping */ + || !(sched_flags & (invalid_flags + | ERTS_PTS_FLGS_FORCE_SCHEDULE_OP))); + +#ifdef USE_VM_PROBES + if(DTRACE_ENABLED(port_command)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(c_p ? c_p->common.id : ERTS_INVALID_PID, prt); + DTRACE4(port_command, process_str, port_str, prt->name, "command"); + } +#endif + + if (drv->outputv) { + ErlIOVec ev; SysIOVec iv[SMALL_WRITE_VEC]; ErlDrvBinary* bv[SMALL_WRITE_VEC]; SysIOVec* ivp; ErlDrvBinary** bvp; - ErlDrvBinary* cbin; - ErlIOVec ev; + int vsize; + Uint csize; + Uint pvsize; + Uint pcsize; + Uint blimit; + size_t iov_offset, binv_offset, alloc_size; - if (io_list_vec_len(list, &vsize, &csize, - &pvsize, &pcsize, &size)) { + if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size)) goto bad_value; + + iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec)); + binv_offset = iov_offset; + binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec)); + alloc_size = binv_offset; + alloc_size += (vsize+1)*sizeof(ErlDrvBinary *); + + if (try_call && vsize < SMALL_WRITE_VEC) { + ivp = ev.iov = iv; + bvp = ev.binv = bv; + evp = &ev; } + else { + char *ptr = erts_alloc((try_call + ? ERTS_ALC_T_TMP + : ERTS_ALC_T_DRV_CMD_DATA), alloc_size); + + evp = (ErlIOVec *) ptr; + ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); + bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); + } + /* To pack or not to pack (small binaries) ...? */ - vsize++; - if (vsize <= SMALL_WRITE_VEC) { + if (vsize < SMALL_WRITE_VEC) { /* Do NOT pack */ blimit = 0; - } else { + } + else { /* Do pack */ vsize = pvsize + 1; csize = pcsize; blimit = ERL_SMALL_IO_BIN_LIMIT; } /* Use vsize and csize from now on */ - if (vsize <= SMALL_WRITE_VEC) { - ivp = iv; - bvp = bv; - } else { - ivp = (SysIOVec *) erts_alloc(ERTS_ALC_T_TMP, - vsize * sizeof(SysIOVec)); - bvp = (ErlDrvBinary**) erts_alloc(ERTS_ALC_T_TMP, - vsize * sizeof(ErlDrvBinary*)); - } + cbin = driver_alloc_binary(csize); if (!cbin) erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); @@ -1182,210 +1940,827 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list) ivp[0].iov_base = NULL; ivp[0].iov_len = 0; bvp[0] = NULL; - ev.vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); - if (ev.vsize < 0) { - if (ivp != iv) { - erts_free(ERTS_ALC_T_TMP, (void *) ivp); - } - if (bvp != bv) { - erts_free(ERTS_ALC_T_TMP, (void *) bvp); - } + evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); + if (evp->vsize < 0) { + if (evp != &ev) + erts_free(try_call ? ERTS_ALC_T_TMP : ERTS_ALC_T_DRV_CMD_DATA, + evp); driver_free_binary(cbin); goto bad_value; } - ev.vsize++; #if 0 /* This assertion may say something useful, but it can be falsified during the emulator test suites. */ - ASSERT(ev.vsize == vsize); -#endif - ev.size = size; /* total size */ - ev.iov = ivp; - ev.binv = bvp; -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(driver_outputv)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) - DTRACE4(driver_outputv, process_str, port_str, p->name, size); - } + ASSERT(evp->vsize == vsize); #endif - fpe_was_unmasked = erts_block_fpe(); - (*drv->outputv)((ErlDrvData)p->drv_data, &ev); - erts_unblock_fpe(fpe_was_unmasked); - if (ivp != iv) { - erts_free(ERTS_ALC_T_TMP, (void *) ivp); - } - if (bvp != bv) { - erts_free(ERTS_ALC_T_TMP, (void *) bvp); + evp->vsize++; + evp->size = size; /* total size */ + + if (!try_call) { + int i; + /* Need to increase refc on all binaries */ + for (i = 1; i < evp->vsize; i++) + if (bvp[i]) + driver_binary_inc_refc(bvp[i]); } - driver_free_binary(cbin); - } else { - int r; - - /* Try with an 8KB buffer first (will often be enough I guess). */ - size = 8*1024; - /* See below why the extra byte is added. */ - buf = erts_alloc(ERTS_ALC_T_TMP, size+1); - r = io_list_to_buf(list, buf, size); + else { + int i; + ErlIOVec *new_evp; + ErtsTryImmDrvCallResult try_call_res; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( + c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + invalid_flags, + !refp, + am_command); + + try_call_state.pre_chk_sched_flags = 0; /* already checked */ + if (force_immediate_call) + try_call_res = force_imm_drv_call(&try_call_state); + else + try_call_res = try_imm_drv_call(&try_call_state); + switch (try_call_res) { + case ERTS_TRY_IMM_DRV_CALL_OK: + call_driver_outputv(flags & ERTS_PORT_SIG_FLG_BANG_OP, + c_p ? c_p->common.id : ERTS_INVALID_PID, + from, + prt, + drv, + evp); + if (force_immediate_call) + finalize_force_imm_drv_call(&try_call_state); + else + finalize_imm_drv_call(&try_call_state); + /* Fall through... */ + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + driver_free_binary(cbin); + if (evp != &ev) + erts_free(ERTS_ALC_T_TMP, evp); + if (try_call_res != ERTS_TRY_IMM_DRV_CALL_OK) + return ERTS_PORT_OP_DROPPED; + if (c_p) + BUMP_REDS(c_p, ERTS_PORT_REDS_CMD_OUTPUTV); + return ERTS_PORT_OP_DONE; + case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: + sched_flags = try_call_state.sched_flags; + if (async_nosuspend + && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { + driver_free_binary(cbin); + if (evp != &ev) + erts_free(ERTS_ALC_T_TMP, evp); + return ((sched_flags & ERTS_PTS_FLG_EXIT) + ? ERTS_PORT_OP_DROPPED + : ERTS_PORT_OP_BUSY); + } + case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: + /* Schedule outputv() call instead... */ + break; + } -#ifdef USE_VM_PROBES - if(DTRACE_ENABLED(port_command)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) - DTRACE4(port_command, process_str, port_str, p->name, "command"); - } + /* Need to increase refc on all binaries */ + for (i = 1; i < evp->vsize; i++) + if (bvp[i]) + driver_binary_inc_refc(bvp[i]); + + new_evp = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, alloc_size); + + if (evp != &ev) { + sys_memcpy((void *) new_evp, (void *) evp, alloc_size); + new_evp->iov = (SysIOVec *) (((char *) new_evp) + + iov_offset); + bvp = new_evp->binv = (ErlDrvBinary **) (((char *) new_evp) + + binv_offset); + +#ifdef DEBUG + ASSERT(new_evp->vsize == evp->vsize); + ASSERT(new_evp->size == evp->size); + for (i = 0; i < evp->vsize; i++) { + ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len); + ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base); + ASSERT(new_evp->binv[i] == evp->binv[i]); + } #endif - if (r >= 0) { - size -= r; -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(driver_output)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) - DTRACE4(driver_output, process_str, port_str, p->name, size); - } + erts_free(ERTS_ALC_T_TMP, evp); + } + else { /* from stack allocated structure; offsets may differ */ + + sys_memcpy((void *) new_evp, (void *) evp, sizeof(ErlIOVec)); + new_evp->iov = (SysIOVec *) (((char *) new_evp) + + iov_offset); + sys_memcpy((void *) new_evp->iov, + (void *) evp->iov, + evp->vsize * sizeof(SysIOVec)); + new_evp->binv = (ErlDrvBinary **) (((char *) new_evp) + + binv_offset); + sys_memcpy((void *) new_evp->binv, + (void *) evp->binv, + evp->vsize * sizeof(ErlDrvBinary *)); + +#ifdef DEBUG + ASSERT(new_evp->vsize == evp->vsize); + ASSERT(new_evp->size == evp->size); + for (i = 0; i < evp->vsize; i++) { + ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len); + ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base); + ASSERT(new_evp->binv[i] == evp->binv[i]); + } #endif - fpe_was_unmasked = erts_block_fpe(); - (*drv->output)((ErlDrvData)p->drv_data, buf, size); - erts_unblock_fpe(fpe_was_unmasked); - erts_free(ERTS_ALC_T_TMP, buf); + + } + + evp = new_evp; } - else if (r == -2) { - erts_free(ERTS_ALC_T_TMP, buf); - goto bad_value; + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; + sigdp->u.outputv.from = from; + sigdp->u.outputv.evp = evp; + sigdp->u.outputv.cbinp = cbin; + port_sig_callback = port_sig_outputv; + } + else { + ErlDrvSizeT r; + + /* + * Apperently there exist code that write 1 byte to + * much in buffer. Where it resides I don't know, but + * we can live with one byte extra allocated... + */ + + if (!try_call) { + if (erts_iolist_size(list, &size)) + goto bad_value; + + buf = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, size + 1); + + r = erts_iolist_to_buf(list, buf, size); + ASSERT(ERTS_IOLIST_TO_BUF_SUCCEEDED(r)); } else { - ASSERT(r == -1); /* Overflow */ + char *new_buf; + ErtsTryImmDrvCallResult try_call_res; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( + c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + invalid_flags, + !refp, + am_command); + + /* Try with an 8KB buffer first (will often be enough I guess). */ + size = 8*1024; + + buf = erts_alloc(ERTS_ALC_T_TMP, size + 1); + r = erts_iolist_to_buf(list, buf, size); + + if (ERTS_IOLIST_TO_BUF_SUCCEEDED(r)) { + ASSERT(r <= size); + size -= r; + } + else { + erts_free(ERTS_ALC_T_TMP, buf); + if (r == ERTS_IOLIST_TO_BUF_TYPE_ERROR) + goto bad_value; + ASSERT(r == ERTS_IOLIST_TO_BUF_OVERFLOW); + if (erts_iolist_size(list, &size)) + goto bad_value; + buf = erts_alloc(ERTS_ALC_T_TMP, size + 1); + r = erts_iolist_to_buf(list, buf, size); + ASSERT(ERTS_IOLIST_TO_BUF_SUCCEEDED(r)); + } + + try_call_state.pre_chk_sched_flags = 0; /* already checked */ + if (force_immediate_call) + try_call_res = force_imm_drv_call(&try_call_state); + else + try_call_res = try_imm_drv_call(&try_call_state); + switch (try_call_res) { + case ERTS_TRY_IMM_DRV_CALL_OK: + call_driver_output(flags & ERTS_PORT_SIG_FLG_BANG_OP, + c_p ? c_p->common.id : ERTS_INVALID_PID, + from, + prt, + drv, + buf, + size); + if (force_immediate_call) + finalize_force_imm_drv_call(&try_call_state); + else + finalize_imm_drv_call(&try_call_state); + /* Fall through... */ + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + erts_free(ERTS_ALC_T_TMP, buf); + if (try_call_res != ERTS_TRY_IMM_DRV_CALL_OK) + return ERTS_PORT_OP_DROPPED; + if (c_p) + BUMP_REDS(c_p, ERTS_PORT_REDS_CMD_OUTPUT); + return ERTS_PORT_OP_DONE; + case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: + sched_flags = try_call_state.sched_flags; + if (async_nosuspend + && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { + erts_free(ERTS_ALC_T_TMP, buf); + return ((sched_flags & ERTS_PTS_FLG_EXIT) + ? ERTS_PORT_OP_DROPPED + : ERTS_PORT_OP_BUSY); + } + case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: + /* Schedule outputv() call instead... */ + break; + } + + new_buf = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, size + 1); + sys_memcpy(new_buf, buf, size); erts_free(ERTS_ALC_T_TMP, buf); - if (erts_iolist_size(list, &size)) { - goto bad_value; + buf = new_buf; + } + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUT; + sigdp->u.output.from = from; + sigdp->u.output.bufp = buf; + sigdp->u.output.size = size; + port_sig_callback = port_sig_output; + } + + task_flags = ERTS_PT_FLG_WAIT_BUSY; + sigdp->flags |= flags; + ns_pthp = NULL; + if (flags & (ERTS_P2P_SIG_DATA_FLG_FORCE|ERTS_P2P_SIG_DATA_FLG_NOSUSPEND)) { + task_flags = 0; + if (flags & ERTS_P2P_SIG_DATA_FLG_FORCE) + sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND; + else if (async_nosuspend) { + ErtsSchedulerData *esdp = (c_p + ? ERTS_PROC_GET_SCHDATA(c_p) + : erts_get_scheduler_data()); + ASSERT(esdp); + ns_pthp = &esdp->nosuspend_port_task_handle; + sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND; + } + else if (flags & ERTS_P2P_SIG_DATA_FLG_NOSUSPEND) + task_flags = ERTS_PT_FLG_NOSUSPEND; + } + + ASSERT(ns_pthp || !async_nosuspend); + ASSERT(async_nosuspend || !ns_pthp); + + res = erts_schedule_proc2port_signal(c_p, + prt, + c_p ? c_p->common.id : ERTS_INVALID_PID, + refp, + sigdp, + task_flags, + ns_pthp, + port_sig_callback); + + if (res != ERTS_PORT_OP_SCHEDULED) { + if (drv->outputv) + cleanup_scheduled_outputv(evp, cbin); + else + cleanup_scheduled_output(buf); + return res; + } + + if (!(flags & ERTS_PORT_SIG_FLG_FORCE)) { + sched_flags = erts_smp_atomic32_read_acqb(&prt->sched.flags); + if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT)) { + if (async_nosuspend) + erts_port_task_tmp_handle_detach(ns_pthp); + } + else { + if (!async_nosuspend) + return ERTS_PORT_OP_BUSY_SCHEDULED; + else { + if (erts_port_task_abort(ns_pthp) == 0) + return ERTS_PORT_OP_BUSY; + else + erts_port_task_tmp_handle_detach(ns_pthp); } + } + } + return res; + +bad_value: + + flags |= ERTS_PORT_SIG_FLG_BAD_OUTPUT; + return bad_port_signal(c_p, flags, prt, from, refp, am_command); +} + +static ERTS_INLINE ErtsPortOpResult +call_deliver_port_exit(int bang_op, + Eterm from, + Port *prt, + erts_aint32_t state, + Eterm reason, + int broken_link) +{ + /* + * if (bang_op) + * we are part of a "Prt ! {From, close}" operation + * else + * we are part of a call to port_close(Port) + * behave accordingly... + */ + + if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) + return ERTS_PORT_OP_DROPPED; + + if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) { + send_badsig(prt); + return ERTS_PORT_OP_DROPPED; + } + + if (broken_link) { + ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from); + if (lnk) + erts_destroy_link(lnk); + else + return ERTS_PORT_OP_DROPPED; + } + + if (!erts_deliver_port_exit(prt, from, reason, bang_op)) + return ERTS_PORT_OP_DROPPED; - /* - * I know drivers that pad space with '\0' this is clearly - * incorrect but I don't feel like fixing them now, insted - * add ONE extra byte. - */ - buf = erts_alloc(ERTS_ALC_T_TMP, size+1); - r = io_list_to_buf(list, buf, size); #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(driver_output)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p) - DTRACE4(driver_output, process_str, port_str, p->name, size); - } + if(DTRACE_ENABLED(port_command) && bang_op) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(from, prt); + DTRACE4(port_command, process_str, port_str, prt->name, "close"); + } #endif - fpe_was_unmasked = erts_block_fpe(); - (*drv->output)((ErlDrvData)p->drv_data, buf, size); - erts_unblock_fpe(fpe_was_unmasked); - erts_free(ERTS_ALC_T_TMP, buf); + + return ERTS_PORT_OP_DONE; +} + +static int +port_sig_exit(Port *prt, + erts_aint32_t state, + int op, + ErtsProc2PortSigData *sigdp) +{ + Eterm msg = am_badarg; + if (op == ERTS_PROC2PORT_SIG_EXEC) { + ErtsPortOpResult res; + int bang_op = sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP; + int broken_link = sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK; + res = call_deliver_port_exit(bang_op, + sigdp->u.exit.from, + prt, + state, + sigdp->u.exit.reason, + broken_link); + + if (res == ERTS_PORT_OP_DONE) + msg = am_true; + } + if (sigdp->u.exit.bp) + free_message_buffer(sigdp->u.exit.bp); + if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) + port_sched_op_reply(sigdp->caller, sigdp->ref, msg); + + return ERTS_PORT_REDS_EXIT; +} + +ErtsPortOpResult +erts_port_exit(Process *c_p, + int flags, + Port *prt, + Eterm from, + Eterm reason, + Eterm *refp) +{ + ErtsPortOpResult res; + ErtsProc2PortSigData *sigdp; + ErlHeapFragment *bp = NULL; + + ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_BROKEN_LINK + | ERTS_PORT_SIG_FLG_FORCE_SCHED)) == 0); + + if (!(flags & ERTS_PORT_SIG_FLG_FORCE_SCHED)) { + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + 0, + !refp, + am_exit); + + + switch (try_imm_drv_call(&try_call_state)) { + case ERTS_TRY_IMM_DRV_CALL_OK: { + res = call_deliver_port_exit(flags & ERTS_PORT_SIG_FLG_BANG_OP, + from, + prt, + try_call_state.state, + reason, + flags & ERTS_PORT_SIG_FLG_BROKEN_LINK); + finalize_imm_drv_call(&try_call_state); + if (res == ERTS_PORT_OP_DONE && c_p) + BUMP_REDS(c_p, ERTS_PORT_REDS_EXIT); + return res; + } + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + return ERTS_PORT_OP_DROPPED; + default: + /* Schedule call instead... */ + break; } } - p->bytes_out += size; - erts_smp_atomic_add_nob(&erts_bytes_out, size); -#ifdef ERTS_SMP - if (p->xports) - erts_smp_xports_unlock(p); - ASSERT(!p->xports); -#endif - p->caller = NIL; - return 0; + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_EXIT | flags; + sigdp->u.exit.from = from; - bad_value: - p->caller = NIL; - { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Bad value on output port '%s'\n", p->name); - erts_send_error_to_logger_nogl(dsbufp); - return 1; + if (is_immed(reason)) { + sigdp->u.exit.reason = reason; + sigdp->u.exit.bp = NULL; + } + else { + Eterm *hp; + Uint hsz = size_object(reason); + bp = new_message_buffer(hsz); + sigdp->u.exit.bp = bp; + hp = bp->mem; + sigdp->u.exit.reason = copy_struct(reason, + hsz, + &hp, + &bp->off_heap); + } + + res = erts_schedule_proc2port_signal(c_p, + prt, + c_p ? c_p->common.id : from, + refp, + sigdp, + 0, + NULL, + port_sig_exit); + + if (res == ERTS_PORT_OP_DROPPED) { + if (bp) + free_message_buffer(bp); } + + return res; } -/* initialize the port array */ -void init_io(void) +static ErtsPortOpResult +set_port_connected(int bang_op, + Eterm from, + Port *prt, + erts_aint32_t state, + Eterm connect) { - int i; - ErlDrvEntry** dp; - char maxports[21]; /* enough for any 64-bit integer */ - size_t maxportssize = sizeof(maxports); - Uint ports_bits = ERTS_PORTS_BITS; - Sint port_extra_shift; + /* + * if (bang_op) + * we are part of a "Prt ! {From, {connect, Connect}}" operation + * else + * we are part of a call to port_connect(Port, Connect) + * behave accordingly... + */ -#ifdef ERTS_SMP - init_xports_list_alloc(); + if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) + return ERTS_PORT_OP_DROPPED; + + if (bang_op) { /* Bang operation */ + if (is_not_internal_pid(connect) || ERTS_PORT_GET_CONNECTED(prt) != from) { + send_badsig(prt); + return ERTS_PORT_OP_DROPPED; + } + + ERTS_PORT_SET_CONNECTED(prt, connect); + deliver_result(prt->common.id, from, am_connected); + +#ifdef USE_VM_PROBES + if(DTRACE_ENABLED(port_command)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(from, prt); + DTRACE4(port_command, process_str, port_str, prt->name, "connect"); + } #endif + } + else { /* Port BIF operation */ + Process *rp = erts_proc_lookup_raw(connect); + if (!rp) + return ERTS_PORT_OP_DROPPED; + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (ERTS_PROC_IS_EXITING(rp)) { + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + return ERTS_PORT_OP_DROPPED; + } - pdl_init(); + erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, prt->common.id); + erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, connect); - if (erts_sys_getenv_raw("ERL_MAX_PORTS", maxports, &maxportssize) == 0) - erts_max_ports = atoi(maxports); - else - erts_max_ports = sys_max_files(); + ERTS_PORT_SET_CONNECTED(prt, connect); - if (erts_max_ports > ERTS_MAX_PORTS) - erts_max_ports = ERTS_MAX_PORTS; - if (erts_max_ports < 1024) - erts_max_ports = 1024; + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (erts_use_r9_pids_ports) { - ports_bits = ERTS_R9_PORTS_BITS; - if (erts_max_ports > ERTS_MAX_R9_PORTS) - erts_max_ports = ERTS_MAX_R9_PORTS; +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_connect)) { + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); + + dtrace_pid_str(connect, process_str); + erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + dtrace_proc_str(rp, newprocess_str); + DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); + } +#endif } - port_extra_shift = erts_fit_in_bits(erts_max_ports - 1); - port_num_mask = (1 << ports_bits) - 1; + return ERTS_PORT_OP_DONE; +} + +static int +port_sig_connect(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) +{ + Eterm msg = am_badarg; + if (op == ERTS_PROC2PORT_SIG_EXEC) { + ErtsPortOpResult res; + res = set_port_connected(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP, + sigdp->u.connect.from, + prt, + state, + sigdp->u.connect.connected); + if (res == ERTS_PORT_OP_DONE) + msg = am_true; + } + if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) + port_sched_op_reply(sigdp->caller, sigdp->ref, msg); + return ERTS_PORT_REDS_CONNECT; +} + +ErtsPortOpResult +erts_port_connect(Process *c_p, + int flags, + Port *prt, + Eterm from, + Eterm connect, + Eterm *refp) +{ + ErtsProc2PortSigData *sigdp; + Eterm connect_id; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + 0, + !refp, + am_connect); + + ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC)) == 0); + + if (is_not_internal_pid(connect)) + connect_id = NIL; /* Fail in op (for signal order) */ + else + connect_id = connect; + + switch (try_imm_drv_call(&try_call_state)) { + case ERTS_TRY_IMM_DRV_CALL_OK: { + ErtsPortOpResult res; + res = set_port_connected(flags & ERTS_PORT_SIG_FLG_BANG_OP, + from, + prt, + try_call_state.state, + connect_id); + finalize_imm_drv_call(&try_call_state); + if (res == ERTS_PORT_OP_DONE) + BUMP_REDS(c_p, ERTS_PORT_REDS_CONNECT); + return res; + } + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + return ERTS_PORT_OP_DROPPED; + default: + /* Schedule call instead... */ + break; + } - erts_port_tab_index_mask = ~(~((Uint) 0) << port_extra_shift); - erts_max_ports = 1 << port_extra_shift; + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_CONNECT | flags; - erts_smp_mtx_init(&erts_driver_list_lock,"driver_list"); - driver_list = NULL; - erts_smp_tsd_key_create(&driver_list_lock_status_key); - erts_smp_tsd_key_create(&driver_list_last_error_key); + sigdp->u.connect.from = from; + sigdp->u.connect.connected = connect_id; + + return erts_schedule_proc2port_signal(c_p, + prt, + c_p->common.id, + refp, + sigdp, + 0, + NULL, + port_sig_connect); +} - if (erts_max_ports * sizeof(Port) <= erts_max_ports) { - /* More memory needed than the whole address space. */ - erts_alloc_enomem(ERTS_ALC_T_PORT_TABLE, ~((Uint) 0)); +static void +port_unlink(Port *prt, Eterm from) +{ + ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from); + if (lnk) + erts_destroy_link(lnk); +} + +static int +port_sig_unlink(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) +{ + if (op == ERTS_PROC2PORT_SIG_EXEC) + port_unlink(prt, sigdp->u.unlink.from); + if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) + port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); + return ERTS_PORT_REDS_UNLINK; +} + +ErtsPortOpResult +erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp) +{ + ErtsProc2PortSigData *sigdp; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p, + prt, + ERTS_PORT_SFLGS_DEAD, + 0, + !refp, + am_unlink); + + switch (try_imm_drv_call(&try_call_state)) { + case ERTS_TRY_IMM_DRV_CALL_OK: + port_unlink(prt, from); + finalize_imm_drv_call(&try_call_state); + BUMP_REDS(c_p, ERTS_PORT_REDS_UNLINK); + return ERTS_PORT_OP_DONE; + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + return ERTS_PORT_OP_DROPPED; + default: + /* Schedule call instead... */ + break; } - erts_port = (Port *) erts_alloc(ERTS_ALC_T_PORT_TABLE, - erts_max_ports * sizeof(Port)); + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_UNLINK; + sigdp->u.unlink.from = from; + + return erts_schedule_proc2port_signal(c_p, + prt, + c_p ? c_p->common.id : from, + refp, + sigdp, + 0, + NULL, + port_sig_unlink); +} - erts_smp_atomic_init_nob(&erts_bytes_out, 0); - erts_smp_atomic_init_nob(&erts_bytes_in, 0); +static void +port_link_failure(Eterm port_id, Eterm linker) +{ + Process *rp; + ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; + ASSERT(is_internal_pid(linker)); + rp = erts_pid2proc(NULL, 0, linker, rp_locks); + if (rp) { + ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id); + if (rlnk) { + int xres = erts_send_exit_signal(NULL, + port_id, + rp, + &rp_locks, + am_noproc, + NIL, + NULL, + 0); + if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { + /* We didn't exit the process and it is traced */ + if (IS_TRACED_FL(rp, F_TRACE_PROCS)) + trace_proc(NULL, rp, am_getting_unlinked, port_id); + } + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } + } +} + +static void +port_link(Port *prt, erts_aint32_t state, Eterm to) +{ + if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) + erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, to); + else + port_link_failure(prt->common.id, to); +} + +static int +port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) +{ + if (op == ERTS_PROC2PORT_SIG_EXEC) + port_link(prt, state, sigdp->u.link.to); + else + port_link_failure(sigdp->u.link.port, sigdp->u.link.to); + if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) + port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); + return ERTS_PORT_REDS_LINK; +} + +ErtsPortOpResult +erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) +{ + ErtsProc2PortSigData *sigdp; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + 0, + !refp, + am_link); + + switch (try_imm_drv_call(&try_call_state)) { + case ERTS_TRY_IMM_DRV_CALL_OK: + port_link(prt, try_call_state.state, to); + finalize_imm_drv_call(&try_call_state); + BUMP_REDS(c_p, ERTS_PORT_REDS_LINK); + return ERTS_PORT_OP_DONE; + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + return ERTS_PORT_OP_BADARG; + default: + /* Schedule call instead... */ + break; + } - for (i = 0; i < erts_max_ports; i++) { - erts_port_task_init_sched(&erts_port[i].sched); + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_LINK; + sigdp->u.link.port = prt->common.id; + sigdp->u.link.to = to; + + return erts_schedule_proc2port_signal(c_p, + prt, + c_p ? c_p->common.id : to, + refp, + sigdp, + 0, + NULL, + port_sig_link); +} + +void erts_init_io(int port_tab_size, + int port_tab_size_ignore_files, + int legacy_port_tab) +{ + ErlDrvEntry** dp; + UWord common_element_size; + erts_smp_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); + common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ)); + common_element_size += 10; /* name */ #ifdef ERTS_SMP - erts_smp_atomic_init_nob(&erts_port[i].refc, 0); - erts_port[i].lock = NULL; - erts_port[i].xports = NULL; - erts_smp_spinlock_init_x(&erts_port[i].state_lck, -#ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) ? "port_state" : NULL, -#else - "port_state", -#endif - make_small(0)); + common_element_size += sizeof(erts_mtx_t); + + init_xports_list_alloc(); #endif - erts_port[i].tracer_proc = NIL; - erts_port[i].trace_flags = 0; - erts_port[i].drv_ptr = NULL; - erts_port[i].status = ERTS_PORT_SFLG_FREE; - erts_port[i].name = NULL; - erts_port[i].nlinks = NULL; - erts_port[i].monitors = NULL; - erts_port[i].linebuf = NULL; - erts_port[i].port_data_lock = NULL; + pdl_init(); + + if (!port_tab_size_ignore_files) { + int max_files = sys_max_files(); + if (port_tab_size < max_files) + port_tab_size = max_files; } - erts_smp_atomic32_init_nob(&erts_ports_snapshot, (erts_aint32_t) 0); - last_port_num = 0; - erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); + if (port_tab_size > ERTS_MAX_PORTS) + port_tab_size = ERTS_MAX_PORTS; + else if (port_tab_size < ERTS_MIN_PORTS) + port_tab_size = ERTS_MIN_PORTS; + + erts_smp_rwmtx_init_opt(&erts_driver_list_lock, + &drv_list_rwmtx_opts, + "driver_list"); + driver_list = NULL; + erts_smp_tsd_key_create(&driver_list_lock_status_key); + erts_smp_tsd_key_create(&driver_list_last_error_key); + + erts_ptab_init_table(&erts_port, + ERTS_ALC_T_PORT_TABLE, + NULL, + (ErtsPTabElementCommon *) &erts_invalid_port.common, + port_tab_size, + common_element_size, /* Doesn't need to be excact */ + "port_table", + legacy_port_tab); + + erts_smp_atomic_init_nob(&erts_bytes_out, 0); + erts_smp_atomic_init_nob(&erts_bytes_in, 0); sys_init_io(); erts_smp_tsd_set(driver_list_lock_status_key, (void *) 1); - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rwlock(&erts_driver_list_lock); init_driver(&fd_driver, &fd_driver_entry, NULL); init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); @@ -1394,27 +2769,66 @@ void init_io(void) erts_add_driver_entry(*dp, NULL, 1); erts_smp_tsd_set(driver_list_lock_status_key, NULL); - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); } #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) -void erts_lcnt_enable_io_lock_count(int enable) { - int i; - for (i = 0; i < erts_max_ports; i++) { - Port* p = &erts_port[i]; - if (enable) { - erts_lcnt_init_lock_x(&p->state_lck.lcnt, "port_state", ERTS_LCNT_LT_SPINLOCK, make_small(i)); - if (p->lock) { - erts_lcnt_init_lock_x(&p->lock->lcnt, "port_lock", ERTS_LCNT_LT_MUTEX, make_small(i)); - } - } else { - erts_lcnt_destroy_lock(&p->state_lck.lcnt); - if (p->lock) { - erts_lcnt_destroy_lock(&p->lock->lcnt); - } - } +static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable) +{ + if (dp->lock) { + if (enable) + erts_lcnt_init_lock_x(&dp->lock->lcnt, + "driver_lock", + ERTS_LCNT_LT_MUTEX, + erts_atom_put((byte*)dp->name, + sys_strlen(dp->name), + ERTS_ATOM_ENC_LATIN1, + 1)); + + else + erts_lcnt_destroy_lock(&dp->lock->lcnt); + + } +} + +static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable) +{ + erts_aint32_t state = erts_atomic32_read_nob(&prt->state); + if (!enable) { + erts_lcnt_destroy_lock(&prt->sched.mtx.lcnt); + if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) + erts_lcnt_destroy_lock(&prt->lock->lcnt); + } + else { + erts_lcnt_init_lock_x(&prt->sched.mtx.lcnt, + "port_sched_lock", + ERTS_LCNT_LT_MUTEX, + prt->common.id); + if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) + erts_lcnt_init_lock_x(&prt->lock->lcnt, + "port_lock", + ERTS_LCNT_LT_MUTEX, + prt->common.id); + } +} + +void erts_lcnt_enable_io_lock_count(int enable) +{ + erts_driver_t *dp; + int i, max = erts_ptab_max(&erts_port); + + for (i = 0; i < max; i++) { + Port *prt = erts_pix2port(i); + if (prt) + lcnt_enable_port_lock_count(prt, enable); } + + lcnt_enable_drv_lock_count(&vanilla_driver, enable); + lcnt_enable_drv_lock_count(&spawn_driver, enable); + lcnt_enable_drv_lock_count(&fd_driver, enable); + for (dp = driver_list; dp; dp = dp->next) + lcnt_enable_drv_lock_count(dp, enable); } #endif @@ -1594,14 +3008,15 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) { Process *rp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_CHK_NO_PROC_LOCKS; - ASSERT(is_internal_port(sender) - && is_internal_pid(pid) - && internal_pid_index(pid) < erts_max_processes); + ASSERT(is_internal_port(sender) && is_internal_pid(pid)); - rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = (scheduler + ? erts_proc_lookup(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (rp) { Eterm tuple; @@ -1609,17 +3024,22 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) ErlOffHeap *ohp; Eterm* hp; Uint sz_res; - sz_res = size_object(res); - hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks); - res = copy_struct(res, sz_res, &hp, ohp); - tuple = TUPLE2(hp, sender, res); + + sz_res = size_object(res); + hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks); + res = copy_struct(res, sz_res, &hp, ohp); + tuple = TUPLE2(hp, sender, res); erts_queue_message(rp, &rp_locks, bp, tuple, NIL #ifdef USE_VM_PROBES , NIL #endif ); - erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (!scheduler) + erts_smp_proc_dec_refc(rp); + } } @@ -1632,7 +3052,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) * len -- length of data */ -static void deliver_read_message(Port* prt, Eterm to, +static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, char *hbuf, ErlDrvSizeT hlen, char *buf, ErlDrvSizeT len, int eol) { @@ -1644,28 +3064,33 @@ static void deliver_read_message(Port* prt, Eterm to, ErlHeapFragment *bp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; need = 3 + 3 + 2*hlen; - if (prt->status & ERTS_PORT_SFLG_LINEBUF_IO) { + + if (state & ERTS_PORT_SFLG_LINEBUF_IO) { need += 3; } - if (prt->status & ERTS_PORT_SFLG_BINARY_IO && buf != NULL) { + if ((state & ERTS_PORT_SFLG_BINARY_IO) && buf != NULL) { need += PROC_BIN_SIZE; } else { need += 2*len; } - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + if (!rp) return; hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); listp = NIL; - if ((prt->status & ERTS_PORT_SFLG_BINARY_IO) == 0) { + if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { listp = buf_to_intlist(&hp, buf, len, listp); } else if (buf != NULL) { ProcBin* pb; @@ -1696,14 +3121,14 @@ static void deliver_read_message(Port* prt, Eterm to, listp = buf_to_intlist(&hp, hbuf, hlen, listp); } - if (prt->status & ERTS_PORT_SFLG_LINEBUF_IO){ + if (state & ERTS_PORT_SFLG_LINEBUF_IO){ listp = TUPLE2(hp, (eol) ? am_eol : am_noeol, listp); hp += 3; } tuple = TUPLE2(hp, am_data, listp); hp += 3; - tuple = TUPLE2(hp, prt->id, tuple); + tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined @@ -1711,15 +3136,18 @@ static void deliver_read_message(Port* prt, Eterm to, , NIL #endif ); - erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } /* * Deliver all lines in a line buffer, repeats calls to * deliver_read_message, and takes the same parameters. */ -static void deliver_linebuf_message(Port* prt, Eterm to, +static void deliver_linebuf_message(Port* prt, erts_aint_t state, + Eterm to, char* hbuf, ErlDrvSizeT hlen, char *buf, ErlDrvSizeT len) { @@ -1728,7 +3156,7 @@ static void deliver_linebuf_message(Port* prt, Eterm to, if(init_linebuf_context(&lc,&(prt->linebuf), buf, len) < 0) return; while((ret = read_linebuf(&lc)) > LINEBUF_EMPTY) - deliver_read_message(prt, to, hbuf, hlen, LINEBUF_DATA(lc), + deliver_read_message(prt, state, to, hbuf, hlen, LINEBUF_DATA(lc), LINEBUF_DATALEN(lc), (ret == LINEBUF_EOL)); } @@ -1739,20 +3167,25 @@ static void deliver_linebuf_message(Port* prt, Eterm to, * Parameters: * prt - Pointer to a Port structure for this port. */ -static void flush_linebuf_messages(Port *prt) +static void flush_linebuf_messages(Port *prt, erts_aint32_t state) { LineBufContext lc; int ret; ERTS_SMP_LC_ASSERT(!prt || erts_lc_is_port_locked(prt)); - if(prt == NULL || !(prt->status & ERTS_PORT_SFLG_LINEBUF_IO)) + + if (!prt) + return; + + if (!(state & ERTS_PORT_SFLG_LINEBUF_IO)) return; if(init_linebuf_context(&lc,&(prt->linebuf), NULL, 0) < 0) return; while((ret = flush_linebuf(&lc)) > LINEBUF_EMPTY) deliver_read_message(prt, - prt->connected, + state, + ERTS_PORT_GET_CONNECTED(prt), NULL, 0, LINEBUF_DATA(lc), @@ -1779,6 +3212,8 @@ deliver_vec_message(Port* prt, /* Port */ ErlHeapFragment *bp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; + erts_aint32_t state; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1787,16 +3222,20 @@ deliver_vec_message(Port* prt, /* Port */ * Check arguments for validity. */ - rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) return; + state = erts_atomic32_read_nob(&prt->state); /* * Calculate the exact number of heap words needed. */ need = 3 + 3; /* Heap space for two tuples */ - if (prt->status & ERTS_PORT_SFLG_BINARY_IO) { + if (state & ERTS_PORT_SFLG_BINARY_IO) { need += (2+PROC_BIN_SIZE)*vsize - 2 + hlen*2; } else { need += (hlen+csize)*2; @@ -1807,7 +3246,7 @@ deliver_vec_message(Port* prt, /* Port */ listp = NIL; iov += vsize; - if ((prt->status & ERTS_PORT_SFLG_BINARY_IO) == 0) { + if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { Eterm* thp = hp; while (vsize--) { iov--; @@ -1860,7 +3299,7 @@ deliver_vec_message(Port* prt, /* Port */ tuple = TUPLE2(hp, am_data, listp); hp += 3; - tuple = TUPLE2(hp, prt->id, tuple); + tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined @@ -1869,7 +3308,8 @@ deliver_vec_message(Port* prt, /* Port */ #endif ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } @@ -1892,7 +3332,7 @@ static void deliver_bin_message(Port* prt, /* port */ /* * Note. * - * The test for (p->status & ERTS_PORT_SFLGS_DEAD) == 0 is important since the + * The test for ERTS_PORT_SFLGS_DEAD is important since the * driver's flush function might call driver_async, which when using no * threads and being short circuited will notice that the io queue is empty * (after calling the driver's async_ready) and recursively call @@ -1908,7 +3348,7 @@ static void flush_port(Port *p) if (p->drv_ptr->flush != NULL) { #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_flush)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(p->connected, p) + DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p) DTRACE3(driver_flush, process_str, port_str, p->name); } #endif @@ -1923,11 +3363,12 @@ static void flush_port(Port *p) } #ifdef ERTS_SMP if (p->xports) - erts_smp_xports_unlock(p); + erts_port_handle_xports(p); ASSERT(!p->xports); #endif } - if ((p->status & ERTS_PORT_SFLGS_DEAD) == 0 && is_port_ioq_empty(p)) { + if ((erts_atomic32_read_nob(&p->state) & ERTS_PORT_SFLGS_DEAD) == 0 + && is_port_ioq_empty(p)) { terminate_port(p); } } @@ -1939,29 +3380,29 @@ terminate_port(Port *prt) Eterm send_closed_port_id; Eterm connected_id = NIL /* Initialize to silence compiler */; erts_driver_t *drv; - int halt; + erts_aint32_t state; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - ASSERT(!prt->nlinks); - ASSERT(!prt->monitors); + ASSERT(!ERTS_P_LINKS(prt)); + ASSERT(!ERTS_P_MONITORS(prt)); - /* prt->status may be altered by kill_port()below */ - halt = (prt->status & ERTS_PORT_SFLG_HALT) != 0; - if (prt->status & ERTS_PORT_SFLG_SEND_CLOSED) { - erts_port_status_band_set(prt, ~ERTS_PORT_SFLG_SEND_CLOSED); - send_closed_port_id = prt->id; - connected_id = prt->connected; + /* state may be altered by kill_port() below */ + state = erts_atomic32_read_band_nob(&prt->state, + ~ERTS_PORT_SFLG_SEND_CLOSED); + if (state & ERTS_PORT_SFLG_SEND_CLOSED) { + send_closed_port_id = prt->common.id; + connected_id = ERTS_PORT_GET_CONNECTED(prt); } else { send_closed_port_id = NIL; } #ifdef ERTS_SMP - erts_cancel_smp_ptimer(prt->ptimer); + erts_cancel_smp_ptimer(prt->common.u.alive.ptimer); #else - erts_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->common.u.alive.tm); #endif drv = prt->drv_ptr; @@ -1969,7 +3410,7 @@ terminate_port(Port *prt) int fpe_was_unmasked = erts_block_fpe(); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_stop)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(prt->connected, prt) + DTRACE_FORMAT_COMMON_PID_AND_PORT(connected_id, prt) DTRACE3(driver_stop, process_str, drv->name, port_str); } #endif @@ -1977,43 +3418,41 @@ terminate_port(Port *prt) erts_unblock_fpe(fpe_was_unmasked); #ifdef ERTS_SMP if (prt->xports) - erts_smp_xports_unlock(prt); + erts_port_handle_xports(prt); ASSERT(!prt->xports); #endif } if(drv->handle != NULL) { - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(drv->handle); - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_runlock(&erts_driver_list_lock); } stopq(prt); /* clear queue memory */ if(prt->linebuf != NULL){ erts_free(ERTS_ALC_T_LINEBUF, (void *) prt->linebuf); prt->linebuf = NULL; } - if (prt->bp != NULL) { - free_message_buffer(prt->bp); - prt->bp = NULL; - prt->data = am_undefined; - } + + erts_cleanup_port_data(prt); if (prt->psd) erts_free(ERTS_ALC_T_PRTSD, prt->psd); + ASSERT(prt->dist_entry == NULL); + kill_port(prt); /* * We don't want to send the closed message until after the * port has been removed from the port table (in kill_port()). */ - if (halt && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { - erts_smp_port_unlock(prt); /* We will exit and never return */ + if ((state & ERTS_PORT_SFLG_HALT) + && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { + erts_port_release(prt); /* We will exit and never return */ erl_exit_flush_async(erts_halt_code, ""); } if (is_internal_port(send_closed_port_id)) deliver_result(send_closed_port_id, connected_id, am_closed); - - ASSERT(prt->dist_entry == NULL); } void @@ -2033,7 +3472,7 @@ static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc) if (!rp) { goto done; } - rmon = erts_remove_monitor(&(rp->monitors),mon->ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon == NULL) { goto done; @@ -2087,7 +3526,7 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) ASSERT(is_internal_pid(lnk->pid)); rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); if (rp) { - ErtsLink *rlnk = erts_remove_link(&(rp->nlinks), psc->port); + ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), psc->port); if (rlnk) { int xres = erts_send_exit_signal(NULL, @@ -2123,11 +3562,13 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) * that is to kill a port till reason kill. Then the port is stopped. * */ -void -erts_do_exit_port(Port *p, Eterm from, Eterm reason) + +int +erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) { ErtsLink *lnk; Eterm rreason; + erts_aint32_t state, set_state_flags; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); @@ -2147,66 +3588,73 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) } #endif - if ((p->status & (ERTS_PORT_SFLGS_DEAD - | ERTS_PORT_SFLG_EXITING - | ERTS_PORT_SFLG_IMMORTAL)) - || ((reason == am_normal) && - ((from != p->connected) && (from != p->id)))) { - return; - } + state = erts_atomic32_read_nob(&p->state); + if (state & (ERTS_PORT_SFLGS_DEAD + | ERTS_PORT_SFLG_EXITING + | ERTS_PORT_SFLG_CLOSING)) + return 0; + + if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(p) && from != p->common.id) + return 0; + + set_state_flags = ERTS_PORT_SFLG_EXITING; + if (send_closed) + set_state_flags |= ERTS_PORT_SFLG_SEND_CLOSED; + + state = erts_atomic32_read_bor_mb(&p->state, set_state_flags); + state |= set_state_flags; if (IS_TRACED_FL(p, F_TRACE_PORTS)) { trace_port(p, am_closed, reason); } - erts_trace_check_exiting(p->id); + erts_trace_check_exiting(p->common.id); - /* - * Setting the port to not busy here, frees the list of pending - * processes and makes them runnable. - */ - set_busy_port((ErlDrvPort)internal_port_index(p->id), 0); + set_busy_port(ERTS_Port2ErlDrvPort(p), 0); - if (p->reg != NULL) - (void) erts_unregister_name(NULL, 0, p, p->reg->name); - - erts_port_status_bor_set(p, ERTS_PORT_SFLG_EXITING); + if (p->common.u.alive.reg != NULL) + (void) erts_unregister_name(NULL, 0, p, p->common.u.alive.reg->name); { - SweepContext sc = {p->id, rreason}; - lnk = p->nlinks; - p->nlinks = NULL; + SweepContext sc = {p->common.id, rreason}; + lnk = ERTS_P_LINKS(p); + ERTS_P_LINKS(p) = NULL; erts_sweep_links(lnk, &sweep_one_link, &sc); } DRV_MONITOR_LOCK_PDL(p); { - ErtsMonitor *moni = p->monitors; - p->monitors = NULL; + ErtsMonitor *moni = ERTS_P_MONITORS(p); + ERTS_P_MONITORS(p) = NULL; erts_sweep_monitors(moni, &sweep_one_monitor, NULL); } DRV_MONITOR_UNLOCK_PDL(p); - if ((p->status & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) { + if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) { erts_do_net_exits(p->dist_entry, rreason); erts_deref_dist_entry(p->dist_entry); - p->dist_entry = NULL; - erts_port_status_band_set(p, ~ERTS_PORT_SFLG_DISTRIBUTION); + p->dist_entry = NULL; + erts_atomic32_read_band_relb(&p->state, + ~ERTS_PORT_SFLG_DISTRIBUTION); } if ((reason != am_kill) && !is_port_ioq_empty(p)) { - erts_port_status_bandor_set(p, - ~ERTS_PORT_SFLG_EXITING, /* must turn it off */ - ERTS_PORT_SFLG_CLOSING); + /* must turn exiting flag off */ + erts_atomic32_read_bset_relb(&p->state, + (ERTS_PORT_SFLG_EXITING + | ERTS_PORT_SFLG_CLOSING), + ERTS_PORT_SFLG_CLOSING); flush_port(p); } else { terminate_port(p); } + + return 1; } /* About the states ERTS_PORT_SFLG_EXITING and ERTS_PORT_SFLG_CLOSING used above. ** -** ERTS_PORT_SFLG_EXITING is a recursion protection for erts_do_exit_port(). +** ERTS_PORT_SFLG_EXITING is a recursion protection for erts_deliver_port_exit(). ** It is unclear whether this state is necessary or not, it might be possible ** to merge it with ERTS_PORT_SFLG_CLOSING. ERTS_PORT_SFLG_EXITING only persists ** over a section of sequential (but highly recursive) code. @@ -2222,234 +3670,892 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) ** {PID, close} ** {PID, {command, io-list}} ** {PID, {connect, New_PID}} -** -** */ -void erts_port_command(Process *proc, - Eterm caller_id, - Port *port, - Eterm command) +ErtsPortOpResult +erts_port_command(Process *c_p, + int flags, + Port *port, + Eterm command, + Eterm *refp) { Eterm *tp; - Eterm pid; - if (!port) - return; + ASSERT(port); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - ERTS_SMP_CHK_NO_PROC_LOCKS; - ASSERT(!INVALID_PORT(port, port->id)); + flags |= ERTS_PORT_SIG_FLG_BANG_OP; + if (!erts_port_synchronous_ops) { + flags |= ERTS_PORT_SIG_FLG_ASYNC; + refp = NULL; + } if (is_tuple_arity(command, 2)) { + Eterm cntd; tp = tuple_val(command); - if ((pid = port->connected) == tp[1]) { - /* PID must be connected */ + cntd = tp[1]; + if (is_internal_pid(cntd)) { if (tp[2] == am_close) { - erts_port_status_bor_set(port, ERTS_PORT_SFLG_SEND_CLOSED); - erts_do_exit_port(port, pid, am_normal); - -#ifdef USE_VM_PROBES - if(DTRACE_ENABLED(port_command)) { - DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port) - DTRACE4(port_command, process_str, port_str, port->name, "close"); - } -#endif - goto done; + flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; + return erts_port_exit(c_p, flags, port, cntd, am_normal, refp); } else if (is_tuple_arity(tp[2], 2)) { tp = tuple_val(tp[2]); if (tp[1] == am_command) { - if (erts_write_to_port(caller_id, port, tp[2]) == 0) - goto done; - } else if ((tp[1] == am_connect) && is_internal_pid(tp[2])) { -#ifdef USE_VM_PROBES - if(DTRACE_ENABLED(port_command)) { - DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port) - DTRACE4(port_command, process_str, port_str, port->name, "connect"); - } -#endif - port->connected = tp[2]; - deliver_result(port->id, pid, am_connected); - goto done; + return erts_port_output(c_p, flags, port, cntd, tp[2], refp); + } + else if (tp[1] == am_connect) { + flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; + return erts_port_connect(c_p, flags, port, cntd, tp[2], refp); } } } } - { - ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - Process* rp = erts_pid2proc_opt(NULL, 0, - port->connected, rp_locks, - ERTS_P2P_FLG_SMP_INC_REFC); - if (rp) { - (void) erts_send_exit_signal(NULL, - port->id, - rp, - &rp_locks, - am_badsig, - NIL, - NULL, - 0); - erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); - } + /* badsig */ + flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; + return bad_port_signal(c_p, flags, port, c_p->common.id, refp, am_command); +} + +static ERTS_INLINE ErtsPortOpResult +call_driver_control(Eterm caller, + Port *prt, + unsigned int command, + char *bufp, + ErlDrvSizeT size, + char **resp_bufp, + ErlDrvSizeT *from_size) +{ + ErlDrvSSizeT cres; + if (!prt->drv_ptr->control) + return ERTS_PORT_OP_BADARG; + + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(port_control) || DTRACE_ENABLED(driver_control)) { + DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt); + DTRACE4(port_control, process_str, port_str, prt->name, command); + DTRACE5(driver_control, process_str, port_str, prt->name, + command, size); } - done: - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); +#endif + + prt->caller = caller; + cres = prt->drv_ptr->control((ErlDrvData) prt->drv_data, + command, + bufp, + size, + resp_bufp, + *from_size); + prt->caller = NIL; + + if (cres < 0) + return ERTS_PORT_OP_BADARG; + + *from_size = (ErlDrvSizeT) cres; + + return ERTS_PORT_OP_DONE; } -/* - * Control a port synchronously. - * Returns either a list or a binary. - */ -Eterm -erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) -{ - byte* to_port = NULL; /* Buffer to write to port. */ - /* Initialization is for shutting up - warning about use before set. */ - Uint to_len = 0; /* Length of buffer. */ - int must_free = 0; /* True if the buffer should be freed. */ - char port_result[ERL_ONHEAP_BIN_LIMIT]; /* Default buffer for result from port. */ - char* port_resp; /* Pointer to result buffer. */ - ErlDrvSSizeT n; - ErlDrvSSizeT (*control) - (ErlDrvData, unsigned, char*, ErlDrvSizeT, char**, ErlDrvSizeT); - int fpe_was_unmasked; +static void +cleanup_scheduled_control(Binary *binp, char *bufp) +{ + if (binp) { + if (erts_refc_dectest(&binp->refc, 0) == 0) + erts_bin_free(binp); + } + else { + if (bufp) + erts_free(ERTS_ALC_T_DRV_CTRL_DATA, bufp); + } +} - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if ((control = prt->drv_ptr->control) == NULL) { - return THE_NON_VALUE; +static ERTS_INLINE Uint +port_control_result_size(int control_flags, + char *resp_bufp, + ErlDrvSizeT *resp_size, + char *pre_alloc_buf) +{ + if (!resp_bufp) + return (Uint) 0; + + if (control_flags & PORT_CONTROL_FLAG_BINARY) { + if (resp_bufp != pre_alloc_buf) { + ErlDrvBinary *dbin = (ErlDrvBinary *) resp_bufp; + *resp_size = dbin->orig_size; + if (*resp_size > ERL_ONHEAP_BIN_LIMIT) + return PROC_BIN_SIZE; + } + ASSERT(*resp_size <= ERL_ONHEAP_BIN_LIMIT); + return (Uint) heap_bin_size((*resp_size)); } - /* - * Convert the iolist to a buffer, pointed to by to_port, - * and with its length in to_len. - */ - if (is_binary(iolist) && binary_bitoffset(iolist) == 0) { + return (Uint) 2*(*resp_size); +} + +static ERTS_INLINE Eterm +write_port_control_result(int control_flags, + char *resp_bufp, + ErlDrvSizeT resp_size, + char *pre_alloc_buf, + Eterm **hpp, + ErlHeapFragment *bp, + ErlOffHeap *ohp) +{ + Eterm res; + if (!resp_bufp) + return NIL; + if (control_flags & PORT_CONTROL_FLAG_BINARY) { + /* Binary result */ + ErlDrvBinary *dbin; + ErlHeapBin *hbin; + + if (resp_bufp == pre_alloc_buf) + dbin = NULL; + else { + dbin = (ErlDrvBinary *) resp_bufp; + if (dbin->orig_size > ERL_ONHEAP_BIN_LIMIT) { + ProcBin* pb = (ProcBin *) *hpp; + *hpp += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = dbin->orig_size; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header *) pb; + pb->val = ErlDrvBinary2Binary(dbin); + pb->bytes = (byte*) dbin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(ohp, dbin->orig_size / sizeof(Eterm)); + return make_binary(pb); + } + resp_bufp = dbin->orig_bytes; + resp_size = dbin->orig_size; + } + + hbin = (ErlHeapBin *) *hpp; + *hpp += heap_bin_size(resp_size); + ASSERT(resp_size <= ERL_ONHEAP_BIN_LIMIT); + hbin->thing_word = header_heap_bin(resp_size); + hbin->size = resp_size; + sys_memcpy(hbin->data, resp_bufp, resp_size); + if (dbin) + driver_free_binary(dbin); + return make_binary(hbin); + } + + /* List result */ + res = buf_to_intlist(hpp, resp_bufp, resp_size, NIL); + if (resp_bufp != pre_alloc_buf) + driver_free(resp_bufp); + return res; +} + +static int +port_sig_control(Port *prt, + erts_aint32_t state, + int op, + ErtsProc2PortSigData *sigdp) +{ + ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); + + if (op == ERTS_PROC2PORT_SIG_EXEC) { + char resp_buf[ERL_ONHEAP_BIN_LIMIT]; + ErlDrvSizeT resp_size = sizeof(resp_buf); + char *resp_bufp = &resp_buf[0]; + ErtsPortOpResult res; + + res = call_driver_control(sigdp->caller, + prt, + sigdp->u.control.command, + sigdp->u.control.bufp, + sigdp->u.control.size, + &resp_bufp, + &resp_size); + + if (res == ERTS_PORT_OP_DONE) { + Eterm msg; + Eterm *hp, *hp_start; + ErlHeapFragment *bp; + ErlOffHeap *ohp; + Process *rp; + ErtsProcLocks rp_locks = 0; + Uint hsz; + int control_flags; + + rp = erts_proc_lookup_raw(sigdp->caller); + if (!rp) + goto done; + + control_flags = prt->control_flags; + + hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; + hsz += port_control_result_size(control_flags, + resp_bufp, + &resp_size, + &resp_buf[0]); + + hp_start = hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + rp, + &rp_locks); + + msg = write_port_control_result(control_flags, + resp_bufp, + resp_size, + &resp_buf[0], + &hp, + bp, + ohp); + + queue_port_sched_op_reply(rp, + &rp_locks, + hp_start, + hp, + hsz, + bp, + sigdp->ref, + msg); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + goto done; + } + } + + /* failure */ + + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + +done: + + cleanup_scheduled_control(sigdp->u.control.binp, + sigdp->u.control.bufp); + + return ERTS_PORT_REDS_CONTROL; +} + + +ErtsPortOpResult +erts_port_control(Process* c_p, + Port *prt, + unsigned int command, + Eterm data, + Eterm *retvalp) +{ + ErtsPortOpResult res; + char *bufp = NULL; + ErlDrvSizeT size = 0; + int try_call; + int tmp_alloced = 0; + erts_aint32_t sched_flags; + Binary *binp; + int copy; + ErtsProc2PortSigData *sigdp; + + sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + if (sched_flags & ERTS_PTS_FLG_EXIT) + return ERTS_PORT_OP_BADARG; + + try_call = !(sched_flags & ERTS_PTS_FLGS_FORCE_SCHEDULE_OP); + + if (is_binary(data) && binary_bitoffset(data) == 0) { + byte *bytep; ERTS_DECLARE_DUMMY(Uint bitoffs); ERTS_DECLARE_DUMMY(Uint bitsize); - ERTS_GET_BINARY_BYTES(iolist, to_port, bitoffs, bitsize); - to_len = binary_size(iolist); + ERTS_GET_BINARY_BYTES(data, bytep, bitoffs, bitsize); + bufp = (char *) bytep; + size = binary_size(data); } else { int r; - /* Try with an 8KB buffer first (will often be enough I guess). */ - to_len = 8*1024; - to_port = erts_alloc(ERTS_ALC_T_TMP, to_len); - must_free = 1; + if (!try_call) { + if (erts_iolist_size(data, &size)) + return ERTS_PORT_OP_BADARG; + bufp = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, size); + r = erts_iolist_to_buf(data, bufp, size); + ASSERT(r == 0); + } + else { + /* Try with an 8KB buffer first (will often be enough I guess). */ + size = 8*1024; + bufp = erts_alloc(ERTS_ALC_T_TMP, size); + tmp_alloced = 1; + + r = erts_iolist_to_buf(data, bufp, size); + if (ERTS_IOLIST_TO_BUF_SUCCEEDED(r)) { + size -= r; + } else { + if (r == ERTS_IOLIST_TO_BUF_TYPE_ERROR) { /* Type error */ + erts_free(ERTS_ALC_T_TMP, bufp); + return ERTS_PORT_OP_BADARG; + } + else { + ASSERT(r == ERTS_IOLIST_TO_BUF_OVERFLOW); /* Overflow */ + erts_free(ERTS_ALC_T_TMP, bufp); + if (erts_iolist_size(data, &size)) + return ERTS_PORT_OP_BADARG; /* Type error */ + } + bufp = erts_alloc(ERTS_ALC_T_TMP, size); + r = erts_iolist_to_buf(data, bufp, size); + ASSERT(r == 0); + } + } + } - /* - * In versions before R10B, we used to reserve random - * amounts of extra memory. From R10B, we allocate the - * exact amount. - */ - r = io_list_to_buf(iolist, (char*) to_port, to_len); - if (r >= 0) { - to_len -= r; - } else if (r == -2) { /* Type error */ - erts_free(ERTS_ALC_T_TMP, (void *) to_port); - return THE_NON_VALUE; - } else { - ASSERT(r == -1); /* Overflow */ - erts_free(ERTS_ALC_T_TMP, (void *) to_port); - if (erts_iolist_size(iolist, &to_len)) { /* Type error */ - return THE_NON_VALUE; + if (try_call) { + char resp_buf[ERL_ONHEAP_BIN_LIMIT]; + char* resp_bufp = &resp_buf[0]; + ErlDrvSizeT resp_size = sizeof(resp_buf); + ErtsTryImmDrvCallResult try_call_res; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( + c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + 0, + 0, + am_control); + + try_call_res = try_imm_drv_call(&try_call_state); + switch (try_call_res) { + case ERTS_TRY_IMM_DRV_CALL_OK: { + Eterm *hp; + Uint hsz; + int control_flags; + + res = call_driver_control(c_p->common.id, + prt, + command, + bufp, + size, + &resp_bufp, + &resp_size); + finalize_imm_drv_call(&try_call_state); + if (tmp_alloced) + erts_free(ERTS_ALC_T_TMP, bufp); + if (res == ERTS_PORT_OP_BADARG) { + return ERTS_PORT_OP_BADARG; } - must_free = 1; - to_port = erts_alloc(ERTS_ALC_T_TMP, to_len); - r = io_list_to_buf(iolist, (char*) to_port, to_len); - ASSERT(r == 0); + + control_flags = prt->control_flags; + + hsz = port_control_result_size(control_flags, + resp_bufp, + &resp_size, + &resp_buf[0]); + hp = HAlloc(c_p, hsz); + *retvalp = write_port_control_result(control_flags, + resp_bufp, + resp_size, + &resp_buf[0], + &hp, + NULL, + &c_p->off_heap); + BUMP_REDS(c_p, ERTS_PORT_REDS_CONTROL); + return ERTS_PORT_OP_DONE; + } + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + if (tmp_alloced) + erts_free(ERTS_ALC_T_TMP, bufp); + return ERTS_PORT_OP_BADARG; + default: + /* Schedule control() call instead... */ + break; } } - prt->caller = p->id; /* Internal pid */ + /* Convert data into something that can be scheduled */ - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - ERTS_SMP_CHK_NO_PROC_LOCKS; + copy = tmp_alloced; + + binp = NULL; + + if (is_binary(data) && binary_bitoffset(data) == 0) { + Eterm *ebinp = binary_val_rel(data, NULL); + ASSERT(!tmp_alloced); + if (*ebinp == HEADER_SUB_BIN) + ebinp = binary_val_rel(((ErlSubBin *) ebinp)->orig, NULL); + if (*ebinp != HEADER_PROC_BIN) + copy = 1; + else { + binp = ((ProcBin *) ebinp)->val; + ASSERT(bufp < bufp + size); + ASSERT(binp->orig_bytes <= bufp + && bufp + size <= binp->orig_bytes + binp->orig_size); + erts_refc_inc(&binp->refc, 1); + } + } + + if (copy) { + char *old_bufp = bufp; + bufp = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, size); + sys_memcpy(bufp, old_bufp, size); + if (tmp_alloced) + erts_free(ERTS_ALC_T_TMP, old_bufp); + } + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_CONTROL; + sigdp->u.control.binp = binp; + sigdp->u.control.command = command; + sigdp->u.control.bufp = bufp; + sigdp->u.control.size = size; + + res = erts_schedule_proc2port_signal(c_p, + prt, + c_p->common.id, + retvalp, + sigdp, + 0, + NULL, + port_sig_control); + if (res != ERTS_PORT_OP_SCHEDULED) { + cleanup_scheduled_control(binp, bufp); + return ERTS_PORT_OP_BADARG; + } + return res; +} + +static ERTS_INLINE ErtsPortOpResult +call_driver_call(Eterm caller, + Port *prt, + unsigned int command, + char *bufp, + ErlDrvSizeT size, + char **resp_bufp, + ErlDrvSizeT *from_size, + unsigned *ret_flagsp) +{ + ErlDrvSSizeT cres; + + if (!prt->drv_ptr->call) + return ERTS_PORT_OP_BADARG; #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(port_control) || DTRACE_ENABLED(driver_control)) { - DTRACE_FORMAT_COMMON_PROC_AND_PORT(p, prt); - DTRACE4(port_control, process_str, port_str, prt->name, command); - DTRACE5(driver_control, process_str, port_str, prt->name, - command, to_len); + if (DTRACE_ENABLED(driver_call)) { + DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); + + dtrace_pid_str(caller, process_str); + dtrace_port_str(prt, port_str); + DTRACE5(driver_call, process_str, port_str, prt->name, command, size); } #endif - /* - * Call the port's control routine. - */ + prt->caller = caller; + cres = prt->drv_ptr->call((ErlDrvData) prt->drv_data, + command, + bufp, + size, + resp_bufp, + *from_size, + ret_flagsp); + prt->caller = NIL; - port_resp = port_result; - fpe_was_unmasked = erts_block_fpe(); - n = control((ErlDrvData)prt->drv_data, command, (char*)to_port, to_len, - &port_resp, sizeof(port_result)); - erts_unblock_fpe(fpe_was_unmasked); - if (must_free) { - erts_free(ERTS_ALC_T_TMP, (void *) to_port); + if (cres <= 0 + || ((byte) (*resp_bufp)[0]) != VERSION_MAGIC) + return ERTS_PORT_OP_BADARG; + + *from_size = (ErlDrvSizeT) cres; + + return ERTS_PORT_OP_DONE; +} + + +static +void cleanup_scheduled_call(char *bufp) +{ + if (bufp) + erts_free(ERTS_ALC_T_DRV_CALL_DATA, bufp); +} + +static int +port_sig_call(Port *prt, + erts_aint32_t state, + int op, + ErtsProc2PortSigData *sigdp) +{ + char resp_buf[256]; + ErlDrvSizeT resp_size = sizeof(resp_buf); + char *resp_bufp = &resp_buf[0]; + unsigned ret_flags = 0U; + + + ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); + + if (op == ERTS_PROC2PORT_SIG_EXEC) { + ErtsPortOpResult res; + + res = call_driver_call(sigdp->caller, + prt, + sigdp->u.call.command, + sigdp->u.call.bufp, + sigdp->u.call.size, + &resp_bufp, + &resp_size, + &ret_flags); + + if (res == ERTS_PORT_OP_DONE) { + Eterm msg; + Eterm *hp; + ErlHeapFragment *bp; + ErlOffHeap *ohp; + Process *rp; + ErtsProcLocks rp_locks = 0; + Sint hsz; + + rp = erts_proc_lookup_raw(sigdp->caller); + if (!rp) + goto done; + + hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size); + if (hsz >= 0) { + Eterm *hp_start; + byte *endp; + + hsz += 3; /* ok tuple */ + hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; + + hp_start = hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + rp, + &rp_locks); + endp = (byte *) resp_bufp; + msg = erts_decode_ext(&hp, ohp, &endp); + if (is_value(msg)) { + msg = TUPLE2(hp, am_ok, msg); + hp += 3; + + queue_port_sched_op_reply(rp, + &rp_locks, + hp_start, + hp, + hsz, + bp, + sigdp->ref, + msg); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + goto done; + } + if (bp) + free_message_buffer(bp); + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } + } } - prt->caller = NIL; -#ifdef ERTS_SMP - if (prt->xports) - erts_smp_xports_unlock(prt); - ASSERT(!prt->xports); -#endif - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - /* - * Handle the result. - */ + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + +done: + + if (resp_bufp != &resp_buf[0] && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) + driver_free(resp_bufp); + + cleanup_scheduled_call(sigdp->u.call.bufp); + + return ERTS_PORT_REDS_CALL; +} + - if (n < 0) { - return THE_NON_VALUE; +ErtsPortOpResult +erts_port_call(Process* c_p, + Port *prt, + unsigned int command, + Eterm data, + Eterm *retvalp) +{ + ErtsPortOpResult res; + char input_buf[256]; + char *bufp; + byte *endp; + ErlDrvSizeT size; + int try_call; + erts_aint32_t sched_flags; + ErtsProc2PortSigData *sigdp; + + sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + if (sched_flags & ERTS_PTS_FLG_EXIT) { + return ERTS_PORT_OP_BADARG; } - if ((prt->control_flags & PORT_CONTROL_FLAG_BINARY) == 0) { /* List result */ - Eterm ret; - Eterm* hp = HAlloc(p, 2*n); - ret = buf_to_intlist(&hp, port_resp, n, NIL); - if (port_resp != port_result) { - driver_free(port_resp); + try_call = !(sched_flags & ERTS_PTS_FLGS_FORCE_SCHEDULE_OP); + + size = erts_encode_ext_size(data); + + if (!try_call) + bufp = erts_alloc(ERTS_ALC_T_DRV_CALL_DATA, size); + else if (size <= sizeof(input_buf)) + bufp = &input_buf[0]; + else + bufp = erts_alloc(ERTS_ALC_T_TMP, size); + + endp = (byte *) bufp; + erts_encode_ext(data, &endp); + + if (endp - (byte *) bufp > size) + ERTS_INTERNAL_ERROR("erts_internal:port_call() - Buffer overflow"); + + size = endp - (byte *) bufp; + + if (try_call) { + char resp_buf[255]; + char* resp_bufp = &resp_buf[0]; + ErlDrvSizeT resp_size = sizeof(resp_buf); + ErtsTryImmDrvCallResult try_call_res; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( + c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + 0, + 0, + am_call); + + try_call_res = try_imm_drv_call(&try_call_state); + switch (try_call_res) { + case ERTS_TRY_IMM_DRV_CALL_OK: { + Eterm *hp, *hp_end; + Sint hsz; + unsigned ret_flags = 0U; + Eterm term; + + res = call_driver_call(c_p->common.id, + prt, + command, + bufp, + size, + &resp_bufp, + &resp_size, + &ret_flags); + + finalize_imm_drv_call(&try_call_state); + if (bufp != &input_buf[0]) + erts_free(ERTS_ALC_T_TMP, bufp); + if (res == ERTS_PORT_OP_BADARG) + return ERTS_PORT_OP_BADARG; + hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size); + if (hsz < 0) + return ERTS_PORT_OP_BADARG; + hsz += 3; + hp = HAlloc(c_p, hsz); + hp_end = hp + hsz; + endp = (byte *) resp_bufp; + term = erts_decode_ext(&hp, &MSO(c_p), &endp); + if (term == THE_NON_VALUE) + return ERTS_PORT_OP_BADARG; + *retvalp = TUPLE2(hp, am_ok, term); + hp += 3; + HRelease(c_p, hp_end, hp); + if (resp_bufp != &resp_buf[0] + && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) + driver_free(resp_bufp); + BUMP_REDS(c_p, ERTS_PORT_REDS_CALL); + return ERTS_PORT_OP_DONE; } - return ret; - } - else if (port_resp == NULL) { - return NIL; - } - else { /* Binary result */ - ErlDrvBinary *dbin; - ErlHeapBin *hbin; - if (port_resp != port_result) { - dbin = (ErlDrvBinary *) port_resp; - if (dbin->orig_size > ERL_ONHEAP_BIN_LIMIT) { - ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = dbin->orig_size; - pb->next = MSO(p).first; - MSO(p).first = (struct erl_off_heap_header*)pb; - pb->val = ErlDrvBinary2Binary(dbin); - pb->bytes = (byte*) dbin->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm)); - return make_binary(pb); - } - port_resp = dbin->orig_bytes; - n = dbin->orig_size; - } else { - dbin = NULL; + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + if (bufp != &input_buf[0]) + erts_free(ERTS_ALC_T_TMP, bufp); + return ERTS_PORT_OP_BADARG; + default: + /* Schedule call() call instead... */ + break; } - hbin = (ErlHeapBin*) HAlloc(p, heap_bin_size(n)); - ASSERT(n <= ERL_ONHEAP_BIN_LIMIT); - hbin->thing_word = header_heap_bin(n); - hbin->size = n; - sys_memcpy(hbin->data, port_resp, n); - if (dbin != NULL) { - driver_free_binary(dbin); + } + + /* Convert data into something that can be scheduled */ + + if (bufp == &input_buf[0] || try_call) { + char *new_bufp = erts_alloc(ERTS_ALC_T_DRV_CALL_DATA, size); + sys_memcpy(new_bufp, bufp, size); + if (bufp != &input_buf[0]) + erts_free(ERTS_ALC_T_TMP, bufp); + bufp = new_bufp; + } + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_CALL; + sigdp->u.call.command = command; + sigdp->u.call.bufp = bufp; + sigdp->u.call.size = size; + + res = erts_schedule_proc2port_signal(c_p, + prt, + c_p->common.id, + retvalp, + sigdp, + 0, + NULL, + port_sig_call); + if (res != ERTS_PORT_OP_SCHEDULED) { + cleanup_scheduled_call(bufp); + return ERTS_PORT_OP_BADARG; + } + return res; +} + +static Eterm +make_port_info_term(Eterm **hpp_start, + Eterm **hpp, + Uint *hszp, + ErlHeapFragment **bpp, + Port *prt, + Eterm item) +{ + ErlOffHeap *ohp; + + if (is_value(item)) { + if (erts_bld_port_info(NULL, NULL, hszp, prt, item) == am_false) + return THE_NON_VALUE; + if (*hszp) { + *bpp = new_message_buffer(*hszp); + *hpp_start = *hpp = (*bpp)->mem; + ohp = &(*bpp)->off_heap; } - return make_binary(hbin); + else { + *bpp = NULL; + *hpp_start = *hpp = NULL; + ohp = NULL; + } + return erts_bld_port_info(hpp, ohp, NULL, prt, item); } + else { + int i; + int len; + int start; + static Eterm item[] = ERTS_PORT_INFO_1_ITEMS; + static Eterm value[sizeof(item)/sizeof(item[0])]; + + start = 0; + len = sizeof(item)/sizeof(item[0]); + + for (i = start; i < sizeof(item)/sizeof(item[0]); i++) { + ASSERT(is_atom(item[i])); + value[i] = erts_bld_port_info(NULL, NULL, hszp, prt, item[i]); + } + + if (value[0] == am_undefined) { + start++; + len--; + } + + erts_bld_list(NULL, hszp, len, &value[start]); + + *bpp = new_message_buffer(*hszp); + *hpp_start = *hpp = (*bpp)->mem; + ohp = &(*bpp)->off_heap; + + for (i = start; i < sizeof(item)/sizeof(item[0]); i++) + value[i] = erts_bld_port_info(hpp, ohp, NULL, prt, item[i]); + + return erts_bld_list(hpp, NULL, len, &value[start]); + } +} + +static int +port_sig_info(Port *prt, + erts_aint32_t state, + int op, + ErtsProc2PortSigData *sigdp) +{ + ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); + if (op != ERTS_PROC2PORT_SIG_EXEC) + port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined); + else { + Eterm *hp, *hp_start; + Uint hsz; + ErlHeapFragment *bp; + Eterm value; + Process *rp; + ErtsProcLocks rp_locks = 0; + + rp = erts_proc_lookup_raw(sigdp->caller); + if (!rp) + return ERTS_PORT_REDS_INFO; + + hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; + value = make_port_info_term(&hp_start, + &hp, + &hsz, + &bp, + prt, + sigdp->u.info.item); + if (is_value(value)) { + queue_port_sched_op_reply(rp, + &rp_locks, + hp_start, + hp, + hsz, + bp, + sigdp->ref, + value); + } + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } + return ERTS_PORT_REDS_INFO; +} + +ErtsPortOpResult +erts_port_info(Process* c_p, + Port *prt, + Eterm item, + Eterm *retvalp) +{ + ErtsProc2PortSigData *sigdp; + ErtsTryImmDrvCallResult try_call_res; + ErtsTryImmDrvCallState try_call_state + = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( + c_p, + prt, + ERTS_PORT_SFLGS_INVALID_LOOKUP, + 0, + 0, + am_info); + + try_call_res = try_imm_drv_call(&try_call_state); + switch (try_call_res) { + case ERTS_TRY_IMM_DRV_CALL_OK: { + Eterm *hp, *hp_start; + ErlHeapFragment *bp; + Uint hsz = 0; + Eterm value = make_port_info_term(&hp_start, &hp, &hsz, &bp, prt, item); + finalize_imm_drv_call(&try_call_state); + if (is_non_value(value)) + return ERTS_PORT_OP_BADARG; + else if (is_immed(value)) + *retvalp = value; + else { + Uint used_h_size = hp - hp_start; + hp = HAlloc(c_p, used_h_size); + *retvalp = copy_struct(value, used_h_size, &hp, &MSO(c_p)); + free_message_buffer(bp); + } + BUMP_REDS(c_p, ERTS_PORT_REDS_INFO); + return ERTS_PORT_OP_DONE; + } + case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: + return ERTS_PORT_OP_DROPPED; + case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: + case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: + /* Schedule call instead... */ + break; + } + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_INFO; + sigdp->u.info.item = item; + + return erts_schedule_proc2port_signal(c_p, + prt, + c_p->common.id, + retvalp, + sigdp, + 0, + NULL, + port_sig_info); } typedef struct { @@ -2470,39 +4576,39 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) } void -print_port_info(int to, void *arg, int i) +print_port_info(Port *p, int to, void *arg) { - Port* p = &erts_port[i]; + erts_aint32_t state = erts_atomic32_read_nob(&p->state); - if (p->status & ERTS_PORT_SFLGS_DEAD) + if (state & ERTS_PORT_SFLGS_DEAD) return; - erts_print(to, arg, "=port:%T\n", p->id); - erts_print(to, arg, "Slot: %d\n", i); - if (p->status & ERTS_PORT_SFLG_CONNECTED) { - erts_print(to, arg, "Connected: %T", p->connected); + erts_print(to, arg, "=port:%T\n", p->common.id); + erts_print(to, arg, "Slot: %d\n", internal_port_index(p->common.id)); + if (state & ERTS_PORT_SFLG_CONNECTED) { + erts_print(to, arg, "Connected: %T", ERTS_PORT_GET_CONNECTED(p)); erts_print(to, arg, "\n"); } - if (p->nlinks != NULL) { + if (ERTS_P_LINKS(p)) { prt_one_lnk_data prtd; prtd.to = to; prtd.arg = arg; erts_print(to, arg, "Links: "); - erts_doforall_links(p->nlinks, &prt_one_lnk, &prtd); + erts_doforall_links(ERTS_P_LINKS(p), &prt_one_lnk, &prtd); erts_print(to, arg, "\n"); } - if (p->monitors != NULL) { + if (ERTS_P_MONITORS(p)) { prt_one_lnk_data prtd; prtd.to = to; prtd.arg = arg; erts_print(to, arg, "Monitors: "); - erts_doforall_monitors(p->monitors, &prt_one_monitor, &prtd); + erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd); erts_print(to, arg, "\n"); } - if (p->reg != NULL) - erts_print(to, arg, "Registered as: %T\n", p->reg->name); + if (p->common.u.alive.reg != NULL) + erts_print(to, arg, "Registered as: %T\n", p->common.u.alive.reg->name); if (p->drv_ptr == &fd_driver) { erts_print(to, arg, "Port is UNIX fd not opened by emulator: %s\n", p->name); @@ -2516,109 +4622,143 @@ print_port_info(int to, void *arg, int i) } void -set_busy_port(ErlDrvPort port_num, int on) +set_busy_port(ErlDrvPort dprt, int on) { + Port *prt; + erts_aint32_t flags; + #ifdef USE_VM_PROBES DTRACE_CHARBUF(port_str, 16); #endif ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[port_num])); + prt = erts_drvport2port(dprt); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return; if (on) { - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_PORT_BUSY); + flags = erts_smp_atomic32_read_bor_acqb(&prt->sched.flags, + ERTS_PTS_FLG_BUSY_PORT); + if (flags & ERTS_PTS_FLG_BUSY_PORT) + return; /* Already busy */ + + if (flags & ERTS_PTS_FLG_HAVE_NS_TASKS) + erts_port_task_abort_nosuspend_tasks(prt); + #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_busy)) { erts_snprintf(port_str, sizeof(port_str), - "%T", erts_port[port_num].id); + "%T", prt->common.id); DTRACE1(port_busy, port_str); } #endif } else { - ErtsProcList* plp = erts_port[port_num].suspended; - erts_port_status_band_set(&erts_port[port_num], - ~ERTS_PORT_SFLG_PORT_BUSY); - erts_port[port_num].suspended = NULL; + flags = erts_smp_atomic32_read_band_acqb(&prt->sched.flags, + ~ERTS_PTS_FLG_BUSY_PORT); + if (!(flags & ERTS_PTS_FLG_BUSY_PORT)) + return; /* Already non-busy */ #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_not_busy)) { erts_snprintf(port_str, sizeof(port_str), - "%T", erts_port[port_num].id); + "%T", prt->common.id); DTRACE1(port_not_busy, port_str); } #endif - if (erts_port[port_num].dist_entry) { + if (prt->dist_entry) { /* * Processes suspended on distribution ports are * normally queued on the dist entry. */ - erts_dist_port_not_busy(&erts_port[port_num]); + erts_dist_port_not_busy(prt); } - /* - * Resume, in a round-robin fashion, all processes waiting on the port. - * - * This version submitted by Tony Rogvall. The earlier version used - * to resume the processes in order, which caused starvation of all but - * the first process. - */ + if (!(flags & ERTS_PTS_FLG_BUSY_PORT_Q)) + erts_port_resume_procs(prt); + } +} + +void +erts_port_resume_procs(Port *prt) +{ + /* + * Resume, in a round-robin fashion, all processes waiting on the port. + * + * This version submitted by Tony Rogvall. The earlier version used + * to resume the processes in order, which caused starvation of all but + * the first process. + */ + ErtsProcList *plp; + + erts_port_task_sched_lock(&prt->sched); + plp = prt->suspended; + prt->suspended = NULL; + erts_port_task_sched_unlock(&prt->sched); + + if (erts_proclist_fetch(&plp, NULL)) { - if (plp) { #ifdef USE_VM_PROBES - /* - * Hrm, for blocked dist ports, plp always seems to be NULL. - * That's not so fun. - * Well, another way to get the same info is using a D - * script to correlate an earlier process-port_blocked+pid - * event with a later process-scheduled event. That's - * subject to the multi-CPU races with how events are - * handled, but hey, that way works most of the time. - */ - if (DTRACE_ENABLED(process_port_unblocked)) { - DTRACE_CHARBUF(pid_str, 16); - ErtsProcList* plp2 = plp; - - erts_snprintf(port_str, sizeof(port_str), - "%T", erts_port[port_num]); - while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); - DTRACE2(process_port_unblocked, pid_str, port_str); - } - } -#endif - /* First proc should be resumed last */ - if (plp->next) { - erts_resume_processes(plp->next); - plp->next = NULL; + /* + * Hrm, for blocked dist ports, plp always seems to be NULL. + * That's not so fun. + * Well, another way to get the same info is using a D + * script to correlate an earlier process-port_blocked+pid + * event with a later process-scheduled event. That's + * subject to the multi-CPU races with how events are + * handled, but hey, that way works most of the time. + */ + if (DTRACE_ENABLED(process_port_unblocked)) { + DTRACE_CHARBUF(port_str, 16); + DTRACE_CHARBUF(pid_str, 16); + ErtsProcList* plp2 = plp; + + erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + while (plp2 != NULL) { + erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + DTRACE2(process_port_unblocked, pid_str, port_str); } - erts_resume_processes(plp); - } + } +#endif + + /* First proc should be resumed last */ + if (plp->next) { + plp->next->prev = NULL; + erts_resume_processes(plp->next); + plp->next = NULL; + } + erts_resume_processes(plp); } } void set_port_control_flags(ErlDrvPort port_num, int flags) { - - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[port_num])); - - erts_port[port_num].control_flags = flags; + Port *prt = erts_drvport2port(port_num); + if (prt != ERTS_INVALID_ERL_DRV_PORT) + prt->control_flags = flags; } -int get_port_flags(ErlDrvPort ix) { - Port* prt = erts_drvport2port(ix); +int get_port_flags(ErlDrvPort ix) +{ + int flags; + Port *prt; + erts_aint32_t state; + + prt = erts_drvport2port_state(ix, &state); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt == NULL) - return 0; + flags = 0; + if (state & ERTS_PORT_SFLG_BINARY_IO) + flags |= PORT_FLAG_BINARY; + if (state & ERTS_PORT_SFLG_LINEBUF_IO) + flags |= PORT_FLAG_LINE; - return (prt->status & ERTS_PORT_SFLG_BINARY_IO ? PORT_FLAG_BINARY : 0) - | (prt->status & ERTS_PORT_SFLG_LINEBUF_IO ? PORT_FLAG_LINE : 0); + return flags; } - void erts_raw_port_command(Port* p, byte* buf, Uint len) { int fpe_was_unmasked; @@ -2655,25 +4795,18 @@ int async_ready(Port *p, void* data) if (p) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); - ASSERT(!(p->status & ERTS_PORT_SFLGS_DEAD)); if (p->drv_ptr->ready_async != NULL) { #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_ready_async)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(p->connected, p) + DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p) DTRACE3(driver_ready_async, process_str, port_str, p->name); } #endif (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data); need_free = 0; -#ifdef ERTS_SMP - if (p->xports) - erts_smp_xports_unlock(p); - ASSERT(!p->xports); -#endif - } - if ((p->status & ERTS_PORT_SFLG_CLOSING) && is_port_ioq_empty(p)) { - terminate_port(p); + } + erts_port_driver_callback_epilogue(p, NULL); } return need_free; } @@ -2681,12 +4814,13 @@ int async_ready(Port *p, void* data) static void report_missing_drv_callback(Port *p, char *drv_type, char *callback) { - ErtsPortNames *pnp = erts_get_port_names(p->id); + ErtsPortNames *pnp = erts_get_port_names(p->common.id, + ERTS_Port2ErlDrvPort(p)); char *unknown = "<unknown>"; char *drv_name = pnp->driver_name ? pnp->driver_name : unknown; char *prt_name = pnp->name ? pnp->name : unknown; erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "%T: %s driver '%s' ", p->id, drv_type, drv_name); + erts_dsprintf(dsbufp, "%T: %s driver '%s' ", p->common.id, drv_type, drv_name); if (sys_strcmp(drv_name, prt_name) != 0) erts_dsprintf(dsbufp, "(%s) ", prt_name); erts_dsprintf(dsbufp, "does not implement the %s callback!\n", callback); @@ -2696,15 +4830,25 @@ report_missing_drv_callback(Port *p, char *drv_type, char *callback) void erts_stale_drv_select(Eterm port, + ErlDrvPort drv_port, ErlDrvEvent hndl, int mode, int deselect) { char *type; - ErlDrvPort drv_port = internal_port_index(port); - ErtsPortNames *pnp = erts_get_port_names(port); + ErtsPortNames *pnp; erts_dsprintf_buf_t *dsbufp; + if (drv_port == ERTS_INVALID_ERL_DRV_PORT) { + Port *prt = erts_port_lookup_raw(port); + if (prt) + drv_port = ERTS_Port2ErlDrvPort(prt); + else + drv_port = ERTS_INVALID_ERL_DRV_PORT; + } + + pnp = erts_get_port_names(port, drv_port); + switch (mode) { case ERL_DRV_READ | ERL_DRV_WRITE: type = "Input/Output"; @@ -2739,23 +4883,28 @@ erts_stale_drv_select(Eterm port, } ErtsPortNames * -erts_get_port_names(Eterm id) +erts_get_port_names(Eterm id, ErlDrvPort drv_port) { + Port *prt; ErtsPortNames *pnp; ASSERT(is_nil(id) || is_internal_port(id)); - - if (is_not_internal_port(id)) { + + prt = ERTS_ErlDrvPort2Port(drv_port); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + prt = erts_port_lookup_raw(id); + + if (!prt) { pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, sizeof(ErtsPortNames)); pnp->name = NULL; pnp->driver_name = NULL; } else { - Port* prt = &erts_port[internal_port_index(id)]; int do_realloc = 1; int len = -1; size_t pnp_len = sizeof(ErtsPortNames); #ifndef DEBUG pnp_len += 100; /* In most cases 100 characters will be enough... */ + ASSERT(prt->common.id == id); #endif pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len); do { @@ -2766,17 +4915,10 @@ erts_get_port_names(Eterm id) pnp_len = sizeof(ErtsPortNames) + len; pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len); } - erts_smp_port_state_lock(prt); - if (id != prt->id) { - len = nlen = 0; - name = driver_name = NULL; - } - else { - name = prt->name; - len = nlen = name ? sys_strlen(name) + 1 : 0; - driver_name = (prt->drv_ptr ? prt->drv_ptr->name : NULL); - len += driver_name ? sys_strlen(driver_name) + 1 : 0; - } + name = prt->name; + len = nlen = name ? sys_strlen(name) + 1 : 0; + driver_name = (prt->drv_ptr ? prt->drv_ptr->name : NULL); + len += driver_name ? sys_strlen(driver_name) + 1 : 0; if (len <= pnp_len - sizeof(ErtsPortNames)) { if (!name) pnp->name = NULL; @@ -2794,7 +4936,6 @@ erts_get_port_names(Eterm id) } do_realloc = 0; } - erts_smp_port_state_unlock(prt); } while (do_realloc); } return pnp; @@ -2819,11 +4960,9 @@ static void schedule_port_timeout(Port *p) * /Rickard */ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); - (void) erts_port_task_schedule(p->id, - &p->timeout_task, - ERTS_PORT_TASK_TIMEOUT, - (ErlDrvEvent) -1, - NULL); + erts_port_task_schedule(p->common.id, + &p->timeout_task, + ERTS_PORT_TASK_TIMEOUT); } ErlDrvTermData driver_mk_term_nil(void) @@ -2831,9 +4970,8 @@ ErlDrvTermData driver_mk_term_nil(void) return driver_term_nil; } -void driver_report_exit(int ix, int status) +void driver_report_exit(ErlDrvPort ix, int status) { - Port* prt = erts_drvport2port(ix); Eterm* hp; Eterm tuple; Process *rp; @@ -2841,13 +4979,21 @@ void driver_report_exit(int ix, int status) ErlHeapFragment *bp = NULL; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; + int scheduler = erts_get_scheduler_id() != 0; + Port* prt = erts_drvport2port(ix); + + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - pid = prt->connected; + pid = ERTS_PORT_GET_CONNECTED(prt); ASSERT(is_internal_pid(pid)); - rp = erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC); + + rp = (scheduler + ? erts_proc_lookup(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) return; @@ -2855,7 +5001,7 @@ void driver_report_exit(int ix, int status) tuple = TUPLE2(hp, am_exit_status, make_small(status)); hp += 3; - tuple = TUPLE2(hp, prt->id, tuple); + tuple = TUPLE2(hp, prt->common.id, tuple); erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined #ifdef USE_VM_PROBES @@ -2864,29 +5010,8 @@ void driver_report_exit(int ix, int status) ); erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); -} - - -static ERTS_INLINE int -deliver_term_check_port(ErlDrvPort drvport) -{ - int res; - int ix = (int) drvport; - if (ix < 0 || erts_max_ports <= ix) - res = -1; /* invalid */ - else { - Port* prt = &erts_port[ix]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_INVALID_LOOKUP)) - res = 1; /* ok */ - else if (prt->status & ERTS_PORT_SFLG_CLOSING) - res = 0; /* closing */ - else - res = -1; /* invalid (dead) */ - erts_smp_port_state_unlock(prt); - } - return res; + if (!scheduler) + erts_smp_proc_dec_refc(rp); } #define ERTS_B2T_STATES_DEF_STATES_SZ 5 @@ -2976,10 +5101,7 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp) */ static int -driver_deliver_term(ErlDrvPort port, - Eterm to, - ErlDrvTermData* data, - int len) +driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) { #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0) Uint need = 0; @@ -2995,6 +5117,7 @@ driver_deliver_term(ErlDrvPort port, ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; + int scheduler = 1; /* Silence erroneous warning... */ init_b2t_states(&b2t); @@ -3180,13 +5303,16 @@ driver_deliver_term(ErlDrvPort port, b2t.ix = 0; /* - * The term is OK. Go ahead and validate the port and process. + * The term is OK. Go ahead and validate the process. */ - res = deliver_term_check_port(port); - if (res <= 0) - goto done; - rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + /* + * Increase refc on proc if done from a non-scheduler thread. + */ + scheduler = erts_get_scheduler_id() != 0; + rp = (scheduler + ? erts_proc_lookup(to) + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); if (!rp) { res = 0; goto done; @@ -3438,7 +5564,8 @@ driver_deliver_term(ErlDrvPort port, if (rp) { if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + if (!scheduler) + erts_smp_proc_dec_refc(rp); } #endif cleanup_b2t_states(&b2t); @@ -3447,25 +5574,119 @@ driver_deliver_term(ErlDrvPort port, #undef ERTS_DDT_FAIL } +static ERTS_INLINE int +deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p) +{ +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); +#endif + erts_aint32_t state; + Port *prt = erts_port_lookup_raw((Eterm) port_id); + if (!prt) + return -1; + state = erts_atomic32_read_nob(&prt->state); + if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_CLOSING)) { + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + return -1; + else + return 0; + } + if (connected_p) { +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + ETHR_MEMBAR(ETHR_LoadLoad); +#endif + *connected_p = ERTS_PORT_GET_CONNECTED(prt); + } +#ifdef ERTS_SMP + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { + erts_thr_progress_unmanaged_continue(dhndl); + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); + } +#endif + ERTS_SMP_LC_ASSERT(dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED + ? erts_lc_is_port_locked(prt) + : !erts_lc_is_port_locked(prt)); + return 1; +} + +int erl_drv_output_term(ErlDrvTermData port_id, ErlDrvTermData* data, int len) +{ + /* May be called from arbitrary thread */ + Eterm connected; + int res = deliver_term_check_port(port_id, &connected); + if (res <= 0) + return res; + return driver_deliver_term(connected, data, len); +} +/* + * driver_output_term() is deprecated, and has been scheduled for + * removal in OTP-R17. It is replaced by erl_drv_output_term() + * above. + */ int -driver_output_term(ErlDrvPort ix, ErlDrvTermData* data, int len) +driver_output_term(ErlDrvPort drvport, ErlDrvTermData* data, int len) { - Port* prt = erts_drvport2port(ix); + erts_aint32_t state; + Port* prt; ERTS_SMP_CHK_NO_PROC_LOCKS; + /* NOTE! It *not* safe to access 'drvport' from unmanaged threads. */ + prt = erts_drvport2port_state(drvport, &state); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return -1; /* invalid (dead) */ + ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + if (state & ERTS_PORT_SFLG_CLOSING) + return 0; - if (prt == NULL) - return -1; - return driver_deliver_term(ix, prt->connected, data, len); + return driver_deliver_term(ERTS_PORT_GET_CONNECTED(prt), data, len); } +int erl_drv_send_term(ErlDrvTermData port_id, + ErlDrvTermData to, + ErlDrvTermData* data, + int len) +{ + /* May be called from arbitrary thread */ + int res = deliver_term_check_port(port_id, NULL); + if (res <= 0) + return res; + return driver_deliver_term(to, data, len); +} +/* + * driver_send_term() is deprecated, and has been scheduled for + * removal in OTP-R17. It is replaced by erl_drv_send_term() above. + */ int -driver_send_term(ErlDrvPort ix, ErlDrvTermData to, ErlDrvTermData* data, int len) +driver_send_term(ErlDrvPort drvport, + ErlDrvTermData to, + ErlDrvTermData* data, + int len) { - return driver_deliver_term(ix, to, data, len); + /* + * NOTE! It is *not* safe to access the 'drvport' parameter + * from unmanaged threads. Also note that it is impossible + * to make this access safe without using a less efficient + * internal data representation for ErlDrvPort. + */ + ERTS_SMP_CHK_NO_PROC_LOCKS; +#ifdef ERTS_SMP + if (erts_thr_progress_is_managed_thread()) +#endif + { + erts_aint32_t state; + Port* prt = erts_drvport2port_state(drvport, &state); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return -1; /* invalid (dead) */ + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + if (state & ERTS_PORT_SFLG_CLOSING) + return 0; + } + return driver_deliver_term(to, data, len); } @@ -3477,26 +5698,27 @@ driver_send_term(ErlDrvPort ix, ErlDrvTermData to, ErlDrvTermData* data, int len int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, ErlDrvBinary* bin, ErlDrvSizeT offs, ErlDrvSizeT len) { - Port* prt = erts_drvport2port(ix); + erts_aint32_t state; + Port* prt = erts_drvport2port_state(ix, &state); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt->status & ERTS_PORT_SFLG_CLOSING) + if (state & ERTS_PORT_SFLG_CLOSING) return 0; prt->bytes_in += (hlen + len); erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); - if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { + if (state & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, (byte*) hbuf, hlen, (byte*) (bin->orig_bytes+offs), len); } else - deliver_bin_message(prt, prt->connected, + deliver_bin_message(prt, ERTS_PORT_GET_CONNECTED(prt), hbuf, hlen, bin, offs, len); return 0; } @@ -3511,21 +5733,21 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, char* buf, ErlDrvSizeT len) { - Port* prt = erts_drvport2port(ix); + erts_aint32_t state; + Port* prt = erts_drvport2port_state(ix, &state); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - - if (prt->status & ERTS_PORT_SFLG_CLOSING) + if (state & ERTS_PORT_SFLG_CLOSING) return 0; prt->bytes_in += (hlen + len); erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); - if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { + if (state & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, prt->dist_entry, @@ -3537,10 +5759,12 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, (byte*) hbuf, hlen, (byte*) buf, len); } - else if(prt->status & ERTS_PORT_SFLG_LINEBUF_IO) - deliver_linebuf_message(prt, prt->connected, hbuf, hlen, buf, len); + else if (state & ERTS_PORT_SFLG_LINEBUF_IO) + deliver_linebuf_message(prt, state, ERTS_PORT_GET_CONNECTED(prt), + hbuf, hlen, buf, len); else - deliver_read_message(prt, prt->connected, hbuf, hlen, buf, len, 0); + deliver_read_message(prt, state, ERTS_PORT_GET_CONNECTED(prt), + hbuf, hlen, buf, len, 0); return 0; } @@ -3561,6 +5785,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, SysIOVec* iov; ErlDrvBinary** binv; Port* prt; + erts_aint32_t state; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -3569,17 +5794,13 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return driver_output2(ix, hbuf, hlen, NULL, 0); size = vec->size - skip; /* Size of remaining bytes in vector */ - ASSERT(hlen >= 0); /* debug only */ - if (hlen < 0) - hlen = 0; - - prt = erts_drvport2port(ix); - if (prt == NULL) + prt = erts_drvport2port_state(ix, &state); + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt->status & ERTS_PORT_SFLG_CLOSING) + if (state & ERTS_PORT_SFLG_CLOSING) return 0; /* size > 0 ! */ @@ -3595,7 +5816,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, binv++; n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -3604,7 +5825,8 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size)); - deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size); + deliver_vec_message(prt, ERTS_PORT_GET_CONNECTED(prt), hbuf, hlen, + binv, iov, n, size); return 0; } @@ -3715,8 +5937,7 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size) } -void driver_free_binary(dbin) -ErlDrvBinary* dbin; +void driver_free_binary(ErlDrvBinary* dbin) { Binary *bin; if (!dbin) { @@ -3812,6 +6033,7 @@ static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl) { ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) == 0); erts_mtx_destroy(&pdl->mtx); + erts_port_dec_refc(pdl->prt); erts_free(ERTS_ALC_T_PORT_DATA_LOCK, pdl); } @@ -3850,15 +6072,17 @@ driver_pdl_create(ErlDrvPort dp) { ErlDrvPDL pdl; Port *pp = erts_drvport2port(dp); - if (!pp || pp->port_data_lock) + if (pp == ERTS_INVALID_ERL_DRV_PORT || pp->port_data_lock) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, sizeof(struct erl_drv_port_data_lock)); erts_mtx_init(&pdl->mtx, "port_data_lock"); pdl_init_refc(pdl); + erts_port_inc_refc(pp); + pdl->prt = pp; pp->port_data_lock = pdl; #ifdef HARDDEBUG - erts_fprintf(stderr, "driver_pdl_create(%T) -> 0x%08X\r\n",pp->id,(unsigned) pdl); + erts_fprintf(stderr, "driver_pdl_create(%T) -> 0x%08X\r\n",pp->common.id,(unsigned) pdl); #endif return pdl; } @@ -4037,7 +6261,7 @@ int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -4102,7 +6326,7 @@ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -4161,7 +6385,7 @@ ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size) q->v_head++; } else { - q->v_head->iov_base += size; + q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size; q->v_head->iov_len -= size; size = 0; } @@ -4296,12 +6520,12 @@ static ERTS_INLINE void drv_cancel_timer(Port *prt) { #ifdef ERTS_SMP - erts_cancel_smp_ptimer(prt->ptimer); + erts_cancel_smp_ptimer(prt->common.u.alive.ptimer); #else - erts_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->common.u.alive.tm); #endif if (erts_port_task_is_scheduled(&prt->timeout_task)) - erts_port_task_abort(prt->id, &prt->timeout_task); + erts_port_task_abort(&prt->timeout_task); } int driver_set_timer(ErlDrvPort ix, unsigned long t) @@ -4310,19 +6534,19 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t) ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + if (prt->drv_ptr->timeout == NULL) return -1; drv_cancel_timer(prt); #ifdef ERTS_SMP - erts_create_smp_ptimer(&prt->ptimer, - prt->id, + erts_create_smp_ptimer(&prt->common.u.alive.ptimer, + prt->common.id, (ErlTimeoutProc) schedule_port_timeout, t); #else - erts_set_timer(&prt->tm, + erts_set_timer(&prt->common.u.alive.tm, (ErlTimeoutProc) schedule_port_timeout, NULL, prt, @@ -4334,7 +6558,7 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t) int driver_cancel_timer(ErlDrvPort ix) { Port* prt = erts_drvport2port(ix); - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); drv_cancel_timer(prt); @@ -4349,13 +6573,15 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t) ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SMP - *t = prt->ptimer ? erts_time_left(&prt->ptimer->timer.tm) : 0; + *t = (prt->common.u.alive.ptimer + ? erts_time_left(&prt->common.u.alive.ptimer->timer.tm) + : 0); #else - *t = erts_time_left(&prt->tm); + *t = erts_time_left(&prt->common.u.alive.tm); #endif return 0; } @@ -4406,8 +6632,8 @@ static int do_driver_monitor_process(Port *prt, } ref = erts_make_ref_in_buffer(buf); - erts_add_monitor(&(prt->monitors), MON_ORIGIN, ref, rp->id, NIL); - erts_add_monitor(&(rp->monitors), MON_TARGET, ref, prt->id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(prt), MON_ORIGIN, ref, rp->common.id, NIL); + erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, prt->common.id, NIL); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); ref_to_driver_monitor(ref,monitor); @@ -4417,34 +6643,19 @@ static int do_driver_monitor_process(Port *prt, /* * This can be called from a non scheduler thread iff a port_data_lock exists */ -int driver_monitor_process(ErlDrvPort port, +int driver_monitor_process(ErlDrvPort drvport, ErlDrvTermData process, ErlDrvMonitor *monitor) { Port *prt; int ret; - Uint32 status; +#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) ErtsSchedulerData *sched = erts_get_scheduler_data(); - int ix = (int) port; - if (ix < 0 || erts_max_ports <= ix) { - return -1; - } - prt = &erts_port[ix]; - - DRV_MONITOR_LOCK_PDL(prt); - - if (sched) { - status = erts_port[ix].status; - } else { - erts_smp_port_state_lock(prt); - status = erts_port[ix].status; - erts_smp_port_state_unlock(prt); - } +#endif - if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { - DRV_MONITOR_UNLOCK_PDL(prt); + prt = DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(drvport); + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - } /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ @@ -4479,7 +6690,7 @@ static int do_driver_demonitor_process(Port *prt, Eterm *buf, memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE); ref = make_internal_ref(buf); - mon = erts_lookup_monitor(prt->monitors, ref); + mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { return 1; } @@ -4491,13 +6702,13 @@ static int do_driver_demonitor_process(Port *prt, Eterm *buf, to, ERTS_PROC_LOCK_LINK, ERTS_P2P_FLG_ALLOW_OTHER_X); - mon = erts_remove_monitor(&(prt->monitors), ref); + mon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref); if (mon) { erts_destroy_monitor(mon); } if (rp) { ErtsMonitor *rmon; - rmon = erts_remove_monitor(&(rp->monitors), ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon != NULL) { erts_destroy_monitor(rmon); @@ -4506,33 +6717,18 @@ static int do_driver_demonitor_process(Port *prt, Eterm *buf, return 0; } -int driver_demonitor_process(ErlDrvPort port, +int driver_demonitor_process(ErlDrvPort drvport, const ErlDrvMonitor *monitor) { Port *prt; int ret; - Uint32 status; +#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) ErtsSchedulerData *sched = erts_get_scheduler_data(); - int ix = (int) port; - if (ix < 0 || erts_max_ports <= ix) { - return -1; - } - prt = &erts_port[ix]; - - DRV_MONITOR_LOCK_PDL(prt); - - if (sched) { - status = erts_port[ix].status; - } else { - erts_smp_port_state_lock(prt); - status = erts_port[ix].status; - erts_smp_port_state_unlock(prt); - } +#endif - if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { - DRV_MONITOR_UNLOCK_PDL(prt); + prt = DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(drvport); + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - } /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ @@ -4565,7 +6761,7 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf, memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE); ref = make_internal_ref(buf); - mon = erts_lookup_monitor(prt->monitors, ref); + mon = erts_lookup_monitor(ERTS_P_MONITORS(prt), ref); if (mon == NULL) { return driver_term_nil; } @@ -4576,33 +6772,18 @@ static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf, } -ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, +ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, const ErlDrvMonitor *monitor) { Port *prt; ErlDrvTermData ret; - Uint32 status; +#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) ErtsSchedulerData *sched = erts_get_scheduler_data(); - int ix = (int) port; - if (ix < 0 || erts_max_ports <= ix) { - return driver_term_nil; - } - prt = &erts_port[ix]; - - DRV_MONITOR_LOCK_PDL(prt); - - if (sched) { - status = erts_port[ix].status; - } else { - erts_smp_port_state_lock(prt); - status = erts_port[ix].status; - erts_smp_port_state_unlock(prt); - } +#endif - if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { - DRV_MONITOR_UNLOCK_PDL(prt); + prt = DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(drvport); + if (prt == ERTS_INVALID_ERL_DRV_PORT) return driver_term_nil; - } /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ @@ -4644,7 +6825,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(prt->drv_ptr != NULL); DRV_MONITOR_LOCK_PDL(prt); - if (erts_lookup_monitor(prt->monitors,ref) == NULL) { + if (erts_lookup_monitor(ERTS_P_MONITORS(prt), ref) == NULL) { DRV_MONITOR_UNLOCK_PDL(prt); return; } @@ -4654,7 +6835,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) DRV_MONITOR_UNLOCK_PDL(prt); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_process_exit)) { - DTRACE_FORMAT_COMMON_PID_AND_PORT(prt->connected, prt) + DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(prt), prt) DTRACE3(driver_process_exit, process_str, port_str, prt->name); } #endif @@ -4663,7 +6844,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) erts_unblock_fpe(fpe_was_unmasked); DRV_MONITOR_LOCK_PDL(prt); /* remove monitor *after* callback */ - rmon = erts_remove_monitor(&(prt->monitors),ref); + rmon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref); DRV_MONITOR_UNLOCK_PDL(prt); if (rmon) { erts_destroy_monitor(rmon); @@ -4674,27 +6855,28 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) static int driver_failure_term(ErlDrvPort ix, Eterm term, int eof) { - Port* prt = erts_drvport2port(ix); + erts_aint32_t state; + Port* prt = erts_drvport2port_state(ix, &state); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (eof) - flush_linebuf_messages(prt); - if (prt->status & ERTS_PORT_SFLG_CLOSING) { + flush_linebuf_messages(prt, state); + if (state & ERTS_PORT_SFLG_CLOSING) { terminate_port(prt); - } else if (eof && (prt->status & ERTS_PORT_SFLG_SOFT_EOF)) { - deliver_result(prt->id, prt->connected, am_eof); + } else if (eof && (state & ERTS_PORT_SFLG_SOFT_EOF)) { + deliver_result(prt->common.id, ERTS_PORT_GET_CONNECTED(prt), am_eof); } else { - /* XXX UGLY WORK AROUND, Let do_exit_port terminate the port */ + /* XXX UGLY WORK AROUND, Let erts_deliver_port_exit() terminate the port */ if (prt->port_data_lock) driver_pdl_lock(prt->port_data_lock); prt->ioq.size = 0; if (prt->port_data_lock) driver_pdl_unlock(prt->port_data_lock); - erts_do_exit_port(prt, prt->id, eof ? am_normal : term); + erts_deliver_port_exit(prt, prt->common.id, eof ? am_normal : term, 0); } return 0; } @@ -4710,20 +6892,20 @@ int driver_exit(ErlDrvPort ix, int err) Port* prt = erts_drvport2port(ix); Process* rp; ErtsLink *lnk, *rlnk = NULL; + Eterm connected; ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - - rp = erts_pid2proc(NULL, 0, prt->connected, ERTS_PROC_LOCK_LINK); + connected = ERTS_PORT_GET_CONNECTED(prt); + rp = erts_pid2proc(NULL, 0, connected, ERTS_PROC_LOCK_LINK); if (rp) { - rlnk = erts_remove_link(&(rp->nlinks),prt->id); + rlnk = erts_remove_link(&ERTS_P_LINKS(rp),prt->common.id); } - lnk = erts_remove_link(&(prt->nlinks),prt->connected); + lnk = erts_remove_link(&ERTS_P_LINKS(prt), connected); #ifdef ERTS_SMP if (rp) @@ -4742,7 +6924,8 @@ int driver_exit(ErlDrvPort ix, int err) return driver_failure_term(ix, am_normal, 0); else { char* err_str = erl_errno_id(err); - Eterm am_err = am_atom_put(err_str, sys_strlen(err_str)); + Eterm am_err = erts_atom_put((byte *) err_str, sys_strlen(err_str), + ERTS_ATOM_ENC_LATIN1, 1); return driver_failure_term(ix, am_err, 0); } } @@ -4755,8 +6938,12 @@ int driver_failure(ErlDrvPort ix, int code) int driver_failure_atom(ErlDrvPort ix, char* string) { - Eterm am = am_atom_put(string, strlen(string)); - return driver_failure_term(ix, am, 0); + return driver_failure_term(ix, + erts_atom_put((byte *) string, + strlen(string), + ERTS_ATOM_ENC_LATIN1, + 1), + 0); } int driver_failure_posix(ErlDrvPort ix, int err) @@ -4773,7 +6960,10 @@ int driver_failure_eof(ErlDrvPort ix) ErlDrvTermData driver_mk_atom(char* string) { - Eterm am = am_atom_put(string, sys_strlen(string)); + Eterm am = erts_atom_put((byte *) string, + sys_strlen(string), + ERTS_ATOM_ENC_LATIN1, + 1); ERTS_SMP_CHK_NO_PROC_LOCKS; return (ErlDrvTermData) am; } @@ -4781,25 +6971,27 @@ ErlDrvTermData driver_mk_atom(char* string) ErlDrvTermData driver_mk_port(ErlDrvPort ix) { Port* prt = erts_drvport2port(ix); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + return (ErlDrvTermData) NIL; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - return (ErlDrvTermData) prt->id; + return (ErlDrvTermData) prt->common.id; } ErlDrvTermData driver_connected(ErlDrvPort ix) { Port* prt = erts_drvport2port(ix); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return NIL; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - return prt->connected; + return ERTS_PORT_GET_CONNECTED(prt); } ErlDrvTermData driver_caller(ErlDrvPort ix) { Port* prt = erts_drvport2port(ix); ERTS_SMP_CHK_NO_PROC_LOCKS; - if (prt == NULL) + if (prt == ERTS_INVALID_ERL_DRV_PORT) return NIL; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); return prt->caller; @@ -4812,20 +7004,18 @@ int driver_lock_driver(ErlDrvPort ix) ERTS_SMP_CHK_NO_PROC_LOCKS; - erts_smp_mtx_lock(&erts_driver_list_lock); - - if (prt == NULL) { - erts_smp_mtx_unlock(&erts_driver_list_lock); + if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - } + + erts_smp_rwmtx_rwlock(&erts_driver_list_lock); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if ((dh = (DE_Handle*)prt->drv_ptr->handle ) == NULL) { - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); return -1; } erts_ddll_lock_driver(dh, prt->drv_ptr->name); - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); return 0; } @@ -4835,7 +7025,7 @@ static int maybe_lock_driver_list(void) void *rec_lock; rec_lock = erts_smp_tsd_get(driver_list_lock_status_key); if (rec_lock == 0) { - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rwlock(&erts_driver_list_lock); return 1; } return 0; @@ -4843,7 +7033,7 @@ static int maybe_lock_driver_list(void) static void maybe_unlock_driver_list(int doit) { if (doit) { - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); } } /* @@ -5012,7 +7202,7 @@ no_event_callback(ErlDrvData drv_data, ErlDrvEvent event, ErlDrvEventData event_ { Port *prt = get_current_port(); report_missing_drv_callback(prt, "Event", "event()"); - driver_event((ErlDrvPort) internal_port_index(prt->id), event, NULL); + driver_event(ERTS_Port2ErlDrvPort(prt), event, NULL); } static void @@ -5020,7 +7210,7 @@ no_ready_input_callback(ErlDrvData drv_data, ErlDrvEvent event) { Port *prt = get_current_port(); report_missing_drv_callback(prt, "Input", "ready_input()"); - driver_select((ErlDrvPort) internal_port_index(prt->id), event, + driver_select(ERTS_Port2ErlDrvPort(prt), event, (ERL_DRV_READ | ERL_DRV_USE_NO_CALLBACK), 0); } @@ -5029,7 +7219,7 @@ no_ready_output_callback(ErlDrvData drv_data, ErlDrvEvent event) { Port *prt = get_current_port(); report_missing_drv_callback(prt, "Output", "ready_output()"); - driver_select((ErlDrvPort) internal_port_index(prt->id), event, + driver_select(ERTS_Port2ErlDrvPort(prt), event, (ERL_DRV_WRITE | ERL_DRV_USE_NO_CALLBACK), 0); } @@ -5064,13 +7254,16 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) drv->lock = NULL; else { drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK, - sizeof(erts_smp_mtx_t)); - erts_smp_mtx_init_x(drv->lock, - "driver_lock", + sizeof(erts_mtx_t)); + erts_mtx_init_x(drv->lock, + "driver_lock", #if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT) - am_atom_put(drv->name, sys_strlen(drv->name)) + erts_atom_put((byte *) drv->name, + sys_strlen(drv->name), + ERTS_ATOM_ENC_LATIN1, + 1) #else - NIL + NIL #endif ); } @@ -5142,7 +7335,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo int res; if (!driver_list_locked) { - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rwlock(&erts_driver_list_lock); } dp->next = driver_list; @@ -5171,7 +7364,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo if (!driver_list_locked) { erts_smp_tsd_set(driver_list_lock_status_key, NULL); - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); } return res; } @@ -5184,7 +7377,7 @@ int remove_driver_entry(ErlDrvEntry *drv) rec_lock = erts_smp_tsd_get(driver_list_lock_status_key); if (rec_lock == NULL) { - erts_smp_mtx_lock(&erts_driver_list_lock); + erts_smp_rwmtx_rwlock(&erts_driver_list_lock); } dp = driver_list; while (dp && dp->entry != drv) @@ -5192,7 +7385,7 @@ int remove_driver_entry(ErlDrvEntry *drv) if (dp) { if (dp->handle) { if (rec_lock == NULL) { - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); } return -1; } @@ -5206,12 +7399,12 @@ int remove_driver_entry(ErlDrvEntry *drv) } erts_destroy_driver(dp); if (rec_lock == NULL) { - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); } return 1; } if (rec_lock == NULL) { - erts_smp_mtx_unlock(&erts_driver_list_lock); + erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); } return 0; } @@ -5241,18 +7434,22 @@ erl_drv_getenv(char *key, char *value, size_t *value_size) * - uses the fact that heart_port is registered when starting heart */ -Port *erts_get_heart_port() { +Port *erts_get_heart_port(void) +{ + int ix, max = erts_ptab_max(&erts_port); - Port* port; - Uint ix; + for (ix = 0; ix < max; ix++) { + struct reg_proc *reg; + Port *port = erts_pix2port(ix); - for(ix = 0; ix < erts_max_ports; ix++) { - port = &erts_port[ix]; + if (!port) + continue; /* only examine undead or alive ports */ - if (port->status & ERTS_PORT_SFLGS_DEAD) + if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD) continue; /* immediate atom compare */ - if (port->reg && port->reg->name == am_heart_port) { + reg = port->common.u.alive.reg; + if (reg && reg->name == am_heart_port) { return port; } } diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index b93b1ad09a..daa6e136c5 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,21 +26,32 @@ #include "global.h" #include "module.h" +#ifdef DEBUG +# define IF_DEBUG(x) x +#else +# define IF_DEBUG(x) +#endif + #define MODULE_SIZE 50 #define MODULE_LIMIT (64*1024) -static IndexTable module_table; +static IndexTable module_tables[ERTS_NUM_CODE_IX]; -/* - * SMP note: We don't need to look accesses to the module table because - * there is one only scheduler thread when we update it. +erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; + +static erts_smp_atomic_t tot_module_bytes; + +/* SMP note: Active module table lookup and current module instance can be + * read without any locks. Old module instances are protected by + * "the_old_code_rwlocks" as purging is done on active module table. + * Staging table is protected by the "code_ix lock". */ #include "erl_smp.h" void module_info(int to, void *to_arg) { - index_info(to, to_arg, &module_table); + index_info(to, to_arg, &module_tables[erts_active_code_ix()]); } @@ -59,45 +70,67 @@ static int module_cmp(Module* tmpl, Module* obj) static Module* module_alloc(Module* tmpl) { Module* obj = (Module*) erts_alloc(ERTS_ALC_T_MODULE, sizeof(Module)); + erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->code = 0; - obj->old_code = 0; - obj->code_length = 0; - obj->old_code_length = 0; + obj->curr.code = 0; + obj->old.code = 0; + obj->curr.code_length = 0; + obj->old.code_length = 0; obj->slot.index = -1; - obj->nif = NULL; - obj->old_nif = NULL; + obj->curr.nif = NULL; + obj->old.nif = NULL; + obj->curr.num_breakpoints = 0; + obj->old.num_breakpoints = 0; + obj->curr.num_traced_exports = 0; + obj->old.num_traced_exports = 0; return obj; } +static void module_free(Module* mod) +{ + erts_free(ERTS_ALC_T_MODULE, mod); + erts_smp_atomic_add_nob(&tot_module_bytes, -sizeof(Module)); +} void init_module_table(void) { HashFunctions f; + int i; f.hash = (H_FUN) module_hash; f.cmp = (HCMP_FUN) module_cmp; f.alloc = (HALLOC_FUN) module_alloc; - f.free = 0; + f.free = (HFREE_FUN) module_free; + + for (i = 0; i < ERTS_NUM_CODE_IX; i++) { + erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code", + MODULE_SIZE, MODULE_LIMIT, f); + } - erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_table, "module_code", - MODULE_SIZE, MODULE_LIMIT, f); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + erts_smp_rwmtx_init_x(&the_old_code_rwlocks[i], "old_code", make_small(i)); + } + erts_smp_atomic_init_nob(&tot_module_bytes, 0); } Module* -erts_get_module(Eterm mod) +erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { Module e; int index; + IndexTable* mod_tab; ASSERT(is_atom(mod)); + + mod_tab = &module_tables[code_ix]; + e.module = atom_val(mod); - index = index_get(&module_table, (void*) &e); + index = index_get(mod_tab, (void*) &e); if (index == -1) { return NULL; } else { - return (Module*) erts_index_lookup(&module_table, index); + return (Module*) erts_index_lookup(mod_tab, index); } } @@ -105,27 +138,101 @@ Module* erts_put_module(Eterm mod) { Module e; - int index; + IndexTable* mod_tab; + int oldsz, newsz; + Module* res; ASSERT(is_atom(mod)); ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_smp_thr_progress_is_blocking()); + || erts_has_code_write_permission()); + + mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); - index = index_put(&module_table, (void*) &e); - return (Module*) erts_index_lookup(&module_table, index); + oldsz = index_table_sz(mod_tab); + res = (Module*) index_put_entry(mod_tab, (void*) &e); + newsz = index_table_sz(mod_tab); + erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + return res; } -Module *module_code(int i) +Module *module_code(int i, ErtsCodeIndex code_ix) { - return (Module*) erts_index_lookup(&module_table, i); + return (Module*) erts_index_lookup(&module_tables[code_ix], i); } -int module_code_size(void) +int module_code_size(ErtsCodeIndex code_ix) { - return module_table.entries; + return module_tables[code_ix].entries; } int module_table_sz(void) { - return index_table_sz(&module_table); + return erts_smp_atomic_read_nob(&tot_module_bytes); +} + +#ifdef DEBUG +static ErtsCodeIndex dbg_load_code_ix = 0; +#endif + +static int entries_at_start_staging = 0; + +void module_start_staging(void) +{ + IndexTable* src = &module_tables[erts_active_code_ix()]; + IndexTable* dst = &module_tables[erts_staging_code_ix()]; + Module* src_mod; + Module* dst_mod; + int i, oldsz, newsz; + + ASSERT(dbg_load_code_ix == -1); + ASSERT(dst->entries <= src->entries); + + /* + * Make sure our existing modules are up-to-date + */ + for (i = 0; i < dst->entries; i++) { + src_mod = (Module*) erts_index_lookup(src, i); + dst_mod = (Module*) erts_index_lookup(dst, i); + ASSERT(src_mod->module == dst_mod->module); + + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + } + + /* + * Copy all new modules from active table + */ + oldsz = index_table_sz(dst); + for (i = dst->entries; i < src->entries; i++) { + src_mod = (Module*) erts_index_lookup(src, i); + dst_mod = (Module*) index_put_entry(dst, src_mod); + ASSERT(dst_mod != src_mod); + + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + } + newsz = index_table_sz(dst); + erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + + entries_at_start_staging = dst->entries; + IF_DEBUG(dbg_load_code_ix = erts_staging_code_ix()); +} + +void module_end_staging(int commit) +{ + ASSERT(dbg_load_code_ix == erts_staging_code_ix()); + + if (!commit) { /* abort */ + IndexTable* tab = &module_tables[erts_staging_code_ix()]; + int oldsz, newsz; + + ASSERT(entries_at_start_staging <= tab->entries); + oldsz = index_table_sz(tab); + index_erase_latest_from(tab, entries_at_start_staging); + newsz = index_table_sz(tab); + erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + } + + IF_DEBUG(dbg_load_code_ix = -1); } + diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 694e4ab72f..5235528e98 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,28 +24,72 @@ #include "index.h" #endif +struct erl_module_instance { + BeamInstr* code; + int code_length; /* Length of loaded code in bytes. */ + unsigned catches; + struct erl_module_nif* nif; + int num_breakpoints; + int num_traced_exports; +}; typedef struct erl_module { IndexSlot slot; /* Must be located at top of struct! */ int module; /* Atom index for module (not tagged). */ - BeamInstr* code; - BeamInstr* old_code; - int code_length; /* Length of loaded code in bytes. */ - int old_code_length; /* Length of old loaded code in bytes */ - unsigned catches, old_catches; - struct erl_module_nif* nif; - struct erl_module_nif* old_nif; + struct erl_module_instance curr; + struct erl_module_instance old; /* protected by "old_code" rwlock */ } Module; -Module* erts_get_module(Eterm mod); +Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); void init_module_table(void); +void module_start_staging(void); +void module_end_staging(int commit); void module_info(int, void *); -Module *module_code(int); -int module_code_size(void); +Module *module_code(int, ErtsCodeIndex); +int module_code_size(ErtsCodeIndex); int module_table_sz(void); +ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex); +ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex); +ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex); +ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex); +#ifdef ERTS_ENABLE_LOCK_CHECK +int erts_is_old_code_rlocked(ErtsCodeIndex); +#endif + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +extern erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; + +ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]); +} +ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]); +} +ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_rlock(&the_old_code_rwlocks[code_ix]); +} +ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex code_ix) +{ + erts_smp_rwmtx_runlock(&the_old_code_rwlocks[code_ix]); +} + +#ifdef ERTS_ENABLE_LOCK_CHECK +ERTS_GLB_INLINE int erts_is_old_code_rlocked(ErtsCodeIndex code_ix) +{ + return erts_smp_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]); +} #endif + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +#endif /* !__MODULE_H__ */ diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 9b168889dd..1e5ae46bfa 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -62,11 +62,8 @@ label L i_func_info I a a I int_code_end -i_trace_breakpoint -i_mtrace_breakpoint +i_generic_breakpoint i_debug_breakpoint -i_count_breakpoint -i_time_breakpoint i_return_time_trace i_return_to_trace i_yield @@ -522,7 +519,6 @@ apply_bif call_nif call_error_handler error_action_code -call_traced_function return_trace # @@ -829,16 +825,20 @@ call_ext_only Ar=u==2 Bif=u$bif:erlang:load_nif/2 => allocate u Ar | i_call_ext # -# The apply/2 and apply/3 BIFs are instructions. +# apply/2 is an instruction, not a BIF. # call_ext u==2 u$func:erlang:apply/2 => i_apply_fun call_ext_last u==2 u$func:erlang:apply/2 D => i_apply_fun_last D call_ext_only u==2 u$func:erlang:apply/2 => i_apply_fun_only -call_ext u==3 u$func:erlang:apply/3 => i_apply -call_ext_last u==3 u$func:erlang:apply/3 D => i_apply_last D -call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only +# +# The apply/3 BIF is an instruction. +# + +call_ext u==3 u$bif:erlang:apply/3 => i_apply +call_ext_last u==3 u$bif:erlang:apply/3 D => i_apply_last D +call_ext_only u==3 u$bif:erlang:apply/3 => i_apply_only # # The exit/1 and throw/1 BIFs never execute the instruction following them; @@ -1021,7 +1021,7 @@ bif0 u$bif:erlang:node/0 Dst=d => node Dst bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst -bif2 Jump=j u$bif:erlang:element/2 S1=s S2=s Dst=d => gen_element(Jump, S1, S2, Dst) +bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst) bif1 Fail Bif Literal=q Dst => move Literal x | bif1 Fail Bif x Dst bif1 p Bif S1 Dst => bif1_body Bif S1 Dst diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index f1cfa8df39..db0e78b1a7 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2011. All Rights Reserved. + * Copyright Ericsson AB 2008-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -67,7 +67,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define HTTP_HDR_HASH_SIZE 53 #define HTTP_METH_HASH_SIZE 13 -#define HTTP_MAX_NAME_LEN 20 +#define HTTP_MAX_NAME_LEN 50 static char tspecial[128]; @@ -460,11 +460,9 @@ int packet_get_length(enum PacketParseType htype, hp = (struct tpkt_head*) ptr; if (hp->vrsn == TPKT_VRSN) { plen = get_int16(hp->packet_length) - hlen; - if (plen < 0) - goto error; - } - else + } else { goto error; + } goto remain; } diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 26d64887d0..c626cb2780 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -93,19 +93,14 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks) reg_write_lock(); } +#endif + static ERTS_INLINE int is_proc_alive(Process *p) { - int res; - erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id); - erts_pix_lock(pixlck); - res = !p->is_exiting; - erts_pix_unlock(pixlck); - return res; + return !ERTS_PROC_IS_EXITING(p); } -#endif - void register_info(int to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; @@ -180,14 +175,14 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) if (is_not_atom(name) || name == am_undefined) return res; - if (c_p->id == id) /* A very common case I think... */ + if (c_p->common.id == id) /* A very common case I think... */ proc = c_p; else { if (is_not_internal_pid(id) && is_not_internal_port(id)) return res; erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); if (is_internal_port(id)) { - port = erts_id2port(id, NULL, 0); + port = erts_id2port(id); if (!port) goto done; } @@ -209,7 +204,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) r.p = proc; if (!proc) goto done; - if (proc->reg) + if (proc->common.u.alive.reg) goto done; r.pt = NULL; } @@ -217,7 +212,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) ASSERT(!INVALID_PORT(port, id)); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); r.pt = port; - if (r.pt->reg) + if (r.pt->common.u.alive.reg) goto done; r.p = NULL; } @@ -229,23 +224,24 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) if (IS_TRACED_FL(proc, F_TRACE_PROCS)) { trace_proc(c_p, proc, am_register, name); } - proc->reg = rp; + proc->common.u.alive.reg = rp; } else if (port && rp->pt == port) { if (IS_TRACED_FL(port, F_TRACE_PORTS)) { trace_port(port, am_register, name); } - port->reg = rp; + port->common.u.alive.reg = rp; } - if ((rp->p && rp->p->id == id) || (rp->pt && rp->pt->id == id)) { + if ((rp->p && rp->p->common.id == id) + || (rp->pt && rp->pt->common.id == id)) { res = 1; } done: reg_write_unlock(); if (port) - erts_smp_port_unlock(port); + erts_port_release(port); if (c_p != proc) { if (proc) erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); @@ -296,9 +292,9 @@ erts_whereis_name_to_id(Process *c_p, Eterm name) * is read only. */ if (rp->p) - res = rp->p->id; + res = rp->p->common.id; else if (rp->pt) - res = rp->pt->id; + res = rp->pt->common.id; break; } b = b->next; @@ -389,8 +385,7 @@ erts_whereis_name(Process *c_p, } #else if (rp->p - && ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - || rp->p->status != P_EXITING)) + && ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) || is_proc_alive(rp->p))) *proc = rp->p; else *proc = NULL; @@ -409,19 +404,19 @@ erts_whereis_name(Process *c_p, if (pending_port) { /* Ahh! Registered port changed while reg lock was unlocked... */ - erts_smp_port_unlock(pending_port); + erts_port_release(pending_port); pending_port = NULL; } if (erts_smp_port_trylock(rp->pt) == EBUSY) { - Eterm id = rp->pt->id; /* id read only... */ + Eterm id = rp->pt->common.id; /* id read only... */ /* Unlock all locks, acquire port lock, and restart... */ if (current_c_p_locks) { erts_smp_proc_unlock(c_p, current_c_p_locks); current_c_p_locks = 0; } reg_read_unlock(); - pending_port = erts_id2port(id, NULL, 0); + pending_port = erts_id2port(id); goto restart; } } @@ -435,7 +430,7 @@ erts_whereis_name(Process *c_p, if (c_p && !current_c_p_locks) erts_smp_proc_lock(c_p, c_p_locks); if (pending_port) - erts_smp_port_unlock(pending_port); + erts_port_release(pending_port); #endif reg_read_unlock(); @@ -497,8 +492,8 @@ int erts_unregister_name(Process *c_p, current_c_p_locks = c_p_locks; } #endif - if (c_p->reg) { - r.name = c_p->reg->name; + if (c_p->common.u.alive.reg) { + r.name = c_p->common.u.alive.reg->name; } else { /* Name got unregistered while main lock was released */ res = 0; @@ -511,20 +506,20 @@ int erts_unregister_name(Process *c_p, if (port != rp->pt) { #ifdef ERTS_SMP if (port) { - ERTS_SMP_LC_ASSERT(port != c_prt); - erts_smp_port_unlock(port); + ASSERT(port != c_prt); + erts_port_release(port); port = NULL; } if (erts_smp_port_trylock(rp->pt) == EBUSY) { - Eterm id = rp->pt->id; /* id read only... */ + Eterm id = rp->pt->common.id; /* id read only... */ /* Unlock all locks, acquire port lock, and restart... */ if (current_c_p_locks) { erts_smp_proc_unlock(c_p, current_c_p_locks); current_c_p_locks = 0; } reg_write_unlock(); - port = erts_id2port(id, NULL, 0); + port = erts_id2port(id); goto restart; } #endif @@ -534,7 +529,7 @@ int erts_unregister_name(Process *c_p, ASSERT(rp->pt == port); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); - rp->pt->reg = NULL; + rp->pt->common.u.alive.reg = NULL; if (IS_TRACED_FL(port, F_TRACE_PORTS)) { trace_port(port, am_unregister, r.name); @@ -551,7 +546,7 @@ int erts_unregister_name(Process *c_p, ERTS_PROC_LOCK_MAIN); current_c_p_locks = c_p_locks; #endif - rp->p->reg = NULL; + rp->p->common.u.alive.reg = NULL; if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) { trace_proc(c_p, rp->p, am_unregister, r.name); } @@ -570,7 +565,7 @@ int erts_unregister_name(Process *c_p, reg_write_unlock(); if (c_prt != port) { if (port) { - erts_smp_port_unlock(port); + erts_port_release(port); } if (c_prt) { erts_smp_port_lock(c_prt); diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 38e8cfbf28..7170463375 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -24,26 +24,19 @@ #ifndef __REGPROC_H__ #define __REGPROC_H__ -#ifndef __SYS_H__ #include "sys.h" -#endif - -#ifndef __HASH_H__ #include "hash.h" -#endif - -#ifndef __PROCESS_H__ #include "erl_process.h" -#endif - -struct port; +#define ERL_PORT_GET_PORT_TYPE_ONLY__ +#include "erl_port.h" +#undef ERL_PORT_GET_PORT_TYPE_ONLY__ typedef struct reg_proc { HashBucket bucket; /* MUST BE LOCATED AT TOP OF STRUCT!!! */ Process *p; /* The process registered (only one of this and 'pt' is non-NULL */ - struct port *pt; /* The port registered */ + Port *pt; /* The port registered */ Eterm name; /* Atom name */ } RegProc; @@ -55,12 +48,12 @@ int erts_register_name(Process *, Eterm, Eterm); Eterm erts_whereis_name_to_id(Process *, Eterm); void erts_whereis_name(Process *, ErtsProcLocks, Eterm, Process**, ErtsProcLocks, int, - struct port**); + Port**); Process *erts_whereis_process(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); -int erts_unregister_name(Process *, ErtsProcLocks, struct port *, Eterm); +int erts_unregister_name(Process *, ErtsProcLocks, Port *, Eterm); #endif diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 41389c8734..096394b878 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,6 @@ # define NO_FPE_SIGNALS #endif -/* xxxP __VXWORKS__ */ -#ifdef VXWORKS -#include <vxWorks.h> -#endif - #ifdef DISABLE_CHILD_WAITER_THREAD #undef ENABLE_CHILD_WAITER_THREAD #endif @@ -39,10 +34,10 @@ #define ENABLE_CHILD_WAITER_THREAD 1 #endif +#define ERTS_I64_LITERAL(X) X##LL + #if defined (__WIN32__) # include "erl_win_sys.h" -#elif defined (VXWORKS) -# include "erl_vxworks_sys.h" #else # include "erl_unix_sys.h" #ifndef UNIX @@ -91,14 +86,22 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # endif #endif -#ifdef __GNUC__ -# if __GNUC__ < 3 && (__GNUC__ != 2 || __GNUC_MINOR__ < 96) -# define ERTS_LIKELY(BOOL) (BOOL) -# define ERTS_UNLIKELY(BOOL) (BOOL) -# else -# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0) -# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0) -# endif +#if !defined(__GNUC__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 +#elif !defined(__GNUC_MINOR__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#elif !defined(__GNUC_PATCHLEVEL__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#else +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#endif + +#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) +# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0) +# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0) #else # define ERTS_LIKELY(BOOL) (BOOL) # define ERTS_UNLIKELY(BOOL) (BOOL) @@ -113,6 +116,16 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_DECLARE_DUMMY(X) X #endif +#if !defined(__func__) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# if !defined(__GNUC__) || __GNUC__ < 2 +# define __func__ "[unknown_function]" +# else +# define __func__ __FUNCTION__ +# endif +# endif +#endif + #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) # undef ERTS_CAN_INLINE # define ERTS_CAN_INLINE 0 @@ -172,10 +185,16 @@ void erl_assert_error(char* expr, char* file, int line); # define const #endif -#ifdef VXWORKS -/* Replace VxWorks' printf with a real one that does fprintf(stdout, ...) */ -int real_printf(const char *fmt, ...); -# define printf real_printf +#undef __deprecated +#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0) +# define __deprecated __attribute__((deprecated)) +#else +# define __deprecated +#endif +#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4) +# define erts_align_attribute(SZ) __attribute__ ((aligned (SZ))) +#else +# define erts_align_attribute(SZ) #endif /* In VC++, noreturn is a declspec that has to be before the types, @@ -185,12 +204,6 @@ int real_printf(const char *fmt, ...); #if __GNUC__ # define __decl_noreturn # define __noreturn __attribute__((noreturn)) -# undef __deprecated -# if __GNUC__ >= 3 -# define __deprecated __attribute__((deprecated)) -# else -# define __deprecated -# endif #else # if defined(__WIN32__) && defined(_MSC_VER) # define __noreturn @@ -199,7 +212,6 @@ int real_printf(const char *fmt, ...); # define __noreturn # define __decl_noreturn # endif -# define __deprecated #endif /* @@ -229,9 +241,11 @@ int real_printf(const char *fmt, ...); #if SIZEOF_VOID_P == 8 #undef ARCH_32 #define ARCH_64 +#define ERTS_SIZEOF_TERM 8 #elif SIZEOF_VOID_P == 4 #define ARCH_32 #undef ARCH_64 +#define ERTS_SIZEOF_TERM 4 #else #error Neither 32 nor 64 bit architecture #endif @@ -239,6 +253,8 @@ int real_printf(const char *fmt, ...); # define HALFWORD_HEAP 1 # define HALFWORD_ASSERT 0 # define ASSERT_HALFWORD(COND) ASSERT(COND) +# undef ERTS_SIZEOF_TERM +# define ERTS_SIZEOF_TERM 4 #else # define HALFWORD_HEAP 0 # define HALFWORD_ASSERT 0 @@ -365,6 +381,27 @@ typedef unsigned char byte; #error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found #endif +#ifdef WORDS_BIGENDIAN +# define ERTS_HUINT_HVAL_HIGH 0 +# define ERTS_HUINT_HVAL_LOW 1 +#else +# define ERTS_HUINT_HVAL_HIGH 1 +# define ERTS_HUINT_HVAL_LOW 0 +#endif +#if ERTS_SIZEOF_TERM == 8 +typedef union { + Uint val; + Uint32 hval[2]; +} HUint; +#elif ERTS_SIZEOF_TERM == 4 +typedef union { + Uint val; + Uint16 hval[2]; +} HUint; +#else +#error "Unsupported size of term" +#endif + # define ERTS_EXTRA_DATA_ALIGN_SZ(X) \ (((size_t) 8) - (((size_t) (X)) & ((size_t) 7))) @@ -471,38 +508,28 @@ static unsigned long zero_value = 0, one_value = 1; # define SET_NONBLOCKING(fd) ioctlsocket((fd), FIONBIO, &one_value) # else -# ifdef VXWORKS -# include <fcntl.h> /* xxxP added for O_WRONLY etc ... macro:s ... */ -# include <ioLib.h> -static const int zero_value = 0, one_value = 1; -# define SET_BLOCKING(fd) ioctl((fd), FIONBIO, (int)&zero_value) -# define SET_NONBLOCKING(fd) ioctl((fd), FIONBIO, (int)&one_value) -# define ERRNO_BLOCK EWOULDBLOCK - -# else -# ifdef NB_FIONBIO /* Old BSD */ -# include <sys/ioctl.h> +# ifdef NB_FIONBIO /* Old BSD */ +# include <sys/ioctl.h> static const int zero_value = 0, one_value = 1; -# define SET_BLOCKING(fd) ioctl((fd), FIONBIO, &zero_value) -# define SET_NONBLOCKING(fd) ioctl((fd), FIONBIO, &one_value) -# define ERRNO_BLOCK EWOULDBLOCK -# else /* !NB_FIONBIO */ -# include <fcntl.h> -# ifdef NB_O_NDELAY /* Nothing needs this? */ -# define NB_FLAG O_NDELAY -# ifndef ERRNO_BLOCK /* allow override (e.g. EAGAIN) via Makefile */ -# define ERRNO_BLOCK EWOULDBLOCK -# endif -# else /* !NB_O_NDELAY */ /* The True Way - POSIX!:-) */ -# define NB_FLAG O_NONBLOCK -# define ERRNO_BLOCK EAGAIN -# endif /* !NB_O_NDELAY */ -# define SET_BLOCKING(fd) fcntl((fd), F_SETFL, \ - fcntl((fd), F_GETFL, 0) & ~NB_FLAG) -# define SET_NONBLOCKING(fd) fcntl((fd), F_SETFL, \ - fcntl((fd), F_GETFL, 0) | NB_FLAG) -# endif /* !NB_FIONBIO */ -# endif /* _WXWORKS_ */ +# define SET_BLOCKING(fd) ioctl((fd), FIONBIO, &zero_value) +# define SET_NONBLOCKING(fd) ioctl((fd), FIONBIO, &one_value) +# define ERRNO_BLOCK EWOULDBLOCK +# else /* !NB_FIONBIO */ +# include <fcntl.h> +# ifdef NB_O_NDELAY /* Nothing needs this? */ +# define NB_FLAG O_NDELAY +# ifndef ERRNO_BLOCK /* allow override (e.g. EAGAIN) via Makefile */ +# define ERRNO_BLOCK EWOULDBLOCK +# endif +# else /* !NB_O_NDELAY */ /* The True Way - POSIX!:-) */ +# define NB_FLAG O_NONBLOCK +# define ERRNO_BLOCK EAGAIN +# endif /* !NB_O_NDELAY */ +# define SET_BLOCKING(fd) fcntl((fd), F_SETFL, \ + fcntl((fd), F_GETFL, 0) & ~NB_FLAG) +# define SET_NONBLOCKING(fd) fcntl((fd), F_SETFL, \ + fcntl((fd), F_GETFL, 0) | NB_FLAG) +# endif /* !NB_FIONBIO */ # endif /* !__WIN32__ */ #endif /* WANT_NONBLOCKING */ @@ -513,6 +540,10 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...); #define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ #define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */ +#define ERTS_INTERNAL_ERROR(What) \ + erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ + __FILE__, __LINE__, __func__, What) + Eterm erts_check_io_info(void *p); /* Size of misc memory allocated from system dependent code */ @@ -587,6 +618,7 @@ typedef struct _SysDriverOpts { char *wd; /* Working directory. */ unsigned spawn_type; /* Bitfield of ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXTERNAL | both*/ + int parallelism; /* Optimize for parallelism */ } SysDriverOpts; extern char *erts_default_arg0; @@ -626,7 +658,7 @@ typedef struct { #define ERTS_SYS_DDLL_ERROR_INIT {NULL} extern void erts_sys_ddll_free_error(ErtsSysDdllError*); extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */ -extern int erts_sys_ddll_open2(char *path, void **handle, ErtsSysDdllError*); +extern int erts_sys_ddll_open2(const char *path, void **handle, ErtsSysDdllError*); #define erts_sys_ddll_open(P,H) erts_sys_ddll_open2(P,H,NULL) extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*); extern int erts_sys_ddll_load_driver_init(void *handle, void **function); @@ -635,7 +667,7 @@ extern int erts_sys_ddll_close2(void *handle, ErtsSysDdllError*); #define erts_sys_ddll_close(H) erts_sys_ddll_close2(H,NULL) extern void *erts_sys_ddll_call_init(void *function); extern void *erts_sys_ddll_call_nif_init(void *function); -extern int erts_sys_ddll_sym2(void *handle, char *name, void **function, ErtsSysDdllError*); +extern int erts_sys_ddll_sym2(void *handle, const char *name, void **function, ErtsSysDdllError*); #define erts_sys_ddll_sym(H,N,F) erts_sys_ddll_sym2(H,N,F,NULL) extern char *erts_sys_ddll_error(int code); @@ -697,10 +729,13 @@ char * getenv_string(GETENV_STATE *); void fini_getenv_state(GETENV_STATE *); /* xxxP */ +#define SYS_DEFAULT_FLOAT_DECIMALS 20 void init_sys_float(void); int sys_chars_to_double(char*, double*); -int sys_double_to_chars(double, char*); -void sys_get_pid(char *); +int sys_double_to_chars(double, char*, size_t); +int sys_double_to_chars_ext(double, char*, size_t, size_t); +int sys_double_to_chars_fast(double, char*, int, int, int); +void sys_get_pid(char *, size_t); /* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */ int erts_sys_putenv(char *key, char *value); @@ -858,13 +893,6 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) extern int erts_use_kernel_poll; #endif -#if defined(VXWORKS) -/* NOTE! sys_calloc2 does not exist on other - platforms than VxWorks and OSE */ -void* sys_calloc2(Uint, Uint); -#endif /* VXWORKS || OSE */ - - #define sys_memcpy(s1,s2,n) memcpy(s1,s2,n) #define sys_memmove(s1,s2,n) memmove(s1,s2,n) #define sys_memcmp(s1,s2,n) memcmp(s1,s2,n) @@ -971,43 +999,6 @@ void erl_bin_write(unsigned char *, int, int); # define DEBUGF(x) #endif - -#ifdef VXWORKS -/* This includes redefines of malloc etc - this should be done after sys_alloc, etc, above */ -# include "reclaim.h" -/*********************Malloc and friends************************ - * There is a problem with the naming of malloc and friends, - * malloc is used throughout sys.c and the resolver to mean save_alloc, - * but it should actually mean either sys_alloc or sys_alloc2, - * so the definitions from reclaim_master.h are not any - * good, i redefine the malloc family here, although it's quite - * ugly, actually it would be preferrable to use the - * names sys_alloc and so on throughout the offending code, but - * that will be saved as an later exercise... - * I also add an own calloc, to make the BSD resolver source happy. - ***************************************************************/ -/* Undefine malloc and friends */ -# ifdef malloc -# undef malloc -# endif -# ifdef calloc -# undef calloc -# endif -# ifdef realloc -# undef realloc -# endif -# ifdef free -# undef free -# endif -/* Redefine malloc and friends */ -# define malloc sys_alloc -# define calloc sys_calloc -# define realloc sys_realloc -# define free sys_free - -#endif - #ifdef __WIN32__ #ifdef ARCH_64 #define ERTS_ALLOC_ALIGN_BYTES 16 @@ -1021,32 +1012,56 @@ void erl_bin_write(unsigned char *, int, int); #define ERTS_SMALL_ABS(Small) labs(Small) #endif +#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +# define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0 +#endif #ifdef __WIN32__ - void call_break_handler(void); char* last_error(void); char* win32_errorstr(int); - - #endif /************************************************************************ * Find out the native filename encoding of the process (look at locale of * Unix processes and just do UTF16 on windows ************************************************************************/ -#define ERL_FILENAME_UNKNOWN 0 -#define ERL_FILENAME_LATIN1 1 -#define ERL_FILENAME_UTF8 2 -#define ERL_FILENAME_UTF8_MAC 3 -#define ERL_FILENAME_WIN_WCHAR 4 +#define ERL_FILENAME_UNKNOWN (0) +#define ERL_FILENAME_LATIN1 (1) +#define ERL_FILENAME_UTF8 (2) +#define ERL_FILENAME_UTF8_MAC (3) +#define ERL_FILENAME_WIN_WCHAR (4) + +/************************************************************************ + * If a filename in for example list_dir is not in the right encoding, it + * will be skipped in the resulting list, but depending on a startup setting + * we will inform the user in different ways. These macros define the + * different reactions to wrongly coded filenames. In the error case an + * exception will be thrown by prim_file. + ************************************************************************/ +#define ERL_FILENAME_WARNING_WARNING (0) +#define ERL_FILENAME_WARNING_IGNORE (1) +#define ERL_FILENAME_WARNING_ERROR (2) + +/*********************************************************************** + * The user can request a range of character that he/she consider + * printable. Currently this can be either latin1 or unicode, but + * in the future a set of ranges, or languages, could be specified. + ***********************************************************************/ +#define ERL_PRINTABLE_CHARACTERS_LATIN1 (0) +#define ERL_PRINTABLE_CHARACTERS_UNICODE (1) int erts_get_native_filename_encoding(void); /* The set function is only to be used by erl_init! */ -void erts_set_user_requested_filename_encoding(int encoding); +void erts_set_user_requested_filename_encoding(int encoding, int warning); int erts_get_user_requested_filename_encoding(void); +int erts_get_filename_warning_type(void); +/* This function is called from erl_init. The setting is read by BIF's + in io/io_lib. Setting is not atomic. */ +void erts_set_printable_characters(int range); +/* Get the setting (ERL_PRINTABLE_CHARACTERS_{LATIN1|UNICODE} */ +int erts_get_printable_characters(void); void erts_init_sys_common_misc(void); #endif - diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 932d157cd8..2fd8e0cf00 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -105,7 +105,14 @@ static ErlTimer *tiw_min_ptr; /* END tiw_lock protected variables */ /* Actual interval time chosen by sys_init_time() */ -static int itime; /* Constant after init */ + +#if SYS_CLOCK_RESOLUTION == 1 +# define TIW_ITIME 1 +# define TIW_ITIME_IS_CONSTANT +#else +static int tiw_itime; /* Constant after init */ +# define TIW_ITIME tiw_itime +#endif erts_smp_atomic32_t do_time; /* set at clock interrupt */ static ERTS_INLINE erts_short_time_t do_time_read(void) @@ -123,7 +130,7 @@ static ERTS_INLINE void do_time_init(void) erts_smp_atomic32_init_nob(&do_time, 0); } -/* get the time (in units of itime) to the next timeout, +/* get the time (in units of TIW_ITIME) to the next timeout, or -1 if there are no timeouts */ static erts_short_time_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ @@ -305,11 +312,18 @@ erts_timer_wheel_memory_size(void) void erts_init_time(void) { - int i; + int i, itime; /* system dependent init; must be done before do_time_init() if timer thread is enabled */ itime = erts_init_time_sup(); +#ifdef TIW_ITIME_IS_CONSTANT + if (itime != TIW_ITIME) { + erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); + } +#else + tiw_itime = itime; +#endif erts_smp_mtx_init(&tiw_lock, "timer_wheel"); @@ -340,7 +354,7 @@ insert_timer(ErlTimer* p, Uint t) * * (x + y - 1)/y is precisely the "number of bins" formula. */ - ticks = (t + itime - 1) / itime; + ticks = (t + (TIW_ITIME - 1)) / TIW_ITIME; /* * Ticks must be a Uint64, or the addition may overflow here, @@ -455,7 +469,7 @@ erts_time_left(ErlTimer *p) erts_smp_mtx_unlock(&tiw_lock); - return (Uint) left * itime; + return (Uint) left * TIW_ITIME; } #ifdef DEBUG diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 9e8b1086d4..bd2be7afca 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -46,6 +46,7 @@ #include "erl_thr_queue.h" #include "erl_sched_spec_pre_alloc.h" #include "beam_bp.h" +#include "erl_ptab.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD @@ -184,15 +185,15 @@ erts_set_hole_marker(Eterm* ptr, Uint sz) * Helper function for the ESTACK macros defined in global.h. */ void -erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end) +erl_grow_stack(ErtsAlcType_t a_type, Eterm** start, Eterm** sp, Eterm** end) { Uint old_size = (*end - *start); Uint new_size = old_size * 2; Uint sp_offs = *sp - *start; if (new_size > 2 * DEF_ESTACK_SIZE) { - *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(Eterm)); + *start = erts_realloc(a_type, (void *) *start, new_size*sizeof(Eterm)); } else { - Eterm* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(Eterm)); + Eterm* new_ptr = erts_alloc(a_type, new_size*sizeof(Eterm)); sys_memcpy(new_ptr, *start, old_size*sizeof(Eterm)); *start = new_ptr; } @@ -203,15 +204,15 @@ erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end) * Helper function for the ESTACK macros defined in global.h. */ void -erl_grow_wstack(UWord** start, UWord** sp, UWord** end) +erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end) { Uint old_size = (*end - *start); Uint new_size = old_size * 2; Uint sp_offs = *sp - *start; if (new_size > 2 * DEF_ESTACK_SIZE) { - *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(UWord)); + *start = erts_realloc(a_type, (void *) *start, new_size*sizeof(UWord)); } else { - UWord* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(UWord)); + UWord* new_ptr = erts_alloc(a_type, new_size*sizeof(UWord)); sys_memcpy(new_ptr, *start, old_size*sizeof(UWord)); *start = new_ptr; } @@ -268,16 +269,42 @@ list_length(Eterm list) return i; } -Uint erts_fit_in_bits(Uint n) +static const struct { + Sint64 mask; + int bits; +} fib_data[] = {{ERTS_I64_LITERAL(0x2), 1}, + {ERTS_I64_LITERAL(0xc), 2}, + {ERTS_I64_LITERAL(0xf0), 4}, + {ERTS_I64_LITERAL(0xff00), 8}, + {ERTS_I64_LITERAL(0xffff0000), 16}, + {ERTS_I64_LITERAL(0xffffffff00000000), 32}}; + +static ERTS_INLINE int +fit_in_bits(Sint64 value, int start) { - Uint i; + int bits = 0; + int i; - i = 0; - while (n > 0) { - i++; - n >>= 1; - } - return i; + for (i = start; i >= 0; i--) { + if (value & fib_data[i].mask) { + value >>= fib_data[i].bits; + bits |= fib_data[i].bits; + } + } + + bits++; + + return bits; +} + +int erts_fit_in_bits_int64(Sint64 value) +{ + return fit_in_bits(value, 5); +} + +int erts_fit_in_bits_int32(Sint32 value) +{ + return fit_in_bits((Sint64) (Uint32) value, 4); } int @@ -344,7 +371,7 @@ Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str) { if (hpp) - return am_atom_put(str, sys_strlen(str)); + return erts_atom_put((byte *) str, sys_strlen(str), ERTS_ATOM_ENC_LATIN1, 1); else return THE_NON_VALUE; } @@ -1292,7 +1319,7 @@ make_hash2(Eterm term) { FloatDef ff; GET_DOUBLE(term, ff); -#if defined(WORDS_BIGENDIAN) +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); #else UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12); @@ -1640,12 +1667,20 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) } #ifndef ERTS_SMP - if ( #ifdef USE_THREADS - !erts_get_scheduler_data() || /* Must be scheduler thread */ + p = NULL; + if (erts_get_scheduler_data()) /* Must be scheduler thread */ #endif - (p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0)) == NULL - || p->status == P_RUNNING) { + { + p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); + if (p) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + if (state & ERTS_PSFLG_RUNNING) + p = NULL; + } + } + + if (!p) { /* buf *always* points to a null terminated string */ erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n", tag, buf); @@ -2678,25 +2713,26 @@ tailrecur_ne: case SMALL_FLOAT: GET_DOUBLE(bw, f2); if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ f1.fd = signed_val(aw); j = float_comp(f1.fd, f2.fd); #if ERTS_SIZEOF_ETERM == 8 } else if (f2.fd > (double) (MAX_SMALL + 1)) { - // Float is a positive bignum, i.e. bigger + /* Float is a positive bignum, i.e. bigger */ j = -1; } else if (f2.fd < (double) (MIN_SMALL - 1)) { - // Float is a negative bignum, i.e. smaller + /* Float is a negative bignum, i.e. smaller */ j = 1; - } else { // Float is a Sint but less precise + } else { + /* Float is a Sint but less precise */ j = signed_val(aw) - (Sint) f2.fd; } #else } else { - // If float is positive it is bigger than small + /* If float is positive it is bigger than small */ j = (f2.fd > 0.0) ? -1 : 1; } -#endif // ERTS_SIZEOF_ETERM == 8 +#endif /* ERTS_SIZEOF_ETERM == 8 */ break; case FLOAT_BIG: { @@ -2708,18 +2744,18 @@ tailrecur_ne: GET_DOUBLE(bw, f2); if ((f2.fd < (double) (MAX_SMALL + 1)) && (f2.fd > (double) (MIN_SMALL - 1))) { - // Float is a Sint + /* Float is a Sint */ j = big_sign(aw) ? -1 : 1; } else if (big_arity(aw) > BIG_ARITY_FLOAT_MAX || pow(2.0,(big_arity(aw)-1)*D_EXP) > fabs(f2.fd)) { - // If bignum size shows that it is bigger than the abs float + /* If bignum size shows that it is bigger than the abs float */ j = big_sign(aw) ? -1 : 1; } else if (big_arity(aw) < BIG_ARITY_FLOAT_MAX && (pow(2.0,(big_arity(aw))*D_EXP)-1.0) < fabs(f2.fd)) { - // If bignum size shows that it is smaller than the abs float + /* If bignum size shows that it is smaller than the abs float */ j = f2.fd < 0 ? 1 : -1; } else if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ if (big_to_double(aw, &f1.fd) < 0) { j = big_sign(aw) ? -1 : 1; } else { @@ -2736,25 +2772,26 @@ tailrecur_ne: case FLOAT_SMALL: GET_DOUBLE(aw, f1); if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ f2.fd = signed_val(bw); j = float_comp(f1.fd, f2.fd); #if ERTS_SIZEOF_ETERM == 8 } else if (f1.fd > (double) (MAX_SMALL + 1)) { - // Float is a positive bignum, i.e. bigger + /* Float is a positive bignum, i.e. bigger */ j = 1; } else if (f1.fd < (double) (MIN_SMALL - 1)) { - // Float is a negative bignum, i.e. smaller + /* Float is a negative bignum, i.e. smaller */ j = -1; - } else { // Float is a Sint but less precise it + } else { + /* Float is a Sint but less precise it */ j = (Sint) f1.fd - signed_val(bw); } #else } else { - // If float is positive it is bigger than small + /* If float is positive it is bigger than small */ j = (f1.fd > 0.0) ? 1 : -1; } -#endif // ERTS_SIZEOF_ETERM == 8 +#endif /* ERTS_SIZEOF_ETERM == 8 */ break; default: j = b_tag - a_tag; @@ -2946,7 +2983,7 @@ char* Sint_to_buf(Sint n, struct Sint_buf *buf) */ Eterm -buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail) +buf_to_intlist(Eterm** hpp, const char *buf, size_t len, Eterm tail) { Eterm* hp = *hpp; size_t i = len; @@ -2982,12 +3019,13 @@ buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail) ** ; ** ** Return remaining bytes in buffer on success -** -1 on overflow -** -2 on type error (including that result would not be a whole number of bytes) +** ERTS_IOLIST_TO_BUF_OVERFLOW on overflow +** ERTS_IOLIST_TO_BUF_TYPE_ERROR on type error (including that result would not be a whole number of bytes) */ -int io_list_to_buf(Eterm obj, char* buf, int len) +ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len) { + ErlDrvSizeT len = (ErlDrvSizeT) alloced_len; Eterm* objp; DECLARE_ESTACK(s); goto L_again; @@ -3080,20 +3118,20 @@ int io_list_to_buf(Eterm obj, char* buf, int len) L_type_error: DESTROY_ESTACK(s); - return -2; + return ERTS_IOLIST_TO_BUF_TYPE_ERROR; L_overflow: DESTROY_ESTACK(s); - return -1; + return ERTS_IOLIST_TO_BUF_OVERFLOW; } /* * Return 0 if successful, and non-zero if unsuccessful. */ -int erts_iolist_size(Eterm obj, Uint* sizep) +int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) { Eterm* objp; - Uint size = 0; + Uint size = 0; /* Intentionally Uint due to halfword heap */ DECLARE_ESTACK(s); goto L_again; @@ -3145,7 +3183,7 @@ int erts_iolist_size(Eterm obj, Uint* sizep) #undef SAFE_ADD DESTROY_ESTACK(s); - *sizep = size; + *sizep = (ErlDrvSizeT) size; return ERTS_IOLIST_OK; L_overflow_error: @@ -3240,7 +3278,7 @@ ptimer_timeout(ErtsSmpPTimer *ptimer) ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, ERTS_P2P_FLG_ALLOW_OTHER_X); if (p) { - if (!p->is_exiting + if (!ERTS_PROC_IS_EXITING(p) && !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) { ASSERT(*ptimer->timer.timer_ref == ptimer); *ptimer->timer.timer_ref = NULL; @@ -3428,6 +3466,291 @@ erts_free_read_env(void *value) erts_free(ERTS_ALC_T_TMP, value); } + +typedef struct { + size_t sz; + char *ptr; +} ErtsEmuArg; + +typedef struct { + int argc; + ErtsEmuArg *arg; + size_t no_bytes; +} ErtsEmuArgs; + +ErtsEmuArgs saved_emu_args = {0}; + +void +erts_save_emu_args(int argc, char **argv) +{ +#ifdef DEBUG + char *end_ptr; +#endif + char *ptr; + int i; + size_t arg_sz[100]; + size_t size; + + ASSERT(!saved_emu_args.argc); + + size = sizeof(ErtsEmuArg)*argc; + for (i = 0; i < argc; i++) { + size_t sz = sys_strlen(argv[i]); + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + arg_sz[i] = sz; + size += sz+1; + } + ptr = (char *) malloc(size); +#ifdef DEBUG + end_ptr = ptr + size; +#endif + saved_emu_args.arg = (ErtsEmuArg *) ptr; + ptr += sizeof(ErtsEmuArg)*argc; + saved_emu_args.argc = argc; + saved_emu_args.no_bytes = 0; + for (i = 0; i < argc; i++) { + size_t sz; + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + sz = arg_sz[i]; + else + sz = sys_strlen(argv[i]); + saved_emu_args.arg[i].ptr = ptr; + saved_emu_args.arg[i].sz = sz; + saved_emu_args.no_bytes += sz; + ptr += sz+1; + sys_strcpy(saved_emu_args.arg[i].ptr, argv[i]); + } + ASSERT(ptr == end_ptr); +} + +Eterm +erts_get_emu_args(Process *c_p) +{ +#ifdef DEBUG + Eterm *end_hp; +#endif + int i; + Uint hsz; + Eterm *hp, res; + + hsz = saved_emu_args.no_bytes*2; + hsz += saved_emu_args.argc*2; + + hp = HAlloc(c_p, hsz); +#ifdef DEBUG + end_hp = hp + hsz; +#endif + res = NIL; + + for (i = saved_emu_args.argc-1; i >= 0; i--) { + Eterm arg = buf_to_intlist(&hp, + saved_emu_args.arg[i].ptr, + saved_emu_args.arg[i].sz, + NIL); + res = CONS(hp, arg, res); + hp += 2; + } + + ASSERT(hp == end_hp); + + return res; +} + + +Eterm +erts_get_ethread_info(Process *c_p) +{ + Uint sz, *szp; + Eterm res, *hp, **hpp, *end_hp = NULL; + + sz = 0; + szp = &sz; + hpp = NULL; + + while (1) { + Eterm tup, list, name; +#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ + || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ + || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) + char buf[1024]; + int i; + char **str; +#endif + + res = NIL; + +#ifdef ETHR_X86_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "sse2"), +#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + erts_bld_string(hpp, szp, + (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + ? "yes" : "no")) +#else + erts_bld_string(hpp, szp, "yes") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, + "x86" +#ifdef ARCH_64 + "_64" +#endif + " OOO"), + erts_bld_string(hpp, szp, +#ifdef ETHR_X86_OUT_OF_ORDER + "yes" +#else + "no" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); +#endif + +#ifdef ETHR_SPARC_V9_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Sparc V9"), + erts_bld_string(hpp, szp, +#if defined(ETHR_SPARC_TSO) + "TSO" +#elif defined(ETHR_SPARC_PSO) + "PSO" +#elif defined(ETHR_SPARC_RMO) + "RMO" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + +#ifdef ETHR_PPC_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "lwsync"), + erts_bld_string(hpp, szp, +#if defined(ETHR_PPC_HAVE_LWSYNC) + "yes" +#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) + "no" +#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) + ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native rw-spinlocks"), +#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native spinlocks"), +#ifdef ETHR_NATIVE_SPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + + list = NIL; +#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL + if (ethr_have_native_dw_atomic()) { + name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL); + str = ethr_native_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + str = ethr_native_su_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + } + else +#endif + name = erts_bld_string(hpp, szp, "no"); + + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "Double word native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC64_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL); + str = ethr_native_atomic64_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "64-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC32_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL); + str = ethr_native_atomic32_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "32-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + if (hpp) { + HRelease(c_p, end_hp, *hpp) + return res; + } + + hp = HAlloc(c_p, sz); + end_hp = hp + sz; + hpp = &hp; + szp = NULL; + } +} + /* * To be used to silence unused result warnings, but do not abuse it. */ @@ -3437,6 +3760,254 @@ void erts_silence_warn_unused_result(long unused) } /* + * Interval counts + */ +void +erts_interval_init(erts_interval_t *icp) +{ +#ifdef ARCH_64 + erts_atomic_init_nob(&icp->counter.atomic, 0); +#else + erts_dw_aint_t dw; +#ifdef ETHR_SU_DW_NAINT_T__ + dw.dw_sint = 0; +#else + dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0; + dw.sint[ERTS_DW_AINT_LOW_WORD] = 0; +#endif + erts_dw_atomic_init_nob(&icp->counter.atomic, &dw); + +#endif +#ifdef DEBUG + icp->smp_api = 0; +#endif +} + +void +erts_smp_interval_init(erts_interval_t *icp) +{ +#ifdef ERTS_SMP + erts_interval_init(icp); +#else + icp->counter.not_atomic = 0; +#endif +#ifdef DEBUG + icp->smp_api = 1; +#endif +} + +static ERTS_INLINE Uint64 +step_interval_nob(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + } +#endif +} + +static ERTS_INLINE Uint64 +step_interval_relb(erts_interval_t *icp) +{ +#ifdef ARCH_64 + return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + } +#endif +} + + +static ERTS_INLINE Uint64 +ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + Uint64 curr_ic; +#ifdef ARCH_64 + curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic); + if (curr_ic > ic) + return curr_ic; + return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + } +#endif +} + + +static ERTS_INLINE Uint64 +ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + Uint64 curr_ic; +#ifdef ARCH_64 + curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); + if (curr_ic > ic) + return curr_ic; + return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic); +#else + erts_dw_aint_t exp; + + erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp); + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + + while (1) { + erts_dw_aint_t new = exp; + +#ifdef ETHR_SU_DW_NAINT_T__ + new.dw_sint++; +#else + new.sint[ERTS_DW_AINT_LOW_WORD]++; + if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) + new.sint[ERTS_DW_AINT_HIGH_WORD]++; +#endif + + if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp)) + return erts_interval_dw_aint_to_val__(&new); + + curr_ic = erts_interval_dw_aint_to_val__(&exp); + if (curr_ic > ic) + return curr_ic; + } +#endif +} + +Uint64 +erts_step_interval_nob(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return step_interval_nob(icp); +} + +Uint64 +erts_step_interval_relb(erts_interval_t *icp) +{ + ASSERT(!icp->smp_api); + return step_interval_relb(icp); +} + +Uint64 +erts_smp_step_interval_nob(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return step_interval_nob(icp); +#else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_smp_step_interval_relb(erts_interval_t *icp) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return step_interval_relb(icp); +#else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(!icp->smp_api); + return ensure_later_interval_nob(icp, ic); +} + +Uint64 +erts_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(!icp->smp_api); + return ensure_later_interval_acqb(icp, ic); +} + +Uint64 +erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return ensure_later_interval_nob(icp, ic); +#else + if (icp->counter.not_atomic > ic) + return icp->counter.not_atomic; + else + return ++icp->counter.not_atomic; +#endif +} + +Uint64 +erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) +{ + ASSERT(icp->smp_api); +#ifdef ERTS_SMP + return ensure_later_interval_acqb(icp, ic); +#else + if (icp->counter.not_atomic > ic) + return icp->counter.not_atomic; + else + return ++icp->counter.not_atomic; +#endif +} + + +/* * A millisecond timestamp without time correction where there's no hrtime * - for tracing on "long" things... */ @@ -3486,7 +4057,7 @@ Process *p; void ppi(Eterm pid) { - pp(erts_pid2proc_unlocked(pid)); + pp(erts_proc_lookup(pid)); } void td(Eterm x) |