diff options
Diffstat (limited to 'erts/emulator')
22 files changed, 860 insertions, 390 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index f8f2e29c95..4716460a6b 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1323,11 +1323,7 @@ void process_main(void) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; - if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule -#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) - && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)) -#endif - ) { + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { BeamInstr *inptr = find_function_from_pc(start_time_i); BeamInstr *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); @@ -1337,7 +1333,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - c_p = schedule(c_p, reds_used); + c_p = erts_schedule(NULL, c_p, reds_used); ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; @@ -3559,12 +3555,10 @@ do { \ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!c_p->scheduler_data) - live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */ - else +#ifdef ERTS_SMP + ASSERT(c_p->scheduler_data); #endif - live_hf_end = c_p->mbuf; + live_hf_end = c_p->mbuf; erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) @@ -3574,10 +3568,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (env.exiting) { - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - goto do_schedule; - } + ASSERT(!env.exiting); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } @@ -5162,6 +5153,278 @@ do { \ } } +/* + * erts_dirty_process_main() is what dirty schedulers execute. Since they handle + * only NIF calls they do not need to be able to execute all BEAM + * instructions. + */ +void erts_dirty_process_main(ErtsSchedulerData *esdp) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Process* c_p = NULL; + ErtsMonotonicTime start_time; +#ifdef DEBUG + ERTS_DECLARE_DUMMY(Eterm pid); +#endif + + /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, + * in all other cases x0 is used. + */ + register Eterm* reg REG_xregs = NULL; + + /* + * Top of heap (next free location); grows upwards. + */ + register Eterm* HTOP REG_htop = NULL; + + /* Stack pointer. Grows downwards; points + * to last item pushed (normally a saved + * continuation pointer). + */ + register Eterm* E REG_stop = NULL; + + /* + * Pointer to next threaded instruction. + */ + register BeamInstr *I REG_I = NULL; + + ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */ + + /* + * start_time always positive for dirty CPU schedulers, + * and negative for dirty I/O schedulers. + */ + + if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) { + start_time = erts_get_monotonic_time(NULL); + ASSERT(start_time >= 0); + } + else { + start_time = ERTS_SINT64_MIN; + ASSERT(start_time < 0); + } + + goto do_dirty_schedule; + + context_switch: + c_p->arity = I[-1]; + c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + + { + int reds_used; + Eterm* argp; + int i; + + /* + * Make sure that there is enough room for the argument registers to be saved. + */ + if (c_p->arity > c_p->max_arg_reg) { + /* + * Yes, this is an expensive operation, but you only pay it the first + * time you call a function with more than 6 arguments which is + * scheduled out. This is better than paying for 26 words of wasted + * space for most processes which never call functions with more than + * 6 arguments. + */ + Uint size = c_p->arity * sizeof(c_p->arg_reg[0]); + if (c_p->arg_reg != c_p->def_arg_reg) { + c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG, + (void *) c_p->arg_reg, + size); + } else { + c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size); + } + c_p->max_arg_reg = c_p->arity; + } + + /* + * Save the argument registers and everything else. + */ + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + argp[i] = reg[i]; + } + SWAPOUT; + c_p->i = I; + + do_dirty_schedule: + + if (start_time < 0) { + /* + * Dirty I/O scheduler: + * One reduction consumed regardless of + * time spent in the dirty NIF. + */ + reds_used = esdp->virtual_reds + 1; + } + else { + /* + * Dirty CPU scheduler: + * Currently two reductions consumed per + * micro second spent in the dirty NIF. + */ + ErtsMonotonicTime time; + time = erts_get_monotonic_time(esdp); + time -= start_time; + time = ERTS_MONOTONIC_TO_USEC(time); + time *= (CONTEXT_REDS-1)/1000 + 1; + ASSERT(time >= 0); + if (time == 0) + time = 1; /* At least one reduction */ + time += esdp->virtual_reds; + reds_used = time > INT_MAX ? INT_MAX : (int) time; + } + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + c_p = erts_schedule(esdp, c_p, reds_used); + + if (start_time >= 0) { + start_time = erts_get_monotonic_time(esdp); + ASSERT(start_time >= 0); + } + } + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); +#ifdef DEBUG + pid = c_p->common.id; /* Save for debugging purposes */ +#endif + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + + ASSERT(!(c_p->flags & F_HIPE_MODE)); + ERTS_MSACC_UPDATE_CACHE_X(); + + reg = esdp->x_reg_array; + { + Eterm* argp; + int i; + + argp = c_p->arg_reg; + for (i = c_p->arity - 1; i >= 0; i--) { + reg[i] = argp[i]; + CHECK_TERM(reg[i]); + } + + /* + * We put the original reduction count in the process structure, to reduce + * the code size (referencing a field in a struct through a pointer stored + * in a register gives smaller code than referencing a global variable). + */ + + I = c_p->i; + + ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I); + + /* + * Set fcalls even though we ignore it, so we don't + * confuse code accessing it... + */ + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + c_p->fcalls = 0; + else + c_p->fcalls = CONTEXT_REDS; + + SWAPIN; + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_scheduled)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE); + dtrace_proc_str(c_p, process_buf); + + if (ERTS_PROC_IS_EXITING(c_p)) { + strcpy(fun_buf, "<exiting>"); + } else { + BeamInstr *fptr = find_function_from_pc(c_p->i); + if (fptr) { + dtrace_fun_decode(c_p, (Eterm)fptr[0], + (Eterm)fptr[1], (Uint)fptr[2], + NULL, fun_buf); + } else { + erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), + "<unknown/%p>", *I); + } + } + + DTRACE2(process_scheduled, process_buf, fun_buf); + } +#endif + } + + { +#ifdef DEBUG + Eterm result; +#endif + Eterm arity; + + { + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF + */ + BifFunction vbf; + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + + DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + c_p->current = I-3; /* current and vbf set to please handle_error */ + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + arity = I[-1]; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + { + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + ASSERT(!c_p->scheduler_data); + + erts_pre_dirty_nif(esdp, &env, c_p, + (struct erl_module_nif*)I[2], NULL); + +#ifdef DEBUG + result = +#else + (void) +#endif + (*fp)(&env, arity, reg); + + erts_post_nif(&env); + + ASSERT(!is_value(result)); + ASSERT(c_p->freason == TRAP); + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (env.exiting) + goto do_dirty_schedule; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + + DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + ERTS_HOLE_CHECK(c_p); + SWAPIN; + I = c_p->i; + goto context_switch; + } + } +#endif /* ERTS_DIRTY_SCHEDULERS */ +} + static BifFunction translate_gc_bif(void* gcf) { diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index fbe4724047..7e239d1f5d 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -123,7 +123,7 @@ struct AOFF_Carrier_t_ { AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ AOFF_RBTree_t* root; /* Root of my block tree */ }; -#define RBT_NODE_TO_MBC(PTR) ((AOFF_Carrier_t*)((char*)(PTR) - offsetof(AOFF_Carrier_t, rbt_node))) +#define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) /* To support carrier migration we keep two kinds of rb-trees: diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 74979f984a..074ac6d64e 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -95,7 +95,8 @@ /* * The following symbols can be manipulated to "tune" the linear hash array */ -#define CHAIN_LEN 6 /* Medium bucket chain len */ +#define GROW_LIMIT(NACTIVE) ((NACTIVE)*2) +#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) /* Number of slots per segment */ #define SEGSZ_EXP 8 @@ -463,7 +464,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) + if (nactive > SEGSZ && NITEMS(tb) < SHRINK_LIMIT(nactive) && !IS_FIXED(tb)) { shrink(tb, nactive); } @@ -670,8 +671,8 @@ int db_create_hash(Process *p, DbTable *tbl) tb->nsegs = NSEG_1; tb->nslots = SEGSZ; - erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP + erts_smp_atomic_init_nob(&tb->is_resizing, 0); if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; @@ -862,7 +863,7 @@ Lnew: WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) { + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nactive); } } @@ -2250,12 +2251,12 @@ static int db_free_table_continue_hash(DbTable *tbl) done /= 2; while(tb->nslots != 0) { - free_seg(tb, 1); + done += 1 + SEGSZ/64 + free_seg(tb, 1); /* * If we have done enough work, get out here. */ - if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) { + if (done >= DELETE_RECORD_LIMIT) { return 0; /* Not done */ } } @@ -2604,23 +2605,22 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1); - else { - if (erts_smp_atomic_read_nob(&tb->is_resizing)) - return 0; - erts_smp_atomic_set_nob(&tb->is_resizing, 1); - return 1; - } + return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); + else + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); +#endif + return 1; } static ERTS_INLINE void done_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->is_resizing, 0); - else - erts_smp_atomic_set_nob(&tb->is_resizing, 0); + erts_atomic_set_relb(&tb->is_resizing, 0); +#endif } /* Grow table with one new bucket. @@ -2871,7 +2871,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { grow(tb, nactive); } } diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e654363cd5..081ff8fafc 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -60,8 +60,8 @@ typedef struct db_table_hash { /* List of slots where elements have been deleted while table was fixed */ erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ erts_smp_atomic_t nactive; /* Number of "active" slots */ - erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ #ifdef ERTS_SMP + erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; #endif #ifdef VALGRIND diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index c7bbbd5ca0..d0d74bbf44 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1183,22 +1183,14 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, adjust_size = p->htop - p->heap; } - goto done; } + else if (need_after > HEAP_SIZE(p)) { + grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); + adjust_size = p->htop - p->heap; + } + /*else: The heap size turned out to be just right. We are done. */ - if (HEAP_SIZE(p) >= need_after) { - /* - * The heap size turned out to be just right. We are done. - */ - goto done; - } - - grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); - adjust_size = p->htop - p->heap; - - done: ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - ASSERT(MBUF(p) == NULL); /* The heap usage during GC should be larger than what we end up after a GC, even if we grow it. If this assertion is not true @@ -1591,6 +1583,9 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, HIGH_WATER(p) = HEAP_TOP(p); +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p); +#endif remove_message_buffers(p); if (p->flags & F_ON_HEAP_MSGQ) @@ -1603,9 +1598,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, adjusted = adjust_after_fullsweep(p, need, objv, nobj); -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif ErtsGcQuickSanityCheck(p); return gc_cost(size_after, adjusted ? size_after : 0); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index ac7b9d6606..71ab92937d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -1345,7 +1345,7 @@ erts_prep_msgq_for_inspection(Process *c_p, Process *rp, mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; - ASSERT((*mpp)->next == bad_mp); + ASSERT(*mpp == bad_mp); erts_msgq_update_internal_pointers(&rp->msg, mpp, &bad_mp->next); @@ -1725,7 +1725,7 @@ void erts_factory_trim_and_close(ErtsHeapFactory* factory, case FACTORY_MESSAGE: { ErtsMessage *mp = factory->message; if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) { - if (!mp->hfrag.next) { + if (!factory->heap_frags) { Uint sz = factory->hp - factory->hp_start; mp = erts_shrink_message(mp, sz, brefs, brefs_size); factory->message = mp; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 2bbb8e3c91..4fd82bad10 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -178,9 +178,6 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, size_t may_need) void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, Process* tracee) { -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerData *esdp; -#endif env->mod_nif = mod_nif; env->proc = p; env->hp = HEAP_TOP(p); @@ -193,57 +190,65 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, ASSERT(p->common.id != ERTS_INVALID_PID); -#ifdef ERTS_DIRTY_SCHEDULERS - esdp = erts_get_scheduler_data(); - ASSERT(esdp); +#if defined(DEBUG) && defined(ERTS_DIRTY_SCHEDULERS) + { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { -#ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - ASSERT(p->scheduler_data == esdp); - ASSERT((state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS)) - && !(state & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + ASSERT(p->scheduler_data == esdp); + ASSERT((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS)) + && !(state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))); + } + } #endif +} - } - else { - Process *sproc; +void erts_pre_dirty_nif(ErtsSchedulerData *esdp, + ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, + Process* tracee) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Process *sproc; #ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - ASSERT(!p->scheduler_data); - ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) - && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(!p->scheduler_data); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); #endif - sproc = esdp->dirty_shadow_process; - ASSERT(sproc); - ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(erts_smp_atomic32_read_nob(&sproc->state) - == (ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_PROXY)); - - sproc->next = p; - sproc->common.id = p->common.id; - sproc->htop = p->htop; - sproc->stop = p->stop; - sproc->hend = p->hend; - sproc->heap = p->heap; - sproc->abandoned_heap = p->abandoned_heap; - sproc->heap_sz = p->heap_sz; - sproc->high_water = p->high_water; - sproc->old_hend = p->old_hend; - sproc->old_htop = p->old_htop; - sproc->old_heap = p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); - env->proc = sproc; - } + erts_pre_nif(env, p, mod_nif, tracee); + + sproc = esdp->dirty_shadow_process; + ASSERT(sproc); + ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + == (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + + sproc->next = p; + sproc->common.id = p->common.id; + sproc->htop = p->htop; + sproc->stop = p->stop; + sproc->hend = p->hend; + sproc->heap = p->heap; + sproc->abandoned_heap = p->abandoned_heap; + sproc->heap_sz = p->heap_sz; + sproc->high_water = p->high_water; + sproc->old_hend = p->old_hend; + sproc->old_htop = p->old_htop; + sproc->old_heap = p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); + env->proc = sproc; #endif } @@ -798,19 +803,13 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; Eterm* hp; -#ifdef SHCOPY - erts_shcopy_t info; - INITIALIZE_SHCOPY(info); - sz = copy_shared_calculate(src_term, &info); - hp = alloc_heap(dst_env, sz); - src_term = copy_shared_perform(src_term, sz, &info, &hp, &MSO(dst_env->proc)); - DESTROY_SHCOPY(info); - return src_term; -#else + /* + * No preserved sharing allowed as long as literals are also preserved. + * Process independent environment can not be reached by purge. + */ sz = size_object(src_term); hp = alloc_heap(dst_env, sz); return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); -#endif } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f8cbe60e76..c0b1d7246c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8197,7 +8197,7 @@ sched_dirty_cpu_thread_func(void *vesdp) #endif erts_thread_init_float(); - process_main(); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Dirty CPU scheduler thread number %beu terminated\n", @@ -8242,7 +8242,7 @@ sched_dirty_io_thread_func(void *vesdp) #endif erts_thread_init_float(); - process_main(); + erts_dirty_process_main(esdp); /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Dirty I/O scheduler thread number %beu terminated\n", @@ -9377,77 +9377,6 @@ scheduler_gc_proc(Process *c_p, int reds_left) return reds; } -static ERTS_INLINE void -clean_dirty_start(Process *p) -{ -#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64) - void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL); - if (ptr) - erts_free(ERTS_ALC_T_DIRTY_START, ptr); -#endif -} - -static ERTS_INLINE void -save_dirty_start(ErtsSchedulerData *esdp, Process *c_p) -{ -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - ErtsMonotonicTime time = erts_get_monotonic_time(esdp); -#ifdef ARCH_64 - ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time); -#else - ErtsMonotonicTime *stimep; - - stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); - if (!stimep) { - stimep = erts_alloc(ERTS_ALC_T_DIRTY_START, - sizeof(ErtsMonotonicTime)); - ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep); - } - *stimep = time; -#endif - } -#endif -} - -static ERTS_INLINE int -get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p) -{ - -#ifndef ERTS_DIRTY_SCHEDULERS - return -1; -#else - ErtsMonotonicTime stime, time; - - if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) - return 1; - -#ifdef ARCH_64 - stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p); -#else - { - ErtsMonotonicTime *stimep; - stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); - ASSERT(stimep); - stime = *stimep; - } -#endif - - time = erts_get_monotonic_time(esdp); - - ASSERT(stime && stime < time); - - time -= stime; - time = ERTS_MONOTONIC_TO_USEC(time); - time *= 2; - - if (time > INT_MAX) - return INT_MAX; - return (int) time; -#endif - -} - /* * schedule() is called from BEAM (process_main()) or HiPE * (hipe_mode_switch()) when the current process is to be @@ -9466,11 +9395,10 @@ get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p) * so that normal processes get to run more frequently. */ -Process *schedule(Process *p, int calls) +Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) { Process *proxy_p = NULL; ErtsRunQueue *rq; - ErtsSchedulerData *esdp; int context_reds; int fcalls; int input_reductions; @@ -9507,8 +9435,19 @@ Process *schedule(Process *p, int calls) * Clean up after the process being scheduled out. */ if (!p) { /* NULL in the very first schedule() call */ +#ifdef ERTS_DIRTY_SCHEDULERS + is_normal_sched = !esdp; + if (is_normal_sched) { + esdp = erts_get_scheduler_data(); + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + } + else { + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + } +#else esdp = erts_get_scheduler_data(); - is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp); + is_normal_sched = 1; +#endif rq = erts_get_runq_current(esdp); ASSERT(esdp); fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); @@ -9517,12 +9456,12 @@ Process *schedule(Process *p, int calls) } else { #ifdef ERTS_SMP #ifdef ERTS_DIRTY_SCHEDULERS - esdp = p->scheduler_data; - is_normal_sched = esdp != NULL; - if (is_normal_sched) + is_normal_sched = !esdp; + if (is_normal_sched) { + esdp = p->scheduler_data; ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + } else { - esdp = erts_get_scheduler_data(); ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } #else @@ -9541,10 +9480,7 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - if (is_normal_sched) - reds = actual_reds = calls - esdp->virtual_reds; - else - reds = actual_reds = get_dirty_reds(esdp, p); + reds = actual_reds = calls - esdp->virtual_reds; ASSERT(actual_reds >= 0); if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST) @@ -9994,17 +9930,10 @@ Process *schedule(Process *p, int calls) calls = 0; reds = context_reds; -#ifdef ERTS_SMP - erts_smp_runq_unlock(rq); -#endif /* ERTS_SMP */ - } - if (!is_normal_sched) - save_dirty_start(esdp, p); - #ifdef ERTS_SMP if (flags & ERTS_RUNQ_FLG_PROTECTED) @@ -11745,8 +11674,6 @@ delete_process(Process* p) if (nif_export) erts_destroy_nif_export(nif_export); - clean_dirty_start(p); - /* Cleanup psd */ psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index b44ac442aa..7c98b60647 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -810,25 +810,13 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_DELAYED_GC_TASK_QS 4 #define ERTS_PSD_NIF_TRAP_EXPORT 5 #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6 -#define ERTS_PSD_DIRTY_CPU_START 7 -#define ERTS_PSD_SIZE 8 +#define ERTS_PSD_SIZE 7 -#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS) +#if !defined(HIPE) # undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF -# undef ERTS_PSD_DIRTY_CPU_START # undef ERTS_PSD_SIZE # define ERTS_PSD_SIZE 6 -#elif !defined(HIPE) -# undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF -# undef ERTS_PSD_DIRTY_CPU_START -# undef ERTS_PSD_SIZE -# define ERTS_PSD_DIRTY_CPU_START 6 -# define ERTS_PSD_SIZE 7 -#elif !defined(ERTS_DIRTY_SCHEDULERS) -# undef ERTS_PSD_DIRTY_CPU_START -# undef ERTS_PSD_SIZE -# define ERTS_PSD_SIZE 7 #endif typedef struct { @@ -1179,6 +1167,9 @@ void erts_check_for_holes(Process* p); * USR_PRIO -> User prio. i.e., prio the user has set. * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently * enqueued in. + * + * Update etp-proc-state-int in $ERL_TOP/erts/etc/unix/etp-commands.in + * when changing ERTS_PSFLG_*. */ #define ERTS_PSFLGS_ACT_PRIO_MASK \ (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET) @@ -1831,7 +1822,7 @@ Eterm erts_get_schedulers_binds(Process *c_p); Eterm erts_set_cpu_topology(Process *c_p, Eterm term); Eterm erts_bind_schedulers(Process *c_p, Eterm how); ErtsRunQueue *erts_schedid2runq(Uint); -Process *schedule(Process*, int); +Process *erts_schedule(ErtsSchedulerData *, Process*, int); void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); void erts_do_exit_process(Process*, Eterm); @@ -2061,13 +2052,6 @@ erts_psd_set(Process *p, int ix, void *data) ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB))) #endif -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PROC_GET_DIRTY_CPU_START(P) \ - ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START)) -#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \ - ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS))) -#endif - ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler); diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 02c24557c1..2a19211987 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -31,7 +31,7 @@ #define EXPORT_INITIAL_SIZE 4000 #define EXPORT_LIMIT (512*1024) -#define EXPORT_HASH(m,f,a) ((m)*(f)+(a)) +#define EXPORT_HASH(m,f,a) ((atom_val(m) * atom_val(f)) ^ (a)) #ifdef DEBUG # define IF_DEBUG(x) x @@ -79,8 +79,7 @@ struct export_templ static struct export_blob* entry_to_blob(struct export_entry* ee) { - return (struct export_blob*) - ((char*)ee->ep - offsetof(struct export_blob,exp)); + return ErtsContainerStruct(ee->ep, struct export_blob, exp); } void diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index b76b9cd874..f3d4ac56cd 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -62,6 +62,9 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); +extern void erts_pre_dirty_nif(ErtsSchedulerData *, + struct enif_environment_t*, Process*, + struct erl_module_nif*, Process* tracee); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -1152,6 +1155,7 @@ void print_pass_through(int, byte*, int); int catchlevel(Process*); void init_emulator(void); void process_main(void); +void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index f303d4f167..9a205d50d3 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -97,7 +97,7 @@ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define ErtsContainerStruct(ptr, type, member) \ - (type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member)) + ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))) #if defined (__WIN32__) # include "erl_win_sys.h" diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index f0418446a8..675fafa726 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -2210,7 +2210,9 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, #ifndef ERTS_SMP #ifdef USE_THREADS - if (erts_get_scheduler_data()) /* Must be scheduler thread */ + if (!erts_get_scheduler_data()) /* Must be scheduler thread */ + *p = NULL; + else #endif { *p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); @@ -2226,18 +2228,10 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, } /* So we have an error logger, lets build the message */ - if (sz <= HeapWordsLeft(*p)) { - *ohp = &MSO(*p); - *hp = HEAP_TOP(*p); - HEAP_TOP(*p) += sz; - } else { -#endif - *bp = new_message_buffer(sz); - *ohp = &(*bp)->off_heap; - *hp = (*bp)->mem; -#ifndef ERTS_SMP - } #endif + *bp = new_message_buffer(sz); + *ohp = &(*bp)->off_heap; + *hp = (*bp)->mem; return (is_nil(gleader) ? am_noproc diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index e87d141ddb..06dfb2dd10 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -34,6 +34,7 @@ #include <stdio.h> #include <stdlib.h> +#include <stddef.h> #include <ctype.h> #include <sys/types.h> #include <errno.h> @@ -58,6 +59,9 @@ #ifdef HAVE_NETPACKET_PACKET_H #include <netpacket/packet.h> #endif +#ifdef HAVE_SYS_UN_H +#include <sys/un.h> +#endif /* All platforms fail on malloc errors. */ #define FATAL_MALLOC @@ -573,7 +577,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #ifdef HAVE_SOCKLEN_T # define SOCKLEN_T socklen_t #else -# define SOCKLEN_T int +# define SOCKLEN_T size_t #endif #include "packet_parser.h" @@ -587,6 +591,22 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) (((unsigned char*) (s))[1] << 8) | \ (((unsigned char*) (s))[0])) +/* strnlen doesn't exist everywhere */ +static size_t my_strnlen(const char *s, size_t maxlen) +{ + size_t i = 0; + while (i < maxlen && s[i] != '\0') + i++; + return i; +} + +/* Check that some character in the buffer != '\0' */ +static int is_nonzero(const char *s, size_t n) +{ + size_t i; + for (i = 0; i < n; i++) if (s[i] != '\0') return !0; + return 0; +} #ifdef VALGRIND # include <valgrind/memcheck.h> @@ -611,6 +631,8 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INET_AF_INET6 2 #define INET_AF_ANY 3 /* INADDR_ANY or IN6ADDR_ANY_INIT */ #define INET_AF_LOOPBACK 4 /* INADDR_LOOPBACK or IN6ADDR_LOOPBACK_INIT */ +#define INET_AF_LOCAL 5 +#define INET_AF_UNDEFINED 6 /* Unknown */ /* open and INET_REQ_GETTYPE enumeration */ #define INET_TYPE_STREAM 1 @@ -896,19 +918,35 @@ typedef union { #ifdef HAVE_IN6 struct sockaddr_in6 sai6; #endif +#ifdef HAVE_SYS_UN_H + struct sockaddr_un sal; +#endif } inet_address; -/* for AF_INET & AF_INET6 */ -#define inet_address_port(x) ((x)->sai.sin_port) +#define inet_address_port(x) \ + ((((x)->sai.sin_family == AF_INET) || \ + ((x)->sai.sin_family == AF_INET6)) ? \ + ((x)->sai.sin_port) : -1) + +#ifdef HAVE_SYS_UN_H +#define localaddrlen(data) \ + ((((unsigned char*)(data))[0] == INET_AF_LOCAL) ? \ + (1 + 1 + ((unsigned char*)(data))[1]) : 1) +#else +#define localaddrlen(data) (1) +#endif #if defined(HAVE_IN6) && defined(AF_INET6) -#define addrlen(family) \ - ((family == AF_INET) ? sizeof(struct in_addr) : \ - ((family == AF_INET6) ? sizeof(struct in6_addr) : 0)) +#define addrlen(data) \ + ((((unsigned char*)(data))[0] == INET_AF_INET) ? \ + (1 + 2 + 4) : \ + ((((unsigned char*)(data))[0] == INET_AF_INET6) ? \ + (1 + 2 + 16) : localaddrlen(data))) #else -#define addrlen(family) \ - ((family == AF_INET) ? sizeof(struct in_addr) : 0) +#define addrlen(data) \ + ((((unsigned char*)(data))[0] == INET_AF_INET) ? \ + (1 + 2 + 4) : localaddrlen(data)) #endif typedef struct _multi_timer_data { @@ -999,8 +1037,10 @@ typedef struct { inet_address peer_addr; /* fake peer address */ inet_address name_addr; /* fake local address */ - inet_address* peer_ptr; /* fake peername or NULL */ - inet_address* name_ptr; /* fake sockname or NULL */ + inet_address* peer_ptr; /* fake peername or NULL */ + inet_address* name_ptr; /* fake sockname or NULL */ + SOCKLEN_T peer_addr_len; /* fake peername size */ + SOCKLEN_T name_addr_len; /* fake sockname size */ int bufsz; /* minimum buffer constraint */ unsigned int hsz; /* the list header size, -1 is large !!! */ @@ -1246,6 +1286,7 @@ static int async_ref = 0; /* async reference id generator */ } while (0) static ErlDrvTermData am_ok; +static ErlDrvTermData am_undefined; static ErlDrvTermData am_tcp; static ErlDrvTermData am_error; static ErlDrvTermData am_einval; @@ -1262,6 +1303,7 @@ static ErlDrvTermData am_ssl_tls; static ErlDrvTermData am_udp; static ErlDrvTermData am_udp_passive; static ErlDrvTermData am_udp_error; +static ErlDrvTermData am_local; #endif #ifdef HAVE_SCTP static ErlDrvTermData am_sctp; @@ -1471,40 +1513,59 @@ static void *realloc_wrapper(void *current, ErlDrvSizeT size){ #endif #ifdef HAVE_UDP -static int load_ip_port(ErlDrvTermData* spec, int i, char* buf) -{ - spec[i++] = ERL_DRV_INT; - spec[i++] = (ErlDrvTermData) get_int16(buf); - return i; -} - -static int load_ip_address(ErlDrvTermData* spec, int i, int family, char* buf) +static int load_address(ErlDrvTermData* spec, int i, char* buf) { int n; - if (family == AF_INET) { - for (n = 0; n < 4; n++) { + switch (*buf++) { /* Family */ + case INET_AF_INET: { + for (n = 2; n < 2+4; n++) { spec[i++] = ERL_DRV_INT; spec[i++] = (ErlDrvTermData) ((unsigned char)buf[n]); } spec[i++] = ERL_DRV_TUPLE; spec[i++] = 4; + spec[i++] = ERL_DRV_INT; + spec[i++] = (ErlDrvTermData) get_int16(buf); + break; } #if defined(HAVE_IN6) && defined(AF_INET6) - else if (family == AF_INET6) { - for (n = 0; n < 16; n += 2) { + case INET_AF_INET6: { + for (n = 2; n < 2+16; n += 2) { spec[i++] = ERL_DRV_INT; spec[i++] = (ErlDrvTermData) get_int16(buf+n); } spec[i++] = ERL_DRV_TUPLE; spec[i++] = 8; + spec[i++] = ERL_DRV_INT; + spec[i++] = (ErlDrvTermData) get_int16(buf); + break; } #endif - else { +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: { + int len = *(unsigned char*)buf++; + i = LOAD_ATOM(spec, i, am_local); + i = LOAD_BUF2BINARY(spec, i, buf, len); + spec[i++] = ERL_DRV_TUPLE; + spec[i++] = 2; + spec[i++] = ERL_DRV_INT; + spec[i++] = 0; + break; + } +#endif + default: { /* INET_AF_UNDEFINED */ + i = LOAD_ATOM(spec, i, am_undefined); + spec[i++] = ERL_DRV_INT; + spec[i++] = 0; spec[i++] = ERL_DRV_TUPLE; + spec[i++] = 2; + spec[i++] = ERL_DRV_INT; spec[i++] = 0; + break; + } } return i; -} + } #endif @@ -1512,10 +1573,13 @@ static int load_ip_address(ErlDrvTermData* spec, int i, int family, char* buf) /* For SCTP, we often need to return {IP, Port} tuples: */ static int inet_get_address(char* dst, inet_address* src, unsigned int* len); -#define LOAD_IP_AND_PORT_CNT \ +/* Max of {{int()*8},int()} | {{int()*4},int()} | + * {{'local',binary()},int()} + */ +#define LOAD_INET_GET_ADDRESS_CNT \ (8*LOAD_INT_CNT + LOAD_TUPLE_CNT + LOAD_INT_CNT + LOAD_TUPLE_CNT) -static int load_ip_and_port +static int load_inet_get_address (ErlDrvTermData* spec, int i, inet_descriptor* desc, struct sockaddr_storage* addr) { @@ -1533,8 +1597,7 @@ static int load_ip_and_port /* NB: the following functions are safe to use, as they create tuples of copied Ints on the "spec", and do not install any String pts -- a ptr to "abuf" would be dangling upon exiting this function: */ - i = load_ip_address(spec, i, desc->sfamily, abuf+3); - i = load_ip_port (spec, i, abuf+1); + i = load_address(spec, i, abuf); /* IP,Port | Family,Addr */ i = LOAD_TUPLE (spec, i, 2); return i; } @@ -2226,7 +2289,6 @@ static ErlDrvTermData am_http_error; static ErlDrvTermData am_abs_path; static ErlDrvTermData am_absoluteURI; static ErlDrvTermData am_star; -static ErlDrvTermData am_undefined; static ErlDrvTermData am_http; static ErlDrvTermData am_https; static ErlDrvTermData am_scheme; @@ -2917,7 +2979,7 @@ static int sctp_parse_async_event ASSERT(sptr->spc_length <= sz); /* No buffer overrun */ i = LOAD_ATOM (spec, i, am_sctp_paddr_change); - i = load_ip_and_port(spec, i, desc, &sptr->spc_aaddr); + i = load_inet_get_address(spec, i, desc, &sptr->spc_aaddr); switch (sptr->spc_state) { @@ -3344,10 +3406,11 @@ static int tcp_error_message(tcp_descriptor* desc, int err) #ifdef HAVE_UDP /* ** active mode message: -** {udp, S, IP, Port, [H1,...Hsz | Data]} or -** {sctp, S, IP, Port, {[AncilData], Event_or_Data}} +** {udp, S, IP, Port, [H1,...Hsz | Data]} or +** {sctp, S, IP, Port, {[AncilData], Event_or_Data}} ** where ** [H1,...,HSz] are msg headers (without IP/Port, UDP only), +** [AddrLen, H2,...,HSz] are msg headers for UDP AF_UNIX only ** Data : List() | Binary() */ static int packet_binary_message @@ -3357,6 +3420,7 @@ static int packet_binary_message ErlDrvTermData spec [PACKET_ERL_DRV_TERM_DATA_LEN]; int i = 0; int alen; + char* data = bin->orig_bytes+offs; DEBUGF(("packet_binary_message(%ld): len = %d\r\n", (long)desc->port, len)); @@ -3366,13 +3430,12 @@ static int packet_binary_message i = LOAD_ATOM(spec, i, am_udp ); /* UDP only */ # endif i = LOAD_PORT(spec, i, desc->dport); /* S */ - - alen = addrlen(desc->sfamily); - i = load_ip_address(spec, i, desc->sfamily, bin->orig_bytes+offs+3); - i = load_ip_port(spec, i, bin->orig_bytes+offs+1); /* IP, Port */ - - offs += (alen + 3); - len -= (alen + 3); + + alen = addrlen(data); + i = load_address(spec, i, data); /* IP,Port | Family,Addr */ + + offs += alen; + len -= alen; # ifdef HAVE_SCTP if (!IS_SCTP(desc)) @@ -3759,6 +3822,7 @@ static int inet_init() # endif INIT_ATOM(ok); + INIT_ATOM(undefined); INIT_ATOM(tcp); #ifdef HAVE_UDP INIT_ATOM(udp); @@ -3775,6 +3839,7 @@ static int inet_init() #ifdef HAVE_UDP INIT_ATOM(udp_passive); INIT_ATOM(udp_error); + INIT_ATOM(local); #endif INIT_ATOM(empty_out_q); INIT_ATOM(ssl_tls); @@ -3787,7 +3852,6 @@ static int inet_init() INIT_ATOM(abs_path); INIT_ATOM(absoluteURI); am_star = driver_mk_atom("*"); - INIT_ATOM(undefined); INIT_ATOM(http); INIT_ATOM(https); INIT_ATOM(scheme); @@ -3883,7 +3947,7 @@ static int inet_init() /* -** Set a inaddr structure: +** Set an inaddr structure: ** src = [P1,P0,X1,X2,.....] ** dst points to a structure large enugh to keep any kind ** of inaddr. @@ -3923,6 +3987,18 @@ static char* inet_set_address(int family, inet_address* dst, return src + 2+16; } #endif +#ifdef HAVE_SYS_UN_H + else if ((family == AF_UNIX) && (*len >= 1)) { + int n = *((unsigned char*)src); + if ((*len < 1+n) || (sizeof(dst->sal.sun_path) < n+1)) + return NULL; + sys_memzero((char*)dst, sizeof(struct sockaddr_un)); + dst->sal.sun_family = family; + sys_memcpy(dst->sal.sun_path, src+1, n); + *len = offsetof(struct sockaddr_un, sun_path) + n; + return src + 1 + n; + } +#endif return NULL; } @@ -3931,7 +4007,7 @@ static char* inet_set_address(int family, inet_address* dst, ** or from argument if source data specifies constant address. ** ** src = [TAG,P1,P0] when TAG = INET_AF_ANY | INET_AF_LOOPBACK -** src = [TAG,P1,P0,X1,X2,...] when TAG = INET_AF_INET | INET_AF_INET6 +** src = [TAG,P1,P0,X1,X2,...] when TAG = INET_AF_INET | INET_AF_INET6 | INET_AF_LOCAL */ static char *inet_set_faddress(int family, inet_address* dst, char *src, ErlDrvSizeT* len) { @@ -3949,6 +4025,12 @@ static char *inet_set_faddress(int family, inet_address* dst, family = AF_INET6; break; # endif +# ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: { + family = AF_UNIX; + break; + } +# endif case INET_AF_ANY: case INET_AF_LOOPBACK: { int port; @@ -4012,7 +4094,6 @@ static char *inet_set_faddress(int family, inet_address* dst, return inet_set_address(family, dst, src, len); } - /* Get a inaddr structure ** src = inaddr structure ** *len is the lenght of structure @@ -4023,6 +4104,7 @@ static char *inet_set_faddress(int family, inet_address* dst, */ static int inet_get_address(char* dst, inet_address* src, unsigned int* len) { + /* Compare the code with inet_address_to_erlang() */ int family; short port; @@ -4045,6 +4127,32 @@ static int inet_get_address(char* dst, inet_address* src, unsigned int* len) return 0; } #endif +#ifdef HAVE_SYS_UN_H + else if (family == AF_UNIX) { + size_t n, m; + if (*len < offsetof(struct sockaddr_un, sun_path)) return -1; + n = *len - offsetof(struct sockaddr_un, sun_path); + if (255 < n) return -1; + /* Portability fix: Assume that the address is a zero terminated + * string, except when the first byte is \0 i.e the + * string length is 0. Then use the reported length instead. + * This fix handles Linux's abstract socket address + * nonportable extension. + */ + m = my_strnlen(src->sal.sun_path, n); + if ((m == 0) && is_nonzero(src->sal.sun_path, n)) + m = n; + dst[0] = INET_AF_LOCAL; + dst[1] = (char) ((unsigned char) m); + sys_memcpy(dst+2, src->sal.sun_path, m); + *len = 1 + 1 + m; + return 0; + } +#endif + else { + dst[0] = INET_AF_UNDEFINED; + *len = 1; + } return -1; } @@ -4053,7 +4161,9 @@ static int inet_get_address(char* dst, inet_address* src, unsigned int* len) ** according to the size of the current, ** and return the resulting encoded size */ -static int inet_address_to_erlang(char *dst, inet_address **src) { +static int +inet_address_to_erlang(char *dst, inet_address **src, SOCKLEN_T sz) { + /* Compare the code with inet_get_address() */ short port; switch ((*src)->sa.sa_family) { @@ -4078,6 +4188,30 @@ static int inet_address_to_erlang(char *dst, inet_address **src) { (*src) = (inet_address *) (&(*src)->sai6 + 1); return 1 + 2 + 16; #endif +#ifdef HAVE_SYS_UN_H + case AF_UNIX: { + size_t n, m; + if (sz < offsetof(struct sockaddr_un, sun_path)) return -1; + n = sz - offsetof(struct sockaddr_un, sun_path); + if (255 < n) return -1; + /* Portability fix: Assume that the address is a zero terminated + * string, except when the first byte is \0 i.e the + * string length is 0. Then use the reported length instead. + * This fix handles Linux's abstract socket address + * nonportable extension. + */ + m = my_strnlen((*src)->sal.sun_path, n); + if ((m == 0) && is_nonzero((*src)->sal.sun_path, n)) + m = n; + if (dst) { + dst[0] = INET_AF_LOCAL; + dst[1] = (char) ((unsigned char) m); + sys_memcpy(dst+2, (*src)->sal.sun_path, m); + } + (*src) = (inet_address *) (&(*src)->sal + 1); + return 1 + 1 + m; + } +#endif default: return -1; } @@ -4086,7 +4220,7 @@ static int inet_address_to_erlang(char *dst, inet_address **src) { /* Encode n encoded addresses from addrs in the result buffer */ static ErlDrvSizeT reply_inet_addrs -(int n, inet_address *addrs, char **rbuf, ErlDrvSizeT rsize) { +(int n, inet_address *addrs, char **rbuf, ErlDrvSizeT rsize, SOCKLEN_T sz) { inet_address *ia; int i, s; ErlDrvSizeT rlen; @@ -4094,11 +4228,19 @@ static ErlDrvSizeT reply_inet_addrs if (IS_SOCKET_ERROR(n)) return ctl_error(sock_errno(), rbuf, rsize); if (n == 0) return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); + /* The sz argument is only used when we have got an actual size + * of addrs[0] from e.g getsockname() and then n == 1 + * so we will loop over 1 element below. Otherwise sz + * would be expected to differ between addresses but that + * can only happen for AF_UNIX and we will only be called with + * n > 1 for SCTP and that will never (?) happen with AF_UNIX + */ + /* Calculate result length */ rlen = 1; ia = addrs; for (i = 0; i < n; i++) { - s = inet_address_to_erlang(NULL, &ia); + s = inet_address_to_erlang(NULL, &ia, sz); if (s < 0) break; rlen += s; } @@ -4109,7 +4251,7 @@ static ErlDrvSizeT reply_inet_addrs rlen = 1; ia = addrs; for (i = 0; i < n; i++) { - s = inet_address_to_erlang((*rbuf)+rlen, &ia); + s = inet_address_to_erlang((*rbuf)+rlen, &ia, sz); if (s < 0) break; rlen += s; } @@ -4180,6 +4322,7 @@ static ErlDrvSSizeT inet_ctl_open(inet_descriptor* desc, int domain, int type, char** rbuf, ErlDrvSizeT rsize) { int save_errno; + int protocol; #ifdef HAVE_SETNS int current_ns, new_ns; current_ns = new_ns = 0; @@ -4218,7 +4361,11 @@ static ErlDrvSSizeT inet_ctl_open(inet_descriptor* desc, int domain, int type, } } #endif - if ((desc->s = sock_open(domain, type, desc->sprotocol)) == INVALID_SOCKET) + protocol = desc->sprotocol; +#ifdef HAVE_SYS_UN_H + if (domain == AF_UNIX) protocol = 0; +#endif + if ((desc->s = sock_open(domain, type, protocol)) == INVALID_SOCKET) save_errno = sock_errno(); #ifdef HAVE_SETNS if (desc->netns != NULL) { @@ -7119,14 +7266,14 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, #ifdef HAVE_SCTP #define LOAD_PADDRINFO_CNT \ - (2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + LOAD_IP_AND_PORT_CNT + \ + (2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + LOAD_INET_GET_ADDRESS_CNT + \ 4*LOAD_INT_CNT + LOAD_TUPLE_CNT) static int load_paddrinfo (ErlDrvTermData * spec, int i, inet_descriptor* desc, struct sctp_paddrinfo* pai) { i = LOAD_ATOM (spec, i, am_sctp_paddrinfo); i = LOAD_ASSOC_ID (spec, i, pai->spinfo_assoc_id); - i = load_ip_and_port(spec, i, desc, &pai->spinfo_address); + i = load_inet_get_address(spec, i, desc, &pai->spinfo_address); switch(pai->spinfo_state) { case SCTP_ACTIVE: @@ -7542,7 +7689,7 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, /* Fill in the response: */ PLACE_FOR(spec, i, 2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + - LOAD_IP_AND_PORT_CNT + 2*LOAD_TUPLE_CNT); + LOAD_INET_GET_ADDRESS_CNT + 2*LOAD_TUPLE_CNT); switch (eopt) { case SCTP_OPT_PRIMARY_ADDR: i = LOAD_ATOM(spec, i, am_sctp_primary_addr); @@ -7556,7 +7703,7 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, ASSERT(0); } i = LOAD_ASSOC_ID (spec, i, sp.sspp_assoc_id); - i = load_ip_and_port(spec, i, desc, &sp.sspp_addr); + i = load_inet_get_address(spec, i, desc, &sp.sspp_addr); i = LOAD_TUPLE (spec, i, 3); i = LOAD_TUPLE (spec, i, 2); break; @@ -7603,11 +7750,11 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, /* Fill in the response: */ PLACE_FOR(spec, i, 2*LOAD_ATOM_CNT + LOAD_ASSOC_ID_CNT + - LOAD_IP_AND_PORT_CNT + 4*LOAD_INT_CNT); + LOAD_INET_GET_ADDRESS_CNT + 4*LOAD_INT_CNT); i = LOAD_ATOM (spec, i, am_sctp_peer_addr_params); i = LOAD_ATOM (spec, i, am_sctp_paddrparams); i = LOAD_ASSOC_ID (spec, i, ap.spp_assoc_id); - i = load_ip_and_port(spec, i, desc, &ap.spp_address); + i = load_inet_get_address(spec, i, desc, &ap.spp_address); i = LOAD_INT (spec, i, ap.spp_hbinterval); i = LOAD_INT (spec, i, ap.spp_pathmaxrxt); @@ -8248,6 +8395,11 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, put_int32(INET_AF_INET6, &tbuf[0]); } #endif +#ifdef HAVE_SYS_UN_H + else if (desc->sfamily == AF_UNIX) { + put_int32(INET_AF_LOCAL, &tbuf[0]); + } +#endif else return ctl_error(EINVAL, rbuf, rsize); @@ -8307,19 +8459,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, assoc_id = get_int32(buf); n = p_sctp_getpaddrs(desc->s, assoc_id, &sa); - rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize); + rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize, 0); if (n > 0) p_sctp_freepaddrs(sa); return rlen; } #endif { /* Fallback to sock_peer */ inet_address addr; - unsigned int sz; + SOCKLEN_T sz; int i; sz = sizeof(addr); i = sock_peer(desc->s, (struct sockaddr *) &addr, &sz); - return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize); + return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize, sz); } } @@ -8327,15 +8479,21 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, char tbuf[sizeof(inet_address)]; inet_address peer; inet_address* ptr; - unsigned int sz = sizeof(peer); + unsigned int sz; DEBUGF(("inet_ctl(%ld): PEER\r\n", (long)desc->port)); if (!(desc->state & INET_F_ACTIVE)) return ctl_error(ENOTCONN, rbuf, rsize); - if ((ptr = desc->peer_ptr) == NULL) { + if ((ptr = desc->peer_ptr) != NULL) { + sz = desc->peer_addr_len; + } + else { ptr = &peer; - if (IS_SOCKET_ERROR(sock_peer(desc->s, (struct sockaddr*)ptr,&sz))) + sz = sizeof(peer); + if (IS_SOCKET_ERROR + (sock_peer + (desc->s, (struct sockaddr*)ptr, &sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(tbuf, ptr, &sz) < 0) @@ -8350,11 +8508,12 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, } else if (len < 2) return ctl_error(EINVAL, rbuf, rsize); - else if (inet_set_address(desc->sfamily, &desc->peer_addr, - buf, &len) == NULL) + else if (inet_set_faddress + (desc->sfamily, &desc->peer_addr, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); else { desc->peer_ptr = &desc->peer_addr; + desc->peer_addr_len = (SOCKLEN_T) len; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } } @@ -8376,19 +8535,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, assoc_id = get_int32(buf); n = p_sctp_getladdrs(desc->s, assoc_id, &sa); - rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize); + rlen = reply_inet_addrs(n, (inet_address *) sa, rbuf, rsize, 0); if (n > 0) p_sctp_freeladdrs(sa); return rlen; } #endif { /* Fallback to sock_name */ inet_address addr; - unsigned int sz; + SOCKLEN_T sz; int i; sz = sizeof(addr); i = sock_name(desc->s, (struct sockaddr *) &addr, &sz); - return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize); + return reply_inet_addrs(i >= 0 ? 1 : i, &addr, rbuf, rsize, sz); } } @@ -8396,16 +8555,21 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, char tbuf[sizeof(inet_address)]; inet_address name; inet_address* ptr; - unsigned int sz = sizeof(name); + unsigned int sz; DEBUGF(("inet_ctl(%ld): NAME\r\n", (long)desc->port)); if (!IS_BOUND(desc)) return ctl_error(EINVAL, rbuf, rsize); /* address is not valid */ - if ((ptr = desc->name_ptr) == NULL) { + if ((ptr = desc->name_ptr) != NULL) { + sz = desc->name_addr_len; + } + else { ptr = &name; - if (IS_SOCKET_ERROR(sock_name(desc->s, (struct sockaddr*)ptr, &sz))) + sz = sizeof(name); + if (IS_SOCKET_ERROR + (sock_name(desc->s, (struct sockaddr*)ptr, &sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(tbuf, ptr, &sz) < 0) @@ -8413,18 +8577,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, return ctl_reply(INET_REP_OK, tbuf, sz, rbuf, rsize); } - case INET_REQ_SETNAME: { /* set fake peername Port Address */ + case INET_REQ_SETNAME: { /* set fake sockname Port Address */ if (len == 0) { desc->name_ptr = NULL; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } else if (len < 2) return ctl_error(EINVAL, rbuf, rsize); - else if (inet_set_address(desc->sfamily, &desc->name_addr, - buf, &len) == NULL) + else if (inet_set_faddress + (desc->sfamily, &desc->name_addr, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); else { desc->name_ptr = &desc->name_addr; + desc->name_addr_len = (SOCKLEN_T) len; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } } @@ -8432,7 +8597,7 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, case INET_REQ_BIND: { /* bind socket */ char tbuf[2]; inet_address local; - short port; + int port; DEBUGF(("inet_ctl(%ld): BIND\r\n", (long)desc->port)); @@ -8449,13 +8614,14 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, desc->state = INET_STATE_BOUND; - if ((port = inet_address_port(&local)) == 0) { + port = inet_address_port(&local); + if (port == 0) { SOCKLEN_T adrlen = sizeof(local); sock_name(desc->s, &local.sa, &adrlen); port = inet_address_port(&local); } - port = sock_ntohs(port); - put_int16(port, tbuf); + else if (port == -1) port = 0; + put_int16(sock_ntohs((Uint16) port), tbuf); return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize); } @@ -8912,6 +9078,11 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: + domain = AF_UNIX; + break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -8938,6 +9109,11 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: + domain = AF_UNIX; + break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -8993,8 +9169,8 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, timeout = get_int32(buf); buf += 4; len -= 4; - if (inet_set_address(desc->inet.sfamily, &desc->inet.remote, - buf, &len) == NULL) + if (inet_set_faddress + (desc->inet.sfamily, &desc->inet.remote, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); code = sock_connect(desc->inet.s, @@ -10777,6 +10953,9 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: af = AF_UNIX; break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -10827,6 +11006,9 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, return ctl_xerror("eafnosupport", rbuf, rsize); break; #endif +#ifdef HAVE_SYS_UN_H + case INET_AF_LOCAL: af = AF_UNIX; break; +#endif default: return ctl_error(EINVAL, rbuf, rsize); } @@ -10909,7 +11091,7 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, /* For SCTP, we do not set the peer's addr in desc->remote, as multiple peers are possible: */ - if (inet_set_address(desc->sfamily, &remote, buf, &len) == NULL) + if (inet_set_faddress(desc->sfamily, &remote, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); sock_select(desc, FD_CONNECT, 1); @@ -10949,8 +11131,8 @@ static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, /* Ignore timeout */ buf += 4; len -= 4; - if (inet_set_address(desc->sfamily, - &desc->remote, buf, &len) == NULL) + if (inet_set_faddress + (desc->sfamily, &desc->remote, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); code = sock_connect(desc->s, @@ -11130,12 +11312,12 @@ static void packet_inet_timeout(ErlDrvData e) /* THIS IS A "send*" REQUEST; on the Erlang side: "port_command". -** input should be: P1 P0 Address buffer . +** input should be: Family Address buffer . ** For UDP, buffer (after Address) is just data to be sent. ** For SCTP, buffer contains a list representing 2 items: ** (1) 6 parms for sctp_sndrcvinfo, as in sctp_get_sendparams(); ** (2) 0+ real data bytes. -** There is no destination address -- SCTYP send is performed over +** There is no destination address -- SCTP send is performed over ** an existing association, using "sctp_sndrcvinfo" specified. */ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) @@ -11210,7 +11392,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) /* UDP socket. Even if it is connected, there is an address prefix here -- ignored for connected sockets: */ sz = len; - qtr = inet_set_address(desc->sfamily, &other, ptr, &sz); + qtr = inet_set_faddress(desc->sfamily, &other, ptr, &sz); if (qtr == NULL) { inet_reply_error(desc, EINVAL); return; @@ -11809,9 +11991,8 @@ int erts_sock_connect(erts_sock_t socket, byte *ip_addr, int len, Uint16 port) if (!inet_set_address(AF_INET, &addr, buf, &blen)) return 0; - if (IS_SOCKET_ERROR(sock_connect(s, - (struct sockaddr *) &addr, - sizeof(struct sockaddr_in)))) + if (IS_SOCKET_ERROR + (sock_connect(s, (struct sockaddr *) &addr, blen))) return 0; return 1; } diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 58b5be3906..3336fded7a 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -697,7 +697,7 @@ static struct nbif nbifs[BIF_SIZE] = { #undef BIF_LIST }; -#define NBIF_HASH(m,f,a) ((m)*(f)+(a)) +#define NBIF_HASH(m,f,a) (atom_val(m) ^ atom_val(f) ^ (a)) static Hash nbif_table; static HashValue nbif_hash(struct nbif *x) @@ -1063,7 +1063,7 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } -#define HIPE_MFA_HASH(M,F,A) ((M) * (F) + (A)) +#define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) { @@ -1144,10 +1144,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; p = hipe_mfa_info_table.bucket[i]; - for (; p; p = p->bucket.next) - /* XXX: do we want to compare p->bucket.hvalue as well? */ - if (p->m == m && p->f == f && p->a == arity) - return p; + for (; p; p = p->bucket.next) { + if (p->bucket.hvalue == h) { + if (p->m == m && p->f == f && p->a == arity) + return p; + } + else ASSERT(!(p->m == m && p->f == f && p->a == arity)); + } return NULL; } @@ -1171,10 +1174,13 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; p = hipe_mfa_info_table.bucket[i]; - for (; p; p = p->bucket.next) - /* XXX: do we want to compare p->bucket.hvalue as well? */ - if (p->m == m && p->f == f && p->a == arity) - return p; + for (; p; p = p->bucket.next) { + if (p->bucket.hvalue == h) { + if (p->m == m && p->f == f && p->a == arity) + return p; + } + else ASSERT(!(p->m == m && p->f == f && p->a == arity)); + } p = hipe_mfa_info_table_alloc(m, f, arity); p->bucket.hvalue = h; p->bucket.next = hipe_mfa_info_table.bucket[i]; diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 884331e969..ed95045292 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -547,7 +547,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) p->flags &= ~F_HIPE_MODE; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p); - p = schedule(p, reds_in - p->fcalls); + p = erts_schedule(NULL, p, reds_in - p->fcalls); ERTS_SMP_REQ_PROC_MAIN_LOCK(p); ASSERT(!(p->flags & F_HIPE_MODE)); #ifdef ERTS_SMP diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 94f3840b5f..f23c7ab03d 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -424,7 +424,7 @@ static ERTS_INLINE int wakeup_cause(ErtsPollSet ps) { int res; - erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); + erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); switch (wakeup_state) { case ERTS_POLL_WOKEN_IO_READY: res = 0; @@ -487,9 +487,8 @@ wake_poller(ErtsPollSet ps, int io_ready) { erts_aint32_t wakeup_state; if (io_ready) { - /* We may set the event multiple times. This is, however, harmless. */ - wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); - erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); + wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN_IO_READY); } else { ERTS_THR_MEMORY_BARRIER; diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index ec6cb6ab72..d31399e4af 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -141,9 +141,11 @@ guard_bifs_in_erl_bif_types(_Config) -> shadow_comments(_Config) -> ensure_erl_bif_types_compiled(), + ErlangList = [{erlang,F,A} || {F,A} <- erlang:module_info(exports), + not is_operator(F,A)], List0 = erlang:system_info(snifs), - List1 = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs], - List = [MFA || MFA <- List1, not is_operator(MFA)], + List1 = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs, M =/= erlang], + List = List1 ++ ErlangList, HasTypes = [MFA || {M,F,A}=MFA <- List, erl_bif_types:is_known(M, F, A)], Path = get_code_path(), @@ -253,12 +255,15 @@ specs(_) -> end. is_operator({erlang,F,A}) -> + is_operator(F,A); +is_operator(_) -> false. + +is_operator(F,A) -> erl_internal:arith_op(F, A) orelse erl_internal:bool_op(F, A) orelse erl_internal:comp_op(F, A) orelse erl_internal:list_op(F, A) orelse - erl_internal:send_op(F, A); -is_operator(_) -> false. + erl_internal:send_op(F, A). extract_specs(M, Abstr) -> [{make_mfa(M, Name),Spec} || {attribute,_,spec,{Name,Spec}} <- Abstr]. diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 941cb435f7..22a1c0b765 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -527,7 +527,7 @@ huge_float_check({'EXIT',{system_limit,_}}) -> ok; huge_float_check({'EXIT',{badarg,_}}) -> ok. huge_binary(Config) when is_list(Config) -> - ct:timetrap({seconds, 30}), + ct:timetrap({seconds, 60}), 16777216 = size(<<0:(id(1 bsl 26)),(-1):(id(1 bsl 26))>>), garbage_collect(), {Shift,Return} = case free_mem() of @@ -561,30 +561,13 @@ huge_binary(Config) when is_list(Config) -> end. free_mem() -> - Cmd = "uname; free", - Output = string:tokens(os:cmd(Cmd), "\n"), - io:format("Output from command ~p\n~p\n",[Cmd,Output]), - case Output of - [OS, ColumnNames, Values | _] -> - case string:str(OS,"Linux") of - 0 -> - io:format("Unknown OS\n",[]), - undefined; - _ -> - case {string:tokens(ColumnNames, " \t"), - string:tokens(Values, " \t")} of - {[_,_,"free"|_],["Mem:",_,_,FreeKb|_]} -> - list_to_integer(FreeKb) div 1024; - _ -> - io:format("Failed to parse output from 'free':\n",[]), - undefined - end - end; - _ -> - io:format("Too few lines in output\n",[]), - undefined + {ok,Apps} = application:ensure_all_started(os_mon), + Mem = memsup:get_system_memory_data(), + [ok = application:stop(App)||App <- Apps], + case proplists:get_value(free_memory,Mem) of + undefined -> undefined; + Val -> Val div 1024 end. - system_limit(Config) when is_list(Config) -> WordSize = erlang:system_info(wordsize), @@ -614,8 +597,7 @@ system_limit_32() -> {'EXIT',{system_limit,_}} = (catch <<42:536870912/unit:8>>), {'EXIT',{system_limit,_}} = (catch <<42:(id(536870912))/unit:8>>), {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:536870912/unit:8>>), - {'EXIT',{system_limit,_}} = - (catch <<0:(id(8)),42:(id(536870912))/unit:8>>), + {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:(id(536870912))/unit:8>>), %% The size would be silently truncated, resulting in a crash. {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 35)>>), @@ -627,16 +609,10 @@ system_limit_32() -> ok. badarg(Config) when is_list(Config) -> - {'EXIT',{badarg,_}} = - (catch <<0:(id(1 bsl 100)),0:(id(-1))>>), - {'EXIT',{badarg,_}} = - (catch <<0:(id(1 bsl 100)),0:(id(-(1 bsl 70)))>>), - {'EXIT',{badarg,_}} = - (catch <<0:(id(-(1 bsl 70))),0:(id(1 bsl 100))>>), - - {'EXIT',{badarg,_}} = - (catch <<(id(<<>>))/binary,0:(id(-(1 bsl 100)))>>), - + {'EXIT',{badarg,_}} = (catch <<0:(id(1 bsl 100)),0:(id(-1))>>), + {'EXIT',{badarg,_}} = (catch <<0:(id(1 bsl 100)),0:(id(-(1 bsl 70)))>>), + {'EXIT',{badarg,_}} = (catch <<0:(id(-(1 bsl 70))),0:(id(1 bsl 100))>>), + {'EXIT',{badarg,_}} = (catch <<(id(<<>>))/binary,0:(id(-(1 bsl 100)))>>), ok. copy_writable_binary(Config) when is_list(Config) -> diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl index c3afbc0803..83b098a704 100644 --- a/erts/emulator/test/dirty_nif_SUITE.erl +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -32,19 +32,23 @@ dirty_nif/1, dirty_nif_send/1, dirty_nif_exception/1, call_dirty_nif_exception/1, dirty_scheduler_exit/1, dirty_call_while_terminated/1, - dirty_heap_access/1]). + dirty_heap_access/1, dirty_process_info/1, + dirty_process_register/1, dirty_process_trace/1]). -define(nif_stub,nif_stub_error(?LINE)). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> +all() -> [dirty_nif, dirty_nif_send, dirty_nif_exception, dirty_scheduler_exit, dirty_call_while_terminated, - dirty_heap_access]. + dirty_heap_access, + dirty_process_info, + dirty_process_register, + dirty_process_trace]. init_per_suite(Config) -> try erlang:system_info(dirty_cpu_schedulers) of @@ -187,7 +191,7 @@ dirty_call_while_terminated(Config) when is_list(Config) -> blipp:blupp(Bin) end, [monitor,link]), - receive {dirty_alive, Pid} -> ok end, + receive {dirty_alive, _Pid} -> ok end, {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2, element(2, process_info(self(), @@ -241,7 +245,7 @@ dirty_heap_access(Config) when is_list(Config) -> end), {N, R} = access_dirty_heap(Dirty, RGL, 0, 0), receive - {Pid, Res} -> + {_Pid, Res} -> 1000 = length(Res), lists:foreach(fun (X) -> Ref = X end, Res) end, @@ -269,12 +273,123 @@ access_dirty_heap(Dirty, RGL, N, R) -> end) end. +%% These tests verify that processes that access a process executing a +%% dirty NIF where the main lock is needed for that access do not get +%% blocked. Each test passes its pid to dirty_sleeper, which sends a +%% 'ready' message when it's running on a dirty scheduler and just before +%% it starts a 6 second sleep. When it receives the message, it verifies +%% that access to the dirty process is as it expects. After the dirty +%% process finishes its 6 second sleep but before it returns from the dirty +%% scheduler, it sends a 'done' message. If the tester already received +%% that message, the test fails because it means attempting to access the +%% dirty process waited for that process to return to a regular scheduler, +%% so verify that we haven't received that message, and also verify that +%% the dirty process is still alive immediately after accessing it. +dirty_process_info(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(NifPid) -> + PI = process_info(NifPid), + {current_function,{?MODULE,dirty_sleeper,1}} = + lists:keyfind(current_function, 1, PI), + ok + end, + fun(_) -> ok end). + +dirty_process_register(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(NifPid) -> + register(test_dirty_process_register, NifPid), + NifPid = whereis(test_dirty_process_register), + unregister(test_dirty_process_register), + false = lists:member(test_dirty_process_register, + registered()), + ok + end, + fun(_) -> ok end). + +dirty_process_trace(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> + erlang:trace_pattern({?MODULE,dirty_sleeper,1}, + [{'_',[],[{return_trace}]}], + [local,meta]), + ok + end, + fun(NifPid) -> + erlang:trace(NifPid, true, [call,timestamp]), + ok + end, + fun(NifPid) -> + receive + done -> + receive + {trace_ts,NifPid,call,{?MODULE,dirty_sleeper,_},_} -> + ok + after + 0 -> + error(missing_trace_call_message) + end, + receive + {trace_ts,NifPid,return_from,{?MODULE,dirty_sleeper,1}, + ok,_} -> + ok + after + 100 -> + error(missing_trace_return_message) + end + after + 6500 -> + error(missing_done_message) + end, + ok + end). + %% %% Internal... %% +access_dirty_process(Config, Start, Test, Finish) -> + {ok, Node} = start_node(Config, ""), + [ok] = mcall(Node, + [fun() -> + Path = ?config(data_dir, Config), + Lib = atom_to_list(?MODULE), + ok = erlang:load_nif(filename:join(Path,Lib), []), + ok = test_dirty_process_access(Start, Test, Finish) + end]), + stop_node(Node), + ok. + +test_dirty_process_access(Start, Test, Finish) -> + ok = Start(), + Self = self(), + NifPid = spawn_link(fun() -> + ok = dirty_sleeper(Self) + end), + ok = receive + ready -> + ok = Test(NifPid), + receive + done -> + error(dirty_process_info_blocked) + after + 0 -> + true = erlang:is_process_alive(NifPid), + ok + end + after + 3000 -> + error(timeout) + end, + ok = Finish(NifPid). + receive_any() -> - receive M -> M end. + receive M -> M end. start_node(Config) -> start_node(Config, ""). @@ -314,13 +429,13 @@ mcall(Node, Funs) -> %% The NIFs: lib_loaded() -> false. -call_nif_schedule(_,_) -> ?nif_stub. call_dirty_nif(_,_,_) -> ?nif_stub. send_from_dirty_nif(_) -> ?nif_stub. call_dirty_nif_exception(_) -> ?nif_stub. call_dirty_nif_zero_args() -> ?nif_stub. dirty_call_while_terminated_nif(_) -> ?nif_stub. dirty_sleeper() -> ?nif_stub. +dirty_sleeper(_) -> ?nif_stub. dirty_heap_access_nif(_) -> ?nif_stub. nif_stub_error(Line) -> diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index 2013c88167..e38bececde 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -19,7 +19,9 @@ */ #include "erl_nif.h" #include <assert.h> -#ifndef __WIN32__ +#ifdef __WIN32__ +#include <windows.h> +#else #include <unistd.h> #endif @@ -146,12 +148,31 @@ static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL static ERL_NIF_TERM dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifPid pid; + ErlNifEnv* msg_env = NULL; + assert(enif_is_on_dirty_scheduler(env)); + + /* If we get a pid argument, it indicates a process involved in the + test wants a message from us. Prior to the sleep we send a 'ready' + message, and then after the sleep, send a 'done' message. */ + if (argc == 1 && enif_get_local_pid(env, argv[0], &pid)) { + msg_env = enif_alloc_env(); + enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "ready")); + } + #ifdef __WIN32__ Sleep(6000); #else sleep(6); #endif + + if (argc == 1) { + assert(msg_env != NULL); + enif_send(env, &pid, msg_env, enif_make_atom(msg_env, "done")); + enif_free_env(msg_env); + } + return enif_make_atom(env, "ok"); } @@ -216,6 +237,7 @@ static ErlNifFunc nif_funcs[] = {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_sleeper", 1, dirty_sleeper, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_call_while_terminated_nif", 1, dirty_call_while_terminated_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_heap_access_nif", 1, dirty_heap_access_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND} }; diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl index c64ddc40da..1360751aee 100644 --- a/erts/emulator/test/lttng_SUITE.erl +++ b/erts/emulator/test/lttng_SUITE.erl @@ -44,6 +44,7 @@ suite() -> all() -> [t_lttng_list, + t_memory_carrier, t_carrier_pool, t_async_io_pool, t_driver_start_stop, @@ -52,8 +53,7 @@ all() -> t_driver_timeout, t_driver_caller, t_driver_flush, - t_scheduler_poll, - t_memory_carrier]. + t_scheduler_poll]. init_per_suite(Config) -> @@ -178,11 +178,13 @@ t_async_io_pool(Config) -> %% com_ericsson_otp:driver_stop t_driver_start_stop(Config) -> ok = lttng_start_event("com_ericsson_otp:driver_*", Config), + timer:sleep(500), Path = proplists:get_value(priv_dir, Config), Name = filename:join(Path, "sometext.txt"), Bin = txt(), ok = file:write_file(Name, Bin), {ok, Bin} = file:read_file(Name), + timer:sleep(500), Res = lttng_stop_and_view(Config), ok = check_tracepoint("com_ericsson_otp:driver_start", Res), ok = check_tracepoint("com_ericsson_otp:driver_stop", Res), @@ -213,6 +215,7 @@ t_driver_control_ready_async(Config) -> %% com_ericsson_otp:driver_ready_output t_driver_ready_input_output(Config) -> ok = lttng_start_event("com_ericsson_otp:driver_ready_*", Config), + timer:sleep(500), Me = self(), Pid = spawn_link(fun() -> tcp_server(Me, active) end), receive {Pid, accept} -> ok end, @@ -225,6 +228,7 @@ t_driver_ready_input_output(Config) -> ok = gen_tcp:close(Sock), receive {Pid, done} -> ok end, + timer:sleep(500), Res = lttng_stop_and_view(Config), ok = check_tracepoint("com_ericsson_otp:driver_ready_input", Res), ok = check_tracepoint("com_ericsson_otp:driver_ready_output", Res), |