From 3b523c25af0df45fbf68ab3cf50c0556f1d4e0a1 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 4 Jun 2012 20:47:08 +0200 Subject: Atomic port state --- erts/emulator/beam/bif.c | 61 +++--- erts/emulator/beam/dist.c | 43 ++-- erts/emulator/beam/dist.h | 3 +- erts/emulator/beam/erl_async.c | 2 +- erts/emulator/beam/erl_bif_ddll.c | 127 +++++------- erts/emulator/beam/erl_bif_info.c | 3 +- erts/emulator/beam/erl_bif_port.c | 27 +-- erts/emulator/beam/erl_bif_trace.c | 4 +- erts/emulator/beam/erl_node_tables.c | 3 +- erts/emulator/beam/erl_port_task.c | 43 ++-- erts/emulator/beam/erl_process.c | 36 ++-- erts/emulator/beam/erl_process.h | 30 +-- erts/emulator/beam/erl_smp.h | 28 +++ erts/emulator/beam/erl_threads.h | 296 +++++++++++++++++++++++++++ erts/emulator/beam/global.h | 102 +++------- erts/emulator/beam/io.c | 343 +++++++++++++++++--------------- erts/emulator/sys/common/erl_check_io.c | 24 +-- 17 files changed, 721 insertions(+), 454 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 89a8f839db..10dcba565b 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1899,30 +1899,34 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) { ERTS_SMP_LC_ASSERT(!pt || erts_lc_is_port_locked(pt)); /* We have waited for locks, trace schedule ports */ - if (pt && IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(pt, am_in, am_command); - } - if (pt && erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) { - profile_runnable_port(pt, am_active); - } - - /* XXX let port_command handle the busy stuff !!! */ - if (pt && (pt->status & ERTS_PORT_SFLG_PORT_BUSY)) { - if (suspend) { - erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); - if (erts_system_monitor_flags.busy_port) { - monitor_generic(p, am_busy_port, portid); - } - } - /* Virtually schedule out the port before releasing */ + if (pt) { + erts_aint32_t state; if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) { - trace_sched_ports_where(pt, am_out, am_command); + trace_sched_ports_where(pt, am_in, am_command); } if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) { - profile_runnable_port(pt, am_inactive); + profile_runnable_port(pt, am_active); + } + + state = erts_smp_atomic32_read_nob(&pt->state); + /* XXX let port_command handle the busy stuff !!! */ + if (state & ERTS_PORT_SFLG_PORT_BUSY) { + if (suspend) { + erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); + if (erts_system_monitor_flags.busy_port) { + monitor_generic(p, am_busy_port, portid); + } + } + /* Virtually schedule out the port before releasing */ + if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) { + trace_sched_ports_where(pt, am_out, am_command); + } + if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) { + profile_runnable_port(pt, am_inactive); + } + erts_port_release(pt); + return SEND_YIELD; } - erts_port_release(pt); - return SEND_YIELD; } if (IS_TRACED(p)) /* trace once only !! */ @@ -3534,14 +3538,17 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) for (i = erts_max_ports-1; i >= 0; i--) { Port* prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD) - && prt->snapshot != next_ss) { - ASSERT(prt->snapshot == next_ss - 1); - *pp++ = prt->id; - prt->snapshot = next_ss; /* Consumed by this snapshot */ + erts_aint32_t state = erts_smp_atomic32_read_acqb(&prt->state); + if (!(state & ERTS_PORT_SFLGS_DEAD)) { + erts_smp_port_minor_lock(prt); + state = erts_smp_atomic32_read_nob(&prt->state); + if (!(state & ERTS_PORT_SFLGS_DEAD) && prt->snapshot != next_ss) { + ASSERT(prt->snapshot == next_ss - 1); + *pp++ = prt->id; + prt->snapshot = next_ss; /* Consumed by this snapshot */ + } + erts_smp_port_minor_unlock(prt); } - erts_smp_port_state_unlock(prt); } dead_ports = (Eterm*)erts_smp_atomic_xchg_nob(&erts_dead_ports_ptr, diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 5de8236a7e..eccef44a11 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -423,7 +423,8 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) prt = erts_id2port(prt_id, NULL, 0); if (prt) { - ASSERT(prt->status & ERTS_PORT_SFLG_DISTRIBUTION); + ASSERT(erts_smp_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLG_DISTRIBUTION); ASSERT(prt->dist_entry); /* will call do_net_exists !!! */ erts_do_exit_port(prt, prt_id, nd_reason); @@ -1907,13 +1908,13 @@ int erts_dist_command(Port *prt, int reds_limit) { Sint reds = ERTS_PORT_REDS_DIST_CMD_START; - int prt_busy; Uint32 status; Uint32 flags; Sint obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = prt->dist_entry; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); + erts_aint32_t state; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -1956,12 +1957,12 @@ erts_dist_command(Port *prt, int reds_limit) dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; + state = erts_smp_atomic32_read_nob(&prt->state); + if (reds > reds_limit) goto preempted; - prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY); - - if (!prt_busy && foq.first) { + if (!(state & ERTS_PORT_SFLG_PORT_BUSY) && foq.first) { int preempt = 0; do { Uint size; @@ -1977,11 +1978,10 @@ erts_dist_command(Port *prt, int reds_limit) obufsize += size_obuf(fob); foq.first = foq.first->next; free_dist_obuf(fob); - preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); - if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - prt_busy = 1; + state = erts_smp_atomic32_read_nob(&prt->state); + preempt = reds > reds_limit || (state & ERTS_PORT_SFLGS_DEAD); + if (state & ERTS_PORT_SFLG_PORT_BUSY) break; - } } while (foq.first && !preempt); if (!foq.first) foq.last = NULL; @@ -1989,7 +1989,7 @@ erts_dist_command(Port *prt, int reds_limit) goto preempted; } - if (prt_busy) { + if (state & ERTS_PORT_SFLG_PORT_BUSY) { if (oq.first) { ErtsDistOutputBuf *ob; int preempt; @@ -2060,12 +2060,10 @@ erts_dist_command(Port *prt, int reds_limit) obufsize += size_obuf(fob); oq.first = oq.first->next; free_dist_obuf(fob); - preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); - if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - prt_busy = 1; - if (oq.first && !preempt) - goto finalize_only; - } + state = erts_smp_atomic32_read_nob(&prt->state); + preempt = reds > reds_limit || (state & ERTS_PORT_SFLGS_DEAD); + if ((state & ERTS_PORT_SFLG_PORT_BUSY) && oq.first && !preempt) + goto finalize_only; } ASSERT(!oq.first || preempt); @@ -2093,7 +2091,7 @@ erts_dist_command(Port *prt, int reds_limit) ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; obufsize = 0; - if (!prt_busy + if (!(state & ERTS_PORT_SFLG_PORT_BUSY) && (dep->qflgs & ERTS_DE_QFLG_BUSY) && dep->qsize < erts_dist_buf_busy_limit) { ErtsProcList *suspendees; @@ -2139,11 +2137,15 @@ erts_dist_command(Port *prt, int reds_limit) return reds; preempted: + /* + * Here we assume that state has been read + * since last call to driver. + */ ASSERT(oq.first || !oq.last); ASSERT(!oq.first || oq.last); - if (prt->status & ERTS_PORT_SFLGS_DEAD) { + if (state & ERTS_PORT_SFLGS_DEAD) { /* * Port died during port command; clean up 'oq' * and 'foq'. Things buffered in dist entry after @@ -2581,7 +2583,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) pp = erts_id2port(BIF_ARG_2, BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_de_rwlock(dep); - if (!pp || (pp->status & ERTS_PORT_SFLG_EXITING)) + if (!pp || (erts_smp_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLG_EXITING)) goto badarg; if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0) @@ -2610,7 +2613,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) if (pp->dist_entry || is_not_nil(dep->cid)) goto badarg; - erts_port_status_bor_set(pp, ERTS_PORT_SFLG_DISTRIBUTION); + erts_smp_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); pp->dist_entry = dep; diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 845151c895..0cefd73459 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -187,7 +187,8 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) if (prt) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - ASSERT((erts_port_status_get(prt) & ERTS_PORT_SFLGS_DEAD) == 0); + ASSERT((erts_smp_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLGS_DEAD) == 0); ASSERT(prt->dist_entry); dep = prt->dist_entry; diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index f321ed21aa..bce9d07a01 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -599,7 +599,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key, sched_id = 1; #endif - prt = erts_drvport2port(ix); + prt = erts_drvport2port(ix, NULL); if (!prt) return -1; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 7f7c975e78..dfb8b453c4 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -114,6 +114,54 @@ static void ddll_no_more_references(void *vdh); #define FREE_PORT_FLAGS (ERTS_PORT_SFLGS_DEAD & (~ERTS_PORT_SFLG_INITIALIZING)) +static void +kill_ports_driver_unloaded(DE_Handle *dh) +{ + int ix; + + for (ix = 0; ix < erts_max_ports; ix++) { + Port* prt = &erts_port[ix]; + erts_aint32_t state; + state = erts_smp_atomic32_read_acqb(&prt->state); + if (state & FREE_PORT_FLAGS) + continue; + + erts_smp_port_minor_lock(prt); + + if (prt->drv_ptr->handle != dh) { + erts_smp_port_minor_unlock(prt); + continue; + } + + erts_smp_atomic_inc_nob(&prt->refc); + erts_smp_port_minor_unlock(prt); + + state = erts_smp_atomic32_read_acqb(&prt->state); + if (!(state & FREE_PORT_FLAGS)) { +#if DDLL_SMP + if (state & ERTS_PORT_SFLG_INITIALIZING) { + /* Extremely rare busy wait */ + do { + ERTS_SPIN_BODY; + state = erts_smp_atomic32_read_acqb(&prt->state); + } while (state & ERTS_PORT_SFLG_INITIALIZING); + } + + erts_smp_mtx_lock(prt->lock); + + if (prt->drv_ptr->handle == dh) { + state = erts_smp_atomic32_read_acqb(&prt->state); + if (!(state & ERTS_PORT_SFLGS_DEAD)) + driver_failure_atom(ix, "driver_unloaded"); + } +#else + driver_failure_atom(ix, "driver_unloaded"); +#endif + erts_port_release(prt); + } + } +} + /* * try_load(Path, Name, OptionList) -> {ok,Status} | * {ok, PendingStatus, Ref} | @@ -358,38 +406,14 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) } assert_drv_list_locked(); if (kill_ports) { - int j; - /* Avoid closing the driver by referencing it */ + /* Avoid closing the driver by referencing it */ erts_ddll_reference_driver(dh); ASSERT(dh->status == ERL_DE_RELOAD); dh->status = ERL_DE_FORCE_RELOAD; #if DDLL_SMP unlock_drv_list(); #endif - for (j = 0; j < erts_max_ports; j++) { - Port* prt = &erts_port[j]; - erts_smp_port_state_lock(prt); - if (!(prt->status & FREE_PORT_FLAGS) && - prt->drv_ptr->handle == dh) { - erts_smp_atomic_inc_nob(&prt->refc); -#if DDLL_SMP - /* Extremely rare spinlock */ - while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { - erts_smp_port_state_unlock(prt); - erts_smp_port_state_lock(prt); - } - erts_smp_port_state_unlock(prt); - erts_smp_mtx_lock(prt->lock); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) { - driver_failure_atom(j, "driver_unloaded"); - } -#else - driver_failure_atom(j, "driver_unloaded"); -#endif - erts_port_release(prt); - } - else erts_smp_port_state_unlock(prt); - } + kill_ports_driver_unloaded(dh); /* Dereference, eventually causing driver destruction */ #if DDLL_SMP lock_drv_list(); @@ -587,37 +611,13 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2) done: assert_drv_list_locked(); if (kill_ports > 1) { - int j; /* Avoid closing the driver by referencing it */ erts_ddll_reference_driver(dh); dh->status = ERL_DE_FORCE_UNLOAD; #if DDLL_SMP unlock_drv_list(); #endif - for (j = 0; j < erts_max_ports; j++) { - Port* prt = &erts_port[j]; - erts_smp_port_state_lock(prt); - if (!(prt->status & FREE_PORT_FLAGS) - && prt->drv_ptr->handle == dh) { - erts_smp_atomic_inc_nob(&prt->refc); -#if DDLL_SMP - /* Extremely rare spinlock */ - while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { - erts_smp_port_state_unlock(prt); - erts_smp_port_state_lock(prt); - } - erts_smp_port_state_unlock(prt); - erts_smp_mtx_lock(prt->lock); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) { - driver_failure_atom(j, "driver_unloaded"); - } -#else - driver_failure_atom(j, "driver_unloaded"); -#endif - erts_port_release(prt); - } - else erts_smp_port_state_unlock(prt); - } + kill_ports_driver_unloaded(dh); #if DDLL_SMP lock_drv_list(); #endif @@ -1047,36 +1047,13 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) } if (!left && drv->handle->port_count > 0) { if (kill_ports) { - int j; DE_Handle *dh = drv->handle; erts_ddll_reference_driver(dh); dh->status = ERL_DE_FORCE_UNLOAD; #if DDLL_SMP unlock_drv_list(); #endif - for (j = 0; j < erts_max_ports; j++) { - Port* prt = &erts_port[j]; - erts_smp_port_state_lock(prt); - if (!(prt->status & FREE_PORT_FLAGS) && - prt->drv_ptr->handle == dh) { - erts_smp_atomic_inc_nob(&prt->refc); -#if DDLL_SMP - while(prt->status & ERTS_PORT_SFLG_INITIALIZING) { - erts_smp_port_state_unlock(prt); - erts_smp_port_state_lock(prt); - } - erts_smp_port_state_unlock(prt); - erts_smp_mtx_lock(prt->lock); - if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) { - driver_failure_atom(j, "driver_unloaded"); - } -#else - driver_failure_atom(j, "driver_unloaded"); -#endif - erts_port_release(prt); - } - else erts_smp_port_state_unlock(prt); - } + kill_ports_driver_unloaded(dh); #if DDLL_SMP lock_drv_list(); /* Needed for future list operations */ #endif diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 874d1606fd..6b71eddc8b 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2975,7 +2975,8 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item) #ifndef ERTS_SMP res = am_false; #else - if (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { + if (erts_smp_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { DECL_AM(port_level); ASSERT(prt->drv_ptr->flags & ERL_DRV_FLAG_USE_PORT_LOCKING); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 5525426824..1a5b05fcfc 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -164,7 +164,7 @@ do_port_command(Process *BIF_P, Eterm arg1, Eterm arg2, Eterm arg3, ERTS_BIF_PREP_ERROR(res, BIF_P, EXC_NOTSUP); } else if (!(flags & ERTS_PORT_COMMAND_FLAG_FORCE) - && p->status & ERTS_PORT_SFLG_PORT_BUSY) { + && (erts_smp_atomic32_read_nob(&p->state) & ERTS_PORT_SFLG_PORT_BUSY)) { if (flags & ERTS_PORT_COMMAND_FLAG_NOSUSPEND) { ERTS_BIF_PREP_RET(res, am_false); } @@ -637,11 +637,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) erts_driver_t* driver; char* name_buf = NULL; SysDriverOpts opts; - int binary_io; - int soft_eof; Sint linebuf; Eterm edir = NIL; byte dir[MAXPATHLEN]; + erts_aint32_t sflgs = 0; /* These are the defaults */ opts.packet_bytes = 0; @@ -655,8 +654,6 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) opts.overlapped_io = 0; opts.spawn_type = ERTS_SPAWN_ANY; opts.argv = NULL; - binary_io = 0; - soft_eof = 0; linebuf = 0; *err_nump = 0; @@ -748,13 +745,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) } else if (*nargs == am_nouse_stdio) { opts.use_stdio = 0; } else if (*nargs == am_binary) { - binary_io = 1; + sflgs |= ERTS_PORT_SFLG_BINARY_IO; } else if (*nargs == am_in) { opts.read_write |= DO_READ; } else if (*nargs == am_out) { opts.read_write |= DO_WRITE; } else if (*nargs == am_eof) { - soft_eof = 1; + sflgs |= ERTS_PORT_SFLG_SOFT_EOF; } else if (*nargs == am_hide) { opts.hide_window = 1; } else if (*nargs == am_exit_status) { @@ -951,19 +948,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) trace_virtual_sched(p, am_in); } - if (binary_io) { - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_BINARY_IO); - } - if (soft_eof) { - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_SOFT_EOF); - } if (linebuf && erts_port[port_num].linebuf == NULL){ - erts_port[port_num].linebuf = allocate_linebuf(linebuf); - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_LINEBUF_IO); + erts_port[port_num].linebuf = allocate_linebuf(linebuf); + sflgs |= ERTS_PORT_SFLG_LINEBUF_IO; } + + if (sflgs) + erts_smp_atomic32_read_bor_relb(&erts_port[port_num].state, sflgs); do_return: if (name_buf) diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 3baac1572a..01d714d57f 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -680,7 +680,9 @@ Eterm trace_3(BIF_ALIST_3) /* tracing of ports */ for (i = 0; i < erts_max_ports; i++) { Port *tracee_port = &erts_port[i]; - if (tracee_port->status & ERTS_PORT_SFLGS_DEAD) continue; + erts_aint32_t state; + state = erts_smp_atomic32_read_nob(&tracee_port->state); + if (state & ERTS_PORT_SFLGS_DEAD) continue; if (tracer != NIL) { if (tracee_port->id == tracer) continue; if (port_already_traced(NULL, tracee_port, tracer)) continue; diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 93122b8317..f64a7fa41a 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1384,7 +1384,8 @@ setup_reference_table(void) /* Insert all ports */ for (i = 0; i < erts_max_ports; i++) { - if (erts_port[i].status & ERTS_PORT_SFLGS_DEAD) + erts_aint32_t state = erts_smp_atomic32_read_nob(&erts_port[i].state); + if (state & ERTS_PORT_SFLGS_DEAD) continue; /* Insert links */ diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 86454fe1fa..3873dd8f99 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -51,7 +51,8 @@ #define ERTS_PORT_TASK_INVALID_PORT(P, ID) \ - ((erts_port_status_get((P)) & ERTS_PORT_SFLGS_DEAD) || (P)->id != (ID)) + ((erts_smp_atomic32_read_acqb(&(P)->state) & ERTS_PORT_SFLGS_DEAD) \ + || (P)->id != (ID)) #define ERTS_PORT_IS_IN_RUNQ(RQ, P) \ ((P)->sched.next || (P)->sched.prev || (RQ)->ports.start == (P)) @@ -615,7 +616,7 @@ erts_port_task_free_port(Port *pp) ErtsPortTaskQueue *ptqp; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); - ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD)); + ASSERT(!(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); runq = erts_port_runq(pp); ASSERT(runq); ERTS_PT_CHK_PRES_PORTQ(runq, pp); @@ -626,11 +627,13 @@ erts_port_task_free_port(Port *pp) ErtsPortTask *ptp; enqueue_free: ptp = port_task_alloc(); - erts_smp_port_state_lock(pp); - pp->status &= ~ERTS_PORT_SFLG_CLOSING; - pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; + erts_smp_port_minor_lock(pp); + erts_smp_atomic32_read_bset_relb(&pp->state, + (ERTS_PORT_SFLG_CLOSING + | ERTS_PORT_SFLG_FREE_SCHEDULED), + ERTS_PORT_SFLG_FREE_SCHEDULED); erts_may_save_closed_port(pp); - erts_smp_port_state_unlock(pp); + erts_smp_port_minor_unlock(pp); ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1); ptp->type = ERTS_PORT_TASK_FREE; ptp->event = (ErlDrvEvent) -1; @@ -648,11 +651,13 @@ erts_port_task_free_port(Port *pp) goto enqueue_free; } ASSERT(!pp->sched.taskq); - erts_smp_port_state_lock(pp); - pp->status &= ~ERTS_PORT_SFLG_CLOSING; - pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED; + erts_smp_port_minor_lock(pp); + erts_smp_atomic32_read_bset_relb(&pp->state, + (ERTS_PORT_SFLG_CLOSING + | ERTS_PORT_SFLG_FREE_SCHEDULED), + ERTS_PORT_SFLG_FREE_SCHEDULED); erts_may_save_closed_port(pp); - erts_smp_port_state_unlock(pp); + erts_smp_port_minor_unlock(pp); erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ handle_remaining_tasks(runq, pp); /* May release runq lock */ @@ -766,7 +771,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_smp_runq_lock(runq); erts_unblock_fpe(fpe_was_unmasked); - ASSERT(pp->status & ERTS_PORT_SFLG_FREE_SCHEDULED); + ASSERT(erts_smp_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLG_FREE_SCHEDULED); if (ptqp->first || (pp->sched.taskq && pp->sched.taskq->first)) handle_remaining_tasks(runq, pp); ASSERT(!ptqp->first @@ -782,14 +788,15 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto tasks_done; case ERTS_PORT_TASK_TIMEOUT: reds += ERTS_PORT_REDS_TIMEOUT; - if (!(pp->status & ERTS_PORT_SFLGS_DEAD)) { + if (!(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)) { DTRACE_DRIVER(driver_timeout, pp); (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); } break; case ERTS_PORT_TASK_INPUT: reds += ERTS_PORT_REDS_INPUT; - ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + ASSERT((erts_smp_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_input, pp); /* NOTE some windows drivers use ->ready_input for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->event); @@ -797,14 +804,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) break; case ERTS_PORT_TASK_OUTPUT: reds += ERTS_PORT_REDS_OUTPUT; - ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + ASSERT((erts_smp_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_output, pp); (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->event); io_tasks_executed++; break; case ERTS_PORT_TASK_EVENT: reds += ERTS_PORT_REDS_EVENT; - ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); + ASSERT((erts_smp_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_event, pp); (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->event, ptp->event_data); io_tasks_executed++; @@ -819,7 +828,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) break; } - if ((pp->status & ERTS_PORT_SFLG_CLOSING) + if ((erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(pp)) { reds += ERTS_PORT_REDS_TERMINATE; erts_terminate_port(pp); @@ -870,7 +879,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ErtsRunQueue *xrunq; #endif - ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD)); + ASSERT(!(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); ASSERT(pp->sched.taskq->first); #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 79614c85db..6170fd1ec8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1486,29 +1486,29 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) erts_smp_atomic32_set_nob(&erts_halt_progress, 1); for (i = 0; i < erts_max_ports; i++) { Port *prt = &erts_port[i]; - erts_smp_port_state_lock(prt); - if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - | ERTS_PORT_SFLG_HALT))) { - erts_smp_port_state_unlock(prt); + erts_aint32_t state = erts_smp_atomic32_read_acqb(&prt->state); + if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT)) continue; - } + erts_smp_port_minor_lock(prt); /* We need to set the halt flag - get the port lock */ #ifdef ERTS_SMP erts_smp_atomic_inc_nob(&prt->refc); #endif - erts_smp_port_state_unlock(prt); + erts_smp_port_minor_unlock(prt); #ifdef ERTS_SMP erts_smp_mtx_lock(prt->lock); #endif - if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - | ERTS_PORT_SFLG_HALT))) { + state = erts_smp_atomic32_read_acqb(&prt->state); + if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT)) { erts_port_release(prt); continue; } - erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT); + state = erts_smp_atomic32_read_bor_relb(&prt->state, + ERTS_PORT_SFLG_HALT); erts_smp_atomic32_inc_nob(&erts_halt_progress); - if (prt->status & (ERTS_PORT_SFLG_EXITING - | ERTS_PORT_SFLG_CLOSING)) { + if (state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING)) { erts_port_release(prt); continue; } @@ -2885,12 +2885,12 @@ resume_run_queue(ErtsRunQueue *rq) erts_smp_runq_lock(rq); - (void) ERTS_RUNQ_FLGS_MASK_SET(rq, - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + (void) ERTS_RUNQ_FLGS_READ_BSET(rq, + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK + | ERTS_RUNQ_FLG_SUSPENDED), + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { @@ -3912,7 +3912,7 @@ erts_fprintf(stderr, "--------------------------------\n"); ERTS_DBG_CHK_FULL_REDS_HISTORY(rq); rq->out_of_work_count = 0; - (void) ERTS_RUNQ_FLGS_MASK_SET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); + (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); rq->max_len = rq->len; for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 17d1ff0bd6..22303973bf 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -205,32 +205,10 @@ extern int erts_sched_thread_suggested_stack_size; ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags)) #define ERTS_RUNQ_FLGS_GET_MB(RQ) \ ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags)) -#define ERTS_RUNQ_FLGS_MASK_SET(RQ, MSK, FLGS) \ - ((Uint32) erts_smp_atomic32_mask_set_relb(&(RQ)->flags, \ - (erts_aint32_t) (MSK), \ - (erts_aint32_t) (FLGS))) - -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p, - erts_aint32_t mask, - erts_aint32_t set); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE erts_aint32_t -erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p, - erts_aint32_t mask, - erts_aint32_t set) -{ - erts_aint32_t act = erts_smp_atomic32_read_nob(a32p); - while (1) { - erts_aint32_t exp = act; - erts_aint32_t new = exp & ~mask; - new |= (mask & set); - act = erts_smp_atomic32_cmpxchg_relb(a32p, new, exp); - if (act == exp) - return act; - } -} -#endif +#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \ + (erts_aint32_t) (MSK), \ + (erts_aint32_t) (FLGS))) typedef enum { ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED, diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index a32e9d9d7c..34c90c0bda 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -274,6 +274,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob #define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob #define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob +#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob #define erts_smp_atomic_init_mb erts_atomic_init_mb #define erts_smp_atomic_set_mb erts_atomic_set_mb @@ -288,6 +289,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb #define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb #define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb +#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb #define erts_smp_atomic_init_acqb erts_atomic_init_acqb #define erts_smp_atomic_set_acqb erts_atomic_set_acqb @@ -302,6 +304,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb #define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb #define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb +#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb #define erts_smp_atomic_init_relb erts_atomic_init_relb #define erts_smp_atomic_set_relb erts_atomic_set_relb @@ -316,6 +319,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb #define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb #define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb +#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb #define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb #define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb @@ -330,6 +334,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb #define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb #define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb +#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb #define erts_smp_atomic_init_rb erts_atomic_init_rb #define erts_smp_atomic_set_rb erts_atomic_set_rb @@ -344,6 +349,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb #define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb #define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb +#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb #define erts_smp_atomic_init_wb erts_atomic_init_wb #define erts_smp_atomic_set_wb erts_atomic_set_wb @@ -358,6 +364,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb #define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb #define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb +#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb /* 32-bit atomics */ @@ -374,6 +381,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob #define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob #define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob +#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob #define erts_smp_atomic32_init_mb erts_atomic32_init_mb #define erts_smp_atomic32_set_mb erts_atomic32_set_mb @@ -388,6 +396,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb #define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb #define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb +#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb #define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb #define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb @@ -402,6 +411,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb #define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb #define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb +#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb #define erts_smp_atomic32_init_relb erts_atomic32_init_relb #define erts_smp_atomic32_set_relb erts_atomic32_set_relb @@ -416,6 +426,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb #define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb #define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb +#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb #define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb #define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb @@ -430,6 +441,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb #define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb #define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb +#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb #define erts_smp_atomic32_init_rb erts_atomic32_init_rb #define erts_smp_atomic32_set_rb erts_atomic32_set_rb @@ -444,6 +456,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb #define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb #define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb +#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb #define erts_smp_atomic32_init_wb erts_atomic32_init_wb #define erts_smp_atomic32_set_wb erts_atomic32_set_wb @@ -458,6 +471,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb #define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb #define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb +#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb #else /* !ERTS_SMP */ @@ -513,6 +527,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_nob erts_no_atomic_read_band #define erts_smp_atomic_xchg_nob erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset #define erts_smp_atomic_init_mb erts_no_atomic_set #define erts_smp_atomic_set_mb erts_no_atomic_set @@ -527,6 +542,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_mb erts_no_atomic_read_band #define erts_smp_atomic_xchg_mb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset #define erts_smp_atomic_init_acqb erts_no_atomic_set #define erts_smp_atomic_set_acqb erts_no_atomic_set @@ -541,6 +557,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band #define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset #define erts_smp_atomic_init_relb erts_no_atomic_set #define erts_smp_atomic_set_relb erts_no_atomic_set @@ -555,6 +572,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_relb erts_no_atomic_read_band #define erts_smp_atomic_xchg_relb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset #define erts_smp_atomic_init_ddrb erts_no_atomic_set #define erts_smp_atomic_set_ddrb erts_no_atomic_set @@ -569,6 +587,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band #define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset #define erts_smp_atomic_init_rb erts_no_atomic_set #define erts_smp_atomic_set_rb erts_no_atomic_set @@ -583,6 +602,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_rb erts_no_atomic_read_band #define erts_smp_atomic_xchg_rb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset #define erts_smp_atomic_init_wb erts_no_atomic_set #define erts_smp_atomic_set_wb erts_no_atomic_set @@ -597,6 +617,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_read_band_wb erts_no_atomic_read_band #define erts_smp_atomic_xchg_wb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg +#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset /* 32-bit atomics */ @@ -613,6 +634,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset #define erts_smp_atomic32_init_mb erts_no_atomic32_set #define erts_smp_atomic32_set_mb erts_no_atomic32_set @@ -627,6 +649,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_acqb erts_no_atomic32_set #define erts_smp_atomic32_set_acqb erts_no_atomic32_set @@ -641,6 +664,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_relb erts_no_atomic32_set #define erts_smp_atomic32_set_relb erts_no_atomic32_set @@ -655,6 +679,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_ddrb erts_no_atomic32_set #define erts_smp_atomic32_set_ddrb erts_no_atomic32_set @@ -669,6 +694,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_rb erts_no_atomic32_set #define erts_smp_atomic32_set_rb erts_no_atomic32_set @@ -683,6 +709,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset #define erts_smp_atomic32_init_wb erts_no_atomic32_set #define erts_smp_atomic32_set_wb erts_no_atomic32_set @@ -697,6 +724,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band #define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset #endif /* !ERTS_SMP */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 17628286bc..0e759a1eec 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -533,6 +533,9 @@ ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp, ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i); ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var); @@ -553,6 +556,9 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, erts_aint32_t new, erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, @@ -612,6 +618,78 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #ifdef USE_THREADS +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_nob(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_ddrb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_rb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_wb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_acqb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_relb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_mb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_nob(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_ddrb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_rb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_wb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_acqb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_relb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_mb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +#define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \ +do { \ + Type act = ReadOp((VarP)); \ + while (1) { \ + Type exp = act; \ + Type new = exp & ~(Mask); \ + new |= ((Mask) & (Set)); \ + act = CmpxchgOp((VarP), new, exp); \ + if (act == exp) \ + return act; \ + } \ +} while (0) +#endif + /* * See "Documentation of atomics and memory barriers" at the top * of this file for info on atomics. @@ -670,6 +748,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_nob ethr_atomic_xchg #define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_nob(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg, + var, mask, set); +} +#endif + #define erts_atomic_init_mb ethr_atomic_init_mb #define erts_atomic_set_mb ethr_atomic_set_mb #define erts_atomic_read_mb ethr_atomic_read_mb @@ -684,6 +775,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_mb ethr_atomic_xchg_mb #define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_mb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_mb, + var, mask, set); +} +#endif + #define erts_atomic_init_acqb ethr_atomic_init_acqb #define erts_atomic_set_acqb ethr_atomic_set_acqb #define erts_atomic_read_acqb ethr_atomic_read_acqb @@ -698,6 +802,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb #define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_acqb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_acqb, + var, mask, set); +} +#endif + #define erts_atomic_init_relb ethr_atomic_init_relb #define erts_atomic_set_relb ethr_atomic_set_relb #define erts_atomic_read_relb ethr_atomic_read_relb @@ -712,6 +829,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb ethr_atomic_xchg_relb #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_relb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_relb, + var, mask, set); +} +#endif + #define erts_atomic_init_ddrb ethr_atomic_init_ddrb #define erts_atomic_set_ddrb ethr_atomic_set_ddrb #define erts_atomic_read_ddrb ethr_atomic_read_ddrb @@ -726,6 +856,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb #define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_ddrb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_ddrb, + var, mask, set); +} +#endif + #define erts_atomic_init_rb ethr_atomic_init_rb #define erts_atomic_set_rb ethr_atomic_set_rb #define erts_atomic_read_rb ethr_atomic_read_rb @@ -740,6 +883,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_rb ethr_atomic_xchg_rb #define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_rb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_rb, + var, mask, set); +} +#endif + #define erts_atomic_init_wb ethr_atomic_init_wb #define erts_atomic_set_wb ethr_atomic_set_wb #define erts_atomic_read_wb ethr_atomic_read_wb @@ -754,6 +910,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_wb ethr_atomic_xchg_wb #define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_bset_wb(erts_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, + ethr_atomic_read, + ethr_atomic_cmpxchg_wb, + var, mask, set); +} +#endif + /* 32-bit atomics */ #define erts_atomic32_init_nob ethr_atomic32_init @@ -770,6 +939,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_nob ethr_atomic32_xchg #define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_nob(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg, + var, mask, set); +} +#endif + #define erts_atomic32_init_mb ethr_atomic32_init_mb #define erts_atomic32_set_mb ethr_atomic32_set_mb #define erts_atomic32_read_mb ethr_atomic32_read_mb @@ -784,6 +966,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb #define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_mb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_mb, + var, mask, set); +} +#endif + #define erts_atomic32_init_acqb ethr_atomic32_init_acqb #define erts_atomic32_set_acqb ethr_atomic32_set_acqb #define erts_atomic32_read_acqb ethr_atomic32_read_acqb @@ -798,6 +993,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb #define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_acqb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_acqb, + var, mask, set); +} +#endif + #define erts_atomic32_init_relb ethr_atomic32_init_relb #define erts_atomic32_set_relb ethr_atomic32_set_relb #define erts_atomic32_read_relb ethr_atomic32_read_relb @@ -812,6 +1020,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_relb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_relb, + var, mask, set); +} +#endif + #define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb #define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb #define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb @@ -826,6 +1047,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb #define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_ddrb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_ddrb, + var, mask, set); +} +#endif + #define erts_atomic32_init_rb ethr_atomic32_init_rb #define erts_atomic32_set_rb ethr_atomic32_set_rb #define erts_atomic32_read_rb ethr_atomic32_read_rb @@ -840,6 +1074,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb #define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_rb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_rb, + var, mask, set); +} +#endif + #define erts_atomic32_init_wb ethr_atomic32_init_wb #define erts_atomic32_set_wb ethr_atomic32_set_wb #define erts_atomic32_read_wb ethr_atomic32_read_wb @@ -854,6 +1101,21 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb #define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_bset_wb(erts_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, + ethr_atomic32_read, + ethr_atomic32_cmpxchg_wb, + var, mask, set); +} +#endif + +#undef ERTS_ATOMIC_BSET_IMPL__ + #else /* !USE_THREADS */ /* Double word size atomics */ @@ -908,6 +1170,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_nob erts_no_atomic_read_band #define erts_atomic_xchg_nob erts_no_atomic_xchg #define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_nob erts_no_atomic_read_bset #define erts_atomic_init_mb erts_no_atomic_set #define erts_atomic_set_mb erts_no_atomic_set @@ -922,6 +1185,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_mb erts_no_atomic_read_band #define erts_atomic_xchg_mb erts_no_atomic_xchg #define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_mb erts_no_atomic_read_bset #define erts_atomic_init_acqb erts_no_atomic_set #define erts_atomic_set_acqb erts_no_atomic_set @@ -936,6 +1200,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_acqb erts_no_atomic_read_band #define erts_atomic_xchg_acqb erts_no_atomic_xchg #define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset #define erts_atomic_init_relb erts_no_atomic_set #define erts_atomic_set_relb erts_no_atomic_set @@ -950,6 +1215,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_relb erts_no_atomic_read_band #define erts_atomic_xchg_relb erts_no_atomic_xchg #define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_relb erts_no_atomic_read_bset #define erts_atomic_init_ddrb erts_no_atomic_set #define erts_atomic_set_ddrb erts_no_atomic_set @@ -964,6 +1230,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_ddrb erts_no_atomic_read_band #define erts_atomic_xchg_ddrb erts_no_atomic_xchg #define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset #define erts_atomic_init_rb erts_no_atomic_set #define erts_atomic_set_rb erts_no_atomic_set @@ -978,6 +1245,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_rb erts_no_atomic_read_band #define erts_atomic_xchg_rb erts_no_atomic_xchg #define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_rb erts_no_atomic_read_bset #define erts_atomic_init_wb erts_no_atomic_set #define erts_atomic_set_wb erts_no_atomic_set @@ -992,6 +1260,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_read_band_wb erts_no_atomic_read_band #define erts_atomic_xchg_wb erts_no_atomic_xchg #define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg +#define erts_atomic_read_bset_wb erts_no_atomic_read_bset /* 32-bit atomics */ @@ -1008,6 +1277,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_nob erts_no_atomic32_read_band #define erts_atomic32_xchg_nob erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset #define erts_atomic32_init_mb erts_no_atomic32_set #define erts_atomic32_set_mb erts_no_atomic32_set @@ -1022,6 +1292,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_mb erts_no_atomic32_read_band #define erts_atomic32_xchg_mb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset #define erts_atomic32_init_acqb erts_no_atomic32_set #define erts_atomic32_set_acqb erts_no_atomic32_set @@ -1036,6 +1307,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_acqb erts_no_atomic32_read_band #define erts_atomic32_xchg_acqb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset #define erts_atomic32_init_relb erts_no_atomic32_set #define erts_atomic32_set_relb erts_no_atomic32_set @@ -1050,6 +1322,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_relb erts_no_atomic32_read_band #define erts_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset #define erts_atomic32_init_ddrb erts_no_atomic32_set #define erts_atomic32_set_ddrb erts_no_atomic32_set @@ -1064,6 +1337,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band #define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset #define erts_atomic32_init_rb erts_no_atomic32_set #define erts_atomic32_set_rb erts_no_atomic32_set @@ -1078,6 +1352,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_rb erts_no_atomic32_read_band #define erts_atomic32_xchg_rb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset #define erts_atomic32_init_wb erts_no_atomic32_set #define erts_atomic32_set_wb erts_no_atomic32_set @@ -1092,6 +1367,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_read_band_wb erts_no_atomic32_read_band #define erts_atomic32_xchg_wb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg +#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset #endif /* !USE_THREADS */ @@ -1856,6 +2132,16 @@ erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, return old; } +ERTS_GLB_INLINE erts_aint_t +erts_no_atomic_read_bset(erts_no_atomic_t *var, + erts_aint_t mask, + erts_aint_t set) +{ + erts_aint_t old = *var; + *var |= (mask & set); + return old; +} + /* atomic32 */ ERTS_GLB_INLINE void @@ -1943,6 +2229,16 @@ erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, return old; } +ERTS_GLB_INLINE erts_aint32_t +erts_no_atomic32_read_bset(erts_no_atomic32_t *var, + erts_aint32_t mask, + erts_aint32_t set) +{ + erts_aint32_t old = *var; + *var |= (mask & set); + return old; +} + /* spinlock */ ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 86f8e3890b..5386e0eb4e 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -183,7 +183,7 @@ struct port { ErtsProcList *suspended; /* List of suspended processes. */ LineBuf *linebuf; /* Buffer to hold data not ready for process to get (line oriented I/O)*/ - Uint32 status; /* Status and type flags */ + erts_smp_atomic32_t state; /* Status and type flags */ int control_flags; /* Flags for port_control() */ erts_aint32_t snapshot; /* Next snapshot that port should be part of */ struct reg_proc *reg; @@ -1202,8 +1202,8 @@ void erts_smp_xports_unlock(Port *); int erts_lc_is_port_locked(Port *); #endif -ERTS_GLB_INLINE void erts_smp_port_state_lock(Port*); -ERTS_GLB_INLINE void erts_smp_port_state_unlock(Port*); +ERTS_GLB_INLINE void erts_smp_port_minor_lock(Port*); +ERTS_GLB_INLINE void erts_smp_port_minor_unlock(Port*); ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt); ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt); @@ -1212,7 +1212,7 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_smp_port_state_lock(Port* prt) +erts_smp_port_minor_lock(Port* prt) { #ifdef ERTS_SMP erts_smp_spin_lock(&prt->state_lck); @@ -1220,7 +1220,7 @@ erts_smp_port_state_lock(Port* prt) } ERTS_GLB_INLINE void -erts_smp_port_state_unlock(Port *prt) +erts_smp_port_minor_unlock(Port *prt) { #ifdef ERTS_SMP erts_smp_spin_unlock(&prt->state_lck); @@ -1274,8 +1274,10 @@ erts_smp_port_unlock(Port *prt) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS) \ - (!(PP) || ((PP)->status & (FLGS)) || (PP)->id != (ID)) +#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS) \ + (!(PP) \ + || (erts_smp_atomic32_read_nob(&(PP)->state) & (FLGS)) \ + || (PP)->id != (ID)) /* port lookup */ @@ -1300,16 +1302,11 @@ Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks); ERTS_GLB_INLINE Port*erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32); ERTS_GLB_INLINE void erts_port_release(Port *); -ERTS_GLB_INLINE Port*erts_drvport2port(ErlDrvPort); +ERTS_GLB_INLINE Port*erts_drvport2port(ErlDrvPort, erts_aint32_t *); ERTS_GLB_INLINE Port*erts_drvportid2port(Eterm); ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm id); ERTS_GLB_INLINE int erts_is_port_alive(Eterm id); ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm id); -ERTS_GLB_INLINE void erts_port_status_bandor_set(Port *, Uint32, Uint32); -ERTS_GLB_INLINE void erts_port_status_band_set(Port *, Uint32); -ERTS_GLB_INLINE void erts_port_status_bor_set(Port *, Uint32); -ERTS_GLB_INLINE void erts_port_status_set(Port *, Uint32); -ERTS_GLB_INLINE Uint32 erts_port_status_get(Port *); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -1326,14 +1323,14 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs prt = &erts_port[internal_port_index(id)]; - erts_smp_port_state_lock(prt); + erts_smp_port_minor_lock(prt); if (ERTS_INVALID_PORT_OPT(prt, id, sflgs)) { - erts_smp_port_state_unlock(prt); + erts_smp_port_minor_unlock(prt); prt = NULL; } else { erts_smp_atomic_inc_nob(&prt->refc); - erts_smp_port_state_unlock(prt); + erts_smp_port_minor_unlock(prt); #ifdef ERTS_SMP if (no_proc_locks) @@ -1347,8 +1344,8 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs /* The id may not have changed... */ ERTS_SMP_LC_ASSERT(prt->id == id); - /* ... but status may have... */ - if (prt->status & sflgs) { + /* ... but state may have... */ + if (erts_smp_atomic32_read_nob(&prt->state) & sflgs) { erts_smp_port_unlock(prt); /* Also decrements refc... */ prt = NULL; } @@ -1366,14 +1363,18 @@ erts_port_release(Port *prt) } ERTS_GLB_INLINE Port* -erts_drvport2port(ErlDrvPort drvport) +erts_drvport2port(ErlDrvPort drvport, erts_aint32_t *statep) { int ix = (int) drvport; + erts_aint32_t state; if (ix < 0 || erts_max_ports <= ix) return NULL; - if (erts_port[ix].status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + state = erts_smp_atomic32_read_nob(&erts_port[ix].state); + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) return NULL; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[ix])); + if (statep) + *statep = state; return &erts_port[ix]; } @@ -1381,12 +1382,14 @@ ERTS_GLB_INLINE Port* erts_drvportid2port(Eterm id) { int ix; + erts_aint32_t state; if (is_not_internal_port(id)) return NULL; ix = (int) internal_port_index(id); if (erts_max_ports <= ix) return NULL; - if (erts_port[ix].status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + state = erts_smp_atomic32_read_nob(&erts_port[ix].state); + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) return NULL; if (erts_port[ix].id != id) return NULL; @@ -1400,17 +1403,14 @@ erts_portid2status(Eterm id) if (is_not_internal_port(id)) return ERTS_PORT_SFLG_INVALID; else { - Uint32 status; + erts_aint32_t state; int ix = internal_port_index(id); if (erts_max_ports <= ix) return ERTS_PORT_SFLG_INVALID; - erts_smp_port_state_lock(&erts_port[ix]); - if (erts_port[ix].id == id) - status = erts_port[ix].status; - else - status = ERTS_PORT_SFLG_INVALID; - erts_smp_port_state_unlock(&erts_port[ix]); - return status; + state = erts_smp_atomic32_read_ddrb(&erts_port[ix].state); + if (erts_port[ix].id != id) + return ERTS_PORT_SFLG_INVALID; + return state; } } @@ -1426,50 +1426,6 @@ erts_is_valid_tracer_port(Eterm id) { return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); } - -ERTS_GLB_INLINE void erts_port_status_bandor_set(Port *prt, - Uint32 band_status, - Uint32 bor_status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status &= band_status; - prt->status |= bor_status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE void erts_port_status_band_set(Port *prt, Uint32 status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status &= status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE void erts_port_status_bor_set(Port *prt, Uint32 status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status |= status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE void erts_port_status_set(Port *prt, Uint32 status) -{ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_port_state_lock(prt); - prt->status = status; - erts_smp_port_state_unlock(prt); -} - -ERTS_GLB_INLINE Uint32 erts_port_status_get(Port *prt) -{ - Uint32 res; - erts_smp_port_state_lock(prt); - res = prt->status; - erts_smp_port_state_unlock(prt); - return res; -} #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ /* erl_drv_thread.c */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index f9e1b62f14..00ecf1fca2 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -90,35 +90,35 @@ static ERTS_INLINE ErlIOQueue* drvport2ioq(ErlDrvPort drvport) { int ix = (int) drvport; - Uint32 status; + erts_aint32_t state; if (ix < 0 || erts_max_ports <= ix) return NULL; + state = erts_smp_atomic32_read_nob(&erts_port[ix].state); + +#ifdef ERTS_ENABLE_LOCK_CHECK + if (erts_get_scheduler_data()) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[ix])); ERTS_LC_ASSERT(!erts_port[ix].port_data_lock || erts_lc_mtx_is_locked( &erts_port[ix].port_data_lock->mtx)); - - status = erts_port[ix].status; } else { - erts_smp_port_state_lock(&erts_port[ix]); - status = erts_port[ix].status; - erts_smp_port_state_unlock(&erts_port[ix]); - - ERTS_LC_ASSERT((status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + ERTS_LC_ASSERT((state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) || erts_port[ix].port_data_lock); ERTS_LC_ASSERT(!erts_port[ix].port_data_lock || erts_lc_mtx_is_locked( &erts_port[ix].port_data_lock->mtx)); - } - return ((status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - ? NULL - : &erts_port[ix].ioq); +#endif + + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + return NULL; + else + return &erts_port[ix].ioq; } static ERTS_INLINE int @@ -216,7 +216,7 @@ kill_port(Port *pp) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); erts_port_task_free_port(pp); - ASSERT(pp->status & ERTS_PORT_SFLGS_DEAD); + ASSERT(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD); } #ifdef ERTS_SMP @@ -242,27 +242,32 @@ get_free_port(void) erts_smp_spin_lock(&get_free_port_lck); num = last_port_num + 1; - for (;; ++num) { + for (;; ++num) { + erts_aint32_t act; + port = &erts_port[num & erts_port_tab_index_mask]; - erts_smp_port_state_lock(port); - if (port->status & ERTS_PORT_SFLG_FREE) { - last_port_num = num; - erts_smp_spin_unlock(&get_free_port_lck); - break; + act = erts_smp_atomic32_read_nob(&port->state); + + while (act & ERTS_PORT_SFLG_FREE) { + erts_aint32_t exp = act; + act = erts_smp_atomic32_cmpxchg_relb(&port->state, + ERTS_PORT_SFLG_INITIALIZING, + exp); + if (act == exp) { + last_port_num = num; + erts_smp_spin_unlock(&get_free_port_lck); + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0); + erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */ + return num & port_num_mask; + } } - erts_smp_port_state_unlock(port); if (--tries == 0) { erts_smp_spin_unlock(&get_free_port_lck); return -1; } } - port->status = ERTS_PORT_SFLG_INITIALIZING; - ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0); - erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */ - erts_smp_port_state_unlock(port); - return num & port_num_mask; } /* @@ -284,13 +289,12 @@ erts_test_next_port(int set, Uint next) Port* port = &erts_port[num & erts_port_tab_index_mask]; - erts_smp_port_state_lock(port); + erts_aint32_t state = erts_smp_atomic32_read_nob(&port->state); - if (port->status & ERTS_PORT_SFLG_FREE) { + if (state & ERTS_PORT_SFLG_FREE) { last_port_num = num - 1; res = num & port_num_mask; } - erts_smp_port_state_unlock(port); } erts_smp_spin_unlock(&get_free_port_lck); return res; @@ -326,38 +330,37 @@ void port_cleanup(Port *prt) { #ifdef ERTS_SMP - Uint32 port_specific; erts_smp_mtx_t *mtx; +#endif +#if defined(ERTS_SMP) || defined(ERTS_ENABLE_LOCK_CHECK) + erts_aint32_t state = erts_smp_atomic32_read_nob(&prt->state); #endif erts_driver_t *driver; - erts_smp_port_state_lock(prt); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); driver = prt->drv_ptr; prt->drv_ptr = NULL; ASSERT(driver); - ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED); - ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0); + ERTS_LC_ASSERT(state & ERTS_PORT_SFLG_FREE_SCHEDULED); + ERTS_LC_ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG); + ERTS_LC_ASSERT(!(state & ERTS_PORT_SFLG_FREE)); - ASSERT(prt->status & ERTS_PORT_SFLG_PORT_DEBUG); - ASSERT(!(prt->status & ERTS_PORT_SFLG_FREE)); - prt->status = ERTS_PORT_SFLG_FREE; + ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0); #ifdef ERTS_SMP - - port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK); - mtx = prt->lock; ASSERT(mtx); prt->lock = NULL; - erts_smp_port_state_unlock(prt); erts_smp_mtx_unlock(mtx); +#endif - if (port_specific) { + erts_smp_atomic32_set_relb(&prt->state, ERTS_PORT_SFLG_FREE); + +#ifdef ERTS_SMP + if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { erts_smp_mtx_destroy(mtx); erts_free(ERTS_ALC_T_PORT_LOCK, mtx); } @@ -422,13 +425,13 @@ static void stopq(Port* prt) static void setup_port(Port* prt, Eterm pid, erts_driver_t *driver, - ErlDrvData drv_data, char *name, Uint32 xstatus) + ErlDrvData drv_data, char *name, erts_aint32_t xstate) { ErtsRunQueue *runq = erts_get_runq_current(NULL); char *new_name, *old_name; #ifdef DEBUG /* Make sure the debug flags survives until port is freed */ - xstatus |= ERTS_PORT_SFLG_PORT_DEBUG; + xstate |= ERTS_PORT_SFLG_PORT_DEBUG; #endif ASSERT(runq); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -437,8 +440,6 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1); sys_strcpy(new_name, name); erts_smp_runq_lock(runq); - erts_smp_port_state_lock(prt); - prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus; prt->snapshot = erts_smp_atomic32_read_nob(&erts_ports_snapshot); old_name = prt->name; prt->name = new_name; @@ -447,7 +448,8 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, #endif ASSERT(!prt->drv_ptr); prt->drv_ptr = driver; - erts_smp_port_state_unlock(prt); + erts_smp_atomic32_set_relb(&prt->state, + ERTS_PORT_SFLG_CONNECTED | xstate); erts_smp_runq_unlock(runq); #ifdef ERTS_SMP ASSERT(!prt->xports); @@ -492,7 +494,7 @@ erts_wake_process_later(Port *prt, Process *process) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt->status & ERTS_PORT_SFLGS_DEAD) + if (erts_smp_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_DEAD) return; for (p = &(prt->suspended); *p != NULL; p = &((*p)->next)) @@ -522,7 +524,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ int port_num; int port_ix; ErlDrvData drv_data = 0; - Uint32 xstatus = 0; + erts_aint32_t xstate = 0; Port *port; int fpe_was_unmasked; @@ -601,11 +603,9 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ *error_number_ptr = BADARG; } /* Need to mark the port as free again */ - erts_smp_port_state_lock(port); - port->status = ERTS_PORT_SFLG_FREE; ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2); erts_smp_atomic_set_nob(&port->refc, 0); - erts_smp_port_state_unlock(port); + erts_smp_atomic32_set_relb(&port->state, ERTS_PORT_SFLG_FREE); return -3; } @@ -623,7 +623,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ erts_smp_mtx_init_x(port->lock, "port_lock", port->id); - xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; + xstate |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } #endif @@ -637,7 +637,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ erts_smp_mtx_lock(port->lock); #endif - setup_port(port, pid, driver, drv_data, name, xstatus); + setup_port(port, pid, driver, drv_data, name, xstate); if (IS_TRACED_FL(port, F_TRACE_PORTS)) { trace_port_open(port, @@ -737,11 +737,11 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ Process *rp; int port_num; Eterm port_id; - Uint32 xstatus = 0; + erts_aint32_t xstate = 0; ERTS_SMP_CHK_NO_PROC_LOCKS; - creator_port = erts_drvport2port(creator_port_ix); + creator_port = erts_drvport2port(creator_port_ix, NULL); if (!creator_port) return (ErlDrvTermData) -1; @@ -780,7 +780,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ port->lock = erts_alloc(ERTS_ALC_T_PORT_LOCK, sizeof(erts_smp_mtx_t)); erts_smp_mtx_init_locked_x(port->lock, "port_lock", port_id); - xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; + xstate |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } #endif @@ -793,7 +793,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); - setup_port(port, pid, driver, drv_data, name, xstatus); + setup_port(port, pid, driver, drv_data, name, xstate); port->id = port_id; erts_add_link(&(port->nlinks), LINK_PID, pid); @@ -1349,7 +1349,7 @@ void init_io(void) ERTS_TRACE_FLAGS(&erts_port[i]) = 0; erts_port[i].drv_ptr = NULL; - erts_port[i].status = ERTS_PORT_SFLG_FREE; + erts_smp_atomic32_init_nob(&erts_port[i].state, ERTS_PORT_SFLG_FREE); erts_port[i].name = NULL; erts_port[i].nlinks = NULL; erts_port[i].monitors = NULL; @@ -1593,7 +1593,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) * len -- length of data */ -static void deliver_read_message(Port* prt, Eterm to, +static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, char *hbuf, ErlDrvSizeT hlen, char *buf, ErlDrvSizeT len, int eol) { @@ -1611,10 +1611,11 @@ static void deliver_read_message(Port* prt, Eterm to, ERTS_SMP_CHK_NO_PROC_LOCKS; need = 3 + 3 + 2*hlen; - if (prt->status & ERTS_PORT_SFLG_LINEBUF_IO) { + + if (state & ERTS_PORT_SFLG_LINEBUF_IO) { need += 3; } - if (prt->status & ERTS_PORT_SFLG_BINARY_IO && buf != NULL) { + if ((state & ERTS_PORT_SFLG_BINARY_IO) && buf != NULL) { need += PROC_BIN_SIZE; } else { need += 2*len; @@ -1630,7 +1631,7 @@ static void deliver_read_message(Port* prt, Eterm to, hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); listp = NIL; - if ((prt->status & ERTS_PORT_SFLG_BINARY_IO) == 0) { + if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { listp = buf_to_intlist(&hp, buf, len, listp); } else if (buf != NULL) { ProcBin* pb; @@ -1661,7 +1662,7 @@ static void deliver_read_message(Port* prt, Eterm to, listp = buf_to_intlist(&hp, hbuf, hlen, listp); } - if (prt->status & ERTS_PORT_SFLG_LINEBUF_IO){ + if (state & ERTS_PORT_SFLG_LINEBUF_IO){ listp = TUPLE2(hp, (eol) ? am_eol : am_noeol, listp); hp += 3; } @@ -1686,7 +1687,8 @@ static void deliver_read_message(Port* prt, Eterm to, * Deliver all lines in a line buffer, repeats calls to * deliver_read_message, and takes the same parameters. */ -static void deliver_linebuf_message(Port* prt, Eterm to, +static void deliver_linebuf_message(Port* prt, erts_aint_t state, + Eterm to, char* hbuf, ErlDrvSizeT hlen, char *buf, ErlDrvSizeT len) { @@ -1695,7 +1697,7 @@ static void deliver_linebuf_message(Port* prt, Eterm to, if(init_linebuf_context(&lc,&(prt->linebuf), buf, len) < 0) return; while((ret = read_linebuf(&lc)) > LINEBUF_EMPTY) - deliver_read_message(prt, to, hbuf, hlen, LINEBUF_DATA(lc), + deliver_read_message(prt, state, to, hbuf, hlen, LINEBUF_DATA(lc), LINEBUF_DATALEN(lc), (ret == LINEBUF_EOL)); } @@ -1706,19 +1708,24 @@ static void deliver_linebuf_message(Port* prt, Eterm to, * Parameters: * prt - Pointer to a Port structure for this port. */ -static void flush_linebuf_messages(Port *prt) +static void flush_linebuf_messages(Port *prt, erts_aint32_t state) { LineBufContext lc; int ret; ERTS_SMP_LC_ASSERT(!prt || erts_lc_is_port_locked(prt)); - if(prt == NULL || !(prt->status & ERTS_PORT_SFLG_LINEBUF_IO)) + + if (!prt) + return; + + if (!(state & ERTS_PORT_SFLG_LINEBUF_IO)) return; if(init_linebuf_context(&lc,&(prt->linebuf), NULL, 0) < 0) return; while((ret = flush_linebuf(&lc)) > LINEBUF_EMPTY) deliver_read_message(prt, + state, prt->connected, NULL, 0, @@ -1747,6 +1754,7 @@ deliver_vec_message(Port* prt, /* Port */ ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; + erts_aint32_t state; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1762,12 +1770,13 @@ deliver_vec_message(Port* prt, /* Port */ if (!rp) return; + state = erts_smp_atomic32_read_nob(&prt->state); /* * Calculate the exact number of heap words needed. */ need = 3 + 3; /* Heap space for two tuples */ - if (prt->status & ERTS_PORT_SFLG_BINARY_IO) { + if (state & ERTS_PORT_SFLG_BINARY_IO) { need += (2+PROC_BIN_SIZE)*vsize - 2 + hlen*2; } else { need += (hlen+csize)*2; @@ -1778,7 +1787,7 @@ deliver_vec_message(Port* prt, /* Port */ listp = NIL; iov += vsize; - if ((prt->status & ERTS_PORT_SFLG_BINARY_IO) == 0) { + if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { Eterm* thp = hp; while (vsize--) { iov--; @@ -1864,7 +1873,7 @@ static void deliver_bin_message(Port* prt, /* port */ /* * Note. * - * The test for (p->status & ERTS_PORT_SFLGS_DEAD) == 0 is important since the + * The test for ERTS_PORT_SFLGS_DEAD is important since the * driver's flush function might call driver_async, which when using no * threads and being short circuited will notice that the io queue is empty * (after calling the driver's async_ready) and recursively call @@ -1899,7 +1908,8 @@ static void flush_port(Port *p) ASSERT(!p->xports); #endif } - if ((p->status & ERTS_PORT_SFLGS_DEAD) == 0 && is_port_ioq_empty(p)) { + if ((erts_smp_atomic32_read_nob(&p->state) & ERTS_PORT_SFLGS_DEAD) == 0 + && is_port_ioq_empty(p)) { terminate_port(p); } } @@ -1911,7 +1921,7 @@ terminate_port(Port *prt) Eterm send_closed_port_id; Eterm connected_id = NIL /* Initialize to silence compiler */; erts_driver_t *drv; - int halt; + erts_aint32_t state; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -1919,10 +1929,10 @@ terminate_port(Port *prt) ASSERT(!prt->nlinks); ASSERT(!prt->monitors); - /* prt->status may be altered by kill_port()below */ - halt = (prt->status & ERTS_PORT_SFLG_HALT) != 0; - if (prt->status & ERTS_PORT_SFLG_SEND_CLOSED) { - erts_port_status_band_set(prt, ~ERTS_PORT_SFLG_SEND_CLOSED); + /* state may be altered by kill_port() below */ + state = erts_smp_atomic32_read_band_nob(&prt->state, + ~ERTS_PORT_SFLG_SEND_CLOSED); + if (state & ERTS_PORT_SFLG_SEND_CLOSED) { send_closed_port_id = prt->id; connected_id = prt->connected; } @@ -1978,7 +1988,8 @@ terminate_port(Port *prt) * We don't want to send the closed message until after the * port has been removed from the port table (in kill_port()). */ - if (halt && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { + if ((state & ERTS_PORT_SFLG_HALT) + && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { erts_smp_port_unlock(prt); /* We will exit and never return */ erl_exit_flush_async(erts_halt_code, ""); } @@ -2100,6 +2111,7 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) { ErtsLink *lnk; Eterm rreason; + erts_aint32_t state; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); @@ -2119,9 +2131,10 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) } #endif - if ((p->status & (ERTS_PORT_SFLGS_DEAD - | ERTS_PORT_SFLG_EXITING - | ERTS_PORT_SFLG_IMMORTAL)) + state = erts_smp_atomic32_read_nob(&p->state); + if ((state & (ERTS_PORT_SFLGS_DEAD + | ERTS_PORT_SFLG_EXITING + | ERTS_PORT_SFLG_IMMORTAL)) || ((reason == am_normal) && ((from != p->connected) && (from != p->id)))) { return; @@ -2142,7 +2155,7 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) if (p->reg != NULL) (void) erts_unregister_name(NULL, 0, p, p->reg->name); - erts_port_status_bor_set(p, ERTS_PORT_SFLG_EXITING); + state = erts_smp_atomic32_read_bor_relb(&p->state, ERTS_PORT_SFLG_EXITING); { SweepContext sc = {p->id, rreason}; @@ -2158,17 +2171,20 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason) } DRV_MONITOR_UNLOCK_PDL(p); - if ((p->status & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) { + if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) { erts_do_net_exits(p->dist_entry, rreason); erts_deref_dist_entry(p->dist_entry); - p->dist_entry = NULL; - erts_port_status_band_set(p, ~ERTS_PORT_SFLG_DISTRIBUTION); + p->dist_entry = NULL; + erts_smp_atomic32_read_band_relb(&p->state, + ~ERTS_PORT_SFLG_DISTRIBUTION); } if ((reason != am_kill) && !is_port_ioq_empty(p)) { - erts_port_status_bandor_set(p, - ~ERTS_PORT_SFLG_EXITING, /* must turn it off */ - ERTS_PORT_SFLG_CLOSING); + /* must turn exiting flag off */ + erts_smp_atomic32_read_bset_relb(&p->state, + (ERTS_PORT_SFLG_EXITING + | ERTS_PORT_SFLG_CLOSING), + ERTS_PORT_SFLG_CLOSING); flush_port(p); } else { @@ -2217,7 +2233,8 @@ void erts_port_command(Process *proc, if ((pid = port->connected) == tp[1]) { /* PID must be connected */ if (tp[2] == am_close) { - erts_port_status_bor_set(port, ERTS_PORT_SFLG_SEND_CLOSED); + erts_smp_atomic32_read_bor_relb(&port->state, + ERTS_PORT_SFLG_SEND_CLOSED); erts_do_exit_port(port, pid, am_normal); #ifdef USE_VM_PROBES @@ -2443,13 +2460,14 @@ void print_port_info(int to, void *arg, int i) { Port* p = &erts_port[i]; + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - if (p->status & ERTS_PORT_SFLGS_DEAD) + if (state & ERTS_PORT_SFLGS_DEAD) return; erts_print(to, arg, "=port:%T\n", p->id); erts_print(to, arg, "Slot: %d\n", i); - if (p->status & ERTS_PORT_SFLG_CONNECTED) { + if (state & ERTS_PORT_SFLG_CONNECTED) { erts_print(to, arg, "Connected: %T", p->connected); erts_print(to, arg, "\n"); } @@ -2497,8 +2515,8 @@ set_busy_port(ErlDrvPort port_num, int on) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[port_num])); if (on) { - erts_port_status_bor_set(&erts_port[port_num], - ERTS_PORT_SFLG_PORT_BUSY); + erts_smp_atomic32_read_bor_relb(&erts_port[port_num].state, + ERTS_PORT_SFLG_PORT_BUSY); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_busy)) { erts_snprintf(port_str, sizeof(port_str), @@ -2508,8 +2526,8 @@ set_busy_port(ErlDrvPort port_num, int on) #endif } else { ErtsProcList* plp = erts_port[port_num].suspended; - erts_port_status_band_set(&erts_port[port_num], - ~ERTS_PORT_SFLG_PORT_BUSY); + erts_smp_atomic32_read_band_relb(&erts_port[port_num].state, + ~ERTS_PORT_SFLG_PORT_BUSY); erts_port[port_num].suspended = NULL; #ifdef USE_VM_PROBES @@ -2576,19 +2594,27 @@ void set_port_control_flags(ErlDrvPort port_num, int flags) erts_port[port_num].control_flags = flags; } -int get_port_flags(ErlDrvPort ix) { - Port* prt = erts_drvport2port(ix); +int get_port_flags(ErlDrvPort ix) +{ + int flags; + Port *prt; + erts_aint32_t state; + + prt = erts_drvport2port(ix, &state); + if (!prt) + return 0; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt == NULL) - return 0; + flags = 0; + if (state & ERTS_PORT_SFLG_BINARY_IO) + flags |= PORT_FLAG_BINARY; + if (state & ERTS_PORT_SFLG_LINEBUF_IO) + flags |= PORT_FLAG_LINE; - return (prt->status & ERTS_PORT_SFLG_BINARY_IO ? PORT_FLAG_BINARY : 0) - | (prt->status & ERTS_PORT_SFLG_LINEBUF_IO ? PORT_FLAG_LINE : 0); + return flags; } - void erts_raw_port_command(Port* p, byte* buf, Uint len) { int fpe_was_unmasked; @@ -2624,8 +2650,9 @@ int async_ready(Port *p, void* data) ERTS_SMP_CHK_NO_PROC_LOCKS; if (p) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); - ASSERT(!(p->status & ERTS_PORT_SFLGS_DEAD)); + ASSERT(!(state & ERTS_PORT_SFLGS_DEAD)); if (p->drv_ptr->ready_async != NULL) { #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_ready_async)) { @@ -2641,7 +2668,7 @@ int async_ready(Port *p, void* data) ASSERT(!p->xports); #endif } - if ((p->status & ERTS_PORT_SFLG_CLOSING) && is_port_ioq_empty(p)) { + if ((state & ERTS_PORT_SFLG_CLOSING) && is_port_ioq_empty(p)) { terminate_port(p); } } @@ -2736,7 +2763,7 @@ erts_get_port_names(Eterm id) pnp_len = sizeof(ErtsPortNames) + len; pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len); } - erts_smp_port_state_lock(prt); + erts_smp_port_minor_lock(prt); if (id != prt->id) { len = nlen = 0; name = driver_name = NULL; @@ -2764,7 +2791,7 @@ erts_get_port_names(Eterm id) } do_realloc = 0; } - erts_smp_port_state_unlock(prt); + erts_smp_port_minor_unlock(prt); } while (do_realloc); } return pnp; @@ -2803,7 +2830,7 @@ ErlDrvTermData driver_mk_term_nil(void) void driver_report_exit(int ix, int status) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); Eterm* hp; Eterm tuple; Process *rp; @@ -2852,14 +2879,13 @@ deliver_term_check_port(ErlDrvPort drvport) res = -1; /* invalid */ else { Port* prt = &erts_port[ix]; - erts_smp_port_state_lock(prt); - if (!(prt->status & ERTS_PORT_SFLGS_INVALID_LOOKUP)) + erts_aint32_t state = erts_smp_atomic32_read_nob(&prt->state); + if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) res = 1; /* ok */ - else if (prt->status & ERTS_PORT_SFLG_CLOSING) + else if (state & ERTS_PORT_SFLG_CLOSING) res = 0; /* closing */ else res = -1; /* invalid (dead) */ - erts_smp_port_state_unlock(prt); } return res; } @@ -3428,7 +3454,7 @@ driver_deliver_term(ErlDrvPort port, int driver_output_term(ErlDrvPort ix, ErlDrvTermData* data, int len) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -3454,19 +3480,20 @@ driver_send_term(ErlDrvPort ix, ErlDrvTermData to, ErlDrvTermData* data, int len int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, ErlDrvBinary* bin, ErlDrvSizeT offs, ErlDrvSizeT len) { - Port* prt = erts_drvport2port(ix); + erts_aint32_t state; + Port* prt = erts_drvport2port(ix, &state); ERTS_SMP_CHK_NO_PROC_LOCKS; if (prt == NULL) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt->status & ERTS_PORT_SFLG_CLOSING) + if (state & ERTS_PORT_SFLG_CLOSING) return 0; prt->bytes_in += (hlen + len); erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); - if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { + if (state & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, (byte*) hbuf, hlen, @@ -3488,7 +3515,8 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, char* buf, ErlDrvSizeT len) { - Port* prt = erts_drvport2port(ix); + erts_aint32_t state; + Port* prt = erts_drvport2port(ix, &state); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -3497,12 +3525,12 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt->status & ERTS_PORT_SFLG_CLOSING) + if (state & ERTS_PORT_SFLG_CLOSING) return 0; prt->bytes_in += (hlen + len); erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); - if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { + if (state & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, prt->dist_entry, @@ -3514,10 +3542,10 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, (byte*) hbuf, hlen, (byte*) buf, len); } - else if(prt->status & ERTS_PORT_SFLG_LINEBUF_IO) - deliver_linebuf_message(prt, prt->connected, hbuf, hlen, buf, len); + else if (state & ERTS_PORT_SFLG_LINEBUF_IO) + deliver_linebuf_message(prt, state, prt->connected, hbuf, hlen, buf, len); else - deliver_read_message(prt, prt->connected, hbuf, hlen, buf, len, 0); + deliver_read_message(prt, state, prt->connected, hbuf, hlen, buf, len, 0); return 0; } @@ -3538,6 +3566,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, SysIOVec* iov; ErlDrvBinary** binv; Port* prt; + erts_aint32_t state; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -3550,13 +3579,13 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, if (hlen < 0) hlen = 0; - prt = erts_drvport2port(ix); + prt = erts_drvport2port(ix, &state); if (prt == NULL) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (prt->status & ERTS_PORT_SFLG_CLOSING) + if (state & ERTS_PORT_SFLG_CLOSING) return 0; /* size > 0 ! */ @@ -3826,7 +3855,7 @@ ErlDrvPDL driver_pdl_create(ErlDrvPort dp) { ErlDrvPDL pdl; - Port *pp = erts_drvport2port(dp); + Port *pp = erts_drvport2port(dp, NULL); if (!pp || pp->port_data_lock) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, @@ -4283,7 +4312,7 @@ drv_cancel_timer(Port *prt) int driver_set_timer(ErlDrvPort ix, unsigned long t) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -4310,7 +4339,7 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t) int driver_cancel_timer(ErlDrvPort ix) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); if (prt == NULL) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -4322,7 +4351,7 @@ int driver_cancel_timer(ErlDrvPort ix) int driver_read_timer(ErlDrvPort ix, unsigned long* t) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -4400,8 +4429,10 @@ int driver_monitor_process(ErlDrvPort port, { Port *prt; int ret; - Uint32 status; + erts_aint32_t state; +#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) ErtsSchedulerData *sched = erts_get_scheduler_data(); +#endif int ix = (int) port; if (ix < 0 || erts_max_ports <= ix) { return -1; @@ -4410,15 +4441,9 @@ int driver_monitor_process(ErlDrvPort port, DRV_MONITOR_LOCK_PDL(prt); - if (sched) { - status = erts_port[ix].status; - } else { - erts_smp_port_state_lock(prt); - status = erts_port[ix].status; - erts_smp_port_state_unlock(prt); - } + state = erts_smp_atomic32_read_nob(&erts_port[ix].state); - if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { DRV_MONITOR_UNLOCK_PDL(prt); return -1; } @@ -4488,8 +4513,10 @@ int driver_demonitor_process(ErlDrvPort port, { Port *prt; int ret; - Uint32 status; + erts_aint32_t state; +#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) ErtsSchedulerData *sched = erts_get_scheduler_data(); +#endif int ix = (int) port; if (ix < 0 || erts_max_ports <= ix) { return -1; @@ -4498,15 +4525,9 @@ int driver_demonitor_process(ErlDrvPort port, DRV_MONITOR_LOCK_PDL(prt); - if (sched) { - status = erts_port[ix].status; - } else { - erts_smp_port_state_lock(prt); - status = erts_port[ix].status; - erts_smp_port_state_unlock(prt); - } + state = erts_smp_atomic32_read_nob(&erts_port[ix].state); - if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { DRV_MONITOR_UNLOCK_PDL(prt); return -1; } @@ -4558,8 +4579,10 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, { Port *prt; ErlDrvTermData ret; - Uint32 status; + erts_aint32_t state; +#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) ErtsSchedulerData *sched = erts_get_scheduler_data(); +#endif int ix = (int) port; if (ix < 0 || erts_max_ports <= ix) { return driver_term_nil; @@ -4568,15 +4591,8 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port, DRV_MONITOR_LOCK_PDL(prt); - if (sched) { - status = erts_port[ix].status; - } else { - erts_smp_port_state_lock(prt); - status = erts_port[ix].status; - erts_smp_port_state_unlock(prt); - } - - if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { + state = erts_smp_atomic32_read_nob(&erts_port[ix].state); + if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) { DRV_MONITOR_UNLOCK_PDL(prt); return driver_term_nil; } @@ -4651,7 +4667,8 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) static int driver_failure_term(ErlDrvPort ix, Eterm term, int eof) { - Port* prt = erts_drvport2port(ix); + erts_aint32_t state; + Port* prt = erts_drvport2port(ix, &state); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -4659,10 +4676,10 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (eof) - flush_linebuf_messages(prt); - if (prt->status & ERTS_PORT_SFLG_CLOSING) { + flush_linebuf_messages(prt, state); + if (state & ERTS_PORT_SFLG_CLOSING) { terminate_port(prt); - } else if (eof && (prt->status & ERTS_PORT_SFLG_SOFT_EOF)) { + } else if (eof && (state & ERTS_PORT_SFLG_SOFT_EOF)) { deliver_result(prt->id, prt->connected, am_eof); } else { /* XXX UGLY WORK AROUND, Let do_exit_port terminate the port */ @@ -4684,7 +4701,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) */ int driver_exit(ErlDrvPort ix, int err) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); Process* rp; ErtsLink *lnk, *rlnk = NULL; @@ -4757,14 +4774,14 @@ ErlDrvTermData driver_mk_atom(char* string) ErlDrvTermData driver_mk_port(ErlDrvPort ix) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); return (ErlDrvTermData) prt->id; } ErlDrvTermData driver_connected(ErlDrvPort ix) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); ERTS_SMP_CHK_NO_PROC_LOCKS; if (prt == NULL) return NIL; @@ -4774,7 +4791,7 @@ ErlDrvTermData driver_connected(ErlDrvPort ix) ErlDrvTermData driver_caller(ErlDrvPort ix) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); ERTS_SMP_CHK_NO_PROC_LOCKS; if (prt == NULL) return NIL; @@ -4784,7 +4801,7 @@ ErlDrvTermData driver_caller(ErlDrvPort ix) int driver_lock_driver(ErlDrvPort ix) { - Port* prt = erts_drvport2port(ix); + Port* prt = erts_drvport2port(ix, NULL); DE_Handle* dh; ERTS_SMP_CHK_NO_PROC_LOCKS; diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index c1336c60d9..20bc1a6f55 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -204,7 +204,7 @@ static void steal_pending_stop_select(erts_dsprintf_buf_t*, ErlDrvPort, static ERTS_INLINE Eterm drvport2id(ErlDrvPort dp) { - Port *pp = erts_drvport2port(dp); + Port *pp = erts_drvport2port(dp, NULL); if (pp) return pp->id; else { @@ -502,8 +502,8 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, DTRACE_CHARBUF(name, 64); #endif - ERTS_SMP_LC_ASSERT(erts_drvport2port(ix) - && erts_lc_is_port_locked(erts_drvport2port(ix))); + ERTS_SMP_LC_ASSERT(erts_drvport2port(ix, NULL) + && erts_lc_is_port_locked(erts_drvport2port(ix, NULL))); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { @@ -529,9 +529,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, if (!on && (mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { if (IS_FD_UNKNOWN(state)) { /* fast track to stop_select callback */ - stop_select_fn = erts_drvport2port(ix)->drv_ptr->stop_select; + stop_select_fn = erts_drvport2port(ix, NULL)->drv_ptr->stop_select; #ifdef USE_VM_PROBES - strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1); + strncpy(name, erts_drvport2port(ix, NULL)->drv_ptr->name, sizeof(name)-1); name[sizeof(name)-1] = '\0'; #endif ret = 0; @@ -664,14 +664,14 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, } } if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { - erts_driver_t* drv_ptr = erts_drvport2port(ix)->drv_ptr; + erts_driver_t* drv_ptr = erts_drvport2port(ix, NULL)->drv_ptr; ASSERT(new_events==0); if (state->remove_cnt == 0 || !wake_poller) { /* Safe to close fd now as it is not in pollset or there was no need to eject fd (kernel poll) */ stop_select_fn = drv_ptr->stop_select; #ifdef USE_VM_PROBES - strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1); + strncpy(name, erts_drvport2port(ix, NULL)->drv_ptr->name, sizeof(name)-1); name[sizeof(name)-1] = '\0'; #endif } @@ -723,8 +723,8 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, int do_wake = 0; int ret; - ERTS_SMP_LC_ASSERT(erts_drvport2port(ix) - && erts_lc_is_port_locked(erts_drvport2port(ix))); + ERTS_SMP_LC_ASSERT(erts_drvport2port(ix, NULL) + && erts_lc_is_port_locked(erts_drvport2port(ix, NULL))); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { @@ -959,7 +959,7 @@ static void print_select_op(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, ErtsSysFdType fd, int mode, int on) { - Port *pp = erts_drvport2port(ix); + Port *pp = erts_drvport2port(ix, NULL); erts_dsprintf(dsbufp, "driver_select(%p, %d,%s%s%s%s, %d) " "by ", @@ -1030,7 +1030,7 @@ steal_pending_stop_select(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, state->driver.drv_ptr = NULL; } else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { - erts_driver_t* drv_ptr = erts_drvport2port(ix)->drv_ptr; + erts_driver_t* drv_ptr = erts_drvport2port(ix, NULL)->drv_ptr; if (drv_ptr != state->driver.drv_ptr) { /* Some other driver wants the stop_select callback */ if (state->driver.drv_ptr->handle) { @@ -1052,7 +1052,7 @@ static void print_event_op(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) { - Port *pp = erts_drvport2port(ix); + Port *pp = erts_drvport2port(ix, NULL); erts_dsprintf(dsbufp, "driver_event(%p, %d, ", ix, (int) fd); if (!event_data) erts_dsprintf(dsbufp, "NULL"); -- cgit v1.2.3