aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--erts/emulator/beam/bif.c61
-rw-r--r--erts/emulator/beam/dist.c43
-rw-r--r--erts/emulator/beam/dist.h3
-rw-r--r--erts/emulator/beam/erl_async.c2
-rw-r--r--erts/emulator/beam/erl_bif_ddll.c127
-rw-r--r--erts/emulator/beam/erl_bif_info.c3
-rw-r--r--erts/emulator/beam/erl_bif_port.c27
-rw-r--r--erts/emulator/beam/erl_bif_trace.c4
-rw-r--r--erts/emulator/beam/erl_node_tables.c3
-rw-r--r--erts/emulator/beam/erl_port_task.c43
-rw-r--r--erts/emulator/beam/erl_process.c36
-rw-r--r--erts/emulator/beam/erl_process.h30
-rw-r--r--erts/emulator/beam/erl_smp.h28
-rw-r--r--erts/emulator/beam/erl_threads.h296
-rw-r--r--erts/emulator/beam/global.h102
-rw-r--r--erts/emulator/beam/io.c343
-rw-r--r--erts/emulator/sys/common/erl_check_io.c24
17 files changed, 721 insertions, 454 deletions
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 89a8f839db..10dcba565b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1899,30 +1899,34 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
ERTS_SMP_LC_ASSERT(!pt || erts_lc_is_port_locked(pt));
/* We have waited for locks, trace schedule ports */
- if (pt && IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(pt, am_in, am_command);
- }
- if (pt && erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) {
- profile_runnable_port(pt, am_active);
- }
-
- /* XXX let port_command handle the busy stuff !!! */
- if (pt && (pt->status & ERTS_PORT_SFLG_PORT_BUSY)) {
- if (suspend) {
- erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt);
- if (erts_system_monitor_flags.busy_port) {
- monitor_generic(p, am_busy_port, portid);
- }
- }
- /* Virtually schedule out the port before releasing */
+ if (pt) {
+ erts_aint32_t state;
if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(pt, am_out, am_command);
+ trace_sched_ports_where(pt, am_in, am_command);
}
if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) {
- profile_runnable_port(pt, am_inactive);
+ profile_runnable_port(pt, am_active);
+ }
+
+ state = erts_smp_atomic32_read_nob(&pt->state);
+ /* XXX let port_command handle the busy stuff !!! */
+ if (state & ERTS_PORT_SFLG_PORT_BUSY) {
+ if (suspend) {
+ erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt);
+ if (erts_system_monitor_flags.busy_port) {
+ monitor_generic(p, am_busy_port, portid);
+ }
+ }
+ /* Virtually schedule out the port before releasing */
+ if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) {
+ trace_sched_ports_where(pt, am_out, am_command);
+ }
+ if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) {
+ profile_runnable_port(pt, am_inactive);
+ }
+ erts_port_release(pt);
+ return SEND_YIELD;
}
- erts_port_release(pt);
- return SEND_YIELD;
}
if (IS_TRACED(p)) /* trace once only !! */
@@ -3534,14 +3538,17 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
for (i = erts_max_ports-1; i >= 0; i--) {
Port* prt = &erts_port[i];
- erts_smp_port_state_lock(prt);
- if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
- && prt->snapshot != next_ss) {
- ASSERT(prt->snapshot == next_ss - 1);
- *pp++ = prt->id;
- prt->snapshot = next_ss; /* Consumed by this snapshot */
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&prt->state);
+ if (!(state & ERTS_PORT_SFLGS_DEAD)) {
+ erts_smp_port_minor_lock(prt);
+ state = erts_smp_atomic32_read_nob(&prt->state);
+ if (!(state & ERTS_PORT_SFLGS_DEAD) && prt->snapshot != next_ss) {
+ ASSERT(prt->snapshot == next_ss - 1);
+ *pp++ = prt->id;
+ prt->snapshot = next_ss; /* Consumed by this snapshot */
+ }
+ erts_smp_port_minor_unlock(prt);
}
- erts_smp_port_state_unlock(prt);
}
dead_ports = (Eterm*)erts_smp_atomic_xchg_nob(&erts_dead_ports_ptr,
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 5de8236a7e..eccef44a11 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -423,7 +423,8 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
prt = erts_id2port(prt_id, NULL, 0);
if (prt) {
- ASSERT(prt->status & ERTS_PORT_SFLG_DISTRIBUTION);
+ ASSERT(erts_smp_atomic32_read_nob(&prt->state)
+ & ERTS_PORT_SFLG_DISTRIBUTION);
ASSERT(prt->dist_entry);
/* will call do_net_exists !!! */
erts_do_exit_port(prt, prt_id, nd_reason);
@@ -1907,13 +1908,13 @@ int
erts_dist_command(Port *prt, int reds_limit)
{
Sint reds = ERTS_PORT_REDS_DIST_CMD_START;
- int prt_busy;
Uint32 status;
Uint32 flags;
Sint obufsize = 0;
ErtsDistOutputQueue oq, foq;
DistEntry *dep = prt->dist_entry;
Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf);
+ erts_aint32_t state;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -1956,12 +1957,12 @@ erts_dist_command(Port *prt, int reds_limit)
dep->finalized_out_queue.first = NULL;
dep->finalized_out_queue.last = NULL;
+ state = erts_smp_atomic32_read_nob(&prt->state);
+
if (reds > reds_limit)
goto preempted;
- prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY);
-
- if (!prt_busy && foq.first) {
+ if (!(state & ERTS_PORT_SFLG_PORT_BUSY) && foq.first) {
int preempt = 0;
do {
Uint size;
@@ -1977,11 +1978,10 @@ erts_dist_command(Port *prt, int reds_limit)
obufsize += size_obuf(fob);
foq.first = foq.first->next;
free_dist_obuf(fob);
- preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD);
- if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) {
- prt_busy = 1;
+ state = erts_smp_atomic32_read_nob(&prt->state);
+ preempt = reds > reds_limit || (state & ERTS_PORT_SFLGS_DEAD);
+ if (state & ERTS_PORT_SFLG_PORT_BUSY)
break;
- }
} while (foq.first && !preempt);
if (!foq.first)
foq.last = NULL;
@@ -1989,7 +1989,7 @@ erts_dist_command(Port *prt, int reds_limit)
goto preempted;
}
- if (prt_busy) {
+ if (state & ERTS_PORT_SFLG_PORT_BUSY) {
if (oq.first) {
ErtsDistOutputBuf *ob;
int preempt;
@@ -2060,12 +2060,10 @@ erts_dist_command(Port *prt, int reds_limit)
obufsize += size_obuf(fob);
oq.first = oq.first->next;
free_dist_obuf(fob);
- preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD);
- if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) {
- prt_busy = 1;
- if (oq.first && !preempt)
- goto finalize_only;
- }
+ state = erts_smp_atomic32_read_nob(&prt->state);
+ preempt = reds > reds_limit || (state & ERTS_PORT_SFLGS_DEAD);
+ if ((state & ERTS_PORT_SFLG_PORT_BUSY) && oq.first && !preempt)
+ goto finalize_only;
}
ASSERT(!oq.first || preempt);
@@ -2093,7 +2091,7 @@ erts_dist_command(Port *prt, int reds_limit)
ASSERT(dep->qsize >= obufsize);
dep->qsize -= obufsize;
obufsize = 0;
- if (!prt_busy
+ if (!(state & ERTS_PORT_SFLG_PORT_BUSY)
&& (dep->qflgs & ERTS_DE_QFLG_BUSY)
&& dep->qsize < erts_dist_buf_busy_limit) {
ErtsProcList *suspendees;
@@ -2139,11 +2137,15 @@ erts_dist_command(Port *prt, int reds_limit)
return reds;
preempted:
+ /*
+ * Here we assume that state has been read
+ * since last call to driver.
+ */
ASSERT(oq.first || !oq.last);
ASSERT(!oq.first || oq.last);
- if (prt->status & ERTS_PORT_SFLGS_DEAD) {
+ if (state & ERTS_PORT_SFLGS_DEAD) {
/*
* Port died during port command; clean up 'oq'
* and 'foq'. Things buffered in dist entry after
@@ -2581,7 +2583,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
pp = erts_id2port(BIF_ARG_2, BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_de_rwlock(dep);
- if (!pp || (pp->status & ERTS_PORT_SFLG_EXITING))
+ if (!pp || (erts_smp_atomic32_read_nob(&pp->state)
+ & ERTS_PORT_SFLG_EXITING))
goto badarg;
if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0)
@@ -2610,7 +2613,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
if (pp->dist_entry || is_not_nil(dep->cid))
goto badarg;
- erts_port_status_bor_set(pp, ERTS_PORT_SFLG_DISTRIBUTION);
+ erts_smp_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION);
pp->dist_entry = dep;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 845151c895..0cefd73459 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -187,7 +187,8 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
if (prt) {
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- ASSERT((erts_port_status_get(prt) & ERTS_PORT_SFLGS_DEAD) == 0);
+ ASSERT((erts_smp_atomic32_read_nob(&prt->state)
+ & ERTS_PORT_SFLGS_DEAD) == 0);
ASSERT(prt->dist_entry);
dep = prt->dist_entry;
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index f321ed21aa..bce9d07a01 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -599,7 +599,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
sched_id = 1;
#endif
- prt = erts_drvport2port(ix);
+ prt = erts_drvport2port(ix, NULL);
if (!prt)
return -1;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 7f7c975e78..dfb8b453c4 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -114,6 +114,54 @@ static void ddll_no_more_references(void *vdh);
#define FREE_PORT_FLAGS (ERTS_PORT_SFLGS_DEAD & (~ERTS_PORT_SFLG_INITIALIZING))
+static void
+kill_ports_driver_unloaded(DE_Handle *dh)
+{
+ int ix;
+
+ for (ix = 0; ix < erts_max_ports; ix++) {
+ Port* prt = &erts_port[ix];
+ erts_aint32_t state;
+ state = erts_smp_atomic32_read_acqb(&prt->state);
+ if (state & FREE_PORT_FLAGS)
+ continue;
+
+ erts_smp_port_minor_lock(prt);
+
+ if (prt->drv_ptr->handle != dh) {
+ erts_smp_port_minor_unlock(prt);
+ continue;
+ }
+
+ erts_smp_atomic_inc_nob(&prt->refc);
+ erts_smp_port_minor_unlock(prt);
+
+ state = erts_smp_atomic32_read_acqb(&prt->state);
+ if (!(state & FREE_PORT_FLAGS)) {
+#if DDLL_SMP
+ if (state & ERTS_PORT_SFLG_INITIALIZING) {
+ /* Extremely rare busy wait */
+ do {
+ ERTS_SPIN_BODY;
+ state = erts_smp_atomic32_read_acqb(&prt->state);
+ } while (state & ERTS_PORT_SFLG_INITIALIZING);
+ }
+
+ erts_smp_mtx_lock(prt->lock);
+
+ if (prt->drv_ptr->handle == dh) {
+ state = erts_smp_atomic32_read_acqb(&prt->state);
+ if (!(state & ERTS_PORT_SFLGS_DEAD))
+ driver_failure_atom(ix, "driver_unloaded");
+ }
+#else
+ driver_failure_atom(ix, "driver_unloaded");
+#endif
+ erts_port_release(prt);
+ }
+ }
+}
+
/*
* try_load(Path, Name, OptionList) -> {ok,Status} |
* {ok, PendingStatus, Ref} |
@@ -358,38 +406,14 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
}
assert_drv_list_locked();
if (kill_ports) {
- int j;
- /* Avoid closing the driver by referencing it */
+ /* Avoid closing the driver by referencing it */
erts_ddll_reference_driver(dh);
ASSERT(dh->status == ERL_DE_RELOAD);
dh->status = ERL_DE_FORCE_RELOAD;
#if DDLL_SMP
unlock_drv_list();
#endif
- for (j = 0; j < erts_max_ports; j++) {
- Port* prt = &erts_port[j];
- erts_smp_port_state_lock(prt);
- if (!(prt->status & FREE_PORT_FLAGS) &&
- prt->drv_ptr->handle == dh) {
- erts_smp_atomic_inc_nob(&prt->refc);
-#if DDLL_SMP
- /* Extremely rare spinlock */
- while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
- erts_smp_port_state_unlock(prt);
- erts_smp_port_state_lock(prt);
- }
- erts_smp_port_state_unlock(prt);
- erts_smp_mtx_lock(prt->lock);
- if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) {
- driver_failure_atom(j, "driver_unloaded");
- }
-#else
- driver_failure_atom(j, "driver_unloaded");
-#endif
- erts_port_release(prt);
- }
- else erts_smp_port_state_unlock(prt);
- }
+ kill_ports_driver_unloaded(dh);
/* Dereference, eventually causing driver destruction */
#if DDLL_SMP
lock_drv_list();
@@ -587,37 +611,13 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2)
done:
assert_drv_list_locked();
if (kill_ports > 1) {
- int j;
/* Avoid closing the driver by referencing it */
erts_ddll_reference_driver(dh);
dh->status = ERL_DE_FORCE_UNLOAD;
#if DDLL_SMP
unlock_drv_list();
#endif
- for (j = 0; j < erts_max_ports; j++) {
- Port* prt = &erts_port[j];
- erts_smp_port_state_lock(prt);
- if (!(prt->status & FREE_PORT_FLAGS)
- && prt->drv_ptr->handle == dh) {
- erts_smp_atomic_inc_nob(&prt->refc);
-#if DDLL_SMP
- /* Extremely rare spinlock */
- while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
- erts_smp_port_state_unlock(prt);
- erts_smp_port_state_lock(prt);
- }
- erts_smp_port_state_unlock(prt);
- erts_smp_mtx_lock(prt->lock);
- if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) {
- driver_failure_atom(j, "driver_unloaded");
- }
-#else
- driver_failure_atom(j, "driver_unloaded");
-#endif
- erts_port_release(prt);
- }
- else erts_smp_port_state_unlock(prt);
- }
+ kill_ports_driver_unloaded(dh);
#if DDLL_SMP
lock_drv_list();
#endif
@@ -1047,36 +1047,13 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
}
if (!left && drv->handle->port_count > 0) {
if (kill_ports) {
- int j;
DE_Handle *dh = drv->handle;
erts_ddll_reference_driver(dh);
dh->status = ERL_DE_FORCE_UNLOAD;
#if DDLL_SMP
unlock_drv_list();
#endif
- for (j = 0; j < erts_max_ports; j++) {
- Port* prt = &erts_port[j];
- erts_smp_port_state_lock(prt);
- if (!(prt->status & FREE_PORT_FLAGS) &&
- prt->drv_ptr->handle == dh) {
- erts_smp_atomic_inc_nob(&prt->refc);
-#if DDLL_SMP
- while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
- erts_smp_port_state_unlock(prt);
- erts_smp_port_state_lock(prt);
- }
- erts_smp_port_state_unlock(prt);
- erts_smp_mtx_lock(prt->lock);
- if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) {
- driver_failure_atom(j, "driver_unloaded");
- }
-#else
- driver_failure_atom(j, "driver_unloaded");
-#endif
- erts_port_release(prt);
- }
- else erts_smp_port_state_unlock(prt);
- }
+ kill_ports_driver_unloaded(dh);
#if DDLL_SMP
lock_drv_list(); /* Needed for future list operations */
#endif
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 874d1606fd..6b71eddc8b 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2975,7 +2975,8 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item)
#ifndef ERTS_SMP
res = am_false;
#else
- if (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
+ if (erts_smp_atomic32_read_nob(&prt->state)
+ & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
DECL_AM(port_level);
ASSERT(prt->drv_ptr->flags
& ERL_DRV_FLAG_USE_PORT_LOCKING);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 5525426824..1a5b05fcfc 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -164,7 +164,7 @@ do_port_command(Process *BIF_P, Eterm arg1, Eterm arg2, Eterm arg3,
ERTS_BIF_PREP_ERROR(res, BIF_P, EXC_NOTSUP);
}
else if (!(flags & ERTS_PORT_COMMAND_FLAG_FORCE)
- && p->status & ERTS_PORT_SFLG_PORT_BUSY) {
+ && (erts_smp_atomic32_read_nob(&p->state) & ERTS_PORT_SFLG_PORT_BUSY)) {
if (flags & ERTS_PORT_COMMAND_FLAG_NOSUSPEND) {
ERTS_BIF_PREP_RET(res, am_false);
}
@@ -637,11 +637,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
erts_driver_t* driver;
char* name_buf = NULL;
SysDriverOpts opts;
- int binary_io;
- int soft_eof;
Sint linebuf;
Eterm edir = NIL;
byte dir[MAXPATHLEN];
+ erts_aint32_t sflgs = 0;
/* These are the defaults */
opts.packet_bytes = 0;
@@ -655,8 +654,6 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
opts.overlapped_io = 0;
opts.spawn_type = ERTS_SPAWN_ANY;
opts.argv = NULL;
- binary_io = 0;
- soft_eof = 0;
linebuf = 0;
*err_nump = 0;
@@ -748,13 +745,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
} else if (*nargs == am_nouse_stdio) {
opts.use_stdio = 0;
} else if (*nargs == am_binary) {
- binary_io = 1;
+ sflgs |= ERTS_PORT_SFLG_BINARY_IO;
} else if (*nargs == am_in) {
opts.read_write |= DO_READ;
} else if (*nargs == am_out) {
opts.read_write |= DO_WRITE;
} else if (*nargs == am_eof) {
- soft_eof = 1;
+ sflgs |= ERTS_PORT_SFLG_SOFT_EOF;
} else if (*nargs == am_hide) {
opts.hide_window = 1;
} else if (*nargs == am_exit_status) {
@@ -951,19 +948,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
trace_virtual_sched(p, am_in);
}
- if (binary_io) {
- erts_port_status_bor_set(&erts_port[port_num],
- ERTS_PORT_SFLG_BINARY_IO);
- }
- if (soft_eof) {
- erts_port_status_bor_set(&erts_port[port_num],
- ERTS_PORT_SFLG_SOFT_EOF);
- }
if (linebuf && erts_port[port_num].linebuf == NULL){
- erts_port[port_num].linebuf = allocate_linebuf(linebuf);
- erts_port_status_bor_set(&erts_port[port_num],
- ERTS_PORT_SFLG_LINEBUF_IO);
+ erts_port[port_num].linebuf = allocate_linebuf(linebuf);
+ sflgs |= ERTS_PORT_SFLG_LINEBUF_IO;
}
+
+ if (sflgs)
+ erts_smp_atomic32_read_bor_relb(&erts_port[port_num].state, sflgs);
do_return:
if (name_buf)
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 3baac1572a..01d714d57f 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -680,7 +680,9 @@ Eterm trace_3(BIF_ALIST_3)
/* tracing of ports */
for (i = 0; i < erts_max_ports; i++) {
Port *tracee_port = &erts_port[i];
- if (tracee_port->status & ERTS_PORT_SFLGS_DEAD) continue;
+ erts_aint32_t state;
+ state = erts_smp_atomic32_read_nob(&tracee_port->state);
+ if (state & ERTS_PORT_SFLGS_DEAD) continue;
if (tracer != NIL) {
if (tracee_port->id == tracer) continue;
if (port_already_traced(NULL, tracee_port, tracer)) continue;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 93122b8317..f64a7fa41a 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1384,7 +1384,8 @@ setup_reference_table(void)
/* Insert all ports */
for (i = 0; i < erts_max_ports; i++) {
- if (erts_port[i].status & ERTS_PORT_SFLGS_DEAD)
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&erts_port[i].state);
+ if (state & ERTS_PORT_SFLGS_DEAD)
continue;
/* Insert links */
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 86454fe1fa..3873dd8f99 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -51,7 +51,8 @@
#define ERTS_PORT_TASK_INVALID_PORT(P, ID) \
- ((erts_port_status_get((P)) & ERTS_PORT_SFLGS_DEAD) || (P)->id != (ID))
+ ((erts_smp_atomic32_read_acqb(&(P)->state) & ERTS_PORT_SFLGS_DEAD) \
+ || (P)->id != (ID))
#define ERTS_PORT_IS_IN_RUNQ(RQ, P) \
((P)->sched.next || (P)->sched.prev || (RQ)->ports.start == (P))
@@ -615,7 +616,7 @@ erts_port_task_free_port(Port *pp)
ErtsPortTaskQueue *ptqp;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
- ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD));
+ ASSERT(!(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD));
runq = erts_port_runq(pp);
ASSERT(runq);
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
@@ -626,11 +627,13 @@ erts_port_task_free_port(Port *pp)
ErtsPortTask *ptp;
enqueue_free:
ptp = port_task_alloc();
- erts_smp_port_state_lock(pp);
- pp->status &= ~ERTS_PORT_SFLG_CLOSING;
- pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
+ erts_smp_port_minor_lock(pp);
+ erts_smp_atomic32_read_bset_relb(&pp->state,
+ (ERTS_PORT_SFLG_CLOSING
+ | ERTS_PORT_SFLG_FREE_SCHEDULED),
+ ERTS_PORT_SFLG_FREE_SCHEDULED);
erts_may_save_closed_port(pp);
- erts_smp_port_state_unlock(pp);
+ erts_smp_port_minor_unlock(pp);
ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1);
ptp->type = ERTS_PORT_TASK_FREE;
ptp->event = (ErlDrvEvent) -1;
@@ -648,11 +651,13 @@ erts_port_task_free_port(Port *pp)
goto enqueue_free;
}
ASSERT(!pp->sched.taskq);
- erts_smp_port_state_lock(pp);
- pp->status &= ~ERTS_PORT_SFLG_CLOSING;
- pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
+ erts_smp_port_minor_lock(pp);
+ erts_smp_atomic32_read_bset_relb(&pp->state,
+ (ERTS_PORT_SFLG_CLOSING
+ | ERTS_PORT_SFLG_FREE_SCHEDULED),
+ ERTS_PORT_SFLG_FREE_SCHEDULED);
erts_may_save_closed_port(pp);
- erts_smp_port_state_unlock(pp);
+ erts_smp_port_minor_unlock(pp);
erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */
ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
handle_remaining_tasks(runq, pp); /* May release runq lock */
@@ -766,7 +771,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
erts_smp_runq_lock(runq);
erts_unblock_fpe(fpe_was_unmasked);
- ASSERT(pp->status & ERTS_PORT_SFLG_FREE_SCHEDULED);
+ ASSERT(erts_smp_atomic32_read_nob(&pp->state)
+ & ERTS_PORT_SFLG_FREE_SCHEDULED);
if (ptqp->first || (pp->sched.taskq && pp->sched.taskq->first))
handle_remaining_tasks(runq, pp);
ASSERT(!ptqp->first
@@ -782,14 +788,15 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
goto tasks_done;
case ERTS_PORT_TASK_TIMEOUT:
reds += ERTS_PORT_REDS_TIMEOUT;
- if (!(pp->status & ERTS_PORT_SFLGS_DEAD)) {
+ if (!(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)) {
DTRACE_DRIVER(driver_timeout, pp);
(*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data);
}
break;
case ERTS_PORT_TASK_INPUT:
reds += ERTS_PORT_REDS_INPUT;
- ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0);
+ ASSERT((erts_smp_atomic32_read_nob(&pp->state)
+ & ERTS_PORT_SFLGS_DEAD) == 0);
DTRACE_DRIVER(driver_ready_input, pp);
/* NOTE some windows drivers use ->ready_input for input and output */
(*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->event);
@@ -797,14 +804,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
break;
case ERTS_PORT_TASK_OUTPUT:
reds += ERTS_PORT_REDS_OUTPUT;
- ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0);
+ ASSERT((erts_smp_atomic32_read_nob(&pp->state)
+ & ERTS_PORT_SFLGS_DEAD) == 0);
DTRACE_DRIVER(driver_ready_output, pp);
(*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->event);
io_tasks_executed++;
break;
case ERTS_PORT_TASK_EVENT:
reds += ERTS_PORT_REDS_EVENT;
- ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0);
+ ASSERT((erts_smp_atomic32_read_nob(&pp->state)
+ & ERTS_PORT_SFLGS_DEAD) == 0);
DTRACE_DRIVER(driver_event, pp);
(*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->event, ptp->event_data);
io_tasks_executed++;
@@ -819,7 +828,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
break;
}
- if ((pp->status & ERTS_PORT_SFLG_CLOSING)
+ if ((erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLG_CLOSING)
&& erts_is_port_ioq_empty(pp)) {
reds += ERTS_PORT_REDS_TERMINATE;
erts_terminate_port(pp);
@@ -870,7 +879,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
ErtsRunQueue *xrunq;
#endif
- ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD));
+ ASSERT(!(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD));
ASSERT(pp->sched.taskq->first);
#ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 79614c85db..6170fd1ec8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1486,29 +1486,29 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
erts_smp_atomic32_set_nob(&erts_halt_progress, 1);
for (i = 0; i < erts_max_ports; i++) {
Port *prt = &erts_port[i];
- erts_smp_port_state_lock(prt);
- if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
- | ERTS_PORT_SFLG_HALT))) {
- erts_smp_port_state_unlock(prt);
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&prt->state);
+ if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+ | ERTS_PORT_SFLG_HALT))
continue;
- }
+ erts_smp_port_minor_lock(prt);
/* We need to set the halt flag - get the port lock */
#ifdef ERTS_SMP
erts_smp_atomic_inc_nob(&prt->refc);
#endif
- erts_smp_port_state_unlock(prt);
+ erts_smp_port_minor_unlock(prt);
#ifdef ERTS_SMP
erts_smp_mtx_lock(prt->lock);
#endif
- if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
- | ERTS_PORT_SFLG_HALT))) {
+ state = erts_smp_atomic32_read_acqb(&prt->state);
+ if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+ | ERTS_PORT_SFLG_HALT)) {
erts_port_release(prt);
continue;
}
- erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT);
+ state = erts_smp_atomic32_read_bor_relb(&prt->state,
+ ERTS_PORT_SFLG_HALT);
erts_smp_atomic32_inc_nob(&erts_halt_progress);
- if (prt->status & (ERTS_PORT_SFLG_EXITING
- | ERTS_PORT_SFLG_CLOSING)) {
+ if (state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING)) {
erts_port_release(prt);
continue;
}
@@ -2885,12 +2885,12 @@ resume_run_queue(ErtsRunQueue *rq)
erts_smp_runq_lock(rq);
- (void) ERTS_RUNQ_FLGS_MASK_SET(rq,
- (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
- | ERTS_RUNQ_FLG_SUSPENDED),
- (ERTS_RUNQ_FLG_OUT_OF_WORK
- | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
+ (void) ERTS_RUNQ_FLGS_READ_BSET(rq,
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_SUSPENDED),
+ (ERTS_RUNQ_FLG_OUT_OF_WORK
+ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
@@ -3912,7 +3912,7 @@ erts_fprintf(stderr, "--------------------------------\n");
ERTS_DBG_CHK_FULL_REDS_HISTORY(rq);
rq->out_of_work_count = 0;
- (void) ERTS_RUNQ_FLGS_MASK_SET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
+ (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags);
rq->max_len = rq->len;
for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 17d1ff0bd6..22303973bf 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -205,32 +205,10 @@ extern int erts_sched_thread_suggested_stack_size;
((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ) \
((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
-#define ERTS_RUNQ_FLGS_MASK_SET(RQ, MSK, FLGS) \
- ((Uint32) erts_smp_atomic32_mask_set_relb(&(RQ)->flags, \
- (erts_aint32_t) (MSK), \
- (erts_aint32_t) (FLGS)))
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p,
- erts_aint32_t mask,
- erts_aint32_t set);
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_mask_set_relb(erts_smp_atomic32_t *a32p,
- erts_aint32_t mask,
- erts_aint32_t set)
-{
- erts_aint32_t act = erts_smp_atomic32_read_nob(a32p);
- while (1) {
- erts_aint32_t exp = act;
- erts_aint32_t new = exp & ~mask;
- new |= (mask & set);
- act = erts_smp_atomic32_cmpxchg_relb(a32p, new, exp);
- if (act == exp)
- return act;
- }
-}
-#endif
+#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS) \
+ ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \
+ (erts_aint32_t) (MSK), \
+ (erts_aint32_t) (FLGS)))
typedef enum {
ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index a32e9d9d7c..34c90c0bda 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -274,6 +274,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
+#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob
#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
@@ -288,6 +289,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
+#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb
#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
@@ -302,6 +304,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
+#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb
#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
@@ -316,6 +319,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
+#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb
#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
@@ -330,6 +334,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
+#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb
#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
@@ -344,6 +349,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
+#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb
#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
@@ -358,6 +364,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
+#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb
/* 32-bit atomics */
@@ -374,6 +381,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
+#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob
#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
@@ -388,6 +396,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
+#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb
#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
@@ -402,6 +411,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
+#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb
#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
@@ -416,6 +426,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
+#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb
#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
@@ -430,6 +441,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
+#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb
#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
@@ -444,6 +456,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
+#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb
#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
@@ -458,6 +471,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
+#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb
#else /* !ERTS_SMP */
@@ -513,6 +527,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset
#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
@@ -527,6 +542,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset
#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
@@ -541,6 +557,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset
#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
@@ -555,6 +572,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset
#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
@@ -569,6 +587,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset
#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
@@ -583,6 +602,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset
#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
@@ -597,6 +617,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset
/* 32-bit atomics */
@@ -613,6 +634,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset
#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
@@ -627,6 +649,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset
#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
@@ -641,6 +664,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset
#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
@@ -655,6 +679,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset
#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
@@ -669,6 +694,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset
#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
@@ -683,6 +709,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset
#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
@@ -697,6 +724,7 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset
#endif /* !ERTS_SMP */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 17628286bc..0e759a1eec 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -533,6 +533,9 @@ ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
erts_aint_t new,
erts_aint_t expected);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
@@ -553,6 +556,9 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
erts_aint32_t new,
erts_aint32_t expected);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
char *name,
@@ -612,6 +618,78 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#ifdef USE_THREADS
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_nob(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_ddrb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_rb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_wb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_acqb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_relb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_mb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_nob(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_rb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_wb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_relb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_mb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+#define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \
+do { \
+ Type act = ReadOp((VarP)); \
+ while (1) { \
+ Type exp = act; \
+ Type new = exp & ~(Mask); \
+ new |= ((Mask) & (Set)); \
+ act = CmpxchgOp((VarP), new, exp); \
+ if (act == exp) \
+ return act; \
+ } \
+} while (0)
+#endif
+
/*
* See "Documentation of atomics and memory barriers" at the top
* of this file for info on atomics.
@@ -670,6 +748,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_nob ethr_atomic_xchg
#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_nob(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
+ ethr_atomic_read,
+ ethr_atomic_cmpxchg,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic_init_mb ethr_atomic_init_mb
#define erts_atomic_set_mb ethr_atomic_set_mb
#define erts_atomic_read_mb ethr_atomic_read_mb
@@ -684,6 +775,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_mb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
+ ethr_atomic_read,
+ ethr_atomic_cmpxchg_mb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic_init_acqb ethr_atomic_init_acqb
#define erts_atomic_set_acqb ethr_atomic_set_acqb
#define erts_atomic_read_acqb ethr_atomic_read_acqb
@@ -698,6 +802,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_acqb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
+ ethr_atomic_read,
+ ethr_atomic_cmpxchg_acqb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic_init_relb ethr_atomic_init_relb
#define erts_atomic_set_relb ethr_atomic_set_relb
#define erts_atomic_read_relb ethr_atomic_read_relb
@@ -712,6 +829,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_relb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
+ ethr_atomic_read,
+ ethr_atomic_cmpxchg_relb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
@@ -726,6 +856,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_ddrb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
+ ethr_atomic_read,
+ ethr_atomic_cmpxchg_ddrb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic_init_rb ethr_atomic_init_rb
#define erts_atomic_set_rb ethr_atomic_set_rb
#define erts_atomic_read_rb ethr_atomic_read_rb
@@ -740,6 +883,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_rb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
+ ethr_atomic_read,
+ ethr_atomic_cmpxchg_rb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic_init_wb ethr_atomic_init_wb
#define erts_atomic_set_wb ethr_atomic_set_wb
#define erts_atomic_read_wb ethr_atomic_read_wb
@@ -754,6 +910,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_bset_wb(erts_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
+ ethr_atomic_read,
+ ethr_atomic_cmpxchg_wb,
+ var, mask, set);
+}
+#endif
+
/* 32-bit atomics */
#define erts_atomic32_init_nob ethr_atomic32_init
@@ -770,6 +939,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_nob ethr_atomic32_xchg
#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_nob(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
+ ethr_atomic32_read,
+ ethr_atomic32_cmpxchg,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic32_init_mb ethr_atomic32_init_mb
#define erts_atomic32_set_mb ethr_atomic32_set_mb
#define erts_atomic32_read_mb ethr_atomic32_read_mb
@@ -784,6 +966,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_mb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
+ ethr_atomic32_read,
+ ethr_atomic32_cmpxchg_mb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
@@ -798,6 +993,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
+ ethr_atomic32_read,
+ ethr_atomic32_cmpxchg_acqb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic32_init_relb ethr_atomic32_init_relb
#define erts_atomic32_set_relb ethr_atomic32_set_relb
#define erts_atomic32_read_relb ethr_atomic32_read_relb
@@ -812,6 +1020,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_relb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
+ ethr_atomic32_read,
+ ethr_atomic32_cmpxchg_relb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
@@ -826,6 +1047,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
+ ethr_atomic32_read,
+ ethr_atomic32_cmpxchg_ddrb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic32_init_rb ethr_atomic32_init_rb
#define erts_atomic32_set_rb ethr_atomic32_set_rb
#define erts_atomic32_read_rb ethr_atomic32_read_rb
@@ -840,6 +1074,19 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_rb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
+ ethr_atomic32_read,
+ ethr_atomic32_cmpxchg_rb,
+ var, mask, set);
+}
+#endif
+
#define erts_atomic32_init_wb ethr_atomic32_init_wb
#define erts_atomic32_set_wb ethr_atomic32_set_wb
#define erts_atomic32_read_wb ethr_atomic32_read_wb
@@ -854,6 +1101,21 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_bset_wb(erts_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
+ ethr_atomic32_read,
+ ethr_atomic32_cmpxchg_wb,
+ var, mask, set);
+}
+#endif
+
+#undef ERTS_ATOMIC_BSET_IMPL__
+
#else /* !USE_THREADS */
/* Double word size atomics */
@@ -908,6 +1170,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_read_band_nob erts_no_atomic_read_band
#define erts_atomic_xchg_nob erts_no_atomic_xchg
#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+#define erts_atomic_read_bset_nob erts_no_atomic_read_bset
#define erts_atomic_init_mb erts_no_atomic_set
#define erts_atomic_set_mb erts_no_atomic_set
@@ -922,6 +1185,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_read_band_mb erts_no_atomic_read_band
#define erts_atomic_xchg_mb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+#define erts_atomic_read_bset_mb erts_no_atomic_read_bset
#define erts_atomic_init_acqb erts_no_atomic_set
#define erts_atomic_set_acqb erts_no_atomic_set
@@ -936,6 +1200,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset
#define erts_atomic_init_relb erts_no_atomic_set
#define erts_atomic_set_relb erts_no_atomic_set
@@ -950,6 +1215,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_read_band_relb erts_no_atomic_read_band
#define erts_atomic_xchg_relb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+#define erts_atomic_read_bset_relb erts_no_atomic_read_bset
#define erts_atomic_init_ddrb erts_no_atomic_set
#define erts_atomic_set_ddrb erts_no_atomic_set
@@ -964,6 +1230,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset
#define erts_atomic_init_rb erts_no_atomic_set
#define erts_atomic_set_rb erts_no_atomic_set
@@ -978,6 +1245,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_read_band_rb erts_no_atomic_read_band
#define erts_atomic_xchg_rb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+#define erts_atomic_read_bset_rb erts_no_atomic_read_bset
#define erts_atomic_init_wb erts_no_atomic_set
#define erts_atomic_set_wb erts_no_atomic_set
@@ -992,6 +1260,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_read_band_wb erts_no_atomic_read_band
#define erts_atomic_xchg_wb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+#define erts_atomic_read_bset_wb erts_no_atomic_read_bset
/* 32-bit atomics */
@@ -1008,6 +1277,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset
#define erts_atomic32_init_mb erts_no_atomic32_set
#define erts_atomic32_set_mb erts_no_atomic32_set
@@ -1022,6 +1292,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset
#define erts_atomic32_init_acqb erts_no_atomic32_set
#define erts_atomic32_set_acqb erts_no_atomic32_set
@@ -1036,6 +1307,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset
#define erts_atomic32_init_relb erts_no_atomic32_set
#define erts_atomic32_set_relb erts_no_atomic32_set
@@ -1050,6 +1322,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset
#define erts_atomic32_init_ddrb erts_no_atomic32_set
#define erts_atomic32_set_ddrb erts_no_atomic32_set
@@ -1064,6 +1337,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset
#define erts_atomic32_init_rb erts_no_atomic32_set
#define erts_atomic32_set_rb erts_no_atomic32_set
@@ -1078,6 +1352,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset
#define erts_atomic32_init_wb erts_no_atomic32_set
#define erts_atomic32_set_wb erts_no_atomic32_set
@@ -1092,6 +1367,7 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset
#endif /* !USE_THREADS */
@@ -1856,6 +2132,16 @@ erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
return old;
}
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_read_bset(erts_no_atomic_t *var,
+ erts_aint_t mask,
+ erts_aint_t set)
+{
+ erts_aint_t old = *var;
+ *var |= (mask & set);
+ return old;
+}
+
/* atomic32 */
ERTS_GLB_INLINE void
@@ -1943,6 +2229,16 @@ erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
return old;
}
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
+ erts_aint32_t mask,
+ erts_aint32_t set)
+{
+ erts_aint32_t old = *var;
+ *var |= (mask & set);
+ return old;
+}
+
/* spinlock */
ERTS_GLB_INLINE void
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 86f8e3890b..5386e0eb4e 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -183,7 +183,7 @@ struct port {
ErtsProcList *suspended; /* List of suspended processes. */
LineBuf *linebuf; /* Buffer to hold data not ready for
process to get (line oriented I/O)*/
- Uint32 status; /* Status and type flags */
+ erts_smp_atomic32_t state; /* Status and type flags */
int control_flags; /* Flags for port_control() */
erts_aint32_t snapshot; /* Next snapshot that port should be part of */
struct reg_proc *reg;
@@ -1202,8 +1202,8 @@ void erts_smp_xports_unlock(Port *);
int erts_lc_is_port_locked(Port *);
#endif
-ERTS_GLB_INLINE void erts_smp_port_state_lock(Port*);
-ERTS_GLB_INLINE void erts_smp_port_state_unlock(Port*);
+ERTS_GLB_INLINE void erts_smp_port_minor_lock(Port*);
+ERTS_GLB_INLINE void erts_smp_port_minor_unlock(Port*);
ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
@@ -1212,7 +1212,7 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
-erts_smp_port_state_lock(Port* prt)
+erts_smp_port_minor_lock(Port* prt)
{
#ifdef ERTS_SMP
erts_smp_spin_lock(&prt->state_lck);
@@ -1220,7 +1220,7 @@ erts_smp_port_state_lock(Port* prt)
}
ERTS_GLB_INLINE void
-erts_smp_port_state_unlock(Port *prt)
+erts_smp_port_minor_unlock(Port *prt)
{
#ifdef ERTS_SMP
erts_smp_spin_unlock(&prt->state_lck);
@@ -1274,8 +1274,10 @@ erts_smp_port_unlock(Port *prt)
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS) \
- (!(PP) || ((PP)->status & (FLGS)) || (PP)->id != (ID))
+#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS) \
+ (!(PP) \
+ || (erts_smp_atomic32_read_nob(&(PP)->state) & (FLGS)) \
+ || (PP)->id != (ID))
/* port lookup */
@@ -1300,16 +1302,11 @@ Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
ERTS_GLB_INLINE Port*erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
ERTS_GLB_INLINE void erts_port_release(Port *);
-ERTS_GLB_INLINE Port*erts_drvport2port(ErlDrvPort);
+ERTS_GLB_INLINE Port*erts_drvport2port(ErlDrvPort, erts_aint32_t *);
ERTS_GLB_INLINE Port*erts_drvportid2port(Eterm);
ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm id);
ERTS_GLB_INLINE int erts_is_port_alive(Eterm id);
ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm id);
-ERTS_GLB_INLINE void erts_port_status_bandor_set(Port *, Uint32, Uint32);
-ERTS_GLB_INLINE void erts_port_status_band_set(Port *, Uint32);
-ERTS_GLB_INLINE void erts_port_status_bor_set(Port *, Uint32);
-ERTS_GLB_INLINE void erts_port_status_set(Port *, Uint32);
-ERTS_GLB_INLINE Uint32 erts_port_status_get(Port *);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -1326,14 +1323,14 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
prt = &erts_port[internal_port_index(id)];
- erts_smp_port_state_lock(prt);
+ erts_smp_port_minor_lock(prt);
if (ERTS_INVALID_PORT_OPT(prt, id, sflgs)) {
- erts_smp_port_state_unlock(prt);
+ erts_smp_port_minor_unlock(prt);
prt = NULL;
}
else {
erts_smp_atomic_inc_nob(&prt->refc);
- erts_smp_port_state_unlock(prt);
+ erts_smp_port_minor_unlock(prt);
#ifdef ERTS_SMP
if (no_proc_locks)
@@ -1347,8 +1344,8 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
/* The id may not have changed... */
ERTS_SMP_LC_ASSERT(prt->id == id);
- /* ... but status may have... */
- if (prt->status & sflgs) {
+ /* ... but state may have... */
+ if (erts_smp_atomic32_read_nob(&prt->state) & sflgs) {
erts_smp_port_unlock(prt); /* Also decrements refc... */
prt = NULL;
}
@@ -1366,14 +1363,18 @@ erts_port_release(Port *prt)
}
ERTS_GLB_INLINE Port*
-erts_drvport2port(ErlDrvPort drvport)
+erts_drvport2port(ErlDrvPort drvport, erts_aint32_t *statep)
{
int ix = (int) drvport;
+ erts_aint32_t state;
if (ix < 0 || erts_max_ports <= ix)
return NULL;
- if (erts_port[ix].status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
+ state = erts_smp_atomic32_read_nob(&erts_port[ix].state);
+ if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
return NULL;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[ix]));
+ if (statep)
+ *statep = state;
return &erts_port[ix];
}
@@ -1381,12 +1382,14 @@ ERTS_GLB_INLINE Port*
erts_drvportid2port(Eterm id)
{
int ix;
+ erts_aint32_t state;
if (is_not_internal_port(id))
return NULL;
ix = (int) internal_port_index(id);
if (erts_max_ports <= ix)
return NULL;
- if (erts_port[ix].status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
+ state = erts_smp_atomic32_read_nob(&erts_port[ix].state);
+ if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
return NULL;
if (erts_port[ix].id != id)
return NULL;
@@ -1400,17 +1403,14 @@ erts_portid2status(Eterm id)
if (is_not_internal_port(id))
return ERTS_PORT_SFLG_INVALID;
else {
- Uint32 status;
+ erts_aint32_t state;
int ix = internal_port_index(id);
if (erts_max_ports <= ix)
return ERTS_PORT_SFLG_INVALID;
- erts_smp_port_state_lock(&erts_port[ix]);
- if (erts_port[ix].id == id)
- status = erts_port[ix].status;
- else
- status = ERTS_PORT_SFLG_INVALID;
- erts_smp_port_state_unlock(&erts_port[ix]);
- return status;
+ state = erts_smp_atomic32_read_ddrb(&erts_port[ix].state);
+ if (erts_port[ix].id != id)
+ return ERTS_PORT_SFLG_INVALID;
+ return state;
}
}
@@ -1426,50 +1426,6 @@ erts_is_valid_tracer_port(Eterm id)
{
return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
}
-
-ERTS_GLB_INLINE void erts_port_status_bandor_set(Port *prt,
- Uint32 band_status,
- Uint32 bor_status)
-{
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- erts_smp_port_state_lock(prt);
- prt->status &= band_status;
- prt->status |= bor_status;
- erts_smp_port_state_unlock(prt);
-}
-
-ERTS_GLB_INLINE void erts_port_status_band_set(Port *prt, Uint32 status)
-{
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- erts_smp_port_state_lock(prt);
- prt->status &= status;
- erts_smp_port_state_unlock(prt);
-}
-
-ERTS_GLB_INLINE void erts_port_status_bor_set(Port *prt, Uint32 status)
-{
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- erts_smp_port_state_lock(prt);
- prt->status |= status;
- erts_smp_port_state_unlock(prt);
-}
-
-ERTS_GLB_INLINE void erts_port_status_set(Port *prt, Uint32 status)
-{
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- erts_smp_port_state_lock(prt);
- prt->status = status;
- erts_smp_port_state_unlock(prt);
-}
-
-ERTS_GLB_INLINE Uint32 erts_port_status_get(Port *prt)
-{
- Uint32 res;
- erts_smp_port_state_lock(prt);
- res = prt->status;
- erts_smp_port_state_unlock(prt);
- return res;
-}
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
/* erl_drv_thread.c */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index f9e1b62f14..00ecf1fca2 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -90,35 +90,35 @@ static ERTS_INLINE ErlIOQueue*
drvport2ioq(ErlDrvPort drvport)
{
int ix = (int) drvport;
- Uint32 status;
+ erts_aint32_t state;
if (ix < 0 || erts_max_ports <= ix)
return NULL;
+ state = erts_smp_atomic32_read_nob(&erts_port[ix].state);
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+
if (erts_get_scheduler_data()) {
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[ix]));
ERTS_LC_ASSERT(!erts_port[ix].port_data_lock
|| erts_lc_mtx_is_locked(
&erts_port[ix].port_data_lock->mtx));
-
- status = erts_port[ix].status;
}
else {
- erts_smp_port_state_lock(&erts_port[ix]);
- status = erts_port[ix].status;
- erts_smp_port_state_unlock(&erts_port[ix]);
-
- ERTS_LC_ASSERT((status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
+ ERTS_LC_ASSERT((state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
|| erts_port[ix].port_data_lock);
ERTS_LC_ASSERT(!erts_port[ix].port_data_lock
|| erts_lc_mtx_is_locked(
&erts_port[ix].port_data_lock->mtx));
-
}
- return ((status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
- ? NULL
- : &erts_port[ix].ioq);
+#endif
+
+ if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
+ return NULL;
+ else
+ return &erts_port[ix].ioq;
}
static ERTS_INLINE int
@@ -216,7 +216,7 @@ kill_port(Port *pp)
{
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp));
erts_port_task_free_port(pp);
- ASSERT(pp->status & ERTS_PORT_SFLGS_DEAD);
+ ASSERT(erts_smp_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD);
}
#ifdef ERTS_SMP
@@ -242,27 +242,32 @@ get_free_port(void)
erts_smp_spin_lock(&get_free_port_lck);
num = last_port_num + 1;
- for (;; ++num) {
+ for (;; ++num) {
+ erts_aint32_t act;
+
port = &erts_port[num & erts_port_tab_index_mask];
- erts_smp_port_state_lock(port);
- if (port->status & ERTS_PORT_SFLG_FREE) {
- last_port_num = num;
- erts_smp_spin_unlock(&get_free_port_lck);
- break;
+ act = erts_smp_atomic32_read_nob(&port->state);
+
+ while (act & ERTS_PORT_SFLG_FREE) {
+ erts_aint32_t exp = act;
+ act = erts_smp_atomic32_cmpxchg_relb(&port->state,
+ ERTS_PORT_SFLG_INITIALIZING,
+ exp);
+ if (act == exp) {
+ last_port_num = num;
+ erts_smp_spin_unlock(&get_free_port_lck);
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0);
+ erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */
+ return num & port_num_mask;
+ }
}
- erts_smp_port_state_unlock(port);
if (--tries == 0) {
erts_smp_spin_unlock(&get_free_port_lck);
return -1;
}
}
- port->status = ERTS_PORT_SFLG_INITIALIZING;
- ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0);
- erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */
- erts_smp_port_state_unlock(port);
- return num & port_num_mask;
}
/*
@@ -284,13 +289,12 @@ erts_test_next_port(int set, Uint next)
Port* port = &erts_port[num & erts_port_tab_index_mask];
- erts_smp_port_state_lock(port);
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&port->state);
- if (port->status & ERTS_PORT_SFLG_FREE) {
+ if (state & ERTS_PORT_SFLG_FREE) {
last_port_num = num - 1;
res = num & port_num_mask;
}
- erts_smp_port_state_unlock(port);
}
erts_smp_spin_unlock(&get_free_port_lck);
return res;
@@ -326,38 +330,37 @@ void
port_cleanup(Port *prt)
{
#ifdef ERTS_SMP
- Uint32 port_specific;
erts_smp_mtx_t *mtx;
#endif
+#if defined(ERTS_SMP) || defined(ERTS_ENABLE_LOCK_CHECK)
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&prt->state);
+#endif
erts_driver_t *driver;
- erts_smp_port_state_lock(prt);
-
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
driver = prt->drv_ptr;
prt->drv_ptr = NULL;
ASSERT(driver);
- ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED);
- ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0);
+ ERTS_LC_ASSERT(state & ERTS_PORT_SFLG_FREE_SCHEDULED);
+ ERTS_LC_ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG);
+ ERTS_LC_ASSERT(!(state & ERTS_PORT_SFLG_FREE));
- ASSERT(prt->status & ERTS_PORT_SFLG_PORT_DEBUG);
- ASSERT(!(prt->status & ERTS_PORT_SFLG_FREE));
- prt->status = ERTS_PORT_SFLG_FREE;
+ ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0);
#ifdef ERTS_SMP
-
- port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK);
-
mtx = prt->lock;
ASSERT(mtx);
prt->lock = NULL;
- erts_smp_port_state_unlock(prt);
erts_smp_mtx_unlock(mtx);
+#endif
- if (port_specific) {
+ erts_smp_atomic32_set_relb(&prt->state, ERTS_PORT_SFLG_FREE);
+
+#ifdef ERTS_SMP
+ if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
erts_smp_mtx_destroy(mtx);
erts_free(ERTS_ALC_T_PORT_LOCK, mtx);
}
@@ -422,13 +425,13 @@ static void stopq(Port* prt)
static void
setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
- ErlDrvData drv_data, char *name, Uint32 xstatus)
+ ErlDrvData drv_data, char *name, erts_aint32_t xstate)
{
ErtsRunQueue *runq = erts_get_runq_current(NULL);
char *new_name, *old_name;
#ifdef DEBUG
/* Make sure the debug flags survives until port is freed */
- xstatus |= ERTS_PORT_SFLG_PORT_DEBUG;
+ xstate |= ERTS_PORT_SFLG_PORT_DEBUG;
#endif
ASSERT(runq);
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -437,8 +440,6 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1);
sys_strcpy(new_name, name);
erts_smp_runq_lock(runq);
- erts_smp_port_state_lock(prt);
- prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus;
prt->snapshot = erts_smp_atomic32_read_nob(&erts_ports_snapshot);
old_name = prt->name;
prt->name = new_name;
@@ -447,7 +448,8 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
#endif
ASSERT(!prt->drv_ptr);
prt->drv_ptr = driver;
- erts_smp_port_state_unlock(prt);
+ erts_smp_atomic32_set_relb(&prt->state,
+ ERTS_PORT_SFLG_CONNECTED | xstate);
erts_smp_runq_unlock(runq);
#ifdef ERTS_SMP
ASSERT(!prt->xports);
@@ -492,7 +494,7 @@ erts_wake_process_later(Port *prt, Process *process)
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- if (prt->status & ERTS_PORT_SFLGS_DEAD)
+ if (erts_smp_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_DEAD)
return;
for (p = &(prt->suspended); *p != NULL; p = &((*p)->next))
@@ -522,7 +524,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
int port_num;
int port_ix;
ErlDrvData drv_data = 0;
- Uint32 xstatus = 0;
+ erts_aint32_t xstate = 0;
Port *port;
int fpe_was_unmasked;
@@ -601,11 +603,9 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
*error_number_ptr = BADARG;
}
/* Need to mark the port as free again */
- erts_smp_port_state_lock(port);
- port->status = ERTS_PORT_SFLG_FREE;
ERTS_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2);
erts_smp_atomic_set_nob(&port->refc, 0);
- erts_smp_port_state_unlock(port);
+ erts_smp_atomic32_set_relb(&port->state, ERTS_PORT_SFLG_FREE);
return -3;
}
@@ -623,7 +623,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
erts_smp_mtx_init_x(port->lock,
"port_lock",
port->id);
- xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
+ xstate |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
}
#endif
@@ -637,7 +637,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
erts_smp_mtx_lock(port->lock);
#endif
- setup_port(port, pid, driver, drv_data, name, xstatus);
+ setup_port(port, pid, driver, drv_data, name, xstate);
if (IS_TRACED_FL(port, F_TRACE_PORTS)) {
trace_port_open(port,
@@ -737,11 +737,11 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
Process *rp;
int port_num;
Eterm port_id;
- Uint32 xstatus = 0;
+ erts_aint32_t xstate = 0;
ERTS_SMP_CHK_NO_PROC_LOCKS;
- creator_port = erts_drvport2port(creator_port_ix);
+ creator_port = erts_drvport2port(creator_port_ix, NULL);
if (!creator_port)
return (ErlDrvTermData) -1;
@@ -780,7 +780,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
port->lock = erts_alloc(ERTS_ALC_T_PORT_LOCK,
sizeof(erts_smp_mtx_t));
erts_smp_mtx_init_locked_x(port->lock, "port_lock", port_id);
- xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
+ xstate |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
}
#endif
@@ -793,7 +793,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port));
- setup_port(port, pid, driver, drv_data, name, xstatus);
+ setup_port(port, pid, driver, drv_data, name, xstate);
port->id = port_id;
erts_add_link(&(port->nlinks), LINK_PID, pid);
@@ -1349,7 +1349,7 @@ void init_io(void)
ERTS_TRACE_FLAGS(&erts_port[i]) = 0;
erts_port[i].drv_ptr = NULL;
- erts_port[i].status = ERTS_PORT_SFLG_FREE;
+ erts_smp_atomic32_init_nob(&erts_port[i].state, ERTS_PORT_SFLG_FREE);
erts_port[i].name = NULL;
erts_port[i].nlinks = NULL;
erts_port[i].monitors = NULL;
@@ -1593,7 +1593,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
* len -- length of data
*/
-static void deliver_read_message(Port* prt, Eterm to,
+static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
char *hbuf, ErlDrvSizeT hlen,
char *buf, ErlDrvSizeT len, int eol)
{
@@ -1611,10 +1611,11 @@ static void deliver_read_message(Port* prt, Eterm to,
ERTS_SMP_CHK_NO_PROC_LOCKS;
need = 3 + 3 + 2*hlen;
- if (prt->status & ERTS_PORT_SFLG_LINEBUF_IO) {
+
+ if (state & ERTS_PORT_SFLG_LINEBUF_IO) {
need += 3;
}
- if (prt->status & ERTS_PORT_SFLG_BINARY_IO && buf != NULL) {
+ if ((state & ERTS_PORT_SFLG_BINARY_IO) && buf != NULL) {
need += PROC_BIN_SIZE;
} else {
need += 2*len;
@@ -1630,7 +1631,7 @@ static void deliver_read_message(Port* prt, Eterm to,
hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks);
listp = NIL;
- if ((prt->status & ERTS_PORT_SFLG_BINARY_IO) == 0) {
+ if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) {
listp = buf_to_intlist(&hp, buf, len, listp);
} else if (buf != NULL) {
ProcBin* pb;
@@ -1661,7 +1662,7 @@ static void deliver_read_message(Port* prt, Eterm to,
listp = buf_to_intlist(&hp, hbuf, hlen, listp);
}
- if (prt->status & ERTS_PORT_SFLG_LINEBUF_IO){
+ if (state & ERTS_PORT_SFLG_LINEBUF_IO){
listp = TUPLE2(hp, (eol) ? am_eol : am_noeol, listp);
hp += 3;
}
@@ -1686,7 +1687,8 @@ static void deliver_read_message(Port* prt, Eterm to,
* Deliver all lines in a line buffer, repeats calls to
* deliver_read_message, and takes the same parameters.
*/
-static void deliver_linebuf_message(Port* prt, Eterm to,
+static void deliver_linebuf_message(Port* prt, erts_aint_t state,
+ Eterm to,
char* hbuf, ErlDrvSizeT hlen,
char *buf, ErlDrvSizeT len)
{
@@ -1695,7 +1697,7 @@ static void deliver_linebuf_message(Port* prt, Eterm to,
if(init_linebuf_context(&lc,&(prt->linebuf), buf, len) < 0)
return;
while((ret = read_linebuf(&lc)) > LINEBUF_EMPTY)
- deliver_read_message(prt, to, hbuf, hlen, LINEBUF_DATA(lc),
+ deliver_read_message(prt, state, to, hbuf, hlen, LINEBUF_DATA(lc),
LINEBUF_DATALEN(lc), (ret == LINEBUF_EOL));
}
@@ -1706,19 +1708,24 @@ static void deliver_linebuf_message(Port* prt, Eterm to,
* Parameters:
* prt - Pointer to a Port structure for this port.
*/
-static void flush_linebuf_messages(Port *prt)
+static void flush_linebuf_messages(Port *prt, erts_aint32_t state)
{
LineBufContext lc;
int ret;
ERTS_SMP_LC_ASSERT(!prt || erts_lc_is_port_locked(prt));
- if(prt == NULL || !(prt->status & ERTS_PORT_SFLG_LINEBUF_IO))
+
+ if (!prt)
+ return;
+
+ if (!(state & ERTS_PORT_SFLG_LINEBUF_IO))
return;
if(init_linebuf_context(&lc,&(prt->linebuf), NULL, 0) < 0)
return;
while((ret = flush_linebuf(&lc)) > LINEBUF_EMPTY)
deliver_read_message(prt,
+ state,
prt->connected,
NULL,
0,
@@ -1747,6 +1754,7 @@ deliver_vec_message(Port* prt, /* Port */
ErlOffHeap *ohp;
ErtsProcLocks rp_locks = 0;
int scheduler = erts_get_scheduler_id() != 0;
+ erts_aint32_t state;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -1762,12 +1770,13 @@ deliver_vec_message(Port* prt, /* Port */
if (!rp)
return;
+ state = erts_smp_atomic32_read_nob(&prt->state);
/*
* Calculate the exact number of heap words needed.
*/
need = 3 + 3; /* Heap space for two tuples */
- if (prt->status & ERTS_PORT_SFLG_BINARY_IO) {
+ if (state & ERTS_PORT_SFLG_BINARY_IO) {
need += (2+PROC_BIN_SIZE)*vsize - 2 + hlen*2;
} else {
need += (hlen+csize)*2;
@@ -1778,7 +1787,7 @@ deliver_vec_message(Port* prt, /* Port */
listp = NIL;
iov += vsize;
- if ((prt->status & ERTS_PORT_SFLG_BINARY_IO) == 0) {
+ if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) {
Eterm* thp = hp;
while (vsize--) {
iov--;
@@ -1864,7 +1873,7 @@ static void deliver_bin_message(Port* prt, /* port */
/*
* Note.
*
- * The test for (p->status & ERTS_PORT_SFLGS_DEAD) == 0 is important since the
+ * The test for ERTS_PORT_SFLGS_DEAD is important since the
* driver's flush function might call driver_async, which when using no
* threads and being short circuited will notice that the io queue is empty
* (after calling the driver's async_ready) and recursively call
@@ -1899,7 +1908,8 @@ static void flush_port(Port *p)
ASSERT(!p->xports);
#endif
}
- if ((p->status & ERTS_PORT_SFLGS_DEAD) == 0 && is_port_ioq_empty(p)) {
+ if ((erts_smp_atomic32_read_nob(&p->state) & ERTS_PORT_SFLGS_DEAD) == 0
+ && is_port_ioq_empty(p)) {
terminate_port(p);
}
}
@@ -1911,7 +1921,7 @@ terminate_port(Port *prt)
Eterm send_closed_port_id;
Eterm connected_id = NIL /* Initialize to silence compiler */;
erts_driver_t *drv;
- int halt;
+ erts_aint32_t state;
ERTS_SMP_CHK_NO_PROC_LOCKS;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -1919,10 +1929,10 @@ terminate_port(Port *prt)
ASSERT(!prt->nlinks);
ASSERT(!prt->monitors);
- /* prt->status may be altered by kill_port()below */
- halt = (prt->status & ERTS_PORT_SFLG_HALT) != 0;
- if (prt->status & ERTS_PORT_SFLG_SEND_CLOSED) {
- erts_port_status_band_set(prt, ~ERTS_PORT_SFLG_SEND_CLOSED);
+ /* state may be altered by kill_port() below */
+ state = erts_smp_atomic32_read_band_nob(&prt->state,
+ ~ERTS_PORT_SFLG_SEND_CLOSED);
+ if (state & ERTS_PORT_SFLG_SEND_CLOSED) {
send_closed_port_id = prt->id;
connected_id = prt->connected;
}
@@ -1978,7 +1988,8 @@ terminate_port(Port *prt)
* We don't want to send the closed message until after the
* port has been removed from the port table (in kill_port()).
*/
- if (halt && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
+ if ((state & ERTS_PORT_SFLG_HALT)
+ && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
erts_smp_port_unlock(prt); /* We will exit and never return */
erl_exit_flush_async(erts_halt_code, "");
}
@@ -2100,6 +2111,7 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
{
ErtsLink *lnk;
Eterm rreason;
+ erts_aint32_t state;
ERTS_SMP_CHK_NO_PROC_LOCKS;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
@@ -2119,9 +2131,10 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
}
#endif
- if ((p->status & (ERTS_PORT_SFLGS_DEAD
- | ERTS_PORT_SFLG_EXITING
- | ERTS_PORT_SFLG_IMMORTAL))
+ state = erts_smp_atomic32_read_nob(&p->state);
+ if ((state & (ERTS_PORT_SFLGS_DEAD
+ | ERTS_PORT_SFLG_EXITING
+ | ERTS_PORT_SFLG_IMMORTAL))
|| ((reason == am_normal) &&
((from != p->connected) && (from != p->id)))) {
return;
@@ -2142,7 +2155,7 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
if (p->reg != NULL)
(void) erts_unregister_name(NULL, 0, p, p->reg->name);
- erts_port_status_bor_set(p, ERTS_PORT_SFLG_EXITING);
+ state = erts_smp_atomic32_read_bor_relb(&p->state, ERTS_PORT_SFLG_EXITING);
{
SweepContext sc = {p->id, rreason};
@@ -2158,17 +2171,20 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
}
DRV_MONITOR_UNLOCK_PDL(p);
- if ((p->status & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) {
+ if ((state & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) {
erts_do_net_exits(p->dist_entry, rreason);
erts_deref_dist_entry(p->dist_entry);
- p->dist_entry = NULL;
- erts_port_status_band_set(p, ~ERTS_PORT_SFLG_DISTRIBUTION);
+ p->dist_entry = NULL;
+ erts_smp_atomic32_read_band_relb(&p->state,
+ ~ERTS_PORT_SFLG_DISTRIBUTION);
}
if ((reason != am_kill) && !is_port_ioq_empty(p)) {
- erts_port_status_bandor_set(p,
- ~ERTS_PORT_SFLG_EXITING, /* must turn it off */
- ERTS_PORT_SFLG_CLOSING);
+ /* must turn exiting flag off */
+ erts_smp_atomic32_read_bset_relb(&p->state,
+ (ERTS_PORT_SFLG_EXITING
+ | ERTS_PORT_SFLG_CLOSING),
+ ERTS_PORT_SFLG_CLOSING);
flush_port(p);
}
else {
@@ -2217,7 +2233,8 @@ void erts_port_command(Process *proc,
if ((pid = port->connected) == tp[1]) {
/* PID must be connected */
if (tp[2] == am_close) {
- erts_port_status_bor_set(port, ERTS_PORT_SFLG_SEND_CLOSED);
+ erts_smp_atomic32_read_bor_relb(&port->state,
+ ERTS_PORT_SFLG_SEND_CLOSED);
erts_do_exit_port(port, pid, am_normal);
#ifdef USE_VM_PROBES
@@ -2443,13 +2460,14 @@ void
print_port_info(int to, void *arg, int i)
{
Port* p = &erts_port[i];
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
- if (p->status & ERTS_PORT_SFLGS_DEAD)
+ if (state & ERTS_PORT_SFLGS_DEAD)
return;
erts_print(to, arg, "=port:%T\n", p->id);
erts_print(to, arg, "Slot: %d\n", i);
- if (p->status & ERTS_PORT_SFLG_CONNECTED) {
+ if (state & ERTS_PORT_SFLG_CONNECTED) {
erts_print(to, arg, "Connected: %T", p->connected);
erts_print(to, arg, "\n");
}
@@ -2497,8 +2515,8 @@ set_busy_port(ErlDrvPort port_num, int on)
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[port_num]));
if (on) {
- erts_port_status_bor_set(&erts_port[port_num],
- ERTS_PORT_SFLG_PORT_BUSY);
+ erts_smp_atomic32_read_bor_relb(&erts_port[port_num].state,
+ ERTS_PORT_SFLG_PORT_BUSY);
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(port_busy)) {
erts_snprintf(port_str, sizeof(port_str),
@@ -2508,8 +2526,8 @@ set_busy_port(ErlDrvPort port_num, int on)
#endif
} else {
ErtsProcList* plp = erts_port[port_num].suspended;
- erts_port_status_band_set(&erts_port[port_num],
- ~ERTS_PORT_SFLG_PORT_BUSY);
+ erts_smp_atomic32_read_band_relb(&erts_port[port_num].state,
+ ~ERTS_PORT_SFLG_PORT_BUSY);
erts_port[port_num].suspended = NULL;
#ifdef USE_VM_PROBES
@@ -2576,19 +2594,27 @@ void set_port_control_flags(ErlDrvPort port_num, int flags)
erts_port[port_num].control_flags = flags;
}
-int get_port_flags(ErlDrvPort ix) {
- Port* prt = erts_drvport2port(ix);
+int get_port_flags(ErlDrvPort ix)
+{
+ int flags;
+ Port *prt;
+ erts_aint32_t state;
+
+ prt = erts_drvport2port(ix, &state);
+ if (!prt)
+ return 0;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- if (prt == NULL)
- return 0;
+ flags = 0;
+ if (state & ERTS_PORT_SFLG_BINARY_IO)
+ flags |= PORT_FLAG_BINARY;
+ if (state & ERTS_PORT_SFLG_LINEBUF_IO)
+ flags |= PORT_FLAG_LINE;
- return (prt->status & ERTS_PORT_SFLG_BINARY_IO ? PORT_FLAG_BINARY : 0)
- | (prt->status & ERTS_PORT_SFLG_LINEBUF_IO ? PORT_FLAG_LINE : 0);
+ return flags;
}
-
void erts_raw_port_command(Port* p, byte* buf, Uint len)
{
int fpe_was_unmasked;
@@ -2624,8 +2650,9 @@ int async_ready(Port *p, void* data)
ERTS_SMP_CHK_NO_PROC_LOCKS;
if (p) {
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
- ASSERT(!(p->status & ERTS_PORT_SFLGS_DEAD));
+ ASSERT(!(state & ERTS_PORT_SFLGS_DEAD));
if (p->drv_ptr->ready_async != NULL) {
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(driver_ready_async)) {
@@ -2641,7 +2668,7 @@ int async_ready(Port *p, void* data)
ASSERT(!p->xports);
#endif
}
- if ((p->status & ERTS_PORT_SFLG_CLOSING) && is_port_ioq_empty(p)) {
+ if ((state & ERTS_PORT_SFLG_CLOSING) && is_port_ioq_empty(p)) {
terminate_port(p);
}
}
@@ -2736,7 +2763,7 @@ erts_get_port_names(Eterm id)
pnp_len = sizeof(ErtsPortNames) + len;
pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len);
}
- erts_smp_port_state_lock(prt);
+ erts_smp_port_minor_lock(prt);
if (id != prt->id) {
len = nlen = 0;
name = driver_name = NULL;
@@ -2764,7 +2791,7 @@ erts_get_port_names(Eterm id)
}
do_realloc = 0;
}
- erts_smp_port_state_unlock(prt);
+ erts_smp_port_minor_unlock(prt);
} while (do_realloc);
}
return pnp;
@@ -2803,7 +2830,7 @@ ErlDrvTermData driver_mk_term_nil(void)
void driver_report_exit(int ix, int status)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
Eterm* hp;
Eterm tuple;
Process *rp;
@@ -2852,14 +2879,13 @@ deliver_term_check_port(ErlDrvPort drvport)
res = -1; /* invalid */
else {
Port* prt = &erts_port[ix];
- erts_smp_port_state_lock(prt);
- if (!(prt->status & ERTS_PORT_SFLGS_INVALID_LOOKUP))
+ erts_aint32_t state = erts_smp_atomic32_read_nob(&prt->state);
+ if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP))
res = 1; /* ok */
- else if (prt->status & ERTS_PORT_SFLG_CLOSING)
+ else if (state & ERTS_PORT_SFLG_CLOSING)
res = 0; /* closing */
else
res = -1; /* invalid (dead) */
- erts_smp_port_state_unlock(prt);
}
return res;
}
@@ -3428,7 +3454,7 @@ driver_deliver_term(ErlDrvPort port,
int
driver_output_term(ErlDrvPort ix, ErlDrvTermData* data, int len)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
ERTS_SMP_CHK_NO_PROC_LOCKS;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -3454,19 +3480,20 @@ driver_send_term(ErlDrvPort ix, ErlDrvTermData to, ErlDrvTermData* data, int len
int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
ErlDrvBinary* bin, ErlDrvSizeT offs, ErlDrvSizeT len)
{
- Port* prt = erts_drvport2port(ix);
+ erts_aint32_t state;
+ Port* prt = erts_drvport2port(ix, &state);
ERTS_SMP_CHK_NO_PROC_LOCKS;
if (prt == NULL)
return -1;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- if (prt->status & ERTS_PORT_SFLG_CLOSING)
+ if (state & ERTS_PORT_SFLG_CLOSING)
return 0;
prt->bytes_in += (hlen + len);
erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
- if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
+ if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
return erts_net_message(prt,
prt->dist_entry,
(byte*) hbuf, hlen,
@@ -3488,7 +3515,8 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
char* buf, ErlDrvSizeT len)
{
- Port* prt = erts_drvport2port(ix);
+ erts_aint32_t state;
+ Port* prt = erts_drvport2port(ix, &state);
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -3497,12 +3525,12 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- if (prt->status & ERTS_PORT_SFLG_CLOSING)
+ if (state & ERTS_PORT_SFLG_CLOSING)
return 0;
prt->bytes_in += (hlen + len);
erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
- if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
+ if (state & ERTS_PORT_SFLG_DISTRIBUTION) {
if (len == 0)
return erts_net_message(prt,
prt->dist_entry,
@@ -3514,10 +3542,10 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
(byte*) hbuf, hlen,
(byte*) buf, len);
}
- else if(prt->status & ERTS_PORT_SFLG_LINEBUF_IO)
- deliver_linebuf_message(prt, prt->connected, hbuf, hlen, buf, len);
+ else if (state & ERTS_PORT_SFLG_LINEBUF_IO)
+ deliver_linebuf_message(prt, state, prt->connected, hbuf, hlen, buf, len);
else
- deliver_read_message(prt, prt->connected, hbuf, hlen, buf, len, 0);
+ deliver_read_message(prt, state, prt->connected, hbuf, hlen, buf, len, 0);
return 0;
}
@@ -3538,6 +3566,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
SysIOVec* iov;
ErlDrvBinary** binv;
Port* prt;
+ erts_aint32_t state;
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -3550,13 +3579,13 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
if (hlen < 0)
hlen = 0;
- prt = erts_drvport2port(ix);
+ prt = erts_drvport2port(ix, &state);
if (prt == NULL)
return -1;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- if (prt->status & ERTS_PORT_SFLG_CLOSING)
+ if (state & ERTS_PORT_SFLG_CLOSING)
return 0;
/* size > 0 ! */
@@ -3826,7 +3855,7 @@ ErlDrvPDL
driver_pdl_create(ErlDrvPort dp)
{
ErlDrvPDL pdl;
- Port *pp = erts_drvport2port(dp);
+ Port *pp = erts_drvport2port(dp, NULL);
if (!pp || pp->port_data_lock)
return NULL;
pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK,
@@ -4283,7 +4312,7 @@ drv_cancel_timer(Port *prt)
int driver_set_timer(ErlDrvPort ix, unsigned long t)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -4310,7 +4339,7 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t)
int driver_cancel_timer(ErlDrvPort ix)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
if (prt == NULL)
return -1;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -4322,7 +4351,7 @@ int driver_cancel_timer(ErlDrvPort ix)
int
driver_read_timer(ErlDrvPort ix, unsigned long* t)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -4400,8 +4429,10 @@ int driver_monitor_process(ErlDrvPort port,
{
Port *prt;
int ret;
- Uint32 status;
+ erts_aint32_t state;
+#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK))
ErtsSchedulerData *sched = erts_get_scheduler_data();
+#endif
int ix = (int) port;
if (ix < 0 || erts_max_ports <= ix) {
return -1;
@@ -4410,15 +4441,9 @@ int driver_monitor_process(ErlDrvPort port,
DRV_MONITOR_LOCK_PDL(prt);
- if (sched) {
- status = erts_port[ix].status;
- } else {
- erts_smp_port_state_lock(prt);
- status = erts_port[ix].status;
- erts_smp_port_state_unlock(prt);
- }
+ state = erts_smp_atomic32_read_nob(&erts_port[ix].state);
- if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+ if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
DRV_MONITOR_UNLOCK_PDL(prt);
return -1;
}
@@ -4488,8 +4513,10 @@ int driver_demonitor_process(ErlDrvPort port,
{
Port *prt;
int ret;
- Uint32 status;
+ erts_aint32_t state;
+#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK))
ErtsSchedulerData *sched = erts_get_scheduler_data();
+#endif
int ix = (int) port;
if (ix < 0 || erts_max_ports <= ix) {
return -1;
@@ -4498,15 +4525,9 @@ int driver_demonitor_process(ErlDrvPort port,
DRV_MONITOR_LOCK_PDL(prt);
- if (sched) {
- status = erts_port[ix].status;
- } else {
- erts_smp_port_state_lock(prt);
- status = erts_port[ix].status;
- erts_smp_port_state_unlock(prt);
- }
+ state = erts_smp_atomic32_read_nob(&erts_port[ix].state);
- if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+ if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
DRV_MONITOR_UNLOCK_PDL(prt);
return -1;
}
@@ -4558,8 +4579,10 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
{
Port *prt;
ErlDrvTermData ret;
- Uint32 status;
+ erts_aint32_t state;
+#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK))
ErtsSchedulerData *sched = erts_get_scheduler_data();
+#endif
int ix = (int) port;
if (ix < 0 || erts_max_ports <= ix) {
return driver_term_nil;
@@ -4568,15 +4591,8 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
DRV_MONITOR_LOCK_PDL(prt);
- if (sched) {
- status = erts_port[ix].status;
- } else {
- erts_smp_port_state_lock(prt);
- status = erts_port[ix].status;
- erts_smp_port_state_unlock(prt);
- }
-
- if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+ state = erts_smp_atomic32_read_nob(&erts_port[ix].state);
+ if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
DRV_MONITOR_UNLOCK_PDL(prt);
return driver_term_nil;
}
@@ -4651,7 +4667,8 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
static int
driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
{
- Port* prt = erts_drvport2port(ix);
+ erts_aint32_t state;
+ Port* prt = erts_drvport2port(ix, &state);
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -4659,10 +4676,10 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
return -1;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
if (eof)
- flush_linebuf_messages(prt);
- if (prt->status & ERTS_PORT_SFLG_CLOSING) {
+ flush_linebuf_messages(prt, state);
+ if (state & ERTS_PORT_SFLG_CLOSING) {
terminate_port(prt);
- } else if (eof && (prt->status & ERTS_PORT_SFLG_SOFT_EOF)) {
+ } else if (eof && (state & ERTS_PORT_SFLG_SOFT_EOF)) {
deliver_result(prt->id, prt->connected, am_eof);
} else {
/* XXX UGLY WORK AROUND, Let do_exit_port terminate the port */
@@ -4684,7 +4701,7 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
*/
int driver_exit(ErlDrvPort ix, int err)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
Process* rp;
ErtsLink *lnk, *rlnk = NULL;
@@ -4757,14 +4774,14 @@ ErlDrvTermData driver_mk_atom(char* string)
ErlDrvTermData driver_mk_port(ErlDrvPort ix)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
return (ErlDrvTermData) prt->id;
}
ErlDrvTermData driver_connected(ErlDrvPort ix)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
ERTS_SMP_CHK_NO_PROC_LOCKS;
if (prt == NULL)
return NIL;
@@ -4774,7 +4791,7 @@ ErlDrvTermData driver_connected(ErlDrvPort ix)
ErlDrvTermData driver_caller(ErlDrvPort ix)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
ERTS_SMP_CHK_NO_PROC_LOCKS;
if (prt == NULL)
return NIL;
@@ -4784,7 +4801,7 @@ ErlDrvTermData driver_caller(ErlDrvPort ix)
int driver_lock_driver(ErlDrvPort ix)
{
- Port* prt = erts_drvport2port(ix);
+ Port* prt = erts_drvport2port(ix, NULL);
DE_Handle* dh;
ERTS_SMP_CHK_NO_PROC_LOCKS;
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index c1336c60d9..20bc1a6f55 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -204,7 +204,7 @@ static void steal_pending_stop_select(erts_dsprintf_buf_t*, ErlDrvPort,
static ERTS_INLINE Eterm
drvport2id(ErlDrvPort dp)
{
- Port *pp = erts_drvport2port(dp);
+ Port *pp = erts_drvport2port(dp, NULL);
if (pp)
return pp->id;
else {
@@ -502,8 +502,8 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
DTRACE_CHARBUF(name, 64);
#endif
- ERTS_SMP_LC_ASSERT(erts_drvport2port(ix)
- && erts_lc_is_port_locked(erts_drvport2port(ix)));
+ ERTS_SMP_LC_ASSERT(erts_drvport2port(ix, NULL)
+ && erts_lc_is_port_locked(erts_drvport2port(ix, NULL)));
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
@@ -529,9 +529,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
if (!on && (mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
if (IS_FD_UNKNOWN(state)) {
/* fast track to stop_select callback */
- stop_select_fn = erts_drvport2port(ix)->drv_ptr->stop_select;
+ stop_select_fn = erts_drvport2port(ix, NULL)->drv_ptr->stop_select;
#ifdef USE_VM_PROBES
- strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1);
+ strncpy(name, erts_drvport2port(ix, NULL)->drv_ptr->name, sizeof(name)-1);
name[sizeof(name)-1] = '\0';
#endif
ret = 0;
@@ -664,14 +664,14 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
}
}
if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
- erts_driver_t* drv_ptr = erts_drvport2port(ix)->drv_ptr;
+ erts_driver_t* drv_ptr = erts_drvport2port(ix, NULL)->drv_ptr;
ASSERT(new_events==0);
if (state->remove_cnt == 0 || !wake_poller) {
/* Safe to close fd now as it is not in pollset
or there was no need to eject fd (kernel poll) */
stop_select_fn = drv_ptr->stop_select;
#ifdef USE_VM_PROBES
- strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1);
+ strncpy(name, erts_drvport2port(ix, NULL)->drv_ptr->name, sizeof(name)-1);
name[sizeof(name)-1] = '\0';
#endif
}
@@ -723,8 +723,8 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
int do_wake = 0;
int ret;
- ERTS_SMP_LC_ASSERT(erts_drvport2port(ix)
- && erts_lc_is_port_locked(erts_drvport2port(ix)));
+ ERTS_SMP_LC_ASSERT(erts_drvport2port(ix, NULL)
+ && erts_lc_is_port_locked(erts_drvport2port(ix, NULL)));
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
@@ -959,7 +959,7 @@ static void
print_select_op(erts_dsprintf_buf_t *dsbufp,
ErlDrvPort ix, ErtsSysFdType fd, int mode, int on)
{
- Port *pp = erts_drvport2port(ix);
+ Port *pp = erts_drvport2port(ix, NULL);
erts_dsprintf(dsbufp,
"driver_select(%p, %d,%s%s%s%s, %d) "
"by ",
@@ -1030,7 +1030,7 @@ steal_pending_stop_select(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix,
state->driver.drv_ptr = NULL;
}
else if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) {
- erts_driver_t* drv_ptr = erts_drvport2port(ix)->drv_ptr;
+ erts_driver_t* drv_ptr = erts_drvport2port(ix, NULL)->drv_ptr;
if (drv_ptr != state->driver.drv_ptr) {
/* Some other driver wants the stop_select callback */
if (state->driver.drv_ptr->handle) {
@@ -1052,7 +1052,7 @@ static void
print_event_op(erts_dsprintf_buf_t *dsbufp,
ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data)
{
- Port *pp = erts_drvport2port(ix);
+ Port *pp = erts_drvport2port(ix, NULL);
erts_dsprintf(dsbufp, "driver_event(%p, %d, ", ix, (int) fd);
if (!event_data)
erts_dsprintf(dsbufp, "NULL");
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-05 17:34:03 +0000