aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/atom.names6
-rw-r--r--erts/emulator/beam/beam_emu.c2
-rw-r--r--erts/emulator/beam/bif.c381
-rw-r--r--erts/emulator/beam/bif.h21
-rw-r--r--erts/emulator/beam/bif.tab32
-rw-r--r--erts/emulator/beam/binary.c6
-rw-r--r--erts/emulator/beam/dist.c152
-rw-r--r--erts/emulator/beam/erl_alloc.types7
-rw-r--r--erts/emulator/beam/erl_bif_ddll.c8
-rw-r--r--erts/emulator/beam/erl_bif_info.c302
-rw-r--r--erts/emulator/beam/erl_bif_port.c791
-rw-r--r--erts/emulator/beam/erl_bif_re.c16
-rw-r--r--erts/emulator/beam/erl_init.c43
-rw-r--r--erts/emulator/beam/erl_nif.c4
-rw-r--r--erts/emulator/beam/erl_port.h207
-rw-r--r--erts/emulator/beam/erl_port_task.c927
-rw-r--r--erts/emulator/beam/erl_port_task.h56
-rw-r--r--erts/emulator/beam/erl_process.c21
-rw-r--r--erts/emulator/beam/erl_thr_progress.h1
-rw-r--r--erts/emulator/beam/erl_trace.h138
-rw-r--r--erts/emulator/beam/global.h155
-rw-r--r--erts/emulator/beam/io.c2803
-rw-r--r--erts/emulator/beam/sys.h3
-rw-r--r--erts/emulator/beam/utils.c17
24 files changed, 4616 insertions, 1483 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index d6d8633f6f..30b891be5d 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -94,6 +94,7 @@ atom asynchronous
atom atom
atom atom_used
atom attributes
+atom await_port_send_result
atom await_proc_exit
atom await_sched_wall_time_modifications
atom awaiting_load
@@ -152,6 +153,7 @@ atom connection_closed
atom cons
atom const
atom context_switches
+atom control
atom copy
atom cpu
atom cpu_timestamp
@@ -204,6 +206,7 @@ atom erlang
atom ERROR='ERROR'
atom error_handler
atom error_logger
+atom erts_internal
atom ets
atom ETS_TRANSFER='ETS-TRANSFER'
atom event
@@ -237,6 +240,7 @@ atom gc_end
atom gc_start
atom Ge='>='
atom generational
+atom get_data
atom get_seq_token
atom get_tcw
atom getenv
@@ -408,6 +412,7 @@ atom overlapped_io
atom owner
atom packet
atom packet_size
+atom parallelism
atom Plus='+'
atom pause
atom pending
@@ -482,6 +487,7 @@ atom sequential_trace_token
atom serial
atom set
atom set_cpu_topology
+atom set_data
atom set_on_first_link
atom set_on_first_spawn
atom set_on_link
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index d8ccbacc26..106f742272 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1135,7 +1135,7 @@ init_emulator(void)
void
dtrace_drvport_str(ErlDrvPort drvport, char *port_buf)
{
- Port *port = erts_drvport2port(drvport);
+ Port *port = erts_drvport2port(drvport, NULL);
erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>",
port_channel_no(port->common.id),
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index dc0dcbabd6..3ab8fdb2c7 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -43,6 +43,7 @@
static Export* flush_monitor_message_trap = NULL;
static Export* set_cpu_topology_trap = NULL;
static Export* await_proc_exit_trap = NULL;
+static Export* await_port_send_result_trap = NULL;
Export* erts_format_cpu_topology_trap = NULL;
static Export *await_sched_wall_time_mod_trap;
@@ -157,22 +158,37 @@ BIF_RETTYPE link_1(BIF_ALIST_1)
}
if (is_internal_port(BIF_ARG_1)) {
- Port *pt = erts_id2port_sflgs(BIF_ARG_1,
- BIF_P,
- ERTS_PROC_LOCK_MAIN,
- ERTS_PORT_SFLGS_INVALID_LOOKUP);
- if (!pt) {
+ int send_link_signal = 0;
+ Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+ if (!prt) {
goto res_no_proc;
}
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
if (erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1) >= 0)
- erts_add_link(&ERTS_P_LINKS(pt), LINK_PID, BIF_P->common.id);
+ send_link_signal = 1;
/* else: already linked */
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
- erts_port_release(pt);
+
+ if (send_link_signal) {
+ Eterm ref;
+ Eterm *refp = erts_port_synchronous_ops ? &ref : NULL;
+
+ switch (erts_port_link(BIF_P, prt, BIF_P->common.id, refp)) {
+ case ERTS_PORT_OP_DROPPED:
+ case ERTS_PORT_OP_BADARG:
+ goto res_no_proc;
+ case ERTS_PORT_OP_SCHEDULED:
+ if (refp) {
+ ASSERT(is_internal_ref(ref));
+ BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
+ }
+ default:
+ break;
+ }
+ }
BIF_RET(am_true);
}
else if (is_external_port(BIF_ARG_1)
@@ -944,36 +960,39 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
}
if (is_internal_port(BIF_ARG_1)) {
- Port *pt = erts_id2port_sflgs(BIF_ARG_1,
- BIF_P,
- ERTS_PROC_LOCK_MAIN,
- ERTS_PORT_SFLGS_DEAD);
-
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
#ifdef ERTS_SMP
- if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
- if (pt)
- erts_port_release(pt);
+ if (ERTS_PROC_PENDING_EXIT(BIF_P))
goto handle_pending_exit;
- }
#endif
l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1);
- ASSERT(pt || !l);
-
- if (pt) {
- rl = erts_remove_link(&ERTS_P_LINKS(pt), BIF_P->common.id);
- erts_port_release(pt);
- if (rl)
- erts_destroy_link(rl);
- }
-
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS);
- if (l)
+ if (l) {
+ Port *prt;
+
erts_destroy_link(l);
+ /* Send unlink signal */
+ prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_DEAD);
+ if (prt) {
+ ErtsPortOpResult res;
+ Eterm ref;
+ Eterm *refp = erts_port_synchronous_ops ? &ref : NULL;
+#ifdef DEBUG
+ ref = NIL;
+#endif
+ res = erts_port_unlink(BIF_P, prt, BIF_P->common.id, refp);
+
+ if (refp && res == ERTS_PORT_OP_SCHEDULED) {
+ ASSERT(is_internal_ref(ref));
+ BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
+ }
+ }
+ }
+
BIF_RET(am_true);
}
else if (is_external_port(BIF_ARG_1)
@@ -1344,15 +1363,28 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
*/
if (is_internal_port(BIF_ARG_1)) {
- Port *prt;
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- prt = erts_id2port(BIF_ARG_1);
+ Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+
if (prt) {
- erts_do_exit_port(prt, BIF_P->common.id, BIF_ARG_2);
- erts_port_release(prt);
+ Eterm ref;
+ Eterm *refp = erts_port_synchronous_ops ? &ref : NULL;
+ ErtsPortOpResult res;
+
+#ifdef DEBUG
+ ref = NIL;
+#endif
+
+ res = erts_port_exit(BIF_P, 0, prt, BIF_P->common.id, BIF_ARG_2, refp);
+
+ ERTS_BIF_CHK_EXITED(BIF_P);
+
+ if (refp && res == ERTS_PORT_OP_SCHEDULED) {
+ ASSERT(is_internal_ref(ref));
+ BIF_TRAP3(await_port_send_result_trap, BIF_P, ref, am_true, am_true);
+ }
+
}
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
- ERTS_BIF_CHK_EXITED(BIF_P);
+
BIF_RET(am_true);
}
else if(is_external_port(BIF_ARG_1)
@@ -1516,22 +1548,24 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
}
/*
* NOTE: It is important that we check for pending exit signals
- * and handle them before flag trap_exit is set to true.
- * For more info, see implementation of erts_send_exit_signal().
+ * and handle them before returning if trap_exit is set to
+ * true. For more info, see implementation of
+ * erts_send_exit_signal().
*/
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_STATUS);
- ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)
- & erts_proc_lc_my_proc_locks(BIF_P));
- ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P,
- ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
if (trap_exit)
- state = erts_smp_atomic32_read_bor_nob(&BIF_P->state,
- ERTS_PSFLG_TRAP_EXIT);
+ state = erts_smp_atomic32_read_bor_mb(&BIF_P->state,
+ ERTS_PSFLG_TRAP_EXIT);
else
- state = erts_smp_atomic32_read_band_nob(&BIF_P->state,
- ~ERTS_PSFLG_TRAP_EXIT);
+ state = erts_smp_atomic32_read_band_mb(&BIF_P->state,
+ ~ERTS_PSFLG_TRAP_EXIT);
+#ifdef ERTS_SMP
+ if (ERTS_PROC_PENDING_EXIT(BIF_P)) {
+ erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
+ ERTS_BIF_EXITED(BIF_P);
+ }
+#endif
+
old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false;
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS);
BIF_RET(old_value);
}
else if (BIF_ARG_1 == am_scheduler) {
@@ -1746,8 +1780,9 @@ ebif_bang_2(BIF_ALIST_2)
#define SEND_BADARG (-4)
#define SEND_USER_ERROR (-5)
#define SEND_INTERNAL_ERROR (-6)
+#define SEND_AWAIT_RESULT (-7)
-Sint do_send(Process *p, Eterm to, Eterm msg, int suspend);
+Sint do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp);
static Sint remote_send(Process *p, DistEntry *dep,
Eterm to, Eterm full_to, Eterm msg, int suspend)
@@ -1801,7 +1836,7 @@ static Sint remote_send(Process *p, DistEntry *dep,
}
Sint
-do_send(Process *p, Eterm to, Eterm msg, int suspend) {
+do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) {
Eterm portid;
Port *pt;
Process* rp;
@@ -1813,13 +1848,10 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
trace_send(p, to, msg);
if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
save_calls(p, &exp_send);
-
- rp = erts_proc_lookup_raw(to);
-
- if (!rp) {
- ERTS_SMP_ASSERT_IS_NOT_EXITING(p);
+
+ rp = erts_proc_lookup_raw(to);
+ if (!rp)
return 0;
- }
} else if (is_external_pid(to)) {
dep = external_pid_dist_entry(to);
if(dep == erts_this_dist_entry) {
@@ -1837,45 +1869,24 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
}
return remote_send(p, dep, to, to, msg, suspend);
} else if (is_atom(to)) {
-
- /* Need to virtual schedule out sending process
- * because of lock wait. This is only necessary
- * for internal port calling but the lock is bundled
- * with name lookup.
- */
-
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(p, am_out);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(p, am_inactive);
- }
- erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
- to,
- &rp, 0, 0,
- &pt);
+ Eterm id = erts_whereis_name_to_id(p, to);
+
+ rp = erts_proc_lookup(id);
+ if (rp)
+ goto send_message;
+ pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP);
if (pt) {
- portid = pt->common.id;
+ portid = id;
goto port_common;
}
-
- /* Not a port virtually schedule the process back in */
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(p, am_in);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(p, am_active);
- }
if (IS_TRACED(p))
trace_send(p, to, msg);
if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
save_calls(p, &exp_send);
- if (!rp) {
- return SEND_BADARG;
- }
+ return SEND_BADARG;
} else if (is_external_port(to)
&& (external_port_dist_entry(to)
== erts_this_dist_entry)) {
@@ -1891,49 +1902,48 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
erts_send_error_to_logger(p->group_leader, dsbufp);
return 0;
} else if (is_internal_port(to)) {
+ int ret_val;
portid = to;
- /* schedule out calling process, waiting for lock*/
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(p, am_out);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(p, am_inactive);
- }
- pt = erts_id2port_sflgs(to,
- p,
- ERTS_PROC_LOCK_MAIN,
- ERTS_PORT_SFLGS_INVALID_LOOKUP);
+
+ pt = erts_port_lookup(portid, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+
port_common:
- ERTS_SMP_LC_ASSERT(!pt || erts_lc_is_port_locked(pt));
+ ret_val = 0;
- /* We have waited for locks, trace schedule ports */
if (pt) {
- erts_aint32_t flags;
- if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(pt, am_in, am_command);
- }
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) {
- profile_runnable_port(pt, am_active);
- }
-
- flags = erts_atomic32_read_rb(&pt->sched.flags);
- /* XXX let port_command handle the busy stuff !!! */
- if (flags & ERTS_PTS_FLG_BUSY) {
+ int ps_flags = suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND;
+ *refp = NIL;
+
+ switch (erts_port_command(p, ps_flags, pt, msg, refp)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ /* We are exiting... */
+ return SEND_USER_ERROR;
+ case ERTS_PORT_OP_BUSY:
+ /* Nothing has been sent */
+ if (suspend)
+ erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt);
+ return SEND_YIELD;
+ case ERTS_PORT_OP_BUSY_SCHEDULED:
+ /* Message was sent */
if (suspend) {
erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt);
- if (erts_system_monitor_flags.busy_port) {
- monitor_generic(p, am_busy_port, portid);
- }
- }
- /* Virtually schedule out the port before releasing */
- if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(pt, am_out, am_command);
+ ret_val = SEND_YIELD_RETURN;
+ break;
}
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) {
- profile_runnable_port(pt, am_inactive);
+ /* Fall through */
+ case ERTS_PORT_OP_SCHEDULED:
+ if (is_not_nil(*refp)) {
+ ASSERT(is_internal_ref(*refp));
+ ret_val = SEND_AWAIT_RESULT;
}
- erts_port_release(pt);
- return SEND_YIELD;
+ break;
+ case ERTS_PORT_OP_DROPPED:
+ case ERTS_PORT_OP_BADARG:
+ case ERTS_PORT_OP_DONE:
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_command() result");
+ break;
}
}
@@ -1952,30 +1962,11 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
SEQ_TRACE_SEND, portid, p);
}
- /* XXX NO GC in port command */
- erts_port_command(p, p->common.id, pt, msg);
- if (pt) {
- /* Virtually schedule out the port before releasing */
- if (IS_TRACED_FL(pt, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(pt, am_out, am_command);
- }
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(pt)) {
- profile_runnable_port(pt, am_inactive);
- }
- erts_port_release(pt);
- }
- /* Virtually schedule in process */
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(p, am_in);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(p, am_active);
- }
if (ERTS_PROC_IS_EXITING(p)) {
KILL_CATCHES(p); /* Must exit */
return SEND_USER_ERROR;
}
- return 0;
+ return ret_val;
} else if (is_tuple(to)) { /* Remote send */
int ret;
tp = tuple_val(to);
@@ -1991,47 +1982,24 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
dep = erts_sysname_to_connected_dist_entry(tp[2]);
if (dep == erts_this_dist_entry) {
+ Eterm id;
erts_deref_dist_entry(dep);
if (IS_TRACED(p))
trace_send(p, to, msg);
if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
save_calls(p, &exp_send);
-
- /* Need to virtual schedule out sending process
- * because of lock wait. This is only necessary
- * for internal port calling but the lock is bundled.
- */
-
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(p, am_out);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(p, am_inactive);
- }
- erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
- tp[1],
- &rp, 0, 0,
- &pt);
+ id = erts_whereis_name_to_id(p, tp[1]);
+
+ rp = erts_proc_lookup_raw(id);
+ if (rp)
+ goto send_message;
+ pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP);
if (pt) {
- portid = pt->common.id;
+ portid = id;
goto port_common;
}
- /* Port lookup failed, virtually schedule the process
- * back in.
- */
-
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(p, am_in);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(p, am_active);
- }
-
- if (!rp) {
- return 0;
- }
- goto send_message;
+ return 0;
}
ret = remote_send(p, dep, tp[1], to, msg, suspend);
@@ -2070,6 +2038,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
BIF_RETTYPE send_3(BIF_ALIST_3)
{
+ Eterm ref;
Process *p = BIF_P;
Eterm to = BIF_ARG_1;
Eterm msg = BIF_ARG_2;
@@ -2093,12 +2062,18 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
if(!is_nil(l)) {
BIF_ERROR(p, BADARG);
}
-
- result = do_send(p, to, msg, suspend);
+
+#ifdef DEBUG
+ ref = NIL;
+#endif
+
+ result = do_send(p, to, msg, suspend, &ref);
if (result > 0) {
ERTS_VBUMP_REDS(p, result);
BIF_RET(am_ok);
- } else switch (result) {
+ }
+
+ switch (result) {
case 0:
BIF_RET(am_ok);
break;
@@ -2121,6 +2096,9 @@ BIF_RETTYPE send_3(BIF_ALIST_3)
ERTS_BIF_YIELD_RETURN(p, am_ok);
else
BIF_RET(am_nosuspend);
+ case SEND_AWAIT_RESULT:
+ ASSERT(is_internal_ref(ref));
+ BIF_TRAP3(await_port_send_result_trap, p, ref, am_nosuspend, am_ok);
case SEND_BADARG:
BIF_ERROR(p, BADARG);
break;
@@ -2145,12 +2123,21 @@ BIF_RETTYPE send_2(BIF_ALIST_2)
Eterm erl_send(Process *p, Eterm to, Eterm msg)
{
- Sint result = do_send(p, to, msg, !0);
+ Eterm ref;
+ Sint result;
+
+#ifdef DEBUG
+ ref = NIL;
+#endif
+
+ result = do_send(p, to, msg, !0, &ref);
if (result > 0) {
ERTS_VBUMP_REDS(p, result);
BIF_RET(msg);
- } else switch (result) {
+ }
+
+ switch (result) {
case 0:
BIF_RET(msg);
break;
@@ -2162,6 +2149,9 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg)
break;
case SEND_YIELD_RETURN:
ERTS_BIF_YIELD_RETURN(p, msg);
+ case SEND_AWAIT_RESULT:
+ ASSERT(is_internal_ref(ref));
+ BIF_TRAP3(await_port_send_result_trap, p, ref, msg, msg);
case SEND_BADARG:
BIF_ERROR(p, BADARG);
break;
@@ -3166,11 +3156,9 @@ static erts_smp_spinlock_t make_ref_lock;
static erts_smp_mtx_t ports_snapshot_mtx;
erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */
-Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE])
+void
+erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS])
{
- Eterm* hp = buffer;
- Uint32 ref0, ref1, ref2;
-
erts_smp_spin_lock(&make_ref_lock);
reference0++;
@@ -3182,24 +3170,36 @@ Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE])
}
}
- ref0 = reference0;
- ref1 = reference1;
- ref2 = reference2;
+ ref[0] = reference0;
+ ref[1] = reference1;
+ ref[2] = reference2;
erts_smp_spin_unlock(&make_ref_lock);
+}
+
+Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE])
+{
+ Eterm* hp = buffer;
+ Uint32 ref[ERTS_MAX_REF_NUMBERS];
- write_ref_thing(hp, ref0, ref1, ref2);
+ erts_make_ref_in_array(ref);
+ write_ref_thing(hp, ref[0], ref[1], ref[2]);
return make_internal_ref(hp);
}
Eterm erts_make_ref(Process *p)
{
Eterm* hp;
+ Uint32 ref[ERTS_MAX_REF_NUMBERS];
ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
hp = HAlloc(p, REF_THING_SIZE);
- return erts_make_ref_in_buffer(hp);
+
+ erts_make_ref_in_array(ref);
+ write_ref_thing(hp, ref[0], ref[1], ref[2]);
+
+ return make_internal_ref(hp);
}
BIF_RETTYPE make_ref_0(BIF_ALIST_0)
@@ -4507,6 +4507,8 @@ void erts_init_bif(void)
am_format_cpu_topology,
1);
await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3);
+ await_port_send_result_trap
+ = erts_export_put(am_erts_internal, am_await_port_send_result, 3);
await_sched_wall_time_mod_trap
= erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2);
erts_smp_atomic32_init_nob(&sched_wall_time, 0);
@@ -4522,19 +4524,18 @@ bif erlang:send_to_logger/2
BIF_RETTYPE send_to_logger_2(BIF_ALIST_2)
{
byte *buf;
- int len;
+ ErlDrvSizeT len;
if (!is_atom(BIF_ARG_1) || !(is_list(BIF_ARG_2) ||
is_nil(BIF_ARG_1))) {
BIF_ERROR(BIF_P,BADARG);
}
- len = io_list_len(BIF_ARG_2);
- if (len < 0)
+ if (erts_iolist_size(BIF_ARG_2, &len) != 0)
BIF_ERROR(BIF_P,BADARG);
else if (len == 0)
buf = "";
else {
#ifdef DEBUG
- int len2;
+ ErlDrvSizeT len2;
#endif
buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, len+1);
#ifdef DEBUG
@@ -4542,7 +4543,7 @@ BIF_RETTYPE send_to_logger_2(BIF_ALIST_2)
#else
(void)
#endif
- io_list_to_buf(BIF_ARG_2, buf, len);
+ erts_iolist_to_buf(BIF_ARG_2, buf, len);
ASSERT(len2 == len);
buf[len] = '\0';
switch (BIF_ARG_1) {
@@ -4636,7 +4637,6 @@ BIF_RETTYPE dt_prepend_vm_tag_data_1(BIF_ALIST_1)
#ifdef USE_VM_PROBES
Eterm b;
Eterm *hp;
- hp = HAlloc(BIF_P,2);
if (is_binary((DT_UTAG(BIF_P)))) {
Uint sz = binary_size(DT_UTAG(BIF_P));
int i;
@@ -4653,6 +4653,7 @@ BIF_RETTYPE dt_prepend_vm_tag_data_1(BIF_ALIST_1)
} else {
b = new_binary(BIF_P,(byte *)"\0",1);
}
+ hp = HAlloc(BIF_P,2);
BIF_RET(CONS(hp,b,BIF_ARG_1));
#else
BIF_RET(BIF_ARG_1);
@@ -4663,7 +4664,6 @@ BIF_RETTYPE dt_append_vm_tag_data_1(BIF_ALIST_1)
#ifdef USE_VM_PROBES
Eterm b;
Eterm *hp;
- hp = HAlloc(BIF_P,2);
if (is_binary((DT_UTAG(BIF_P)))) {
Uint sz = binary_size(DT_UTAG(BIF_P));
int i;
@@ -4680,6 +4680,7 @@ BIF_RETTYPE dt_append_vm_tag_data_1(BIF_ALIST_1)
} else {
b = new_binary(BIF_P,(byte *)"\0",1);
}
+ hp = HAlloc(BIF_P,2);
BIF_RET(CONS(hp,BIF_ARG_1,b));
#else
BIF_RET(BIF_ARG_1);
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index d20089a9fb..086dd9a5fd 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -322,27 +322,6 @@ do { \
ERTS_BIF_EXITED((PROC)); \
} while (0)
-#ifdef ERTS_SMP
-#define ERTS_SMP_BIF_CHK_PENDING_EXIT(P, L) \
-do { \
- ERTS_SMP_LC_ASSERT((L) == erts_proc_lc_my_proc_locks((P))); \
- ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & (L)); \
- if (!((L) & ERTS_PROC_LOCK_STATUS)) \
- erts_smp_proc_lock((P), ERTS_PROC_LOCK_STATUS); \
- if (ERTS_PROC_PENDING_EXIT((P))) { \
- erts_handle_pending_exit((P), (L)|ERTS_PROC_LOCK_STATUS); \
- erts_smp_proc_unlock((P), \
- (((L)|ERTS_PROC_LOCK_STATUS) \
- & ~ERTS_PROC_LOCK_MAIN)); \
- ERTS_BIF_EXITED((P)); \
- } \
- if (!((L) & ERTS_PROC_LOCK_STATUS)) \
- erts_smp_proc_unlock((P), ERTS_PROC_LOCK_STATUS); \
-} while (0)
-#else
-#define ERTS_SMP_BIF_CHK_PENDING_EXIT(P, L)
-#endif
-
/*
* The ERTS_BIF_*_AWAIT_X_*_TRAP makros either exits the caller, or
* sets up a trap to erlang:await_proc_exit/3.
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 8a85e102d1..c0a83c6f96 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -184,10 +184,6 @@ bif 'erl.lang.port':open/2 ebif_open_port_2 open_port_2
bif erlang:pid_to_list/1
bif 'erl.lang.proc':pid_to_string/1 ebif_pid_to_string_1 pid_to_list_1
-bif erlang:port_info/1
-bif 'erl.lang.port':info/1 ebif_port_info_1
-bif erlang:port_info/2
-bif 'erl.lang.port':info/2 ebif_port_info_2
bif erlang:ports/0
bif 'erl.lang.node':ports/0 ebif_ports_0
bif erlang:pre_loaded/0
@@ -256,24 +252,16 @@ bif erlang:setnode/2
bif erlang:setnode/3
bif erlang:dist_exit/3
-bif erlang:port_call/2
-bif 'erl.lang.port':call/2 ebif_port_call_2
-bif erlang:port_call/3
-bif 'erl.lang.port':call/3 ebif_port_call_3
-bif erlang:port_command/2
-bif 'erl.lang.port':command/2 ebif_port_command_2
-bif erlang:port_command/3
-bif 'erl.lang.port':command/3 ebif_port_command_3
-bif erlang:port_control/3
-bif 'erl.lang.port':control/3 ebif_port_control_3
-bif erlang:port_close/1
-bif 'erl.lang.port':close/1 ebif_port_close_1
-bif erlang:port_connect/2
-bif 'erl.lang.port':connect/2 ebif_port_connect_2
-bif erlang:port_set_data/2
-bif 'erl.lang.port':set_data/2 ebif_port_set_data_2
-bif erlang:port_get_data/1
-bif 'erl.lang.port':get_data/1 ebif_port_get_data_1
+# Static native functions in erts_internal
+bif erts_internal:port_info/1
+bif erts_internal:port_info/2
+bif erts_internal:port_call/3
+bif erts_internal:port_command/3
+bif erts_internal:port_control/3
+bif erts_internal:port_close/1
+bif erts_internal:port_connect/2
+bif erts_internal:port_set_data/2
+bif erts_internal:port_get_data/1
# Tracing & debugging.
bif erlang:trace_pattern/2
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 3d2725e239..4441dab181 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -355,10 +355,10 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
{
Eterm bin;
- Uint size;
+ ErlDrvSizeT size;
byte* bytes;
#ifdef DEBUG
- int offset;
+ ErlDrvSizeT offset;
#endif
if (is_nil(arg)) {
@@ -377,7 +377,7 @@ BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
#ifdef DEBUG
offset =
#endif
- io_list_to_buf(arg, (char*) bytes, size);
+ erts_iolist_to_buf(arg, (char*) bytes, size);
ASSERT(offset == 0);
BIF_RET(bin);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index f28d9dff09..a939b90eac 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -124,6 +124,13 @@ static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm);
static void init_nodes_monitors(void);
static erts_smp_atomic_t no_caches;
+static erts_smp_atomic_t no_nodes;
+
+struct {
+ Eterm reason;
+ ErlHeapFragment *bp;
+} nodedown;
+
static void
delete_cache(ErtsAtomCache *cache)
@@ -393,6 +400,47 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
erts_destroy_link(lnk);
}
+static void
+set_node_not_alive(void *unused)
+{
+ ErlHeapFragment *bp;
+ Eterm nodename = erts_this_dist_entry->sysname;
+
+ ASSERT(erts_smp_atomic_read_nob(&no_nodes) == 0);
+
+ erts_smp_thr_progress_block();
+ erts_set_this_node(am_Noname, 0);
+ erts_is_alive = 0;
+ send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nodedown.reason);
+ nodedown.reason = NIL;
+ bp = nodedown.bp;
+ nodedown.bp = NULL;
+ erts_smp_thr_progress_unblock();
+ if (bp)
+ free_message_buffer(bp);
+}
+
+static ERTS_INLINE void
+dec_no_nodes(void)
+{
+ erts_aint_t no = erts_smp_atomic_dec_read_mb(&no_nodes);
+ ASSERT(no >= 0);
+ ASSERT(erts_get_scheduler_id()); /* Need to be a scheduler */
+ if (no == 0)
+ erts_schedule_misc_aux_work(erts_get_scheduler_id(),
+ set_node_not_alive,
+ NULL);
+}
+
+static ERTS_INLINE void
+inc_no_nodes(void)
+{
+#ifdef DEBUG
+ erts_aint_t no = erts_smp_atomic_read_nob(&no_nodes);
+ ASSERT(erts_is_alive ? no > 0 : no == 0);
+#endif
+ erts_smp_atomic_inc_mb(&no_nodes);
+}
/*
* proc is currently running or exiting process.
@@ -402,48 +450,76 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
Eterm nodename;
if (dep == erts_this_dist_entry) { /* Net kernel has died (clean up!!) */
+ DistEntry *tdep;
+ int no_dist_port = 0;
Eterm nd_reason = (reason == am_no_network
? am_no_network
: am_net_kernel_terminated);
erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+ for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next)
+ no_dist_port++;
+ for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next)
+ no_dist_port++;
+
/* KILL all port controllers */
- while(erts_visible_dist_entries || erts_hidden_dist_entries) {
- DistEntry *tdep;
- Eterm prt_id;
- Port *prt;
- if(erts_hidden_dist_entries)
- tdep = erts_hidden_dist_entries;
+ if (no_dist_port == 0)
+ erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+ else {
+ Eterm def_buf[128];
+ int i = 0;
+ Eterm *dist_port;
+
+ if (no_dist_port <= sizeof(def_buf)/sizeof(def_buf[0]))
+ dist_port = &def_buf[0];
else
- tdep = erts_visible_dist_entries;
- prt_id = tdep->cid;
- ASSERT(is_internal_port(prt_id));
+ dist_port = erts_alloc(ERTS_ALC_T_TMP,
+ sizeof(Eterm)*no_dist_port);
+ for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) {
+ ASSERT(is_internal_port(tdep->cid));
+ dist_port[i++] = tdep->cid;
+ }
+ for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) {
+ ASSERT(is_internal_port(tdep->cid));
+ dist_port[i++] = tdep->cid;
+ }
erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
- prt = erts_id2port(prt_id);
- if (prt) {
+ for (i = 0; i < no_dist_port; i++) {
+ Port *prt = erts_port_lookup(dist_port[i],
+ ERTS_PORT_SFLGS_INVALID_LOOKUP);
+ if (!prt)
+ continue;
ASSERT(erts_atomic32_read_nob(&prt->state)
& ERTS_PORT_SFLG_DISTRIBUTION);
- ASSERT(prt->dist_entry);
- /* will call do_net_exists !!! */
- erts_do_exit_port(prt, prt_id, nd_reason);
- erts_port_release(prt);
+
+ erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED,
+ prt, dist_port[i], nd_reason, NULL);
}
- erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx);
+ if (dist_port != &def_buf[0])
+ erts_free(ERTS_ALC_T_TMP, dist_port);
}
- erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
-
- nodename = erts_this_dist_entry->sysname;
- erts_smp_thr_progress_block();
- erts_set_this_node(am_Noname, 0);
- erts_is_alive = 0;
- send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nd_reason);
- erts_smp_thr_progress_unblock();
-
+ /*
+ * When last dist port exits, node will be taken
+ * from alive to not alive.
+ */
+ ASSERT(is_nil(nodedown.reason) && !nodedown.bp);
+ if (is_immed(nd_reason))
+ nodedown.reason = nd_reason;
+ else {
+ Eterm *hp;
+ Uint sz = size_object(nd_reason);
+ nodedown.bp = new_message_buffer(sz);
+ hp = nodedown.bp->mem;
+ nodedown.reason = copy_struct(nd_reason,
+ sz,
+ &hp,
+ &nodedown.bp->off_heap);
+ }
}
- else { /* recursive call via erts_do_exit_port() will end up here */
+ else { /* Call from distribution port */
NetExitsContext nec = {dep};
ErtsLink *nlinks;
ErtsLink *node_links;
@@ -503,6 +579,9 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
clear_dist_entry(dep);
}
+
+ dec_no_nodes();
+
return 1;
}
@@ -516,6 +595,10 @@ void init_dist(void)
{
init_nodes_monitors();
+ nodedown.reason = NIL;
+ nodedown.bp = NULL;
+
+ erts_smp_atomic_init_nob(&no_nodes, 0);
erts_smp_atomic_init_nob(&no_caches, 0);
/* Lookup/Install all references to trap functions */
@@ -1601,7 +1684,7 @@ int erts_net_message(Port *prt,
}
#endif
UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
- erts_do_exit_port(prt, dep->cid, am_killed);
+ erts_deliver_port_exit(prt, dep->cid, am_killed, 0);
ERTS_SMP_CHK_NO_PROC_LOCKS;
return -1;
}
@@ -1924,7 +2007,7 @@ erts_dist_command(Port *prt, int reds_limit)
erts_smp_de_runlock(dep);
if (status & ERTS_DE_SFLG_EXITING) {
- erts_do_exit_port(prt, prt->common.id, am_killed);
+ erts_deliver_port_exit(prt, prt->common.id, am_killed, 0);
erts_deref_dist_entry(dep);
return reds + ERTS_PORT_REDS_DIST_CMD_EXIT;
}
@@ -2484,6 +2567,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
+ inc_no_nodes();
erts_set_this_node(BIF_ARG_1, (Uint32) creation);
erts_is_alive = 1;
send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL);
@@ -2638,6 +2722,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
erts_smp_de_rwunlock(dep);
dep = NULL; /* inc of refc transferred to port (dist_entry field) */
+ inc_no_nodes();
+
send_nodes_mon_msgs(BIF_P,
am_nodeup,
BIF_ARG_1,
@@ -2723,11 +2809,17 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3)
#endif
erts_smp_proc_unlock(lp, lp_locks);
if (lp == BIF_P) {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&BIF_P->state);
/*
* We may have exited current process and may have to take action.
*/
- ERTS_BIF_CHK_EXITED(BIF_P);
- ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN);
+ if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
+#ifdef ERTS_SMP
+ if (state & ERTS_PSFLG_PENDING_EXIT)
+ erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
+#endif
+ ERTS_BIF_EXITED(BIF_P);
+ }
}
}
else if (is_external_pid(local)
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 36beec9553..264cbf8efe 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -190,6 +190,9 @@ type TIMER_WHEEL LONG_LIVED SYSTEM timer_wheel
type DRV DRIVER SYSTEM drv_internal
type DRV_BINARY BINARY BINARIES drv_binary
type DRIVER DRIVER SYSTEM driver
+type DRV_CMD_DATA DRIVER SYSTEM driver_command_data
+type DRV_CTRL_DATA DRIVER SYSTEM driver_control_data
+type DRV_CALL_DATA DRIVER SYSTEM driver_call_data
type NIF DRIVER SYSTEM nif_internal
type BINARY BINARY BINARIES binary
type NBIF_TABLE SYSTEM SYSTEM nbif_tab
@@ -203,7 +206,6 @@ type BITS_BUF STANDARD SYSTEM bits_buf
type TMP_DIST_BUF TEMPORARY SYSTEM tmp_dist_buf
type ASYNC_DATA LONG_LIVED SYSTEM internal_async_data
type ESTACK TEMPORARY SYSTEM estack
-type PORT_CALL_BUF TEMPORARY SYSTEM port_call_buf
type DB_TABLE ETS ETS db_tab
type DB_FIXATION SHORT_LIVED ETS db_fixation
type DB_FIX_DEL SHORT_LIVED ETS fixed_del
@@ -233,6 +235,7 @@ type DDLL_HANDLE STANDARD SYSTEM ddll_handle
type DDLL_ERRCODES LONG_LIVED SYSTEM ddll_errcodes
type DDLL_TMP_BUF TEMPORARY SYSTEM ddll_tmp_buf
type PORT_TASK SHORT_LIVED SYSTEM port_task
+type PT_HNDL_LIST SHORT_LIVED SYSTEM port_task_handle_list
type MISC_OP_LIST SHORT_LIVED SYSTEM misc_op_list
type PORT_NAMES SHORT_LIVED SYSTEM port_names
type PORT_DATA_LOCK DRIVER SYSTEM port_data_lock
@@ -262,6 +265,8 @@ type ZLIB STANDARD SYSTEM zlib
type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map
type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts
type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q
+type BUSY_CALLER_TAB SHORT_LIVED SYSTEM busy_caller_table
+type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller
+if threads_no_smp
# Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 2b6deae7be..7cbea55eac 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -182,7 +182,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
Eterm name_term = BIF_ARG_2;
Eterm options = BIF_ARG_3;
char *path = NULL;
- Uint path_len;
+ ErlDrvSizeT path_len;
char *name = NULL;
DE_Handle *dh;
erts_driver_t *drv;
@@ -261,7 +261,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
goto error;
}
path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1);
- if (io_list_to_buf(path_term, path, path_len) != 0) {
+ if (erts_iolist_to_buf(path_term, path, path_len) != 0) {
goto error;
}
while (path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/')) {
@@ -1841,7 +1841,7 @@ static Eterm mkatom(char *str)
static char *pick_list_or_atom(Eterm name_term)
{
char *name = NULL;
- Uint name_len;
+ ErlDrvSizeT name_len;
if (is_atom(name_term)) {
Atom *ap = atom_tab(atom_val(name_term));
if (ap->len == 0) {
@@ -1857,7 +1857,7 @@ static char *pick_list_or_atom(Eterm name_term)
goto error;
}
name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1);
- if (io_list_to_buf(name_term, name, name_len) != 0) {
+ if (erts_iolist_to_buf(name_term, name, name_len) != 0) {
goto error;
}
name[name_len] = '\0';
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 44c29252b2..f5a355a336 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -137,8 +137,6 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
static Eterm os_type_tuple;
static Eterm os_version_tuple;
-static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item);
-
static Eterm
current_function(Process* p, Process* rp, Eterm** hpp, int full_info);
static Eterm current_stacktrace(Process* p, Process* rp, Eterm** hpp);
@@ -1864,17 +1862,17 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */
# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML
# endif
#endif
- Uint buf_size = 8*1024; /* Try with 8KB first */
+ ErlDrvSizeT buf_size = 8*1024; /* Try with 8KB first */
char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
- int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
- if (r < 0) {
+ ErlDrvSizeT r = erts_iolist_to_buf(*tp, (char*) buf, buf_size - 1);
+ if (ERTS_IOLIST_TO_BUF_FAILED(r)) {
erts_free(ERTS_ALC_T_TMP, (void *) buf);
if (erts_iolist_size(*tp, &buf_size)) {
goto badarg;
}
buf_size++;
buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
- r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
+ r = erts_iolist_to_buf(*tp, (char*) buf, buf_size - 1);
ASSERT(r == buf_size - 1);
}
buf[buf_size - 1 - r] = '\0';
@@ -2593,6 +2591,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
} else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
res = make_small(erts_no_run_queues);
BIF_RET(res);
+ } else if (ERTS_IS_ATOM_STR("port_parallelism", BIF_ARG_1)) {
+ res = erts_port_parallelism ? am_true : am_false;
+ BIF_RET(res);
} else if (ERTS_IS_ATOM_STR("c_compiler_used", BIF_ARG_1)) {
Eterm *hp = NULL;
Uint sz = 0;
@@ -2760,65 +2761,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
BIF_ERROR(BIF_P, BADARG);
}
-BIF_RETTYPE
-port_info_1(BIF_ALIST_1)
-{
- Process* p = BIF_P;
- Eterm pid = BIF_ARG_1;
- static Eterm keys[] = {
- am_name,
- am_links,
- am_id,
- am_connected,
- am_input,
- am_output
- };
- Eterm items[ASIZE(keys)];
- Eterm result = NIL;
- Eterm reg_name;
- Eterm* hp;
- Uint need;
- int i;
-
- /*
- * Collect all information about the port.
- */
-
- for (i = 0; i < ASIZE(keys); i++) {
- Eterm item;
-
- item = port_info(p, pid, keys[i]);
- if (is_non_value(item)) {
- return THE_NON_VALUE;
- }
- if (item == am_undefined) {
- return am_undefined;
- }
- items[i] = item;
- }
- reg_name = port_info(p, pid, am_registered_name);
-
- /*
- * Build the resulting list.
- */
-
- need = 2*ASIZE(keys);
- if (is_tuple(reg_name)) {
- need += 2;
- }
- hp = HAlloc(p, need);
- for (i = ASIZE(keys) - 1; i >= 0; i--) {
- result = CONS(hp, items[i], result);
- hp += 2;
- }
- if (is_tuple(reg_name)) {
- result = CONS(hp, reg_name, result);
- }
-
- return result;
-}
-
-
/**********************************************************************/
/* Return information on ports */
/* Info:
@@ -2830,41 +2772,20 @@ port_info_1(BIF_ALIST_1)
** output Number of bytes output to the port program
*/
-BIF_RETTYPE port_info_2(BIF_ALIST_2)
-{
- return port_info(BIF_P, BIF_ARG_1, BIF_ARG_2);
-}
-
-static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item)
+Eterm
+erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm item)
{
- BIF_RETTYPE ret;
- Port *prt;
- Eterm res;
- Eterm* hp;
- int count;
-
- if (is_internal_port(portid))
- prt = erts_id2port_sflgs(portid,
- p,
- ERTS_PROC_LOCK_MAIN,
- ERTS_PORT_SFLGS_INVALID_LOOKUP);
- else if (is_atom(portid))
- erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
- portid, NULL, 0, 0, &prt);
- else if (is_external_port(portid)
- && external_port_dist_entry(portid) == erts_this_dist_entry)
- BIF_RET(am_undefined);
- else {
- BIF_ERROR(p, BADARG);
- }
+ Eterm res = THE_NON_VALUE;
- if (!prt) {
- BIF_RET(am_undefined);
- }
+ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
if (item == am_id) {
- hp = HAlloc(p, 3);
- res = make_small(internal_port_number(portid));
+ if (hpp)
+ res = make_small(internal_port_index(prt->common.id));
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_links) {
MonitorInfoCollection mic;
@@ -2875,15 +2796,24 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item)
erts_doforall_links(ERTS_P_LINKS(prt), &collect_one_link, &mic);
- hp = HAlloc(p, 3 + mic.sz);
- res = NIL;
- for (i = 0; i < mic.mi_i; i++) {
- item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity);
- res = CONS(hp, item, res);
- hp += 2;
+ if (szp)
+ *szp += mic.sz;
+
+ if (hpp) {
+ res = NIL;
+ for (i = 0; i < mic.mi_i; i++) {
+ item = STORE_NC(hpp, ohp, mic.mi[i].entity);
+ res = CONS(*hpp, item, res);
+ *hpp += 2;
+ }
}
+
DESTROY_MONITOR_INFOS(mic);
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_monitors) {
MonitorInfoCollection mic;
@@ -2894,64 +2824,84 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item)
erts_doforall_monitors(ERTS_P_MONITORS(prt), &collect_one_origin_monitor, &mic);
- hp = HAlloc(p, 3 + mic.sz);
- res = NIL;
- for (i = 0; i < mic.mi_i; i++) {
- Eterm t;
- item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity);
- t = TUPLE2(hp, am_process, item);
- hp += 3;
- res = CONS(hp, t, res);
- hp += 2;
+ if (szp)
+ *szp += mic.sz;
+
+ if (hpp) {
+ res = NIL;
+ for (i = 0; i < mic.mi_i; i++) {
+ Eterm t;
+ item = STORE_NC(hpp, ohp, mic.mi[i].entity);
+ t = TUPLE2(*hpp, am_process, item);
+ *hpp += 3;
+ res = CONS(*hpp, t, res);
+ *hpp += 2;
+ }
}
+
DESTROY_MONITOR_INFOS(mic);
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_name) {
- count = sys_strlen(prt->name);
+ int count = sys_strlen(prt->name);
- hp = HAlloc(p, 3 + 2*count);
- res = buf_to_intlist(&hp, prt->name, count, NIL);
+ if (hpp)
+ res = buf_to_intlist(hpp, prt->name, count, NIL);
+
+ if (szp) {
+ *szp += 2*count;
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_connected) {
- hp = HAlloc(p, 3);
- res = ERTS_PORT_GET_CONNECTED(prt); /* internal pid */
+ if (hpp)
+ res = ERTS_PORT_GET_CONNECTED(prt); /* internal pid */
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_input) {
- Uint hsz = 3;
- Uint n = prt->bytes_in;
- (void) erts_bld_uint(NULL, &hsz, n);
- hp = HAlloc(p, hsz);
- res = erts_bld_uint(&hp, NULL, n);
+ res = erts_bld_uint(hpp, szp, prt->bytes_in);
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_output) {
- Uint hsz = 3;
- Uint n = prt->bytes_out;
- (void) erts_bld_uint(NULL, &hsz, n);
- hp = HAlloc(p, hsz);
- res = erts_bld_uint(&hp, NULL, n);
+ res = erts_bld_uint(hpp, szp, prt->bytes_out);
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_registered_name) {
- RegProc *reg;
- reg = prt->common.u.alive.reg;
- if (reg == NULL) {
- ERTS_BIF_PREP_RET(ret, NIL);
- goto done;
- } else {
- hp = HAlloc(p, 3);
+ RegProc *reg = prt->common.u.alive.reg;
+ if (reg) {
res = reg->name;
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
+ }
+ else {
+ if (szp)
+ return am_undefined;
+ return NIL;
}
}
else if (item == am_memory) {
/* All memory consumed in bytes (the Port struct should not be
included though).
*/
- Uint hsz = 3;
Uint size = 0;
ErlHeapFragment* bp;
- hp = HAlloc(p, 3);
-
erts_doforall_links(ERTS_P_LINKS(prt), &one_link_size, &size);
for (bp = prt->bp; bp; bp = bp->next)
@@ -2966,52 +2916,72 @@ static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item)
/* All memory allocated by the driver should be included, but it is
hard to retrieve... */
- (void) erts_bld_uint(NULL, &hsz, size);
- hp = HAlloc(p, hsz);
- res = erts_bld_uint(&hp, NULL, size);
+ res = erts_bld_uint(hpp, szp, size);
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (item == am_queue_size) {
Uint ioq_size = erts_port_ioq_size(prt);
- Uint hsz = 3;
- (void) erts_bld_uint(NULL, &hsz, ioq_size);
- hp = HAlloc(p, hsz);
- res = erts_bld_uint(&hp, NULL, ioq_size);
+ res = erts_bld_uint(hpp, szp, ioq_size);
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
}
else if (ERTS_IS_ATOM_STR("locking", item)) {
- hp = HAlloc(p, 3);
+ if (hpp) {
#ifndef ERTS_SMP
- res = am_false;
+ res = am_false;
#else
- if (erts_atomic32_read_nob(&prt->state)
- & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
- DECL_AM(port_level);
- ASSERT(prt->drv_ptr->flags
- & ERL_DRV_FLAG_USE_PORT_LOCKING);
- res = AM_port_level;
+ if (erts_atomic32_read_nob(&prt->state)
+ & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) {
+ DECL_AM(port_level);
+ ASSERT(prt->drv_ptr->flags
+ & ERL_DRV_FLAG_USE_PORT_LOCKING);
+ res = AM_port_level;
+ }
+ else {
+ DECL_AM(driver_level);
+ ASSERT(!(prt->drv_ptr->flags
+ & ERL_DRV_FLAG_USE_PORT_LOCKING));
+ res = AM_driver_level;
+ }
+#endif
}
- else {
- DECL_AM(driver_level);
- ASSERT(!(prt->drv_ptr->flags
- & ERL_DRV_FLAG_USE_PORT_LOCKING));
- res = AM_driver_level;
+ if (szp) {
+ res = am_true;
+ goto done;
}
-#endif
+ }
+ else if (item == am_parallelism) {
+ if (szp) {
+ res = am_true;
+ goto done;
+ }
+ res = ((ERTS_PTS_FLG_PARALLELISM &
+ erts_smp_atomic32_read_nob(&prt->sched.flags))
+ ? am_true
+ : am_false);
}
else {
- ERTS_BIF_PREP_ERROR(ret, p, BADARG);
- goto done;
+ if (szp)
+ return am_false;
+ return THE_NON_VALUE;
}
- ERTS_BIF_PREP_RET(ret, TUPLE2(hp, item, res));
-
- done:
-
- erts_port_release(prt);
+done:
+ if (szp)
+ *szp += 3;
+ if (hpp) {
+ res = TUPLE2(*hpp, item, res);
+ *hpp += 3;
+ }
- return ret;
+ return res;
}
-
BIF_RETTYPE
fun_info_2(BIF_ALIST_2)
{
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 0d0733b8d0..97f147132e 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -49,9 +49,6 @@ static void free_args(char **);
char *erts_default_arg0 = "default";
-static BIF_RETTYPE
-port_call(Process* p, Eterm arg1, Eterm arg2, Eterm arg3);
-
BIF_RETTYPE open_port_2(BIF_ALIST_2)
{
Port *port;
@@ -86,539 +83,397 @@ BIF_RETTYPE open_port_2(BIF_ALIST_2)
BIF_RET(port_id);
}
-/****************************************************************************
-
- PORT BIFS:
-
- port_command/2 -- replace Port ! {..., {command, Data}}
- port_command(Port, Data) -> true
- when port(Port), io-list(Data)
-
- port_control/3 -- new port_control(Port, Ctl, Data) -> Reply
- port_control(Port, Ctl, Data) -> Reply
- where integer(Ctl), io-list(Data), io-list(Reply)
-
- port_close/1 -- replace Port ! {..., close}
- port_close(Port) -> true
- when port(Port)
-
- port_connect/2 -- replace Port ! {..., {connect, Pid}}
- port_connect(Port, Pid)
- when port(Port), pid(Pid)
-
- ***************************************************************************/
-
-static Port*
-id_or_name2port(Process *c_p, Eterm id)
+static ERTS_INLINE Port *
+lookup_port(Process *c_p, Eterm id_or_name)
{
- Port *port;
- if (is_not_atom(id))
- port = erts_id2port_sflgs(id,
- c_p,
- ERTS_PROC_LOCK_MAIN,
- ERTS_PORT_SFLGS_INVALID_LOOKUP);
+ /* TODO: Implement nicer lookup in register... */
+ Eterm id;
+ if (is_atom(id_or_name))
+ id = erts_whereis_name_to_id(c_p, id_or_name);
else
- erts_whereis_name(c_p, ERTS_PROC_LOCK_MAIN, id, NULL, 0, 0, &port);
- return port;
+ id = id_or_name;
+ return erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP);
}
-#define ERTS_PORT_COMMAND_FLAG_FORCE (((Uint32) 1) << 0)
-#define ERTS_PORT_COMMAND_FLAG_NOSUSPEND (((Uint32) 1) << 1)
+/*
+ * erts_internal:port_command/3 is used by the
+ * erlang:port_command/2 and erlang:port_command/3
+ * BIFs.
+ */
-static BIF_RETTYPE
-do_port_command(Process *BIF_P, Eterm arg1, Eterm arg2, Eterm arg3,
- Uint32 flags)
+BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3)
{
BIF_RETTYPE res;
- Port *p;
-
- /* Trace sched out before lock check wait */
- if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(BIF_P, am_out);
- }
-
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(BIF_P, am_inactive);
- }
-
- p = id_or_name2port(BIF_P, arg1);
- if (!p) {
- if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(BIF_P, am_in);
+ Port *prt;
+ int flags = 0;
+ Eterm ref;
+
+ if (is_not_nil(BIF_ARG_3)) {
+ Eterm l = BIF_ARG_3;
+ while (is_list(l)) {
+ Eterm* cons = list_val(l);
+ Eterm car = CAR(cons);
+ if (car == am_force)
+ flags |= ERTS_PORT_SIG_FLG_FORCE;
+ else if (car == am_nosuspend)
+ flags |= ERTS_PORT_SIG_FLG_NOSUSPEND;
+ else
+ BIF_RET(am_badarg);
+ l = CDR(cons);
}
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(BIF_P, am_active);
- }
- BIF_ERROR(BIF_P, BADARG);
+ if (!is_nil(l))
+ BIF_RET(am_badarg);
}
-
- /* Trace port in, id_or_name2port causes wait */
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(p, am_in, am_command);
- }
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
- profile_runnable_port(p, am_active);
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_badarg);
+
+ if (flags & ERTS_PORT_SIG_FLG_FORCE) {
+ if (!(prt->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY))
+ BIF_RET(am_notsup);
}
- ERTS_BIF_PREP_RET(res, am_true);
+#ifdef DEBUG
+ ref = NIL;
+#endif
- if ((flags & ERTS_PORT_COMMAND_FLAG_FORCE)
- && !(p->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY)) {
- ERTS_BIF_PREP_ERROR(res, BIF_P, EXC_NOTSUP);
- }
- else if (!(flags & ERTS_PORT_COMMAND_FLAG_FORCE)
- && (erts_smp_atomic32_read_nob(&p->sched.flags)
- & ERTS_PTS_FLG_BUSY)) {
- if (flags & ERTS_PORT_COMMAND_FLAG_NOSUSPEND) {
+ switch (erts_port_output(BIF_P, flags, prt, prt->common.id, BIF_ARG_2, &ref)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ case ERTS_PORT_OP_DROPPED:
+ ERTS_BIF_PREP_RET(res, am_badarg);
+ break;
+ case ERTS_PORT_OP_BUSY:
+ ASSERT(!(flags & ERTS_PORT_SIG_FLG_FORCE));
+ if (flags & ERTS_PORT_SIG_FLG_NOSUSPEND)
ERTS_BIF_PREP_RET(res, am_false);
- }
else {
- erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, p);
- if (erts_system_monitor_flags.busy_port) {
- monitor_generic(BIF_P, am_busy_port, p->common.id);
- }
- ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_port_command_3], BIF_P,
- arg1, arg2, arg3);
- }
- } else {
- int wres;
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- ERTS_SMP_CHK_NO_PROC_LOCKS;
- wres = erts_write_to_port(BIF_P->common.id, p, arg2);
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
- if (wres != 0) {
- ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
+ erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, prt);
+ ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_erts_internal_port_command_3],
+ BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
}
- }
-
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(p, am_out, am_command);
- }
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
- profile_runnable_port(p, am_inactive);
+ break;
+ case ERTS_PORT_OP_BUSY_SCHEDULED:
+ ASSERT(!(flags & ERTS_PORT_SIG_FLG_FORCE));
+ /* Fall through... */
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(ref));
+ ERTS_BIF_PREP_RET(res, ref);
+ break;
+ case ERTS_PORT_OP_DONE:
+ ERTS_BIF_PREP_RET(res, am_true);
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_output() result");
+ break;
}
- erts_port_release(p);
- /* Trace sched in after port release */
- if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(BIF_P, am_in);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(BIF_P, am_active);
- }
-
if (ERTS_PROC_IS_EXITING(BIF_P)) {
KILL_CATCHES(BIF_P); /* Must exit */
ERTS_BIF_PREP_ERROR(res, BIF_P, EXC_ERROR);
}
- return res;
-}
-BIF_RETTYPE port_command_2(BIF_ALIST_2)
-{
- return do_port_command(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL, 0);
+ return res;
}
-BIF_RETTYPE port_command_3(BIF_ALIST_3)
+BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3)
{
- Eterm l = BIF_ARG_3;
- Uint32 flags = 0;
- while (is_list(l)) {
- Eterm* cons = list_val(l);
- Eterm car = CAR(cons);
- if (car == am_force) {
- flags |= ERTS_PORT_COMMAND_FLAG_FORCE;
- } else if (car == am_nosuspend) {
- flags |= ERTS_PORT_COMMAND_FLAG_NOSUSPEND;
- } else {
- BIF_ERROR(BIF_P, BADARG);
- }
- l = CDR(cons);
- }
- if(!is_nil(l)) {
- BIF_ERROR(BIF_P, BADARG);
+ Port* prt;
+ Eterm retval;
+ Uint uint_op;
+ unsigned int op;
+ erts_aint32_t state;
+
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_badarg);
+
+ if (!term_to_Uint(BIF_ARG_2, &uint_op))
+ BIF_RET(am_badarg);
+
+ if (uint_op > (Uint) UINT_MAX)
+ BIF_RET(am_badarg);
+
+ op = (unsigned int) uint_op;
+
+ switch (erts_port_call(BIF_P, prt, op, BIF_ARG_3, &retval)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_DROPPED:
+ case ERTS_PORT_OP_BADARG:
+ retval = am_badarg;
+ break;
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(retval));
+ break;
+ case ERTS_PORT_OP_DONE:
+ ASSERT(is_not_internal_ref(retval));
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_call() result");
+ retval = am_internal_error;
+ break;
}
- return do_port_command(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, flags);
-}
-BIF_RETTYPE port_call_2(BIF_ALIST_2)
-{
- return port_call(BIF_P,BIF_ARG_1, make_small(0), BIF_ARG_2);
-}
+ state = erts_smp_atomic32_read_acqb(&BIF_P->state);
+ if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
+#ifdef ERTS_SMP
+ if (state & ERTS_PSFLG_PENDING_EXIT)
+ erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
+#endif
+ ERTS_BIF_EXITED(BIF_P);
+ }
-BIF_RETTYPE port_call_3(BIF_ALIST_3)
-{
- return port_call(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+ BIF_RET(retval);
}
-static BIF_RETTYPE
-port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3)
+BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3)
{
- Uint op;
- Port *p;
- Uint size;
- byte *bytes;
- byte *endp;
- ErlDrvSizeT real_size;
- erts_driver_t *drv;
- byte port_input[256]; /* Default input buffer to encode in */
- byte port_result[256]; /* Buffer for result from port. */
- byte* port_resp; /* Pointer to result buffer. */
- char *prc;
- ErlDrvSSizeT ret;
- Eterm res;
- Sint result_size;
- Eterm *hp;
- Eterm *hp_end; /* To satisfy hybrid heap architecture */
- unsigned ret_flags = 0U;
- int fpe_was_unmasked;
-
- bytes = &port_input[0];
- port_resp = port_result;
- /* trace of port scheduling with virtual process descheduling
- * lock wait
- */
- if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(c_p, am_out);
- }
-
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(c_p, am_inactive);
+ Port* prt;
+ Eterm retval;
+ Uint uint_op;
+ unsigned int op;
+ erts_aint32_t state;
+
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_badarg);
+
+ if (!term_to_Uint(BIF_ARG_2, &uint_op))
+ BIF_RET(am_badarg);
+
+ if (uint_op > (Uint) UINT_MAX)
+ BIF_RET(am_badarg);
+
+ op = (unsigned int) uint_op;
+
+ switch (erts_port_control(BIF_P, prt, op, BIF_ARG_3, &retval)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ case ERTS_PORT_OP_DROPPED:
+ retval = am_badarg;
+ break;
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(retval));
+ break;
+ case ERTS_PORT_OP_DONE:
+ ASSERT(is_not_internal_ref(retval));
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_control() result");
+ retval = am_internal_error;
+ break;
}
- p = id_or_name2port(c_p, arg1);
- if (!p) {
- error:
- if (port_resp != port_result &&
- !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) {
- driver_free(port_resp);
- }
- if (bytes != &port_input[0])
- erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes);
- /* Need to virtual schedule in the process if there
- * was an error.
- */
- if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(c_p, am_in);
- }
-
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(c_p, am_active);
- }
-
- if (p)
- erts_port_release(p);
+ state = erts_smp_atomic32_read_acqb(&BIF_P->state);
+ if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) {
#ifdef ERTS_SMP
- ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN);
-#else
- ERTS_BIF_CHK_EXITED(c_p);
+ if (state & ERTS_PSFLG_PENDING_EXIT)
+ erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN);
#endif
- BIF_ERROR(c_p, BADARG);
+ ERTS_BIF_EXITED(BIF_P);
}
- if ((drv = p->drv_ptr) == NULL) {
- goto error;
- }
- if (drv->call == NULL) {
- goto error;
- }
- if (!term_to_Uint(arg2, &op)) {
- goto error;
- }
- p->caller = c_p->common.id;
-
- /* Lock taken, virtual schedule of port */
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(p, am_in, am_call);
- }
-
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
- profile_runnable_port(p, am_active);
- }
- size = erts_encode_ext_size(arg3);
- if (size > sizeof(port_input))
- bytes = erts_alloc(ERTS_ALC_T_PORT_CALL_BUF, size);
-
- endp = bytes;
- erts_encode_ext(arg3, &endp);
+ BIF_RET(retval);
+}
- real_size = endp - bytes;
- if (real_size > size) {
- erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
- __FILE__, __LINE__, endp - (bytes + size));
- }
- erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
-#ifdef USE_VM_PROBES
- if (DTRACE_ENABLED(driver_call)) {
- DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
- DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+/*
+ * erts_internal:port_close/1 is used by the
+ * erlang:port_close/1 BIF.
+ */
+BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1)
+{
+ Eterm ref;
+ Port *prt;
- dtrace_pid_str(ERTS_PORT_GET_CONNECTED(p), process_str);
- dtrace_port_str(p, port_str);
- DTRACE5(driver_call, process_str, port_str, p->name, op, real_size);
- }
-#endif
- prc = (char *) port_resp;
- fpe_was_unmasked = erts_block_fpe();
- ret = drv->call((ErlDrvData)p->drv_data,
- (unsigned) op,
- (char *) bytes,
- (int) real_size,
- &prc,
- (int) sizeof(port_result),
- &ret_flags);
- erts_unblock_fpe(fpe_was_unmasked);
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(p, am_out, am_call);
- }
-
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
- profile_runnable_port(p, am_inactive);
- }
-
- port_resp = (byte *) prc;
- p->caller = NIL;
- erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
-#ifdef HARDDEBUG
- {
- ErlDrvSizeT z;
- printf("real_size = %ld,%d, ret = %ld,%d\r\n", (unsigned long) real_size,
- (int) real_size, (unsigned long)ret, (int) ret);
- printf("[");
- for(z = 0; z < real_size; ++z) {
- printf("%d, ",(int) bytes[z]);
- }
- printf("]\r\n");
- printf("[");
- for(z = 0; z < ret; ++z) {
- printf("%d, ",(int) port_resp[z]);
- }
- printf("]\r\n");
- }
-#endif
- if (ret <= 0 || port_resp[0] != VERSION_MAGIC) {
- /* Error or a binary without magic/ with wrong magic */
- goto error;
- }
- result_size = erts_decode_ext_size(port_resp, ret);
- if (result_size < 0) {
- goto error;
- }
- hp = HAlloc(c_p, result_size);
- hp_end = hp + result_size;
- endp = port_resp;
- res = erts_decode_ext(&hp, &MSO(c_p), &endp);
- if (res == THE_NON_VALUE) {
- goto error;
- }
- HRelease(c_p, hp_end, hp);
- if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) {
- driver_free(port_resp);
- }
- if (bytes != &port_input[0])
- erts_free(ERTS_ALC_T_PORT_CALL_BUF, bytes);
- if (p)
- erts_port_release(p);
-#ifdef ERTS_SMP
- ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN);
-#else
- ERTS_BIF_CHK_EXITED(c_p);
+#ifdef DEBUG
+ ref = NIL;
#endif
- if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(c_p, am_in);
- }
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(c_p, am_active);
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_badarg);
+
+
+ switch (erts_port_exit(BIF_P, 0, prt, prt->common.id, am_normal, &ref)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ case ERTS_PORT_OP_DROPPED:
+ BIF_RET(am_badarg);
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(ref));
+ BIF_RET(ref);
+ case ERTS_PORT_OP_DONE:
+ BIF_RET(am_true);
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_exit() result");
+ BIF_RET(am_internal_error);
}
-
- return res;
}
-
-BIF_RETTYPE port_control_3(BIF_ALIST_3)
+
+/*
+ * erts_internal:port_connect/2 is used by the
+ * erlang:port_connect/2 BIF.
+ */
+BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2)
{
- Port* p;
- Uint op;
- Eterm res = THE_NON_VALUE;
-
- /* Virtual schedule out calling process before lock wait */
- if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(BIF_P, am_out);
- }
+ Eterm ref;
+ Port* prt;
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(BIF_P, am_inactive);
- }
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_badarg);
- p = id_or_name2port(BIF_P, BIF_ARG_1);
- if (!p) {
- /* Schedule the process before exiting */
- if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(BIF_P, am_in);
- }
-
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(BIF_P, am_active);
- }
-
- BIF_ERROR(BIF_P, BADARG);
- }
-
- /* Trace the port for scheduling in */
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(p, am_in, am_control);
- }
-
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
- profile_runnable_port(p, am_active);
- }
+#ifdef DEBUG
+ ref = NIL;
+#endif
- if (term_to_Uint(BIF_ARG_2, &op))
- res = erts_port_control(BIF_P, p, op, BIF_ARG_3);
-
- /* Trace the port for scheduling out */
- if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
- trace_sched_ports_where(p, am_out, am_control);
+ switch (erts_port_connect(BIF_P, 0, prt, prt->common.id, BIF_ARG_2, &ref)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ case ERTS_PORT_OP_DROPPED:
+ BIF_RET(am_badarg);
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(ref));
+ BIF_RET(ref);
+ break;
+ case ERTS_PORT_OP_DONE:
+ BIF_RET(am_true);
+ break;
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_connect() result");
+ BIF_RET(am_internal_error);
}
+}
- if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
- profile_runnable_port(p, am_inactive);
- }
+BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1)
+{
+ Eterm retval;
+ Port* prt;
- erts_port_release(p);
-#ifdef ERTS_SMP
- ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN);
-#else
- ERTS_BIF_CHK_EXITED(BIF_P);
-#endif
-
- if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
- trace_virtual_sched(BIF_P, am_in);
+ if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) {
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_undefined);
}
-
- if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
- profile_runnable_proc(BIF_P, am_active);
+ else if (is_external_port(BIF_ARG_1)) {
+ if (external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry)
+ BIF_RET(am_undefined);
+ else
+ BIF_RET(am_badarg);
}
-
- if (is_non_value(res)) {
- BIF_ERROR(BIF_P, BADARG);
+ else {
+ BIF_RET(am_badarg);
+ }
+
+ switch (erts_port_info(BIF_P, prt, THE_NON_VALUE, &retval)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ BIF_RET(am_badarg);
+ case ERTS_PORT_OP_DROPPED:
+ BIF_RET(am_undefined);
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(retval));
+ BIF_RET(retval);
+ case ERTS_PORT_OP_DONE:
+ ASSERT(is_not_internal_ref(retval));
+ BIF_RET(retval);
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_info() result");
+ BIF_RET(am_internal_error);
}
- BIF_RET(res);
}
-BIF_RETTYPE port_close_1(BIF_ALIST_1)
-{
- Port* p;
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- p = id_or_name2port(NULL, BIF_ARG_1);
- if (!p) {
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
- BIF_ERROR(BIF_P, BADARG);
- }
- erts_do_exit_port(p, ERTS_PORT_GET_CONNECTED(p), am_normal);
- /* if !ERTS_SMP: since we terminate port with reason normal
- we SHOULD never get an exit signal ourselves
- */
- erts_port_release(p);
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
- BIF_RET(am_true);
-}
-BIF_RETTYPE port_connect_2(BIF_ALIST_2)
+BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2)
{
+ Eterm retval;
Port* prt;
- Process* rp;
- Eterm pid = BIF_ARG_2;
- if (is_not_internal_pid(pid)) {
- error:
- BIF_ERROR(BIF_P, BADARG);
- }
- prt = id_or_name2port(BIF_P, BIF_ARG_1);
- if (!prt) {
- goto error;
+ if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) {
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_undefined);
}
-
- rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
- pid, ERTS_PROC_LOCK_LINK);
- if (!rp) {
- erts_port_release(prt);
- ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P);
- goto error;
+ else if (is_external_port(BIF_ARG_1)) {
+ if (external_port_dist_entry(BIF_ARG_1) == erts_this_dist_entry)
+ BIF_RET(am_undefined);
+ else
+ BIF_RET(am_badarg);
}
-
- erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, prt->common.id);
- erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, pid);
-
- erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
-
- ERTS_PORT_SET_CONNECTED_RELB(prt, pid); /* internal pid */
- erts_port_release(prt);
-#ifdef USE_VM_PROBES
- if (DTRACE_ENABLED(port_connect)) {
- DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
- DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
- DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE);
-
- dtrace_pid_str(pid, process_str);
- erts_snprintf(port_str, sizeof(port_str), "%T", prt->id);
- dtrace_proc_str(rp, newprocess_str);
- DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str);
+ else {
+ BIF_RET(am_badarg);
+ }
+
+ switch (erts_port_info(BIF_P, prt, BIF_ARG_2, &retval)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ BIF_RET(am_badarg);
+ case ERTS_PORT_OP_DROPPED:
+ BIF_RET(am_undefined);
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(retval));
+ BIF_RET(retval);
+ case ERTS_PORT_OP_DONE:
+ ASSERT(is_not_internal_ref(retval));
+ BIF_RET(retval);
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_info() result");
+ BIF_RET(am_internal_error);
}
-#endif
- BIF_RET(am_true);
}
-BIF_RETTYPE port_set_data_2(BIF_ALIST_2)
+
+BIF_RETTYPE erts_internal_port_set_data_2(BIF_ALIST_2)
{
+ Eterm ref;
Port* prt;
- Eterm portid = BIF_ARG_1;
- Eterm data = BIF_ARG_2;
- prt = id_or_name2port(BIF_P, portid);
- if (!prt) {
- BIF_ERROR(BIF_P, BADARG);
- }
- if (prt->bp != NULL) {
- free_message_buffer(prt->bp);
- prt->bp = NULL;
- }
- if (IS_CONST(data)) {
- prt->data = data;
- } else {
- Uint size;
- ErlHeapFragment* bp;
- Eterm* hp;
-
- size = size_object(data);
- prt->bp = bp = new_message_buffer(size);
- hp = bp->mem;
- prt->data = copy_struct(data, size, &hp, &bp->off_heap);
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_badarg);
+
+ switch (erts_port_set_data(BIF_P, prt, BIF_ARG_2, &ref)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ case ERTS_PORT_OP_DROPPED:
+ BIF_RET(am_badarg);
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(ref));
+ BIF_RET(ref);
+ case ERTS_PORT_OP_DONE:
+ BIF_RET(am_true);
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_set_data() result");
+ BIF_RET(am_internal_error);
}
- erts_port_release(prt);
- BIF_RET(am_true);
}
-BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
+BIF_RETTYPE erts_internal_port_get_data_1(BIF_ALIST_1)
{
- BIF_RETTYPE res;
+ Eterm retval;
Port* prt;
- Eterm portid = BIF_ARG_1;
- prt = id_or_name2port(BIF_P, portid);
- if (!prt) {
- BIF_ERROR(BIF_P, BADARG);
- }
- if (prt->bp == NULL) { /* MUST be CONST! */
- res = prt->data;
- } else {
- Eterm* hp = HAlloc(BIF_P, prt->bp->used_size);
- res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P));
+ prt = lookup_port(BIF_P, BIF_ARG_1);
+ if (!prt)
+ BIF_RET(am_badarg);
+
+ switch (erts_port_get_data(BIF_P, prt, &retval)) {
+ case ERTS_PORT_OP_CALLER_EXIT:
+ case ERTS_PORT_OP_BADARG:
+ case ERTS_PORT_OP_DROPPED:
+ BIF_RET(am_badarg);
+ case ERTS_PORT_OP_SCHEDULED:
+ ASSERT(is_internal_ref(retval));
+ BIF_RET(retval);
+ case ERTS_PORT_OP_DONE:
+ ASSERT(is_not_internal_ref(retval));
+ BIF_RET(retval);
+ default:
+ ERTS_INTERNAL_ERROR("Unexpected erts_port_get_data() result");
+ BIF_RET(am_internal_error);
}
- erts_port_release(prt);
- BIF_RET(res);
}
/*
@@ -659,6 +514,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
opts.overlapped_io = 0;
opts.spawn_type = ERTS_SPAWN_ANY;
opts.argv = NULL;
+ opts.parallelism = erts_port_parallelism;
linebuf = 0;
*err_nump = 0;
@@ -736,6 +592,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
}
} else if (option == am_cd) {
edir = *tp;
+ } else if (option == am_parallelism) {
+ if (*tp == am_true)
+ opts.parallelism = 1;
+ else if (*tp == am_false)
+ opts.parallelism = 0;
+ else
+ goto badarg;
} else {
goto badarg;
}
@@ -904,9 +767,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump)
heap[2] = make_small(0);
heap[3] = NIL;
iolist = make_list(heap);
- r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
+ r = erts_iolist_to_buf(iolist, (char*) dir, MAXPATHLEN);
UnUseTmpHeap(4,p);
- if (r < 0) {
+ if (ERTS_IOLIST_TO_BUF_FAILED(r)) {
goto badarg;
}
opts.wd = (char *) dir;
@@ -1035,7 +898,7 @@ static byte* convert_environment(Process* p, Eterm env)
Eterm* hp;
Uint heap_size;
int n;
- Uint size;
+ ErlDrvSizeT size;
byte* bytes;
if ((n = list_length(env)) < 0) {
@@ -1087,7 +950,7 @@ static byte* convert_environment(Process* p, Eterm env)
* Put the result in a binary (no risk for a memory leak that way).
*/
(void) erts_new_heap_binary(p, NULL, size, &bytes);
- io_list_to_buf(all, (char*)bytes, size);
+ erts_iolist_to_buf(all, (char*)bytes, size);
done:
erts_free(ERTS_ALC_T_TMP, temp_heap);
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 6b843d2e08..ace128f045 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -418,7 +418,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
static BIF_RETTYPE
re_compile(Process* p, Eterm arg1, Eterm arg2)
{
- Uint slen;
+ ErlDrvSizeT slen;
char *expr;
pcre *result;
int errcode = 0;
@@ -449,7 +449,7 @@ re_compile(Process* p, Eterm arg1, Eterm arg2)
BIF_ERROR(p,BADARG);
}
expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
- if (io_list_to_buf(arg1, expr, slen) != 0) {
+ if (erts_iolist_to_buf(arg1, expr, slen) != 0) {
erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
BIF_ERROR(p,BADARG);
}
@@ -802,7 +802,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
memcpy(tmpb,ap->name,ap->len);
tmpb[ap->len] = '\0';
} else {
- Uint slen;
+ ErlDrvSizeT slen;
if (erts_iolist_size(val, &slen)) {
goto error;
}
@@ -814,7 +814,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
(tmpbsiz = slen + 1));
}
}
- if (io_list_to_buf(val, tmpb, slen) != 0) {
+ if (erts_iolist_to_buf(val, tmpb, slen) != 0) {
goto error;
}
tmpb[slen] = '\0';
@@ -858,7 +858,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
const pcre *code_tmp;
RestartContext restart;
byte *temp_alloc = NULL;
- Uint slength;
+ ErlDrvSizeT slength;
int startoffset = 0;
int options = 0, comp_options = 0;
int ovsize;
@@ -882,7 +882,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 4)) {
if (is_binary(arg2) || is_list(arg2) || is_nil(arg2)) {
/* Compile from textual RE */
- Uint slen;
+ ErlDrvSizeT slen;
char *expr;
pcre *result;
int errcode = 0;
@@ -901,7 +901,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
}
expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
- if (io_list_to_buf(arg2, expr, slen) != 0) {
+ if (erts_iolist_to_buf(arg2, expr, slen) != 0) {
erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
BIF_ERROR(p,BADARG);
}
@@ -1044,7 +1044,7 @@ handle_iolist:
}
restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength);
- if (io_list_to_buf(arg1, restart.subject, slength) != 0) {
+ if (erts_iolist_to_buf(arg1, restart.subject, slength) != 0) {
erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
erts_free(ERTS_ALC_T_RE_SUBJECT, restart.subject);
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index f83e380b02..af7c2b91a0 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -564,7 +564,8 @@ void erts_usage(void)
/* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */
erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n");
-
+ erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n");
+ erts_fprintf(stderr, " Note that this flag is deprecated!\n");
erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n");
erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n");
@@ -592,6 +593,7 @@ void erts_usage(void)
erts_fprintf(stderr, " valid range is [%d-%d]\n",
ERTS_SCHED_THREAD_MIN_STACK_SIZE,
ERTS_SCHED_THREAD_MAX_STACK_SIZE);
+ erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n");
erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n");
erts_fprintf(stderr, " schedulers online (n2), valid range for both\n");
erts_fprintf(stderr, " numbers are [1-%d]\n",
@@ -1228,6 +1230,30 @@ erl_start(int argc, char **argv)
arg);
break;
+ case 'n':
+ arg = get_arg(argv[i]+2, argv[i+1], &i);
+ switch (arg[0]) {
+ case 's': /* synchronous */
+ erts_port_synchronous_ops = 1;
+ erts_port_schedule_all_ops = 0;
+ break;
+ case 'a': /* asynchronous */
+ erts_port_synchronous_ops = 0;
+ erts_port_schedule_all_ops = 1;
+ break;
+ case 'd': /* Default - schedule on conflict (asynchronous) */
+ erts_port_synchronous_ops = 0;
+ erts_port_schedule_all_ops = 0;
+ break;
+ default:
+ bad_n_option:
+ erts_fprintf(stderr, "bad -n option %s\n", arg);
+ erts_usage();
+ }
+ if (arg[1] != '\0')
+ goto bad_n_option;
+ break;
+
case 'P': /* set maximum number of processes */
arg = get_arg(argv[i]+2, argv[i+1], &i);
errno = 0;
@@ -1342,6 +1368,19 @@ erl_start(int argc, char **argv)
erts_usage();
}
}
+ else if (has_prefix("pp", sub_param)) {
+ arg = get_arg(sub_param+2, argv[i+1], &i);
+ if (sys_strcmp(arg, "true") == 0)
+ erts_port_parallelism = 1;
+ else if (sys_strcmp(arg, "false") == 0)
+ erts_port_parallelism = 0;
+ else {
+ erts_fprintf(stderr,
+ "bad port parallelism scheduling hint %s\n",
+ arg);
+ erts_usage();
+ }
+ }
else if (sys_strcmp("nsp", sub_param) == 0)
erts_use_sender_punish = 0;
else if (sys_strcmp("wt", sub_param) == 0) {
@@ -1467,8 +1506,6 @@ erl_start(int argc, char **argv)
}
break;
}
- case 'n': /* XXX obsolete */
- break;
case 'c':
if (argv[i][2] == 0) { /* -c: documented option */
erts_disable_tolerant_timeofday = 1;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8ef6e8f846..04721cee7c 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -501,7 +501,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
{
struct enif_tmp_obj_t* tobj;
ErtsAlcType_t allocator;
- Uint sz;
+ ErlDrvSizeT sz;
if (is_binary(term)) {
return enif_inspect_binary(env,term,bin);
}
@@ -527,7 +527,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
bin->size = sz;
bin->bin_term = THE_NON_VALUE;
bin->ref_bin = NULL;
- io_list_to_buf(term, (char*) bin->data, sz);
+ erts_iolist_to_buf(term, (char*) bin->data, sz);
ADD_READONLY_CHECK(env, bin->data, bin->size);
return 1;
}
diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h
index 6ad92dcd7d..beeddd09a0 100644
--- a/erts/emulator/beam/erl_port.h
+++ b/erts/emulator/beam/erl_port.h
@@ -20,6 +20,7 @@
#ifndef ERL_PORT_TYPE__
#define ERL_PORT_TYPE__
typedef struct _erl_drv_port Port;
+typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData;
#endif
#if !defined(ERL_PORT_H__) && !defined(ERL_PORT_GET_PORT_TYPE_ONLY__)
@@ -33,6 +34,10 @@ typedef struct _erl_drv_port Port;
#define ERTS_DEFAULT_MAX_PORTS (1 << 16)
#define ERTS_MIN_PORTS 1024
+extern int erts_port_synchronous_ops;
+extern int erts_port_schedule_all_ops;
+extern int erts_port_parallelism;
+
typedef struct erts_driver_t_ erts_driver_t;
#define ERTS_INVALID_ERL_DRV_PORT ((ErlDrvPort) (SWord) -1)
@@ -63,6 +68,19 @@ typedef struct line_buf { /* Buffer used in line oriented I/O */
The rest is the overflow buffer. */
} LineBuf;
+/*
+ * Items part of erlang:port_info/1 result. Note am_registered_name
+ * *need* to be first.
+ */
+
+#define ERTS_PORT_INFO_1_ITEMS \
+ { am_registered_name, /* Needs to be first */ \
+ am_name, \
+ am_links, \
+ am_id, \
+ am_connected, \
+ am_input, \
+ am_output }
/*
* Port Specific Data.
@@ -113,7 +131,7 @@ struct _erl_drv_port {
erts_atomic32_t refc;
int cleanup;
#endif
- erts_smp_atomic_t connected;/* A connected process */
+ erts_atomic_t connected; /* A connected process */
Eterm caller; /* Current caller. */
Eterm data; /* Data associated with port. */
ErlHeapFragment* bp; /* Heap fragment holding data (NULL if imm data). */
@@ -135,6 +153,13 @@ struct _erl_drv_port {
ErtsPrtSD *psd; /* Port specific data */
};
+#define ERTS_PORT_GET_CONNECTED(PRT) \
+ ((Eterm) erts_atomic_read_nob(&(PRT)->connected))
+#define ERTS_PORT_SET_CONNECTED(PRT, PID) \
+ erts_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID))
+#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \
+ erts_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID))
+
struct erl_drv_port_data_lock {
erts_mtx_t mtx;
@@ -240,6 +265,7 @@ extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */
(ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID)
#define ERTS_PORT_SFLGS_INVALID_LOOKUP \
(ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP \
+ | ERTS_PORT_SFLG_EXITING \
| ERTS_PORT_SFLG_CLOSING)
#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP \
(ERTS_PORT_SFLGS_INVALID_LOOKUP \
@@ -358,17 +384,6 @@ erts_smp_port_unlock(Port *prt)
#define ERTS_PORT_SCHED_ID(P, ID) \
((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))
-#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \
- erts_smp_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID))
-#define ERTS_PORT_SET_CONNECTED(PRT, PID) \
- erts_smp_atomic_set_nob(&(PRT)->connected, (erts_aint_t) (PID))
-#define ERTS_PORT_SET_CONNECTED_RELB(PRT, PID) \
- erts_smp_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID))
-#define ERTS_PORT_GET_CONNECTED(PRT) \
- ((Eterm) erts_smp_atomic_read_nob(&(PRT)->connected))
-#define ERTS_PORT_GET_CONNECTED_ACQB(PRT) \
- ((Eterm) erts_smp_atomic_read_acqb(&(PRT)->connected))
-
extern const Port erts_invalid_port;
#define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port)
@@ -424,8 +439,11 @@ ERTS_GLB_INLINE Port *
erts_port_lookup(Eterm id, Uint32 invalid_sflgs)
{
Port *prt = erts_port_lookup_raw(id);
- erts_aint32_t state = erts_atomic32_read_nob(&prt->state);
- return (state & invalid_sflgs) ? NULL : prt;
+ return (!prt
+ ? NULL
+ : ((invalid_sflgs & erts_atomic32_read_nob(&prt->state))
+ ? NULL
+ : prt));
}
@@ -584,14 +602,16 @@ erts_thr_drvport2port_raw(ErlDrvPort drvport)
else {
Port *prt = (Port *) drvport;
#if ERTS_ENABLE_LOCK_CHECK
- if (emu_thread) {
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- ERTS_LC_ASSERT(!prt->port_data_lock
- || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
- }
- else {
- ERTS_LC_ASSERT(prt->port_data_lock);
- ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
+ if (!ERTS_IS_CRASH_DUMPING) {
+ if (emu_thread) {
+ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+ ERTS_LC_ASSERT(!prt->port_data_lock
+ || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
+ }
+ else {
+ ERTS_LC_ASSERT(prt->port_data_lock);
+ ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
+ }
}
#endif
return prt;
@@ -606,7 +626,8 @@ erts_drvport2port_raw(ErlDrvPort drvport)
return NULL;
else {
Port *prt = (Port *) drvport;
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
+ || ERTS_IS_CRASH_DUMPING);
return prt;
}
}
@@ -637,7 +658,8 @@ erts_drvportid2port(Eterm id)
internal_port_index(id));
if (!prt)
return NULL;
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
+ || ERTS_IS_CRASH_DUMPING);
if (prt->common.id != id)
return NULL;
state = erts_atomic32_read_nob(&prt->state);
@@ -680,4 +702,141 @@ erts_is_valid_tracer_port(Eterm id)
}
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+struct binary;
+
+#define ERTS_P2P_SIG_DATA_FLG_BANG_OP (1 << 0)
+#define ERTS_P2P_SIG_DATA_FLG_REPLY (1 << 1)
+#define ERTS_P2P_SIG_DATA_FLG_NOSUSPEND (1 << 2)
+#define ERTS_P2P_SIG_DATA_FLG_FORCE (1 << 3)
+#define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT (1 << 4)
+#define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK (1 << 5)
+#define ERTS_P2P_SIG_DATA_FLG_SCHED (1 << 6)
+
+struct ErtsProc2PortSigData_ {
+ int flags;
+ Eterm caller;
+ Uint32 ref[ERTS_MAX_REF_NUMBERS];
+ union {
+ struct {
+ Eterm from;
+ ErlIOVec *evp;
+ ErlDrvBinary *cbinp;
+ } outputv;
+ struct {
+ Eterm from;
+ char *bufp;
+ ErlDrvSizeT size;
+ } output;
+ struct {
+ Eterm from;
+ Eterm connected;
+ } connect;
+ struct {
+ Eterm from;
+ Eterm reason;
+ ErlHeapFragment *bp;
+ } exit;
+ struct {
+ struct binary *binp;
+ unsigned int command;
+ char *bufp;
+ ErlDrvSizeT size;
+ } control;
+ struct {
+ unsigned int command;
+ char *bufp;
+ ErlDrvSizeT size;
+ } call;
+ struct {
+ Eterm item;
+ } info;
+ struct {
+ Eterm port;
+ Eterm to;
+ } link;
+ struct {
+ Eterm from;
+ } unlink;
+ struct {
+ ErlHeapFragment *bp;
+ Eterm data;
+ } set_data;
+ } u;
+} ;
+
+#define ERTS_PROC2PORT_SIG_EXEC 0
+#define ERTS_PROC2PORT_SIG_ABORT 1
+#define ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND 2
+#define ERTS_PROC2PORT_SIG_ABORT_CLOSED 3
+
+typedef int (*ErtsProc2PortSigCallback)(Port *,
+ erts_aint32_t,
+ int,
+ ErtsProc2PortSigData *);
+
+#define ERTS_PORT_REDS_CMD_OUTPUTV 400
+#define ERTS_PORT_REDS_CMD_OUTPUT 400
+#define ERTS_PORT_REDS_EXIT 300
+#define ERTS_PORT_REDS_CONNECT 40
+#define ERTS_PORT_REDS_UNLINK 40
+#define ERTS_PORT_REDS_LINK 40
+#define ERTS_PORT_REDS_BADSIG 40
+#define ERTS_PORT_REDS_CONTROL 400
+#define ERTS_PORT_REDS_CALL 400
+#define ERTS_PORT_REDS_INFO 100
+#define ERTS_PORT_REDS_SET_DATA 40
+#define ERTS_PORT_REDS_GET_DATA 40
+
+typedef enum {
+ ERTS_PORT_OP_BADARG,
+ ERTS_PORT_OP_CALLER_EXIT,
+ ERTS_PORT_OP_BUSY,
+ ERTS_PORT_OP_BUSY_SCHEDULED,
+ ERTS_PORT_OP_SCHEDULED,
+ ERTS_PORT_OP_DROPPED,
+ ERTS_PORT_OP_DONE
+} ErtsPortOpResult;
+
+ErtsPortOpResult
+erts_schedule_proc2port_signal(Process *,
+ Port *,
+ Eterm,
+ Eterm *,
+ ErtsProc2PortSigData *,
+ int,
+ ErtsProc2PortSigCallback);
+
+int erts_deliver_port_exit(Port *, Eterm, Eterm, int);
+
+/*
+ * Port signal flags
+ */
+#define ERTS_PORT_SIG_FLG_BANG_OP ERTS_P2P_SIG_DATA_FLG_BANG_OP
+#define ERTS_PORT_SIG_FLG_NOSUSPEND ERTS_P2P_SIG_DATA_FLG_NOSUSPEND
+#define ERTS_PORT_SIG_FLG_FORCE ERTS_P2P_SIG_DATA_FLG_FORCE
+#define ERTS_PORT_SIG_FLG_BROKEN_LINK ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK
+#define ERTS_PORT_SIG_FLG_BAD_OUTPUT ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT
+#define ERTS_PORT_SIG_FLG_FORCE_SCHED ERTS_P2P_SIG_DATA_FLG_SCHED
+
+/*
+ * Port ! {Owner, {command, Data}}
+ * Port ! {Owner, {connect, NewOwner}}
+ * Port ! {Owner, close}
+ */
+ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *);
+
+/*
+ * Signals from processes to ports.
+ */
+ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *);
+ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *);
+ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *);
+ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *);
+ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *);
+ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *);
+ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
+ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);
+ErtsPortOpResult erts_port_set_data(Process *, Port *, Eterm, Eterm *);
+ErtsPortOpResult erts_port_get_data(Process *, Port *, Eterm *);
+
#endif
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index e613ad0144..e8ed00daee 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -36,9 +36,20 @@
#include <stdarg.h>
#if defined(DEBUG) && 0
-#define HARD_DEBUG
+#define ERTS_HARD_DEBUG_TASK_QUEUES
+#else
+#undef ERTS_HARD_DEBUG_TASK_QUEUES
+#endif
+
+#ifdef ERTS_HARD_DEBUG_TASK_QUEUES
+static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue);
+#define ERTS_PT_DBG_CHK_TASK_QS(PP, EQ, PBQ) \
+ chk_task_queues((PP), (EQ), (PBQ))
+#else
+#define ERTS_PT_DBG_CHK_TASK_QS(PP, EQ, PBQ)
#endif
+
/*
* Costs in reductions for some port operations.
*/
@@ -52,7 +63,7 @@
#ifdef USE_VM_PROBES
#define DTRACE_DRIVER(PROBE_NAME, PP) \
- if (DTRACE_ENABLED(driver_ready_input)) { \
+ if (DTRACE_ENABLED(PROBE_NAME)) { \
DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE); \
DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); \
\
@@ -70,6 +81,17 @@ erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
#define ERTS_PT_STATE_ABORTED 1
#define ERTS_PT_STATE_EXECUTING 2
+typedef union {
+ struct { /* I/O tasks */
+ ErlDrvEvent event;
+ ErlDrvEventData event_data;
+ } io;
+ struct {
+ ErtsProc2PortSigCallback callback;
+ ErtsProc2PortSigData data;
+ } psig;
+} ErtsPortTaskTypeData;
+
struct ErtsPortTask_ {
erts_smp_atomic32_t state;
ErtsPortTaskType type;
@@ -77,17 +99,39 @@ struct ErtsPortTask_ {
struct {
ErtsPortTask *next;
ErtsPortTaskHandle *handle;
- union {
- struct { /* I/O tasks */
- ErlDrvEvent event;
- ErlDrvEventData event_data;
- } io;
- } u;
+ int flags;
+ Uint32 ref[ERTS_MAX_REF_NUMBERS];
+ ErtsPortTaskTypeData td;
} alive;
ErtsThrPrgrLaterOp release;
} u;
};
+struct ErtsPortTaskHandleList_ {
+ ErtsPortTaskHandle handle;
+ union {
+ ErtsPortTaskHandleList *next;
+#ifdef ERTS_SMP
+ ErtsThrPrgrLaterOp release;
+#endif
+ } u;
+};
+
+typedef struct ErtsPortTaskBusyCaller_ ErtsPortTaskBusyCaller;
+struct ErtsPortTaskBusyCaller_ {
+ ErtsPortTaskBusyCaller *next;
+ Eterm caller;
+ SWord count;
+ ErtsPortTask *last;
+};
+
+#define ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS 17
+struct ErtsPortTaskBusyCallerTable_ {
+ ErtsPortTaskBusyCaller *bucket[ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS];
+ ErtsPortTaskBusyCaller pre_alloc_busy_caller;
+};
+
+
static void begin_port_cleanup(Port *pp, ErtsPortTask **execq);
ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_task,
@@ -95,6 +139,11 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_task,
1000,
ERTS_ALC_T_PORT_TASK)
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(busy_caller_table,
+ ErtsPortTaskBusyCallerTable,
+ 50,
+ ERTS_ALC_T_BUSY_CALLER_TAB)
+
#ifdef ERTS_SMP
static void
call_port_task_free(void *vptp)
@@ -115,6 +164,358 @@ schedule_port_task_free(ErtsPortTask *ptp)
#endif
}
+static ERTS_INLINE ErtsPortTask *
+p2p_sig_data_to_task(ErtsProc2PortSigData *sigdp)
+{
+ ErtsPortTask *ptp;
+ char *ptr = (char *) sigdp;
+ ptr -= offsetof(ErtsPortTask, u.alive.td.psig.data);
+ ptp = (ErtsPortTask *) ptr;
+ ASSERT(ptp->type == ERTS_PORT_TASK_PROC_SIG);
+ return ptp;
+}
+
+ErtsProc2PortSigData *
+erts_port_task_alloc_p2p_sig_data(void)
+{
+ ErtsPortTask *ptp = port_task_alloc();
+
+ ptp->type = ERTS_PORT_TASK_PROC_SIG;
+ ptp->u.alive.flags = ERTS_PT_FLG_SIG_DEP;
+ erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED);
+
+ ASSERT(ptp == p2p_sig_data_to_task(&ptp->u.alive.td.psig.data));
+
+ return &ptp->u.alive.td.psig.data;
+}
+
+static ERTS_INLINE Eterm
+task_caller(ErtsPortTask *ptp)
+{
+ Eterm caller;
+
+ ASSERT(ptp->type == ERTS_PORT_TASK_PROC_SIG);
+
+ caller = ptp->u.alive.td.psig.data.caller;
+
+ ASSERT(is_internal_pid(caller) || is_internal_port(caller));
+
+ return caller;
+}
+
+/*
+ * Busy queue management
+ */
+
+static ERTS_INLINE int
+caller2bix(Eterm caller)
+{
+ ASSERT(is_internal_pid(caller) || is_internal_port(caller));
+ return (int) (_GET_PID_DATA(caller) % ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS);
+}
+
+
+static void
+popped_from_busy_queue(Port *pp, ErtsPortTask *ptp, int last)
+{
+ ErtsPortTaskBusyCaller **prev_bcpp = NULL, *bcp;
+ ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table;
+ Eterm caller = task_caller(ptp);
+ int bix = caller2bix(caller);
+
+ ASSERT(is_internal_pid(caller));
+
+ ASSERT(tabp);
+ bcp = tabp->bucket[bix];
+ prev_bcpp = &tabp->bucket[bix];
+ ASSERT(bcp);
+ while (bcp->caller != caller) {
+ prev_bcpp = &bcp->next;
+ bcp = bcp->next;
+ ASSERT(bcp);
+ }
+ ASSERT(bcp->count > 0);
+ if (--bcp->count != 0) {
+ ASSERT(!last);
+ }
+ else {
+ *prev_bcpp = bcp->next;
+ if (bcp == &tabp->pre_alloc_busy_caller)
+ bcp->caller = am_undefined;
+ else
+ erts_free(ERTS_ALC_T_BUSY_CALLER, bcp);
+ if (last) {
+#ifdef DEBUG
+ erts_aint32_t flags =
+#endif
+ erts_smp_atomic32_read_band_nob(
+ &pp->sched.flags,
+ ~ERTS_PTS_FLG_HAVE_BUSY_TASKS);
+ ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS);
+#ifdef DEBUG
+ for (bix = 0; bix < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; bix++) {
+ ASSERT(!tabp->bucket[bix]);
+ }
+#endif
+ busy_caller_table_free(tabp);
+ pp->sched.taskq.local.busy.first = NULL;
+ pp->sched.taskq.local.busy.last = NULL;
+ pp->sched.taskq.local.busy.table = NULL;
+ }
+ }
+}
+
+static void
+busy_wait_move_to_busy_queue(Port *pp, ErtsPortTask *ptp)
+{
+ ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table;
+ Eterm caller = task_caller(ptp);
+ ErtsPortTaskBusyCaller *bcp;
+ int bix;
+
+ ASSERT(is_internal_pid(caller));
+ /*
+ * Port is busy and this task type needs to wait until not busy.
+ */
+
+ ASSERT(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY);
+
+ ptp->u.alive.next = NULL;
+ if (pp->sched.taskq.local.busy.last) {
+ ASSERT(pp->sched.taskq.local.busy.first);
+ pp->sched.taskq.local.busy.last->u.alive.next = ptp;
+ }
+ else {
+ int i;
+ erts_aint32_t flags;
+
+ pp->sched.taskq.local.busy.first = ptp;
+ flags = erts_smp_atomic32_read_bor_nob(&pp->sched.flags,
+ ERTS_PTS_FLG_HAVE_BUSY_TASKS);
+ ASSERT(!(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS));
+
+ ASSERT(!tabp);
+
+ tabp = busy_caller_table_alloc();
+ pp->sched.taskq.local.busy.table = tabp;
+ for (i = 0; i < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; i++)
+ tabp->bucket[i] = NULL;
+ tabp->pre_alloc_busy_caller.caller = am_undefined;
+ }
+ pp->sched.taskq.local.busy.last = ptp;
+
+ bix = caller2bix(caller);
+ ASSERT(tabp);
+ bcp = tabp->bucket[bix];
+
+ while (bcp && bcp->caller != caller)
+ bcp = bcp->next;
+
+ if (bcp)
+ bcp->count++;
+ else {
+ if (tabp->pre_alloc_busy_caller.caller == am_undefined)
+ bcp = &tabp->pre_alloc_busy_caller;
+ else
+ bcp = erts_alloc(ERTS_ALC_T_BUSY_CALLER,
+ sizeof(ErtsPortTaskBusyCaller));
+ bcp->caller = caller;
+ bcp->count = 1;
+ bcp->next = tabp->bucket[bix];
+ tabp->bucket[bix] = bcp;
+ }
+
+ bcp->last = ptp;
+}
+
+static ERTS_INLINE int
+check_sig_dep_move_to_busy_queue(Port *pp, ErtsPortTask *ptp)
+{
+ ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table;
+ ErtsPortTask *last_ptp;
+ ErtsPortTaskBusyCaller *bcp;
+ int bix;
+ Eterm caller;
+
+ ASSERT(ptp->u.alive.flags & ERTS_PT_FLG_SIG_DEP);
+ ASSERT(pp->sched.taskq.local.busy.last);
+ ASSERT(tabp);
+
+
+ /*
+ * We are either not busy, or the task does not imply wait on busy port.
+ * However, due to the signaling order requirements the task might depend
+ * on other tasks in the busy queue.
+ */
+
+ caller = task_caller(ptp);
+ bix = caller2bix(caller);
+ bcp = tabp->bucket[bix];
+ while (bcp && bcp->caller != caller)
+ bcp = bcp->next;
+
+ if (!bcp)
+ return 0;
+
+ /*
+ * There are other tasks that we depend on in the busy queue;
+ * move into busy queue.
+ */
+
+ bcp->count++;
+ last_ptp = bcp->last;
+ ptp->u.alive.next = last_ptp->u.alive.next;
+ if (!ptp->u.alive.next) {
+ ASSERT(pp->sched.taskq.local.busy.last == last_ptp);
+ pp->sched.taskq.local.busy.last = ptp;
+ }
+ last_ptp->u.alive.next = ptp;
+ bcp->last = ptp;
+
+ return 1;
+}
+
+static void
+no_sig_dep_move_from_busyq(Port *pp)
+{
+ ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table;
+ ErtsPortTask *first_ptp, *last_ptp, *ptp;
+ ErtsPortTaskBusyCaller **prev_bcpp = NULL, *bcp = NULL;
+
+ /*
+ * Move tasks at the head of the busy queue that no longer
+ * have any dependencies to busy wait tasks into the ordinary
+ * queue.
+ */
+
+ first_ptp = ptp = pp->sched.taskq.local.busy.first;
+
+ ASSERT(ptp && !(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY));
+ ASSERT(tabp);
+
+ do {
+ Eterm caller = task_caller(ptp);
+
+ if (!bcp || bcp->caller != caller) {
+ int bix = caller2bix(caller);
+
+ prev_bcpp = &tabp->bucket[bix];
+ bcp = tabp->bucket[bix];
+ ASSERT(bcp);
+ while (bcp->caller != caller) {
+ ASSERT(bcp);
+ prev_bcpp = &bcp->next;
+ bcp = bcp->next;
+ }
+ }
+
+ ASSERT(bcp->caller == caller);
+ ASSERT(bcp->count > 0);
+
+ if (--bcp->count == 0) {
+ *prev_bcpp = bcp->next;
+ if (bcp == &tabp->pre_alloc_busy_caller)
+ bcp->caller = am_undefined;
+ else
+ erts_free(ERTS_ALC_T_BUSY_CALLER, bcp);
+ }
+
+ last_ptp = ptp;
+ ptp = ptp->u.alive.next;
+ } while (ptp && !(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY));
+
+ pp->sched.taskq.local.busy.first = last_ptp->u.alive.next;
+ if (!pp->sched.taskq.local.busy.first) {
+#ifdef DEBUG
+ int bix;
+ erts_aint32_t flags =
+#endif
+ erts_smp_atomic32_read_band_nob(
+ &pp->sched.flags,
+ ~ERTS_PTS_FLG_HAVE_BUSY_TASKS);
+ ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS);
+#ifdef DEBUG
+ for (bix = 0; bix < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; bix++) {
+ ASSERT(!tabp->bucket[bix]);
+ }
+#endif
+ busy_caller_table_free(tabp);
+ pp->sched.taskq.local.busy.last = NULL;
+ pp->sched.taskq.local.busy.table = NULL;
+ }
+ last_ptp->u.alive.next = pp->sched.taskq.local.first;
+ pp->sched.taskq.local.first = first_ptp;
+}
+
+#ifdef ERTS_HARD_DEBUG_TASK_QUEUES
+
+static void
+chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue)
+{
+ Sint tot_count, tot_table_count;
+ int bix;
+ ErtsPortTask *ptp, *last;
+ ErtsPortTask *first = processing_busy_queue ? execq : pp->sched.taskq.local.busy.first;
+ ErtsPortTask *nb_task_queue = processing_busy_queue ? pp->sched.taskq.local.first : execq;
+ ErtsPortTaskBusyCallerTable *tabp = pp->sched.taskq.local.busy.table;
+ ErtsPortTaskBusyCaller *bcp;
+
+ if (!first) {
+ ASSERT(!tabp);
+ ASSERT(!pp->sched.taskq.local.busy.last);
+ ASSERT(!(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS));
+ return;
+ }
+
+ ASSERT(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS);
+ ASSERT(tabp);
+
+ tot_count = 0;
+ ptp = first;
+ while (ptp) {
+ Sint count = 0;
+ Eterm caller = task_caller(ptp);
+ int bix = caller2bix(caller);
+ for (bcp = tabp->bucket[bix]; bcp; bcp = bcp->next)
+ if (bcp->caller == caller)
+ break;
+ ASSERT(bcp && bcp->caller == caller);
+
+ ASSERT(bcp->last);
+ while (1) {
+ ErtsPortTask *ptp2;
+
+ ASSERT(caller == task_caller(ptp));
+ count++;
+ tot_count++;
+ last = ptp;
+
+ for (ptp2 = nb_task_queue; ptp2; ptp2 = ptp2->u.alive.next) {
+ ASSERT(ptp != ptp2);
+ }
+
+ if (ptp == bcp->last)
+ break;
+ ptp = ptp->u.alive.next;
+ }
+
+ ASSERT(count == bcp->count);
+ ptp = ptp->u.alive.next;
+ }
+
+ tot_table_count = 0;
+ for (bix = 0; bix < ERTS_PORT_TASK_BUSY_CALLER_TABLE_BUCKETS; bix++) {
+ for (bcp = tabp->bucket[bix]; bcp; bcp = bcp->next)
+ tot_table_count += bcp->count;
+ }
+
+ ASSERT(tot_count == tot_table_count);
+
+ ASSERT(last == pp->sched.taskq.local.busy.last);
+}
+
+#endif /* ERTS_HARD_DEBUG_TASK_QUEUES */
+
/*
* Task handle manipulation.
*/
@@ -151,6 +552,141 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
}
/*
+ * No-suspend handles.
+ */
+
+#ifdef ERTS_SMP
+static void
+free_port_task_handle_list(void *vpthlp)
+{
+ erts_free(ERTS_ALC_T_PT_HNDL_LIST, vpthlp);
+}
+#endif
+
+static void
+schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp)
+{
+#ifdef ERTS_SMP
+ erts_schedule_thr_prgr_later_op(free_port_task_handle_list,
+ (void *) pthlp,
+ &pthlp->u.release);
+#else
+ erts_free(ERTS_ALC_T_PT_HNDL_LIST, pthlp);
+#endif
+}
+
+static ERTS_INLINE void
+abort_nosuspend_task(ErtsPortTaskType type,
+ ErtsPortTaskTypeData *tdp)
+{
+
+ if (type != ERTS_PORT_TASK_PROC_SIG)
+ ERTS_INTERNAL_ERROR("Invalid no-suspend port task type");
+
+ tdp->psig.callback(NULL,
+ ERTS_PORT_SFLG_INVALID,
+ ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND,
+ &tdp->psig.data);
+}
+
+
+static void
+save_nosuspend_handle(Port *pp, ErtsPortTask *ptp)
+{
+ erts_aint32_t act;
+ ErtsPortTaskHandleList *pthlp = erts_alloc(ERTS_ALC_T_PT_HNDL_LIST,
+ sizeof(ErtsPortTaskHandleList));
+
+ set_handle(ptp, &pthlp->handle);
+
+ ASSERT(ptp == handle2task(&pthlp->handle));
+ ASSERT(ptp->u.alive.handle == &pthlp->handle);
+
+ act = erts_smp_atomic32_read_nob(&pp->sched.flags);
+
+ if (!(act & ERTS_PTS_FLG_BUSY)) {
+
+ erts_port_task_sched_lock(&pp->sched);
+ pthlp->u.next = pp->sched.taskq.local.busy.nosuspend;
+ pp->sched.taskq.local.busy.nosuspend = pthlp;
+ erts_port_task_sched_unlock(&pp->sched);
+
+ act = erts_smp_atomic32_read_nob(&pp->sched.flags);
+
+ while (1) {
+ erts_aint32_t exp, new;
+
+ if ((act & (ERTS_PTS_FLG_BUSY|ERTS_PTS_FLG_HAVE_NS_TASKS))
+ == ERTS_PTS_FLG_HAVE_NS_TASKS)
+ return;
+
+ if (act & ERTS_PTS_FLG_BUSY) {
+ erts_aint32_t s;
+ s = erts_smp_atomic32_cmpxchg_nob(&ptp->state,
+ ERTS_PT_STATE_ABORTED,
+ ERTS_PT_STATE_SCHEDULED);
+ if (s == ERTS_PT_STATE_SCHEDULED) {
+ reset_port_task_handle(&pthlp->handle);
+ break; /* Abort task */
+ }
+ /* Else: someone else handled it */
+ return;
+ }
+
+ new = exp = act;
+
+ new |= ERTS_PTS_FLG_HAVE_NS_TASKS;
+
+ act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp);
+ if (act == exp)
+ return;
+
+ }
+ }
+
+
+ abort_nosuspend_task(ptp->type, &ptp->u.alive.td);
+}
+
+
+static ErtsPortTaskHandleList *
+get_free_nosuspend_handles(Port *pp)
+{
+ ErtsPortTaskHandleList *nshp, *last_nshp = NULL;
+
+ ERTS_SMP_LC_ASSERT(erts_port_task_sched_lock_is_locked(&pp->sched));
+
+ nshp = pp->sched.taskq.local.busy.nosuspend;
+
+ while (nshp && !erts_port_task_is_scheduled(&nshp->handle)) {
+ last_nshp = nshp;
+ nshp = nshp->u.next;
+ }
+
+ if (!last_nshp)
+ nshp = NULL;
+ else {
+ nshp = pp->sched.taskq.local.busy.nosuspend;
+ pp->sched.taskq.local.busy.nosuspend = last_nshp->u.next;
+ last_nshp->u.next = NULL;
+ if (!pp->sched.taskq.local.busy.nosuspend)
+ erts_smp_atomic32_read_band_nob(&pp->sched.flags,
+ ~ERTS_PTS_FLG_HAVE_NS_TASKS);
+ }
+ return nshp;
+}
+
+static void
+free_nosuspend_handles(ErtsPortTaskHandleList *free_nshp)
+{
+ while (free_nshp) {
+ ErtsPortTaskHandleList *nshp = free_nshp;
+ free_nshp = free_nshp->u.next;
+ schedule_port_task_handle_list_free(nshp);
+ }
+}
+
+/*
* Port queue operations
*/
@@ -225,42 +761,21 @@ enqueue_task(Port *pp, ErtsPortTask *ptp)
return flags;
}
-static ERTS_INLINE ErtsPortTask *
-pop_task(Port *pp, ErtsPortTask **execqp)
+static ERTS_INLINE void
+prepare_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
{
- ErtsPortTask *ptp;
+ erts_aint32_t act = erts_smp_atomic32_read_nob(&pp->sched.flags);
- ptp = *execqp;
- if (ptp) {
- *execqp = ptp->u.alive.next;
- return ptp;
+ if (!pp->sched.taskq.local.busy.first || (act & ERTS_PTS_FLG_BUSY)) {
+ *execqp = pp->sched.taskq.local.first;
+ *processing_busy_q_p = 0;
+ }
+ else {
+ *execqp = pp->sched.taskq.local.busy.first;
+ *processing_busy_q_p = 1;
}
- ASSERT(!pp->sched.taskq.local);
-
- erts_port_task_sched_lock(&pp->sched);
- ptp = pp->sched.taskq.in.first;
- pp->sched.taskq.in.first = NULL;
- pp->sched.taskq.in.last = NULL;
- if (ptp)
- *execqp = ptp->u.alive.next;
- else
- erts_smp_atomic32_read_band_nob(&pp->sched.flags,
- ~ERTS_PTS_FLG_HAVE_TASKS);
- erts_port_task_sched_unlock(&pp->sched);
-
-
- return ptp;
-}
-
-static ERTS_INLINE void
-prepare_exec(Port *pp, ErtsPortTask **execqp)
-{
- erts_aint32_t act;
- *execqp = pp->sched.taskq.local;
-
- /* guess a likely value */
- act = ERTS_PTS_FLG_HAVE_TASKS|ERTS_PTS_FLG_IN_RUNQ;
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p);
while (1) {
erts_aint32_t new, exp;
@@ -281,11 +796,19 @@ prepare_exec(Port *pp, ErtsPortTask **execqp)
/* finalize_exec() return value != 0 if port should remain active */
static ERTS_INLINE int
-finalize_exec(Port *pp, ErtsPortTask **execq)
+finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q)
{
erts_aint32_t act;
- pp->sched.taskq.local = *execq;
+ if (!processing_busy_q)
+ pp->sched.taskq.local.first = *execq;
+ else {
+ pp->sched.taskq.local.busy.first = *execq;
+ ASSERT(*execq);
+ }
+
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execq, processing_busy_q);
+
*execq = NULL;
/* guess a likely value */
@@ -313,6 +836,155 @@ finalize_exec(Port *pp, ErtsPortTask **execq)
return (act & ERTS_PTS_FLG_HAVE_TASKS) != 0;
}
+static ERTS_INLINE erts_aint32_t
+select_queue_for_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p)
+{
+ erts_aint32_t flags = erts_smp_atomic32_read_nob(&pp->sched.flags);
+
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p);
+
+ if (flags & ERTS_PTS_FLG_BUSY) {
+ if (*processing_busy_q_p) {
+ ErtsPortTask *ptp;
+
+ ptp = pp->sched.taskq.local.busy.first = *execqp;
+ if (!ptp)
+ pp->sched.taskq.local.busy.last = NULL;
+ else if (!(ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY))
+ no_sig_dep_move_from_busyq(pp);
+
+ *execqp = pp->sched.taskq.local.first;
+ *processing_busy_q_p = 0;
+
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p);
+ }
+
+ return flags;
+ }
+
+ /* Not busy */
+
+ if (!*processing_busy_q_p && pp->sched.taskq.local.busy.first) {
+ pp->sched.taskq.local.first = *execqp;
+ *execqp = pp->sched.taskq.local.busy.first;
+ *processing_busy_q_p = 1;
+
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p);
+ }
+
+ return flags;
+}
+
+/*
+ * check_task_for_exec() returns a value !0 if the task
+ * is ok to execute; otherwise 0.
+ */
+static ERTS_INLINE int
+check_task_for_exec(Port *pp,
+ erts_aint32_t flags,
+ ErtsPortTask **execqp,
+ int *processing_busy_q_p,
+ ErtsPortTask *ptp)
+{
+
+ if (!*processing_busy_q_p) {
+ /* Processing normal queue */
+
+ ERTS_PT_DBG_CHK_TASK_QS(pp, ptp, *processing_busy_q_p);
+
+ if ((flags & ERTS_PTS_FLG_BUSY)
+ && (ptp->u.alive.flags & ERTS_PT_FLG_WAIT_BUSY)) {
+
+ busy_wait_move_to_busy_queue(pp, ptp);
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p);
+
+ return 0;
+ }
+
+ if (pp->sched.taskq.local.busy.last
+ && (ptp->u.alive.flags & ERTS_PT_FLG_SIG_DEP)) {
+
+ int res = !check_sig_dep_move_to_busy_queue(pp, ptp);
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p);
+
+ return res;
+ }
+
+ }
+ else {
+ /* Processing busy queue */
+
+ ASSERT(!(flags & ERTS_PTS_FLG_BUSY));
+
+ ERTS_PT_DBG_CHK_TASK_QS(pp, ptp, *processing_busy_q_p);
+
+ popped_from_busy_queue(pp, ptp, !*execqp);
+
+ if (!*execqp) {
+ *execqp = pp->sched.taskq.local.first;
+ *processing_busy_q_p = 0;
+ }
+
+ ERTS_PT_DBG_CHK_TASK_QS(pp, *execqp, *processing_busy_q_p);
+
+ }
+
+ return 1;
+}
+
+static ErtsPortTask *
+fetch_in_queue(Port *pp, ErtsPortTask **execqp)
+{
+ ErtsPortTask *ptp;
+ ErtsPortTaskHandleList *free_nshp = NULL;
+
+ erts_port_task_sched_lock(&pp->sched);
+
+ ptp = pp->sched.taskq.in.first;
+ pp->sched.taskq.in.first = NULL;
+ pp->sched.taskq.in.last = NULL;
+ if (ptp)
+ *execqp = ptp->u.alive.next;
+ else
+ erts_smp_atomic32_read_band_nob(&pp->sched.flags,
+ ~ERTS_PTS_FLG_HAVE_TASKS);
+
+
+ if (pp->sched.taskq.local.busy.nosuspend)
+ free_nshp = get_free_nosuspend_handles(pp);
+
+ erts_port_task_sched_unlock(&pp->sched);
+
+ if (free_nshp)
+ free_nosuspend_handles(free_nshp);
+
+ return ptp;
+}
+
+static ERTS_INLINE ErtsPortTask *
+select_task_for_exec(Port *pp,
+ ErtsPortTask **execqp,
+ int *processing_busy_q_p)
+{
+ ErtsPortTask *ptp;
+ erts_aint32_t flags;
+
+ flags = select_queue_for_exec(pp, execqp, processing_busy_q_p);
+
+ while (1) {
+ ptp = *execqp;
+ if (ptp)
+ *execqp = ptp->u.alive.next;
+ else {
+ ptp = fetch_in_queue(pp, execqp);
+ if (!ptp)
+ return NULL;
+ }
+ if (check_task_for_exec(pp, flags, execqp, processing_busy_q_p, ptp))
+ return ptp;
+ }
+}
+
/*
* Abort a scheduled task.
*/
@@ -358,6 +1030,12 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
&erts_port_task_outstanding_io_tasks) > 0);
erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
break;
+ case ERTS_PORT_TASK_PROC_SIG:
+ ptp->u.alive.td.psig.callback(NULL,
+ ERTS_PORT_SFLG_INVALID,
+ ERTS_PROC2PORT_SIG_ABORT,
+ &ptp->u.alive.td.psig.data);
+ break;
default:
break;
}
@@ -373,6 +1051,88 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp)
return res;
}
+void
+erts_port_task_abort_nosuspend_tasks(Port *pp)
+{
+ ErtsPortTaskHandleList *abort_list;
+#ifdef ERTS_SMP
+ ErtsThrPrgrDelayHandle dhndl = ERTS_THR_PRGR_DHANDLE_INVALID;
+#endif
+
+ erts_port_task_sched_lock(&pp->sched);
+ erts_smp_atomic32_read_band_nob(&pp->sched.flags,
+ ~ERTS_PTS_FLG_HAVE_NS_TASKS);
+ abort_list = pp->sched.taskq.local.busy.nosuspend;
+ pp->sched.taskq.local.busy.nosuspend = NULL;
+ erts_port_task_sched_unlock(&pp->sched);
+
+ while (abort_list) {
+#ifdef DEBUG
+ ErtsPortTaskHandle *saved_pthp;
+#endif
+ ErtsPortTaskType type;
+ ErtsPortTaskTypeData td;
+ ErtsPortTaskHandle *pthp;
+ ErtsPortTask *ptp;
+ ErtsPortTaskHandleList *pthlp;
+ erts_aint32_t old_state;
+
+ pthlp = abort_list;
+ abort_list = pthlp->u.next;
+
+#ifdef ERTS_SMP
+ if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
+ dhndl = erts_thr_progress_unmanaged_delay();
+#endif
+
+ pthp = &pthlp->handle;
+ ptp = handle2task(pthp);
+ if (!ptp) {
+#ifdef ERTS_SMP
+ if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
+ erts_thr_progress_unmanaged_continue(dhndl);
+#endif
+ schedule_port_task_handle_list_free(pthlp);
+ continue;
+ }
+
+#ifdef DEBUG
+ saved_pthp = ptp->u.alive.handle;
+ ERTS_SMP_READ_MEMORY_BARRIER;
+ old_state = erts_smp_atomic32_read_nob(&ptp->state);
+ if (old_state == ERTS_PT_STATE_SCHEDULED) {
+ ASSERT(saved_pthp == pthp);
+ }
+#endif
+
+ old_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state,
+ ERTS_PT_STATE_ABORTED,
+ ERTS_PT_STATE_SCHEDULED);
+ if (old_state != ERTS_PT_STATE_SCHEDULED) {
+ /* Task already aborted, executing, or executed */
+#ifdef ERTS_SMP
+ if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
+ erts_thr_progress_unmanaged_continue(dhndl);
+#endif
+ schedule_port_task_handle_list_free(pthlp);
+ continue;
+ }
+
+ reset_port_task_handle(pthp);
+
+ type = ptp->type;
+ td = ptp->u.alive.td;
+
+#ifdef ERTS_SMP
+ if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
+ erts_thr_progress_unmanaged_continue(dhndl);
+#endif
+ schedule_port_task_handle_list_free(pthlp);
+
+ abort_nosuspend_task(type, &td);
+ }
+}
+
/*
* Schedule a task.
*/
@@ -416,18 +1176,23 @@ erts_port_task_schedule(Eterm id,
if (!pp)
goto fail;
- ptp = port_task_alloc();
+ if (type != ERTS_PORT_TASK_PROC_SIG) {
+ ptp = port_task_alloc();
- ptp->type = type;
+ ptp->type = type;
+ ptp->u.alive.flags = 0;
- erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED);
+ erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED);
+
+ set_handle(ptp, pthp);
+ }
switch (type) {
case ERTS_PORT_TASK_INPUT:
case ERTS_PORT_TASK_OUTPUT: {
va_list argp;
va_start(argp, type);
- ptp->u.alive.u.io.event = va_arg(argp, ErlDrvEvent);
+ ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent);
va_end(argp);
erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
break;
@@ -435,18 +1200,32 @@ erts_port_task_schedule(Eterm id,
case ERTS_PORT_TASK_EVENT: {
va_list argp;
va_start(argp, type);
- ptp->u.alive.u.io.event = va_arg(argp, ErlDrvEvent);
- ptp->u.alive.u.io.event_data = va_arg(argp, ErlDrvEventData);
+ ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent);
+ ptp->u.alive.td.io.event_data = va_arg(argp, ErlDrvEventData);
va_end(argp);
erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
break;
}
+ case ERTS_PORT_TASK_PROC_SIG: {
+ ErtsProc2PortSigData *sigdp;
+ va_list argp;
+ ASSERT(!pthp);
+ va_start(argp, type);
+ sigdp = va_arg(argp, ErtsProc2PortSigData *);
+ ptp = p2p_sig_data_to_task(sigdp);
+ ptp->u.alive.td.psig.callback = va_arg(argp, ErtsProc2PortSigCallback);
+ ptp->u.alive.flags |= va_arg(argp, int);
+ va_end(argp);
+ if (ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND)
+ save_nosuspend_handle(pp, ptp);
+ else
+ set_handle(ptp, pthp);
+ break;
+ }
default:
break;
}
- set_handle(ptp, pthp);
-
act = enqueue_task(pp, ptp);
if (act & ERTS_PTS_FLG_EXIT) {
reset_handle(ptp);
@@ -572,6 +1351,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
{
Port *pp;
ErtsPortTask *execq;
+ int processing_busy_q;
int res = 0;
int reds = ERTS_PORT_REDS_EXECUTE;
erts_aint_t io_tasks_executed = 0;
@@ -606,7 +1386,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
erts_smp_spin_unlock(&erts_sched_stat.lock);
}
- prepare_exec(pp, &execq);
+ prepare_exec(pp, &execq, &processing_busy_q);
erts_smp_port_lock(pp);
@@ -617,14 +1397,14 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
fpe_was_unmasked = erts_block_fpe();
- state = erts_atomic_read_nob(&pp->state);
+ state = erts_atomic32_read_nob(&pp->state);
goto begin_handle_tasks;
while (1) {
erts_aint32_t task_state;
ErtsPortTask *ptp;
- ptp = pop_task(pp, &execq);
+ ptp = select_task_for_exec(pp, &execq, &processing_busy_q);
if (!ptp)
break;
@@ -645,40 +1425,44 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
switch (ptp->type) {
case ERTS_PORT_TASK_TIMEOUT:
reds += ERTS_PORT_REDS_TIMEOUT;
- if (!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)) {
+ if (!(state & ERTS_PORT_SFLGS_DEAD)) {
DTRACE_DRIVER(driver_timeout, pp);
(*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data);
}
break;
case ERTS_PORT_TASK_INPUT:
reds += ERTS_PORT_REDS_INPUT;
- ASSERT((erts_atomic32_read_nob(&pp->state)
- & ERTS_PORT_SFLGS_DEAD) == 0);
+ ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
DTRACE_DRIVER(driver_ready_input, pp);
/* NOTE some windows drivers use ->ready_input for input and output */
(*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data,
- ptp->u.alive.u.io.event);
+ ptp->u.alive.td.io.event);
io_tasks_executed++;
break;
case ERTS_PORT_TASK_OUTPUT:
reds += ERTS_PORT_REDS_OUTPUT;
- ASSERT((erts_atomic32_read_nob(&pp->state)
- & ERTS_PORT_SFLGS_DEAD) == 0);
+ ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
DTRACE_DRIVER(driver_ready_output, pp);
(*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data,
- ptp->u.alive.u.io.event);
+ ptp->u.alive.td.io.event);
io_tasks_executed++;
break;
case ERTS_PORT_TASK_EVENT:
reds += ERTS_PORT_REDS_EVENT;
- ASSERT((erts_atomic32_read_nob(&pp->state)
- & ERTS_PORT_SFLGS_DEAD) == 0);
+ ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
DTRACE_DRIVER(driver_event, pp);
(*pp->drv_ptr->event)((ErlDrvData) pp->drv_data,
- ptp->u.alive.u.io.event,
- ptp->u.alive.u.io.event_data);
+ ptp->u.alive.td.io.event,
+ ptp->u.alive.td.io.event_data);
io_tasks_executed++;
break;
+ case ERTS_PORT_TASK_PROC_SIG:
+ ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0);
+ reds += ptp->u.alive.td.psig.callback(pp,
+ state,
+ ERTS_PROC2PORT_SIG_EXEC,
+ &ptp->u.alive.td.psig.data);
+ break;
case ERTS_PORT_TASK_DIST_CMD:
reds += erts_dist_command(pp, CONTEXT_REDS-reds);
break;
@@ -740,7 +1524,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
#endif
- active = finalize_exec(pp, &execq);
+ active = finalize_exec(pp, &execq, processing_busy_q);
erts_port_release(pp);
@@ -858,24 +1642,30 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp)
break;
case ERTS_PORT_TASK_INPUT:
erts_stale_drv_select(pp->common.id,
- ptp->u.alive.u.io.event,
+ ptp->u.alive.td.io.event,
DO_READ,
1);
break;
case ERTS_PORT_TASK_OUTPUT:
erts_stale_drv_select(pp->common.id,
- ptp->u.alive.u.io.event,
+ ptp->u.alive.td.io.event,
DO_WRITE,
1);
break;
case ERTS_PORT_TASK_EVENT:
erts_stale_drv_select(pp->common.id,
- ptp->u.alive.u.io.event,
+ ptp->u.alive.td.io.event,
0,
1);
break;
case ERTS_PORT_TASK_DIST_CMD:
break;
+ case ERTS_PORT_TASK_PROC_SIG:
+ ptp->u.alive.td.psig.callback(NULL,
+ ERTS_PORT_SFLG_INVALID,
+ ERTS_PROC2PORT_SIG_ABORT_CLOSED,
+ &ptp->u.alive.td.psig.data);
+ break;
default:
erl_exit(ERTS_ABORT_EXIT,
"Invalid port task type: %d\n",
@@ -944,4 +1734,5 @@ erts_port_task_init(void)
erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks,
(erts_aint_t) 0);
init_port_task_alloc();
+ init_busy_caller_table_alloc();
}
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 71b1cc6e25..c41f8104c7 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -27,6 +27,9 @@
#define ERTS_PORT_TASK_H_BASIC_TYPES__
#include "erl_sys_driver.h"
#include "erl_smp.h"
+#define ERL_PORT_GET_PORT_TYPE_ONLY__
+#include "erl_port.h"
+#undef ERL_PORT_GET_PORT_TYPE_ONLY__
typedef erts_smp_atomic_t ErtsPortTaskHandle;
#endif
@@ -43,12 +46,19 @@ typedef erts_smp_atomic_t ErtsPortTaskHandle;
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
+#define ERTS_PT_FLG_WAIT_BUSY (1 << 0)
+#define ERTS_PT_FLG_SIG_DEP (1 << 1)
+#define ERTS_PT_FLG_NOSUSPEND (1 << 2)
+#define ERTS_PT_FLG_REF (1 << 3)
+#define ERTS_PT_FLG_BAD_OUTPUT (1 << 4)
+
typedef enum {
ERTS_PORT_TASK_INPUT,
ERTS_PORT_TASK_OUTPUT,
ERTS_PORT_TASK_EVENT,
ERTS_PORT_TASK_TIMEOUT,
- ERTS_PORT_TASK_DIST_CMD
+ ERTS_PORT_TASK_DIST_CMD,
+ ERTS_PORT_TASK_PROC_SIG
} ErtsPortTaskType;
#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
@@ -61,14 +71,35 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
#define ERTS_PTS_FLG_HAVE_TASKS (((erts_aint32_t) 1) << 2)
#define ERTS_PTS_FLG_EXIT (((erts_aint32_t) 1) << 3)
#define ERTS_PTS_FLG_BUSY (((erts_aint32_t) 1) << 4)
+#define ERTS_PTS_FLG_HAVE_BUSY_TASKS (((erts_aint32_t) 1) << 5)
+#define ERTS_PTS_FLG_HAVE_NS_TASKS (((erts_aint32_t) 1) << 6)
+#define ERTS_PTS_FLG_PARALLELISM (((erts_aint32_t) 1) << 7)
+#define ERTS_PTS_FLG_FORCE_SCHED (((erts_aint32_t) 1) << 8)
+
+#define ERTS_PTS_FLGS_FORCE_SCHEDULE_OP \
+ (ERTS_PTS_FLG_EXIT \
+ | ERTS_PTS_FLG_HAVE_BUSY_TASKS \
+ | ERTS_PTS_FLG_HAVE_TASKS \
+ | ERTS_PTS_FLG_EXEC \
+ | ERTS_PTS_FLG_FORCE_SCHED)
typedef struct ErtsPortTask_ ErtsPortTask;
+typedef struct ErtsPortTaskBusyCallerTable_ ErtsPortTaskBusyCallerTable;
+typedef struct ErtsPortTaskHandleList_ ErtsPortTaskHandleList;
typedef struct {
Port *next;
struct {
- ErtsPortTask *local;
+ struct {
+ struct {
+ ErtsPortTask *first;
+ ErtsPortTask *last;
+ ErtsPortTaskBusyCallerTable *table;
+ ErtsPortTaskHandleList *nosuspend;
+ } busy;
+ ErtsPortTask *first;
+ } local;
struct {
ErtsPortTask *first;
ErtsPortTask *last;
@@ -87,6 +118,7 @@ ERTS_GLB_INLINE void erts_port_task_init_sched(ErtsPortTaskSched *ptsp,
ERTS_GLB_INLINE void erts_port_task_fini_sched(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_lock(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp);
+ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp);
#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
@@ -110,7 +142,11 @@ ERTS_GLB_INLINE void
erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
{
ptsp->next = NULL;
- ptsp->taskq.local = NULL;
+ ptsp->taskq.local.busy.first = NULL;
+ ptsp->taskq.local.busy.last = NULL;
+ ptsp->taskq.local.busy.table = NULL;
+ ptsp->taskq.local.busy.nosuspend = NULL;
+ ptsp->taskq.local.first = NULL;
ptsp->taskq.in.first = NULL;
ptsp->taskq.in.last = NULL;
erts_smp_atomic32_init_nob(&ptsp->flags, 0);
@@ -135,6 +171,17 @@ erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp)
#endif
}
+ERTS_GLB_INLINE int
+erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp)
+{
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+ return erts_lc_mtx_is_locked(&ptsp->mtx);
+#else
+ return 0;
+#endif
+}
+
+
ERTS_GLB_INLINE void
erts_port_task_fini_sched(ErtsPortTaskSched *ptsp)
{
@@ -162,12 +209,15 @@ void erts_port_task_init(void);
#endif
int erts_port_task_abort(ErtsPortTaskHandle *);
+void erts_port_task_abort_nosuspend_tasks(Port *);
+
int erts_port_task_schedule(Eterm,
ErtsPortTaskHandle *,
ErtsPortTaskType,
...);
void erts_port_task_free_port(Port *);
int erts_port_is_scheduled(Port *);
+ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void);
#ifdef ERTS_SMP
void erts_enqueue_port(ErtsRunQueue *rq, Port *pp);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6cf267b01e..332cc81d48 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1490,7 +1490,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
ERTS_PORT_SFLG_HALT);
erts_smp_atomic32_inc_nob(&erts_halt_progress);
if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING)))
- erts_do_exit_port(prt, prt->common.id, am_killed);
+ erts_deliver_port_exit(prt, prt->common.id, am_killed, 0);
}
erts_port_release(prt);
@@ -6086,6 +6086,8 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port)
if (!(c_p_locks & ERTS_PROC_LOCK_STATUS))
erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS);
+ if (suspend && busy_port && erts_system_monitor_flags.busy_port)
+ monitor_generic(c_p, am_busy_port, busy_port->common.id);
}
void
@@ -8183,14 +8185,15 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
switch(lnk->type) {
case LINK_PID:
if(is_internal_port(item)) {
- Port *prt = erts_id2port(item);
- if (prt) {
- rlnk = erts_remove_link(&ERTS_P_LINKS(prt), p->common.id);
- if (rlnk)
- erts_destroy_link(rlnk);
- erts_do_exit_port(prt, p->common.id, reason);
- erts_port_release(prt);
- }
+ Port *prt = erts_port_lookup(item, ERTS_PORT_SFLGS_INVALID_LOOKUP);
+ if (prt)
+ erts_port_exit(NULL,
+ (ERTS_PORT_SIG_FLG_FORCE_SCHED
+ | ERTS_PORT_SIG_FLG_BROKEN_LINK),
+ prt,
+ p->common.id,
+ reason,
+ NULL);
}
else if(is_external_port(item)) {
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 0e63d41fda..1416aa6166 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -134,6 +134,7 @@ typedef struct {
typedef int ErtsThrPrgrDelayHandle;
#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
+#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)
extern ErtsThrPrgr erts_thr_prgr__;
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
new file mode 100644
index 0000000000..0d6df699ff
--- /dev/null
+++ b/erts/emulator/beam/erl_trace.h
@@ -0,0 +1,138 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+
+#ifndef ERL_TRACE_H__
+#define ERL_TRACE_H__
+
+struct binary;
+
+/* erl_bif_trace.c */
+Eterm erl_seq_trace_info(Process *p, Eterm arg1);
+void erts_system_monitor_clear(Process *c_p);
+void erts_system_profile_clear(Process *c_p);
+
+/* erl_trace.c */
+void erts_init_trace(void);
+void erts_trace_check_exiting(Eterm exiting);
+Eterm erts_set_system_seq_tracer(Process *c_p,
+ ErtsProcLocks c_p_locks,
+ Eterm new);
+Eterm erts_get_system_seq_tracer(void);
+void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp);
+void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp);
+void erts_set_system_monitor(Eterm monitor);
+Eterm erts_get_system_monitor(void);
+
+#ifdef ERTS_SMP
+void erts_check_my_tracer_proc(Process *);
+void erts_block_sys_msg_dispatcher(void);
+void erts_release_sys_msg_dispatcher(void);
+void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
+ Eterm,
+ Eterm,
+ ErlHeapFragment *));
+void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
+#endif
+
+void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
+void trace_send(Process*, Eterm, Eterm);
+void trace_receive(Process*, Eterm);
+Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, Eterm* args,
+ int local, Eterm *tracer_pid);
+void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid);
+void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
+ Eterm *tracer);
+void erts_trace_return_to(Process *p, BeamInstr *pc);
+void trace_sched(Process*, Eterm);
+void trace_proc(Process*, Process*, Eterm, Eterm);
+void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
+void save_calls(Process *p, Export *);
+void trace_gc(Process *p, Eterm what);
+/* port tracing */
+void trace_virtual_sched(Process*, Eterm);
+void trace_sched_ports(Port *pp, Eterm);
+void trace_sched_ports_where(Port *pp, Eterm, Eterm);
+void trace_port(Port *, Eterm what, Eterm data);
+void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name);
+
+/* system_profile */
+void erts_set_system_profile(Eterm profile);
+Eterm erts_get_system_profile(void);
+void profile_scheduler(Eterm scheduler_id, Eterm);
+void profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us);
+void profile_runnable_proc(Process* p, Eterm status);
+void profile_runnable_port(Port* p, Eterm status);
+void erts_system_profile_setup_active_schedulers(void);
+
+/* system_monitor */
+void monitor_long_gc(Process *p, Uint time);
+void monitor_large_heap(Process *p);
+void monitor_generic(Process *p, Eterm type, Eterm spec);
+Uint erts_trace_flag2bit(Eterm flag);
+int erts_trace_flags(Eterm List,
+ Uint *pMask, Eterm *pTracer, int *pCpuTimestamp);
+Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);
+
+#ifdef ERTS_SMP
+void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
+#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) \
+do { \
+ if ((ESDP)->pending_trace_msgs) \
+ erts_send_pending_trace_msgs((ESDP)); \
+} while (0)
+#else
+#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)
+#endif
+
+#define seq_trace_output(token, msg, type, receiver, process) \
+seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL)
+#define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \
+seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom))
+void seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
+ Eterm receiver, Process *process, Eterm exitfrom);
+
+int seq_trace_update_send(Process *process);
+
+Eterm erts_seq_trace(Process *process,
+ Eterm atom_type, Eterm atom_true_or_false,
+ int build_result);
+
+struct trace_pattern_flags {
+ unsigned int breakpoint : 1; /* Set if any other is set */
+ unsigned int local : 1; /* Local call trace breakpoint */
+ unsigned int meta : 1; /* Metadata trace breakpoint */
+ unsigned int call_count : 1; /* Fast call count breakpoint */
+ unsigned int call_time : 1; /* Fast call time breakpoint */
+};
+extern const struct trace_pattern_flags erts_trace_pattern_flags_off;
+extern int erts_call_time_breakpoint_tracing;
+int erts_set_trace_pattern(Eterm* mfa, int specified,
+ struct binary* match_prog_set, struct binary *meta_match_prog_set,
+ int on, struct trace_pattern_flags,
+ Eterm meta_tracer_pid);
+void
+erts_get_default_trace_pattern(int *trace_pattern_is_on,
+ struct binary **match_spec,
+ struct binary **meta_match_spec,
+ struct trace_pattern_flags *trace_pattern_flags,
+ Eterm *meta_tracer_pid);
+void erts_bif_trace_init(void);
+
+#endif /* ERL_TRACE_H__ */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 6713481a04..d91dd7d9ec 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -578,11 +578,35 @@ Eterm erts_realloc_binary(Eterm bin, size_t size);
/* erl_bif_info.c */
+Eterm
+erts_bld_port_info(Eterm **hpp,
+ ErlOffHeap *ohp,
+ Uint *szp,
+ Port *prt,
+ Eterm item);
+
void erts_bif_info_init(void);
/* bif.c */
Eterm erts_make_ref(Process *);
Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]);
+void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+
+ERTS_GLB_INLINE Eterm
+erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Eterm
+erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS])
+{
+ Eterm *hp = HAlloc(c_p, REF_THING_SIZE);
+ write_ref_thing(hp, ref[0], ref[1], ref[2]);
+ return make_internal_ref(hp);
+}
+
+#endif
+
void erts_queue_monitor_message(Process *,
ErtsProcLocks*,
Eterm,
@@ -596,13 +620,6 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg);
Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
-/* erl_bif_port.c */
-
-/* erl_bif_trace.c */
-Eterm erl_seq_trace_info(Process *p, Eterm arg1);
-void erts_system_monitor_clear(Process *c_p);
-void erts_system_profile_clear(Process *c_p);
-
/* beam_load.c */
typedef struct {
BeamInstr* current; /* Pointer to: Mod, Name, Arity */
@@ -892,7 +909,6 @@ typedef struct {
#define ERTS_SPAWN_DRIVER 1
#define ERTS_SPAWN_EXECUTABLE 2
#define ERTS_SPAWN_ANY (ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXECUTABLE)
-
int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_locked);
void erts_destroy_driver(erts_driver_t *drv);
int erts_save_suspend_process_on_port(Port*, Process*);
@@ -900,10 +916,6 @@ Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int
int erts_is_port_ioq_empty(Port *);
void erts_terminate_port(Port *);
void erts_init_io(int, int);
-void erts_do_exit_port(Port *, Eterm, Eterm);
-void erts_port_command(Process *, Eterm, Port *, Eterm);
-Eterm erts_port_control(Process*, Port*, Uint, Eterm);
-int erts_write_to_port(Eterm caller_id, Port *p, Eterm list);
void erts_raw_port_command(Port*, byte*, Uint);
void driver_report_exit(ErlDrvPort, int);
LineBuf* allocate_linebuf(int);
@@ -963,79 +975,6 @@ char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int
#define ERTS_UTF8_ERROR 2
#define ERTS_UTF8_ANALYZE_MORE 3
-/* erl_trace.c */
-void erts_init_trace(void);
-void erts_trace_check_exiting(Eterm exiting);
-Eterm erts_set_system_seq_tracer(Process *c_p,
- ErtsProcLocks c_p_locks,
- Eterm new);
-Eterm erts_get_system_seq_tracer(void);
-void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp);
-void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp);
-void erts_set_system_monitor(Eterm monitor);
-Eterm erts_get_system_monitor(void);
-
-#ifdef ERTS_SMP
-void erts_check_my_tracer_proc(Process *);
-void erts_block_sys_msg_dispatcher(void);
-void erts_release_sys_msg_dispatcher(void);
-void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
- Eterm,
- Eterm,
- ErlHeapFragment *));
-void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
-#endif
-
-void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
-void trace_send(Process*, Eterm, Eterm);
-void trace_receive(Process*, Eterm);
-Uint32 erts_call_trace(Process *p, BeamInstr mfa[], Binary *match_spec, Eterm* args,
- int local, Eterm *tracer_pid);
-void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid);
-void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
- Eterm *tracer);
-void erts_trace_return_to(Process *p, BeamInstr *pc);
-void trace_sched(Process*, Eterm);
-void trace_proc(Process*, Process*, Eterm, Eterm);
-void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
-void save_calls(Process *p, Export *);
-void trace_gc(Process *p, Eterm what);
-/* port tracing */
-void trace_virtual_sched(Process*, Eterm);
-void trace_sched_ports(Port *pp, Eterm);
-void trace_sched_ports_where(Port *pp, Eterm, Eterm);
-void trace_port(Port *, Eterm what, Eterm data);
-void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name);
-
-/* system_profile */
-void erts_set_system_profile(Eterm profile);
-Eterm erts_get_system_profile(void);
-void profile_scheduler(Eterm scheduler_id, Eterm);
-void profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us);
-void profile_runnable_proc(Process* p, Eterm status);
-void profile_runnable_port(Port* p, Eterm status);
-void erts_system_profile_setup_active_schedulers(void);
-
-/* system_monitor */
-void monitor_long_gc(Process *p, Uint time);
-void monitor_large_heap(Process *p);
-void monitor_generic(Process *p, Eterm type, Eterm spec);
-Uint erts_trace_flag2bit(Eterm flag);
-int erts_trace_flags(Eterm List,
- Uint *pMask, Eterm *pTracer, int *pCpuTimestamp);
-Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);
-
-#ifdef ERTS_SMP
-void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
-#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) \
-do { \
- if ((ESDP)->pending_trace_msgs) \
- erts_send_pending_trace_msgs((ESDP)); \
-} while (0)
-#else
-#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)
-#endif
-
void bin_write(int, void*, byte*, size_t);
int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */
@@ -1053,9 +992,16 @@ char* Sint_to_buf(Sint, struct Sint_buf*);
#define ERTS_IOLIST_TYPE 2
Eterm buf_to_intlist(Eterm**, char*, size_t, Eterm); /* most callers pass plain char*'s */
-int io_list_to_buf(Eterm, char*, int);
-int io_list_to_buf2(Eterm, char*, int);
-int erts_iolist_size(Eterm, Uint *);
+
+#define ERTS_IOLIST_TO_BUF_OVERFLOW (~((ErlDrvSizeT) 0))
+#define ERTS_IOLIST_TO_BUF_TYPE_ERROR (~((ErlDrvSizeT) 1))
+#define ERTS_IOLIST_TO_BUF_FAILED(R) \
+ (((R) & (~((ErlDrvSizeT) 1))) == (~((ErlDrvSizeT) 1)))
+#define ERTS_IOLIST_TO_BUF_SUCCEEDED(R) \
+ (!ERTS_IOLIST_TO_BUF_FAILED((R)))
+
+ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT);
+int erts_iolist_size(Eterm, ErlDrvSizeT *);
int is_string(Eterm);
void erl_at_exit(void (*) (void*), void*);
Eterm collect_memory(Process *);
@@ -1100,39 +1046,6 @@ Uint erts_current_reductions(Process* current, Process *p);
int erts_print_system_version(int to, void *arg, Process *c_p);
int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
-#define seq_trace_output(token, msg, type, receiver, process) \
-seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL)
-#define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \
-seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom))
-void seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
- Eterm receiver, Process *process, Eterm exitfrom);
-
-int seq_trace_update_send(Process *process);
-
-Eterm erts_seq_trace(Process *process,
- Eterm atom_type, Eterm atom_true_or_false,
- int build_result);
-
-struct trace_pattern_flags {
- unsigned int breakpoint : 1; /* Set if any other is set */
- unsigned int local : 1; /* Local call trace breakpoint */
- unsigned int meta : 1; /* Metadata trace breakpoint */
- unsigned int call_count : 1; /* Fast call count breakpoint */
- unsigned int call_time : 1; /* Fast call time breakpoint */
-};
-extern const struct trace_pattern_flags erts_trace_pattern_flags_off;
-extern int erts_call_time_breakpoint_tracing;
-int erts_set_trace_pattern(Eterm* mfa, int specified,
- Binary* match_prog_set, Binary *meta_match_prog_set,
- int on, struct trace_pattern_flags,
- Eterm meta_tracer_pid);
-void
-erts_get_default_trace_pattern(int *trace_pattern_is_on,
- Binary **match_spec,
- Binary **meta_match_spec,
- struct trace_pattern_flags *trace_pattern_flags,
- Eterm *meta_tracer_pid);
-void erts_bif_trace_init(void);
/*
** Call_trace uses this API for the parameter matching functions
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 6776f72894..2a55a7c09f 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -43,6 +43,8 @@
#include "erl_version.h"
#include "error.h"
#include "erl_async.h"
+#define ERTS_WANT_EXTERNAL_TAGS
+#include "external.h"
#include "dtrace-wrapper.h"
extern ErlDrvEntry fd_driver_entry;
@@ -69,6 +71,11 @@ erts_driver_t vanilla_driver;
erts_driver_t spawn_driver;
erts_driver_t fd_driver;
+int erts_port_synchronous_ops = 0;
+int erts_port_schedule_all_ops = 0;
+int erts_port_parallelism = 0;
+
+static void deliver_result(Eterm sender, Eterm pid, Eterm res);
static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *);
static void terminate_port(Port *p);
static void pdl_init(void);
@@ -254,9 +261,12 @@ static void insert_port_struct(void *vprt, Eterm data)
erts_atomic32_init_relb(&prt->state, ERTS_PORT_SFLG_INITIALIZING);
}
+#define ERTS_CREATE_PORT_FLAG_PARALLELISM (1 << 0)
+
static Port *create_port(char *name,
erts_driver_t *driver,
erts_mtx_t *driver_lock,
+ int create_flags,
Eterm pid,
int *enop)
{
@@ -264,6 +274,7 @@ static Port *create_port(char *name,
char *p;
size_t port_size, size;
erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED;
+ erts_aint32_t x_pts_flgs = 0;
#ifdef DEBUG
/* Make sure the debug flags survives until port is freed */
state |= ERTS_PORT_SFLG_PORT_DEBUG;
@@ -373,6 +384,15 @@ static Port *create_port(char *name,
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+ if (erts_port_schedule_all_ops)
+ x_pts_flgs |= ERTS_PTS_FLG_FORCE_SCHED;
+
+ if (create_flags & ERTS_CREATE_PORT_FLAG_PARALLELISM)
+ x_pts_flgs |= ERTS_PTS_FLG_PARALLELISM;
+
+ if (x_pts_flgs)
+ erts_smp_atomic32_read_bor_nob(&prt->sched.flags, x_pts_flgs);
+
erts_atomic32_set_relb(&prt->state, state);
return prt;
}
@@ -525,6 +545,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
int error_type, error_number;
int port_errno = 0;
erts_mtx_t *driver_lock = NULL;
+ int cprt_flgs = 0;
ERTS_SMP_CHK_NO_PROC_LOCKS;
@@ -599,7 +620,10 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
* to allow message sending and setting timers in the start function.
*/
- port = create_port(name, driver, driver_lock, pid, &port_errno);
+ if (opts->parallelism)
+ cprt_flgs |= ERTS_CREATE_PORT_FLAG_PARALLELISM;
+
+ port = create_port(name, driver, driver_lock, cprt_flgs, pid, &port_errno);
if (!port) {
if (driver->handle) {
erts_smp_rwmtx_rlock(&erts_driver_list_lock);
@@ -713,6 +737,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
char* name, /* Driver name */
ErlDrvData drv_data) /* Driver data */
{
+ int cprt_flgs = 0;
Port *creator_port;
Port* port;
erts_driver_t *driver;
@@ -753,7 +778,11 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
erts_smp_rwmtx_runlock(&erts_driver_list_lock);
- port = create_port(name, driver, driver_lock, pid, NULL);
+ /* Inherit parallelism flag from parent */
+ if (ERTS_PTS_FLG_PARALLELISM &
+ erts_smp_atomic32_read_nob(&creator_port->sched.flags))
+ cprt_flgs |= ERTS_CREATE_PORT_FLAG_PARALLELISM;
+ port = create_port(name, driver, driver_lock, cprt_flgs, pid, NULL);
if (!port) {
if (driver->handle) {
erts_smp_rwmtx_rlock(&erts_driver_list_lock);
@@ -806,10 +835,15 @@ erts_smp_xports_unlock(Port *prt)
xplp = prt->xports;
ASSERT(xplp);
while (xplp) {
+ Port *rprt = xplp->port;
ErtsXPortsList *free_xplp;
- if (xplp->port->xports)
- erts_smp_xports_unlock(xplp->port);
- erts_port_release(xplp->port);
+ erts_aint32_t state;
+ if (rprt->xports)
+ erts_smp_xports_unlock(rprt);
+ state = erts_atomic32_read_nob(&rprt->state);
+ if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(rprt))
+ terminate_port(rprt);
+ erts_port_release(rprt);
free_xplp = xplp;
xplp = xplp->next;
xports_list_free(free_xplp);
@@ -849,8 +883,8 @@ io_list_to_vec(Eterm obj, /* io-list */
DECLARE_ESTACK(s);
Eterm* objp;
char *buf = cbin->orig_bytes;
- ErlDrvSizeT len = cbin->orig_size;
- ErlDrvSizeT csize = 0;
+ Uint len = cbin->orig_size;
+ Uint csize = 0;
int vlen = 0;
char* cptr = buf;
@@ -965,7 +999,7 @@ io_list_to_vec(Eterm obj, /* io-list */
#define IO_LIST_VEC_COUNT(obj) \
do { \
- ErlDrvSizeT _size = binary_size(obj); \
+ Uint _size = binary_size(obj); \
Eterm _real; \
ERTS_DECLARE_DUMMY(Uint _offset); \
int _bitoffs; \
@@ -1016,8 +1050,9 @@ do { \
*/
static int
-io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
- Uint* pvsize, Uint* pcsize, Uint* total_size)
+io_list_vec_len(Eterm obj, int* vsize, Uint* csize,
+ Uint* pvsize, Uint* pcsize,
+ ErlDrvSizeT* total_size)
{
DECLARE_ESTACK(s);
Eterm* objp;
@@ -1028,7 +1063,7 @@ io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
Uint p_v_size = 0;
Uint p_c_size = 0;
Uint p_in_clist = 0;
- Uint total;
+ Uint total; /* Uint due to halfword emulator */
goto L_jump_start; /* avoid a push */
@@ -1088,7 +1123,7 @@ io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
if (total < c_size) {
goto L_overflow_error;
}
- *total_size = total;
+ *total_size = (ErlDrvSizeT) total;
DESTROY_ESTACK(s);
*vsize = v_size;
@@ -1103,56 +1138,691 @@ io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
return 1;
}
-/* write data to a port */
-int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
-{
- char *buf;
- erts_driver_t *drv = p->drv_ptr;
- Uint size;
+typedef enum {
+ ERTS_TRY_IMM_DRV_CALL_OK,
+ ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK,
+ ERTS_TRY_IMM_DRV_CALL_INVALID_PORT,
+ ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS
+} ErtsTryImmDrvCallResult;
+
+typedef struct {
+ Process *c_p; /* Currently executing process (unlocked) */
+ Port *port; /* Port to operate on */
+ Eterm port_op; /* port operation as an atom */
+ erts_aint32_t state; /* in: invalid state; out: read state (if read) */
+ erts_aint32_t sched_flags; /* in: invalid flags; out: read flags (if read) */
+ int async; /* Asynchronous operation */
+ int pre_chk_sched_flags; /* Check sched flags before lock? */
int fpe_was_unmasked;
-
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
+} ErtsTryImmDrvCallState;
+
+#define ERTS_INIT_TRY_IMM_DRV_CALL_STATE(C_P, PRT, SFLGS, PTS_FLGS, A, PRT_OP) \
+ {(C_P), (PRT), (PRT_OP), (SFLGS), (PTS_FLGS), (A), 1, 0}
+
+/*
+ * Try doing an immediate driver callback call from a process. If
+ * this fail, the operation should be scheduled in the normal case...
+ *
+ */
+static ERTS_INLINE ErtsTryImmDrvCallResult
+try_imm_drv_call(ErtsTryImmDrvCallState *sp)
+{
+ ErtsTryImmDrvCallResult res;
+ erts_aint32_t invalid_state, invalid_sched_flags;
+ Port *prt = sp->port;
+ Process *c_p = sp->c_p;
+
+ ASSERT(is_atom(sp->port_op));
+
+ invalid_sched_flags = ERTS_PTS_FLGS_FORCE_SCHEDULE_OP;
+ invalid_sched_flags |= sp->sched_flags;
+ if (sp->async)
+ invalid_sched_flags |= ERTS_PTS_FLG_PARALLELISM;
+
+ if (sp->pre_chk_sched_flags) {
+ sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+ if (sp->sched_flags & invalid_sched_flags)
+ return ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
+ }
+
+ if (erts_smp_port_trylock(prt) == EBUSY)
+ return ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK;
+
+ invalid_state = sp->state;
+ sp->state = erts_atomic32_read_nob(&prt->state);
+ if (sp->state & invalid_state) {
+ res = ERTS_TRY_IMM_DRV_CALL_INVALID_PORT;
+ goto locked_fail;
+ }
+
+ sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+ if (sp->sched_flags & invalid_sched_flags) {
+ res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
+ goto locked_fail;
+ }
+
+ if (c_p) {
+ if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
+ trace_virtual_sched(c_p, am_out);
+ if (erts_system_profile_flags.runnable_procs
+ && erts_system_profile_flags.exclusive)
+ profile_runnable_proc(c_p, am_inactive);
+
+ erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+ }
+
ERTS_SMP_CHK_NO_PROC_LOCKS;
- p->caller = caller_id;
- if (drv->outputv != NULL) {
- Uint vsize;
- Uint csize;
- Uint pvsize;
- Uint pcsize;
- ErlDrvSizeT blimit;
+ if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+ trace_sched_ports_where(prt, am_in, sp->port_op);
+ if (erts_system_profile_flags.runnable_ports
+ && !erts_port_is_scheduled(prt))
+ profile_runnable_port(prt, am_active);
+
+ sp->fpe_was_unmasked = erts_block_fpe();
+
+ return ERTS_TRY_IMM_DRV_CALL_OK;
+
+locked_fail:
+ erts_port_release(prt);
+ return res;
+}
+
+static ERTS_INLINE void
+finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
+{
+ Port *prt = sp->port;
+ Process *c_p = sp->c_p;
+
+ erts_unblock_fpe(sp->fpe_was_unmasked);
+
+#ifdef ERTS_SMP
+ if (prt->xports)
+ erts_smp_xports_unlock(prt);
+ ASSERT(!prt->xports);
+#endif
+
+ if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+ trace_sched_ports_where(prt, am_out, sp->port_op);
+ if (erts_system_profile_flags.runnable_ports
+ && !erts_port_is_scheduled(prt))
+ profile_runnable_port(prt, am_inactive);
+
+ erts_port_release(prt);
+
+ if (c_p) {
+ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+
+ if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
+ trace_virtual_sched(c_p, am_in);
+ if (erts_system_profile_flags.runnable_procs
+ && erts_system_profile_flags.exclusive)
+ profile_runnable_proc(c_p, am_active);
+ }
+}
+
+#define ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE (REF_THING_SIZE + 3)
+
+static ERTS_INLINE void
+queue_port_sched_op_reply(Process *rp,
+ ErtsProcLocks *rp_locksp,
+ Eterm *hp_start,
+ Eterm *hp,
+ Uint h_size,
+ ErlHeapFragment* bp,
+ Uint32 *ref_num,
+ Eterm msg)
+{
+ Eterm ref = make_internal_ref(hp);
+ write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]);
+ hp += REF_THING_SIZE;
+
+ msg = TUPLE2(hp, ref, msg);
+ hp += 3;
+
+ if (!bp) {
+ HRelease(rp, hp_start + h_size, hp);
+ }
+ else {
+ Uint used_h_size = hp - hp_start;
+ ASSERT(h_size >= used_h_size);
+ if (h_size > used_h_size)
+ bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1);
+ }
+
+ erts_queue_message(rp,
+ rp_locksp,
+ bp,
+ msg,
+ NIL
+#ifdef USE_VM_PROBES
+ , NIL
+#endif
+ );
+}
+
+static void
+port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg)
+{
+ Process *rp = erts_proc_lookup_raw(to);
+ if (rp) {
+ ErlOffHeap *ohp;
+ ErlHeapFragment* bp;
+ Eterm msg_copy;
+ Uint hsz, msg_sz;
+ Eterm *hp, *hp_start;
+ ErtsProcLocks rp_locks = 0;
+
+ hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+ if (is_immed(msg))
+ msg_sz = 0;
+ else {
+ msg_sz = size_object(msg);
+ hsz += msg_sz;
+ }
+
+ hp_start = hp = erts_alloc_message_heap(hsz,
+ &bp,
+ &ohp,
+ rp,
+ &rp_locks);
+ if (is_immed(msg))
+ msg_copy = msg;
+ else
+ msg_copy = copy_struct(msg, msg_sz, &hp, ohp);
+
+ queue_port_sched_op_reply(rp,
+ &rp_locks,
+ hp_start,
+ hp,
+ hsz,
+ bp,
+ ref_num,
+ msg_copy);
+
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ }
+}
+
+
+ErtsPortOpResult
+erts_schedule_proc2port_signal(Process *c_p,
+ Port *prt,
+ Eterm caller,
+ Eterm *refp,
+ ErtsProc2PortSigData *sigdp,
+ int task_flags,
+ ErtsProc2PortSigCallback callback)
+{
+ int sched_res;
+ if (!refp) {
+ if (c_p)
+ erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+ }
+ else {
+ ASSERT(c_p);
+ sigdp->flags |= ERTS_P2P_SIG_DATA_FLG_REPLY;
+ erts_make_ref_in_array(sigdp->ref);
+ *refp = erts_proc_store_ref(c_p, sigdp->ref);
+
+ /*
+ * Caller needs to wait for a message containing
+ * the ref that we just created. No such message
+ * can exist in callers message queue at this time.
+ * We therefore move the save pointer of the
+ * callers message queue to the end of the queue.
+ *
+ * NOTE: It is of vital importance that the caller
+ * immediately do a receive unconditionaly
+ * waiting for the message with the reference;
+ * otherwise, next receive will *not* work
+ * as expected!
+ */
+ erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+
+ if (ERTS_PROC_PENDING_EXIT(c_p)) {
+ /* need to exit caller instead */
+ erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
+ KILL_CATCHES(c_p);
+ c_p->freason = EXC_EXIT;
+ return ERTS_PORT_OP_CALLER_EXIT;
+ }
+
+ ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+ c_p->msg.save = c_p->msg.last;
+
+ erts_smp_proc_unlock(c_p,
+ (ERTS_PROC_LOCK_MAIN
+ | ERTS_PROC_LOCKS_MSG_RECEIVE));
+ }
+
+
+ sigdp->caller = caller;
+
+ /* Schedule port close call for later execution... */
+ sched_res = erts_port_task_schedule(prt->common.id,
+ NULL,
+ ERTS_PORT_TASK_PROC_SIG,
+ sigdp,
+ callback,
+ task_flags);
+
+ if (c_p)
+ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+
+ if (sched_res != 0) {
+ if (refp)
+ *refp = NIL;
+ return ERTS_PORT_OP_DROPPED;
+ }
+ return ERTS_PORT_OP_SCHEDULED;
+}
+
+static ERTS_INLINE void
+send_badsig(Port *prt)
+{
+ ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
+ Process* rp;
+ Eterm connected = ERTS_PORT_GET_CONNECTED(prt);
+
+ ERTS_SMP_CHK_NO_PROC_LOCKS;
+ ERTS_LC_ASSERT(erts_get_scheduler_id());
+
+ ASSERT(is_internal_pid(connected));
+
+ rp = erts_proc_lookup_raw(connected);
+ if (rp) {
+ erts_smp_proc_lock(rp, rp_locks);
+ if (!ERTS_PROC_IS_EXITING(rp))
+ (void) erts_send_exit_signal(NULL,
+ prt->common.id,
+ rp,
+ &rp_locks,
+ am_badsig,
+ NIL,
+ NULL,
+ 0);
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ }
+}
+
+static void
+badsig_received(int bang_op,
+ Port *prt,
+ erts_aint32_t state,
+ int bad_output_value)
+{
+ /*
+ * if (bang_op)
+ * we are part of a "Prt ! Something" operation
+ * else
+ * we are part of a call to a port BIF
+ * behave accordingly...
+ */
+ if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP)) {
+ if (bad_output_value) {
+ erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+ erts_dsprintf(dsbufp, "Bad value on output port '%s'\n", prt->name);
+ erts_send_error_to_logger_nogl(dsbufp);
+ }
+ if (bang_op)
+ send_badsig(prt);
+ }
+}
+
+static int
+port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
+{
+ if (op == ERTS_PROC2PORT_SIG_EXEC)
+ badsig_received(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP,
+ prt,
+ state,
+ sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT);
+ if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+ return ERTS_PORT_REDS_BADSIG;
+}
+
+
+/*
+ * bad_port_signal() will
+ * - preserve signal order of signals.
+ * - send a 'badsig' exit signal to connected process if 'from' is an
+ * internal pid and the port is alive when the bad signal reaches
+ * it.
+ */
+static ErtsPortOpResult
+bad_port_signal(Process *c_p,
+ int flags,
+ Port *prt,
+ Eterm from,
+ Eterm *refp,
+ Eterm port_op)
+{
+ ErtsProc2PortSigData *sigdp;
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ !refp,
+ port_op);
+
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK:
+ badsig_received(flags & ERTS_PORT_SIG_FLG_BANG_OP,
+ prt,
+ try_call_state.state,
+ flags & ERTS_PORT_SIG_FLG_BAD_OUTPUT);
+ finalize_imm_drv_call(&try_call_state);
+ return ERTS_PORT_OP_BADARG;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_DROPPED;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS:
+ case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK:
+ /* Schedule badsig() call instead... */
+ break;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = flags;
+
+ return erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ refp,
+ sigdp,
+ 0,
+ port_badsig);
+}
+
+
+/*
+ * Driver outputv() callback
+ */
+
+static ERTS_INLINE void
+call_driver_outputv(int bang_op,
+ Eterm caller,
+ Eterm from,
+ Port *prt,
+ erts_driver_t *drv,
+ ErlIOVec *evp)
+{
+ /*
+ * if (bang_op)
+ * we are part of a "Prt ! {From, {command, Data}}" operation
+ * else
+ * we are part of a call to port_command/[2,3]
+ * behave accordingly...
+ */
+ if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt))
+ send_badsig(prt);
+ else {
+ ErlDrvSizeT size = evp->size;
+
+ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+
+#ifdef USE_VM_PROBES
+ if (DTRACE_ENABLED(driver_outputv)) {
+ DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt);
+ DTRACE4(driver_outputv, process_str, port_str, prt->name, size);
+ }
+#endif
+
+ prt->caller = caller;
+ (*drv->outputv)((ErlDrvData) prt->drv_data, evp);
+ prt->caller = NIL;
+
+ prt->bytes_out += size;
+ erts_smp_atomic_add_nob(&erts_bytes_out, size);
+ }
+}
+
+static ERTS_INLINE void
+cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp)
+{
+ int i;
+ /* Need to free all binaries */
+ for (i = 1; i < ev->vsize; i++)
+ if (ev->binv[i])
+ driver_free_binary(ev->binv[i]);
+ if (cbinp)
+ driver_free_binary(cbinp);
+ erts_free(ERTS_ALC_T_DRV_CMD_DATA, ev);
+}
+
+static int
+port_sig_outputv(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
+{
+ Eterm reply;
+
+ switch (op) {
+ case ERTS_PROC2PORT_SIG_EXEC:
+ /* Execution of a scheduled outputv() call */
+
+ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+
+ if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
+ reply = am_badarg;
+ else {
+ call_driver_outputv(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP,
+ sigdp->caller,
+ sigdp->u.outputv.from,
+ prt,
+ prt->drv_ptr,
+ sigdp->u.outputv.evp);
+ reply = am_true;
+ }
+ break;
+ case ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND:
+ reply = am_false;
+ break;
+ default:
+ reply = am_badarg;
+ break;
+ }
+
+ if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, reply);
+
+ cleanup_scheduled_outputv(sigdp->u.outputv.evp,
+ sigdp->u.outputv.cbinp);
+
+ return ERTS_PORT_REDS_CMD_OUTPUTV;
+}
+
+/*
+ * Driver output() callback
+ */
+
+static ERTS_INLINE void
+call_driver_output(int bang_op,
+ Eterm caller,
+ Eterm from,
+ Port *prt,
+ erts_driver_t *drv,
+ char *bufp,
+ ErlDrvSizeT size)
+{
+ /*
+ * if (bang_op)
+ * we are part of a "Prt ! {From, {command, Data}}" operation
+ * else
+ * we are part of a call to port_command/[2,3]
+ * behave accordingly...
+ */
+ if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt))
+ send_badsig(prt);
+ else {
+
+#ifdef USE_VM_PROBES
+ if (DTRACE_ENABLED(driver_output)) {
+ DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt);
+ DTRACE4(driver_output, process_str, port_str, prt->name, size);
+ }
+#endif
+
+ prt->caller = caller;
+ (*drv->output)((ErlDrvData) prt->drv_data, bufp, size);
+ prt->caller = NIL;
+
+ prt->bytes_out += size;
+ erts_smp_atomic_add_nob(&erts_bytes_out, size);
+ }
+}
+
+static ERTS_INLINE void
+cleanup_scheduled_output(char *bufp)
+{
+ erts_free(ERTS_ALC_T_DRV_CMD_DATA, bufp);
+}
+
+static int
+port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
+{
+ Eterm reply;
+
+ switch (op) {
+ case ERTS_PROC2PORT_SIG_EXEC:
+ /* Execution of a scheduled output() call */
+
+ ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
+
+ if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
+ reply = am_badarg;
+ else {
+ call_driver_output(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP,
+ sigdp->caller,
+ sigdp->u.output.from,
+ prt,
+ prt->drv_ptr,
+ sigdp->u.output.bufp,
+ sigdp->u.output.size);
+ reply = am_true;
+ }
+ break;
+ case ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND:
+ reply = am_false;
+ break;
+ default:
+ reply = am_badarg;
+ break;
+ }
+
+ if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, reply);
+
+ cleanup_scheduled_output(sigdp->u.output.bufp);
+
+ return ERTS_PORT_REDS_CMD_OUTPUT;
+}
+
+ErtsPortOpResult
+erts_port_output(Process *c_p,
+ int flags,
+ Port *prt,
+ Eterm from,
+ Eterm list,
+ Eterm *refp)
+{
+ ErtsPortOpResult res;
+ ErtsProc2PortSigData *sigdp;
+ erts_driver_t *drv = prt->drv_ptr;
+ size_t size;
+ int try_call;
+ erts_aint32_t sched_flags, busy_flag, invalid_flags;
+ int task_flags;
+ ErtsProc2PortSigCallback port_sig_callback;
+ ErlDrvBinary *cbin = NULL;
+ ErlIOVec *evp = NULL;
+ char *buf = NULL;
+
+ ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP
+ | ERTS_PORT_SIG_FLG_NOSUSPEND
+ | ERTS_PORT_SIG_FLG_FORCE)) == 0);
+
+ busy_flag = ((flags & ERTS_PORT_SIG_FLG_FORCE)
+ ? ((erts_aint32_t) 0)
+ : ERTS_PTS_FLG_BUSY);
+ invalid_flags = busy_flag;
+ if (!refp)
+ invalid_flags |= ERTS_PTS_FLG_PARALLELISM;
+
+ /*
+ * Assumes caller have checked that port is valid...
+ */
+
+ sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+ if (sched_flags & (busy_flag|ERTS_PTS_FLG_EXIT))
+ return ((sched_flags & busy_flag)
+ ? ERTS_PORT_OP_BUSY
+ : ERTS_PORT_OP_DROPPED);
+
+ try_call = !(sched_flags & (invalid_flags|ERTS_PTS_FLGS_FORCE_SCHEDULE_OP));
+
+#ifdef USE_VM_PROBES
+ if(DTRACE_ENABLED(port_command)) {
+ DTRACE_FORMAT_COMMON_PID_AND_PORT(c_p->common.id, prt);
+ DTRACE4(port_command, process_str, port_str, prt->name, "command");
+ }
+#endif
+
+ if (drv->outputv) {
+ ErlIOVec ev;
SysIOVec iv[SMALL_WRITE_VEC];
ErlDrvBinary* bv[SMALL_WRITE_VEC];
SysIOVec* ivp;
ErlDrvBinary** bvp;
- ErlDrvBinary* cbin;
- ErlIOVec ev;
+ int vsize;
+ Uint csize;
+ Uint pvsize;
+ Uint pcsize;
+ Uint blimit;
+ size_t iov_offset, binv_offset, alloc_size;
- if (io_list_vec_len(list, &vsize, &csize,
- &pvsize, &pcsize, &size)) {
+ if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size))
goto bad_value;
+
+ iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec));
+ binv_offset = iov_offset;
+ binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec));
+ alloc_size = binv_offset;
+ alloc_size += (vsize+1)*sizeof(ErlDrvBinary *);
+
+ if (try_call && vsize < SMALL_WRITE_VEC) {
+ ivp = ev.iov = iv;
+ bvp = ev.binv = bv;
+ evp = &ev;
}
+ else {
+ char *ptr = erts_alloc((try_call
+ ? ERTS_ALC_T_TMP
+ : ERTS_ALC_T_DRV_CMD_DATA), alloc_size);
+
+ evp = (ErlIOVec *) ptr;
+ ivp = evp->iov = (SysIOVec *) (ptr + iov_offset);
+ bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset);
+ }
+
/* To pack or not to pack (small binaries) ...? */
- vsize++;
- if (vsize <= SMALL_WRITE_VEC) {
+ if (vsize < SMALL_WRITE_VEC) {
/* Do NOT pack */
blimit = 0;
- } else {
+ }
+ else {
/* Do pack */
vsize = pvsize + 1;
csize = pcsize;
blimit = ERL_SMALL_IO_BIN_LIMIT;
}
/* Use vsize and csize from now on */
- if (vsize <= SMALL_WRITE_VEC) {
- ivp = iv;
- bvp = bv;
- } else {
- ivp = (SysIOVec *) erts_alloc(ERTS_ALC_T_TMP,
- vsize * sizeof(SysIOVec));
- bvp = (ErlDrvBinary**) erts_alloc(ERTS_ALC_T_TMP,
- vsize * sizeof(ErlDrvBinary*));
- }
+
cbin = driver_alloc_binary(csize);
if (!cbin)
erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize));
@@ -1161,120 +1831,695 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
ivp[0].iov_base = NULL;
ivp[0].iov_len = 0;
bvp[0] = NULL;
- ev.vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit);
- if (ev.vsize < 0) {
- if (ivp != iv) {
- erts_free(ERTS_ALC_T_TMP, (void *) ivp);
- }
- if (bvp != bv) {
- erts_free(ERTS_ALC_T_TMP, (void *) bvp);
- }
+ evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit);
+ if (evp->vsize < 0) {
+ if (evp != &ev)
+ erts_free(try_call ? ERTS_ALC_T_TMP : ERTS_ALC_T_DRV_CMD_DATA,
+ evp);
driver_free_binary(cbin);
goto bad_value;
}
- ev.vsize++;
#if 0
/* This assertion may say something useful, but it can
be falsified during the emulator test suites. */
- ASSERT(ev.vsize == vsize);
+ ASSERT(evp->vsize == vsize);
#endif
- ev.size = size; /* total size */
- ev.iov = ivp;
- ev.binv = bvp;
-#ifdef USE_VM_PROBES
- if (DTRACE_ENABLED(driver_outputv)) {
- DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
- DTRACE4(driver_outputv, process_str, port_str, p->name, size);
- }
-#endif
- fpe_was_unmasked = erts_block_fpe();
- (*drv->outputv)((ErlDrvData)p->drv_data, &ev);
- erts_unblock_fpe(fpe_was_unmasked);
- if (ivp != iv) {
- erts_free(ERTS_ALC_T_TMP, (void *) ivp);
- }
- if (bvp != bv) {
- erts_free(ERTS_ALC_T_TMP, (void *) bvp);
+ evp->vsize++;
+ evp->size = size; /* total size */
+
+ if (!try_call) {
+ int i;
+ /* Need to increase refc on all binaries */
+ for (i = 1; i < evp->vsize; i++)
+ if (bvp[i])
+ driver_binary_inc_refc(bvp[i]);
}
- driver_free_binary(cbin);
- } else {
- int r;
-
- /* Try with an 8KB buffer first (will often be enough I guess). */
- size = 8*1024;
- /* See below why the extra byte is added. */
- buf = erts_alloc(ERTS_ALC_T_TMP, size+1);
- r = io_list_to_buf(list, buf, size);
+ else {
+ int i;
+ ErlIOVec *new_evp;
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ invalid_flags,
+ !refp,
+ am_command);
+ try_call_state.pre_chk_sched_flags = 0; /* already checked */
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK:
+ call_driver_outputv(flags & ERTS_PORT_SIG_FLG_BANG_OP,
+ c_p->common.id,
+ from,
+ prt,
+ drv,
+ evp);
+ finalize_imm_drv_call(&try_call_state);
+ /* Fall through... */
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ driver_free_binary(cbin);
+ if (evp != &ev)
+ erts_free(ERTS_ALC_T_TMP, evp);
+ if (try_call_res == ERTS_TRY_IMM_DRV_CALL_OK)
+ return ERTS_PORT_OP_DONE;
+ else
+ return ERTS_PORT_OP_DROPPED;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS:
+ sched_flags = try_call_state.sched_flags;
+ case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK:
+ /* Schedule outputv() call instead... */
+ break;
+ }
-#ifdef USE_VM_PROBES
- if(DTRACE_ENABLED(port_command)) {
- DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
- DTRACE4(port_command, process_str, port_str, p->name, "command");
- }
+ /* Need to increase refc on all binaries */
+ for (i = 1; i < evp->vsize; i++)
+ if (bvp[i])
+ driver_binary_inc_refc(bvp[i]);
+
+ new_evp = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, alloc_size);
+
+ if (evp != &ev) {
+ sys_memcpy((void *) new_evp, (void *) evp, alloc_size);
+ new_evp->iov = (SysIOVec *) (((char *) new_evp)
+ + iov_offset);
+ bvp = new_evp->binv = (ErlDrvBinary **) (((char *) new_evp)
+ + binv_offset);
+
+#ifdef DEBUG
+ ASSERT(new_evp->vsize == evp->vsize);
+ ASSERT(new_evp->size == evp->size);
+ for (i = 0; i < evp->vsize; i++) {
+ ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len);
+ ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base);
+ ASSERT(new_evp->binv[i] == evp->binv[i]);
+ }
#endif
- if (r >= 0) {
- size -= r;
-#ifdef USE_VM_PROBES
- if (DTRACE_ENABLED(driver_output)) {
- DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
- DTRACE4(driver_output, process_str, port_str, p->name, size);
- }
+ erts_free(ERTS_ALC_T_TMP, evp);
+ }
+ else { /* from stack allocated structure; offsets may differ */
+
+ sys_memcpy((void *) new_evp, (void *) evp, sizeof(ErlIOVec));
+ new_evp->iov = (SysIOVec *) (((char *) new_evp)
+ + iov_offset);
+ sys_memcpy((void *) new_evp->iov,
+ (void *) evp->iov,
+ evp->vsize * sizeof(SysIOVec));
+ new_evp->binv = (ErlDrvBinary **) (((char *) new_evp)
+ + binv_offset);
+ sys_memcpy((void *) new_evp->binv,
+ (void *) evp->binv,
+ evp->vsize * sizeof(ErlDrvBinary *));
+
+#ifdef DEBUG
+ ASSERT(new_evp->vsize == evp->vsize);
+ ASSERT(new_evp->size == evp->size);
+ for (i = 0; i < evp->vsize; i++) {
+ ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len);
+ ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base);
+ ASSERT(new_evp->binv[i] == evp->binv[i]);
+ }
#endif
- fpe_was_unmasked = erts_block_fpe();
- (*drv->output)((ErlDrvData)p->drv_data, buf, size);
- erts_unblock_fpe(fpe_was_unmasked);
- erts_free(ERTS_ALC_T_TMP, buf);
+
+ }
+
+ evp = new_evp;
}
- else if (r == -2) {
- erts_free(ERTS_ALC_T_TMP, buf);
- goto bad_value;
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->u.outputv.from = from;
+ sigdp->u.outputv.evp = evp;
+ sigdp->u.outputv.cbinp = cbin;
+ port_sig_callback = port_sig_outputv;
+ }
+ else {
+ ErlDrvSizeT r;
+
+ /*
+ * Apperently there exist code that write 1 byte to
+ * much in buffer. Where it resides I don't know, but
+ * we can live with one byte extra allocated...
+ */
+
+ if (!try_call) {
+ if (erts_iolist_size(list, &size))
+ goto bad_value;
+
+ buf = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, size + 1);
+
+ r = erts_iolist_to_buf(list, buf, size);
+ ASSERT(ERTS_IOLIST_TO_BUF_SUCCEEDED(r));
}
else {
- ASSERT(r == -1); /* Overflow */
- erts_free(ERTS_ALC_T_TMP, buf);
- if (erts_iolist_size(list, &size)) {
- goto bad_value;
+ char *new_buf;
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ invalid_flags,
+ !refp,
+ am_command);
+
+ /* Try with an 8KB buffer first (will often be enough I guess). */
+ size = 8*1024;
+
+ buf = erts_alloc(ERTS_ALC_T_TMP, size + 1);
+ r = erts_iolist_to_buf(list, buf, size);
+
+ if (ERTS_IOLIST_TO_BUF_SUCCEEDED(r)) {
+ ASSERT(r <= size);
+ size -= r;
+ }
+ else {
+ erts_free(ERTS_ALC_T_TMP, buf);
+ if (r == ERTS_IOLIST_TO_BUF_TYPE_ERROR)
+ goto bad_value;
+ ASSERT(r == ERTS_IOLIST_TO_BUF_OVERFLOW);
+ if (erts_iolist_size(list, &size))
+ goto bad_value;
+ buf = erts_alloc(ERTS_ALC_T_TMP, size + 1);
+ r = erts_iolist_to_buf(list, buf, size);
+ ASSERT(ERTS_IOLIST_TO_BUF_SUCCEEDED(r));
+ }
+ try_call_state.pre_chk_sched_flags = 0; /* already checked */
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK:
+ call_driver_output(flags & ERTS_PORT_SIG_FLG_BANG_OP,
+ c_p->common.id,
+ from,
+ prt,
+ drv,
+ buf,
+ size);
+ finalize_imm_drv_call(&try_call_state);
+ /* Fall through... */
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ erts_free(ERTS_ALC_T_TMP, buf);
+ if (try_call_res == ERTS_TRY_IMM_DRV_CALL_OK)
+ return ERTS_PORT_OP_DONE;
+ else
+ return ERTS_PORT_OP_DROPPED;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS:
+ sched_flags = try_call_state.sched_flags;
+ case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK:
+ /* Schedule outputv() call instead... */
+ break;
}
- /*
- * I know drivers that pad space with '\0' this is clearly
- * incorrect but I don't feel like fixing them now, insted
- * add ONE extra byte.
- */
- buf = erts_alloc(ERTS_ALC_T_TMP, size+1);
- r = io_list_to_buf(list, buf, size);
+ new_buf = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, size + 1);
+ sys_memcpy(new_buf, buf, size);
+ erts_free(ERTS_ALC_T_TMP, buf);
+ buf = new_buf;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->u.output.from = from;
+ sigdp->u.output.bufp = buf;
+ sigdp->u.output.size = size;
+ port_sig_callback = port_sig_output;
+ }
+
+ task_flags = ERTS_PT_FLG_WAIT_BUSY;
+ sigdp->flags = flags;
+ if (flags & (ERTS_P2P_SIG_DATA_FLG_FORCE|ERTS_P2P_SIG_DATA_FLG_NOSUSPEND)) {
+ task_flags = 0;
+ if (flags & ERTS_P2P_SIG_DATA_FLG_FORCE)
+ sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND;
+ else if (flags & ERTS_P2P_SIG_DATA_FLG_NOSUSPEND)
+ task_flags = ERTS_PT_FLG_NOSUSPEND;
+ }
+
+ res = erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ refp,
+ sigdp,
+ task_flags,
+ port_sig_callback);
+
+ if (res != ERTS_PORT_OP_SCHEDULED) {
+ if (drv->outputv)
+ cleanup_scheduled_outputv(evp, cbin);
+ else
+ cleanup_scheduled_output(buf);
+ return res;
+ }
+
+ if ((sched_flags & (busy_flag|ERTS_PTS_FLG_EXIT)) == ERTS_PTS_FLG_BUSY)
+ return ERTS_PORT_OP_BUSY_SCHEDULED;
+
+ return res;
+
+bad_value:
+
+ flags |= ERTS_PORT_SIG_FLG_BAD_OUTPUT;
+ return bad_port_signal(c_p, flags, prt, from, refp, am_command);
+}
+
+static ERTS_INLINE ErtsPortOpResult
+call_deliver_port_exit(int bang_op,
+ Eterm from,
+ Port *prt,
+ erts_aint32_t state,
+ Eterm reason,
+ int broken_link)
+{
+ /*
+ * if (bang_op)
+ * we are part of a "Prt ! {From, close}" operation
+ * else
+ * we are part of a call to port_close(Port)
+ * behave accordingly...
+ */
+
+ if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
+ return ERTS_PORT_OP_DROPPED;
+
+ if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) {
+ send_badsig(prt);
+ return ERTS_PORT_OP_DROPPED;
+ }
+
+ if (broken_link) {
+ ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from);
+ if (lnk)
+ erts_destroy_link(lnk);
+ else
+ return ERTS_PORT_OP_DROPPED;
+ }
+
+ if (!erts_deliver_port_exit(prt, from, reason, bang_op))
+ return ERTS_PORT_OP_DROPPED;
+
#ifdef USE_VM_PROBES
- if (DTRACE_ENABLED(driver_output)) {
- DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
- DTRACE4(driver_output, process_str, port_str, p->name, size);
- }
+ if(DTRACE_ENABLED(port_command) && bang_op) {
+ DTRACE_FORMAT_COMMON_PID_AND_PORT(from, prt);
+ DTRACE4(port_command, process_str, port_str, prt->name, "close");
+ }
#endif
- fpe_was_unmasked = erts_block_fpe();
- (*drv->output)((ErlDrvData)p->drv_data, buf, size);
- erts_unblock_fpe(fpe_was_unmasked);
- erts_free(ERTS_ALC_T_TMP, buf);
+
+ return ERTS_PORT_OP_DONE;
+}
+
+static int
+port_sig_exit(Port *prt,
+ erts_aint32_t state,
+ int op,
+ ErtsProc2PortSigData *sigdp)
+{
+ Eterm msg = am_badarg;
+ if (op == ERTS_PROC2PORT_SIG_EXEC) {
+ ErtsPortOpResult res;
+ int bang_op = sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP;
+ int broken_link = sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK;
+ res = call_deliver_port_exit(bang_op,
+ sigdp->u.exit.from,
+ prt,
+ state,
+ sigdp->u.exit.reason,
+ broken_link);
+
+ if (res == ERTS_PORT_OP_DONE)
+ msg = am_true;
+ }
+ if (sigdp->u.exit.bp)
+ free_message_buffer(sigdp->u.exit.bp);
+ if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, msg);
+
+ return ERTS_PORT_REDS_EXIT;
+}
+
+ErtsPortOpResult
+erts_port_exit(Process *c_p,
+ int flags,
+ Port *prt,
+ Eterm from,
+ Eterm reason,
+ Eterm *refp)
+{
+ ErtsPortOpResult res;
+ ErtsProc2PortSigData *sigdp;
+ ErlHeapFragment *bp = NULL;
+
+ ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP
+ | ERTS_PORT_SIG_FLG_BROKEN_LINK
+ | ERTS_PORT_SIG_FLG_FORCE_SCHED)) == 0);
+
+ if (!(flags & ERTS_PORT_SIG_FLG_FORCE_SCHED)) {
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ !refp,
+ am_exit);
+
+
+ switch (try_imm_drv_call(&try_call_state)) {
+ case ERTS_TRY_IMM_DRV_CALL_OK: {
+ res = call_deliver_port_exit(flags & ERTS_PORT_SIG_FLG_BANG_OP,
+ from,
+ prt,
+ try_call_state.state,
+ reason,
+ flags & ERTS_PORT_SIG_FLG_BROKEN_LINK);
+ finalize_imm_drv_call(&try_call_state);
+ return res;
+ }
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_DROPPED;
+ default:
+ /* Schedule call instead... */
+ break;
}
}
- p->bytes_out += size;
- erts_smp_atomic_add_nob(&erts_bytes_out, size);
-#ifdef ERTS_SMP
- if (p->xports)
- erts_smp_xports_unlock(p);
- ASSERT(!p->xports);
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = flags;
+ sigdp->u.exit.from = from;
+
+ if (is_immed(reason)) {
+ sigdp->u.exit.reason = reason;
+ sigdp->u.exit.bp = NULL;
+ }
+ else {
+ Eterm *hp;
+ Uint hsz = size_object(reason);
+ bp = new_message_buffer(hsz);
+ sigdp->u.exit.bp = bp;
+ hp = bp->mem;
+ sigdp->u.exit.reason = copy_struct(reason,
+ hsz,
+ &hp,
+ &bp->off_heap);
+ }
+
+ res = erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p ? c_p->common.id : from,
+ refp,
+ sigdp,
+ 0,
+ port_sig_exit);
+
+ if (res == ERTS_PORT_OP_DROPPED) {
+ if (bp)
+ free_message_buffer(bp);
+ }
+
+ return res;
+}
+
+static ErtsPortOpResult
+set_port_connected(int bang_op,
+ Eterm from,
+ Port *prt,
+ erts_aint32_t state,
+ Eterm connect)
+{
+ /*
+ * if (bang_op)
+ * we are part of a "Prt ! {From, {connect, Connect}}" operation
+ * else
+ * we are part of a call to port_connect(Port, Connect)
+ * behave accordingly...
+ */
+
+ if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP)
+ return ERTS_PORT_OP_DROPPED;
+
+ if (bang_op) { /* Bang operation */
+ if (is_not_internal_pid(connect) || ERTS_PORT_GET_CONNECTED(prt) != from) {
+ send_badsig(prt);
+ return ERTS_PORT_OP_DROPPED;
+ }
+
+ ERTS_PORT_SET_CONNECTED(prt, connect);
+ deliver_result(prt->common.id, from, am_connected);
+
+#ifdef USE_VM_PROBES
+ if(DTRACE_ENABLED(port_command)) {
+ DTRACE_FORMAT_COMMON_PID_AND_PORT(from, prt);
+ DTRACE4(port_command, process_str, port_str, prt->name, "connect");
+ }
#endif
- p->caller = NIL;
- return 0;
+ }
+ else { /* Port BIF operation */
+ Process *rp = erts_proc_lookup_raw(connect);
+ if (!rp)
+ return ERTS_PORT_OP_DROPPED;
+ erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK);
+ if (ERTS_PROC_IS_EXITING(rp)) {
+ erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+ return ERTS_PORT_OP_DROPPED;
+ }
- bad_value:
- p->caller = NIL;
- {
- erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
- erts_dsprintf(dsbufp, "Bad value on output port '%s'\n", p->name);
- erts_send_error_to_logger_nogl(dsbufp);
- return 1;
+ erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, prt->common.id);
+ erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, connect);
+
+ ERTS_PORT_SET_CONNECTED(prt, connect);
+
+ erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
+
+#ifdef USE_VM_PROBES
+ if (DTRACE_ENABLED(port_connect)) {
+ DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
+ DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+ DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE);
+
+ dtrace_pid_str(connect, process_str);
+ erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
+ dtrace_proc_str(rp, newprocess_str);
+ DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str);
+ }
+#endif
+ }
+
+ return ERTS_PORT_OP_DONE;
+}
+
+static int
+port_sig_connect(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
+{
+ Eterm msg = am_badarg;
+ if (op == ERTS_PROC2PORT_SIG_EXEC) {
+ ErtsPortOpResult res;
+ res = set_port_connected(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP,
+ sigdp->u.connect.from,
+ prt,
+ state,
+ sigdp->u.connect.connected);
+ if (res == ERTS_PORT_OP_DONE)
+ msg = am_true;
+ }
+ if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, msg);
+ return ERTS_PORT_REDS_CONNECT;
+}
+
+ErtsPortOpResult
+erts_port_connect(Process *c_p,
+ int flags,
+ Port *prt,
+ Eterm from,
+ Eterm connect,
+ Eterm *refp)
+{
+ ErtsProc2PortSigData *sigdp;
+ Eterm connect_id;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ !refp,
+ am_connect);
+
+ ASSERT((flags & ~ERTS_PORT_SIG_FLG_BANG_OP) == 0);
+
+ if (is_not_internal_pid(connect))
+ connect_id = NIL; /* Fail in op (for signal order) */
+ else
+ connect_id = connect;
+
+ switch (try_imm_drv_call(&try_call_state)) {
+ case ERTS_TRY_IMM_DRV_CALL_OK: {
+ ErtsPortOpResult res;
+ res = set_port_connected(flags & ERTS_PORT_SIG_FLG_BANG_OP,
+ from,
+ prt,
+ try_call_state.state,
+ connect_id);
+ finalize_imm_drv_call(&try_call_state);
+ return res;
+ }
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_DROPPED;
+ default:
+ /* Schedule call instead... */
+ break;
}
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = flags;
+
+ sigdp->u.connect.from = from;
+ sigdp->u.connect.connected = connect_id;
+
+ return erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ refp,
+ sigdp,
+ 0,
+ port_sig_connect);
+}
+
+static void
+port_unlink(Port *prt, Eterm from)
+{
+ ErtsLink *lnk = erts_remove_link(&ERTS_P_LINKS(prt), from);
+ if (lnk)
+ erts_destroy_link(lnk);
+}
+
+static int
+port_sig_unlink(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
+{
+ if (op == ERTS_PROC2PORT_SIG_EXEC)
+ port_unlink(prt, sigdp->u.unlink.from);
+ if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_true);
+ return ERTS_PORT_REDS_UNLINK;
+}
+
+ErtsPortOpResult
+erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp)
+{
+ ErtsProc2PortSigData *sigdp;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p,
+ prt,
+ ERTS_PORT_SFLGS_DEAD,
+ 0,
+ !refp,
+ am_unlink);
+
+ switch (try_imm_drv_call(&try_call_state)) {
+ case ERTS_TRY_IMM_DRV_CALL_OK:
+ port_unlink(prt, from);
+ finalize_imm_drv_call(&try_call_state);
+ return ERTS_PORT_OP_DONE;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_DROPPED;
+ default:
+ /* Schedule call instead... */
+ break;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = 0;
+ sigdp->u.unlink.from = from;
+
+ return erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p ? c_p->common.id : from,
+ refp,
+ sigdp,
+ 0,
+ port_sig_unlink);
+}
+
+static void
+port_link_failure(Eterm port_id, Eterm linker)
+{
+ Process *rp;
+ ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
+ ASSERT(is_internal_pid(linker));
+ rp = erts_pid2proc(NULL, 0, linker, rp_locks);
+ if (rp) {
+ ErtsLink *rlnk = erts_remove_link(&ERTS_P_LINKS(rp), port_id);
+ if (rlnk) {
+ int xres = erts_send_exit_signal(NULL,
+ port_id,
+ rp,
+ &rp_locks,
+ am_noproc,
+ NIL,
+ NULL,
+ 0);
+ if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) {
+ /* We didn't exit the process and it is traced */
+ if (IS_TRACED_FL(rp, F_TRACE_PROCS))
+ trace_proc(NULL, rp, am_getting_unlinked, port_id);
+ }
+ }
+ }
+}
+
+static void
+port_link(Port *prt, erts_aint32_t state, Eterm to)
+{
+ if (!(state & ERTS_PORT_SFLGS_INVALID_LOOKUP))
+ erts_add_link(&ERTS_P_LINKS(prt), LINK_PID, to);
+ else
+ port_link_failure(prt->common.id, to);
+}
+
+static int
+port_sig_link(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp)
+{
+ if (op == ERTS_PROC2PORT_SIG_EXEC)
+ port_link(prt, state, sigdp->u.link.to);
+ else
+ port_link_failure(sigdp->u.link.port, sigdp->u.link.to);
+ if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_true);
+ return ERTS_PORT_REDS_LINK;
+}
+
+ErtsPortOpResult
+erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp)
+{
+ ErtsProc2PortSigData *sigdp;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ !refp,
+ am_link);
+
+ switch (try_imm_drv_call(&try_call_state)) {
+ case ERTS_TRY_IMM_DRV_CALL_OK:
+ port_link(prt, try_call_state.state, to);
+ finalize_imm_drv_call(&try_call_state);
+ return ERTS_PORT_OP_DONE;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_BADARG;
+ default:
+ /* Schedule call instead... */
+ break;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = 0;
+ sigdp->u.link.port = prt->common.id;
+ sigdp->u.link.to = to;
+
+ return erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p ? c_p->common.id : to,
+ refp,
+ sigdp,
+ 0,
+ port_sig_link);
}
void erts_init_io(int port_tab_size,
@@ -1526,16 +2771,19 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
ErlOffHeap *ohp;
Eterm* hp;
Uint sz_res;
- sz_res = size_object(res);
- hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks);
- res = copy_struct(res, sz_res, &hp, ohp);
- tuple = TUPLE2(hp, sender, res);
+
+ sz_res = size_object(res);
+ hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks);
+ res = copy_struct(res, sz_res, &hp, ohp);
+ tuple = TUPLE2(hp, sender, res);
erts_queue_message(rp, &rp_locks, bp, tuple, NIL
#ifdef USE_VM_PROBES
, NIL
#endif
);
- erts_smp_proc_unlock(rp, rp_locks);
+
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
if (!scheduler)
erts_smp_proc_dec_refc(rp);
@@ -2064,8 +3312,9 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc)
* that is to kill a port till reason kill. Then the port is stopped.
*
*/
-void
-erts_do_exit_port(Port *p, Eterm from, Eterm reason)
+
+int
+erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
{
ErtsLink *lnk;
Eterm rreason;
@@ -2090,11 +3339,17 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
#endif
state = erts_atomic32_read_nob(&p->state);
- if ((state & (ERTS_PORT_SFLGS_DEAD|ERTS_PORT_SFLG_EXITING))
- || ((reason == am_normal) &&
- ((from != ERTS_PORT_GET_CONNECTED(p)) && (from != p->common.id)))) {
- return;
- }
+ if (state & (ERTS_PORT_SFLGS_DEAD
+ | ERTS_PORT_SFLG_EXITING
+ | ERTS_PORT_SFLG_CLOSING))
+ return 0;
+
+ if (reason == am_normal && from != ERTS_PORT_GET_CONNECTED(p) && from != p->common.id)
+ return 0;
+
+ if (send_closed)
+ erts_atomic32_read_bor_relb(&p->state,
+ ERTS_PORT_SFLG_SEND_CLOSED);
if (IS_TRACED_FL(p, F_TRACE_PORTS)) {
trace_port(p, am_closed, reason);
@@ -2146,11 +3401,13 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
else {
terminate_port(p);
}
+
+ return 1;
}
/* About the states ERTS_PORT_SFLG_EXITING and ERTS_PORT_SFLG_CLOSING used above.
**
-** ERTS_PORT_SFLG_EXITING is a recursion protection for erts_do_exit_port().
+** ERTS_PORT_SFLG_EXITING is a recursion protection for erts_deliver_port_exit().
** It is unclear whether this state is necessary or not, it might be possible
** to merge it with ERTS_PORT_SFLG_CLOSING. ERTS_PORT_SFLG_EXITING only persists
** over a section of sequential (but highly recursive) code.
@@ -2166,233 +3423,1108 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
** {PID, close}
** {PID, {command, io-list}}
** {PID, {connect, New_PID}}
-**
-**
*/
-void erts_port_command(Process *proc,
- Eterm caller_id,
- Port *port,
- Eterm command)
+ErtsPortOpResult
+erts_port_command(Process *c_p,
+ int flags,
+ Port *port,
+ Eterm command,
+ Eterm *refp)
{
Eterm *tp;
- Eterm pid;
- if (!port)
- return;
+ ASSERT(port);
- erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
- ERTS_SMP_CHK_NO_PROC_LOCKS;
- ASSERT(!INVALID_PORT(port, port->common.id));
+ flags |= ERTS_PORT_SIG_FLG_BANG_OP;
if (is_tuple_arity(command, 2)) {
+ Eterm cntd;
tp = tuple_val(command);
- if ((pid = ERTS_PORT_GET_CONNECTED(port)) == tp[1]) {
- /* PID must be connected */
+ cntd = tp[1];
+ if (is_internal_pid(cntd)) {
if (tp[2] == am_close) {
- erts_atomic32_read_bor_relb(&port->state,
- ERTS_PORT_SFLG_SEND_CLOSED);
- erts_do_exit_port(port, pid, am_normal);
-
-#ifdef USE_VM_PROBES
- if(DTRACE_ENABLED(port_command)) {
- DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port)
- DTRACE4(port_command, process_str, port_str, port->name, "close");
- }
-#endif
- goto done;
+ if (!erts_port_synchronous_ops)
+ refp = NULL;
+ flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND;
+ return erts_port_exit(c_p, flags, port, cntd, am_normal, refp);
} else if (is_tuple_arity(tp[2], 2)) {
tp = tuple_val(tp[2]);
if (tp[1] == am_command) {
- if (erts_write_to_port(caller_id, port, tp[2]) == 0)
- goto done;
- } else if ((tp[1] == am_connect) && is_internal_pid(tp[2])) {
-#ifdef USE_VM_PROBES
- if(DTRACE_ENABLED(port_command)) {
- DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port)
- DTRACE4(port_command, process_str, port_str, port->name, "connect");
- }
-#endif
- ERTS_PORT_SET_CONNECTED_RELB(port, tp[2]);
- deliver_result(port->common.id, pid, am_connected);
- goto done;
+ if (!(flags & ERTS_PORT_SIG_FLG_NOSUSPEND)
+ && !erts_port_synchronous_ops)
+ refp = NULL;
+ return erts_port_output(c_p, flags, port, cntd, tp[2], refp);
+ }
+ else if (tp[1] == am_connect) {
+ if (!erts_port_synchronous_ops)
+ refp = NULL;
+ flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND;
+ return erts_port_connect(c_p, flags, port, cntd, tp[2], refp);
}
}
}
}
- {
- ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
- Process* rp = erts_pid2proc(NULL, 0,
- ERTS_PORT_GET_CONNECTED(port), rp_locks);
- if (rp) {
- (void) erts_send_exit_signal(NULL,
- port->common.id,
- rp,
- &rp_locks,
- am_badsig,
- NIL,
- NULL,
- 0);
- erts_smp_proc_unlock(rp, rp_locks);
- }
+ /* badsig */
+ if (!erts_port_synchronous_ops)
+ refp = NULL;
+ flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND;
+ return bad_port_signal(c_p, flags, port, c_p->common.id, refp, am_command);
+}
+
+static ERTS_INLINE ErtsPortOpResult
+call_driver_control(Eterm caller,
+ Port *prt,
+ unsigned int command,
+ char *bufp,
+ ErlDrvSizeT size,
+ char **resp_bufp,
+ ErlDrvSizeT *from_size)
+{
+ ErlDrvSSizeT cres;
+
+ if (!prt->drv_ptr->control)
+ return ERTS_PORT_OP_BADARG;
+
+#ifdef USE_VM_PROBES
+ if (DTRACE_ENABLED(port_control) || DTRACE_ENABLED(driver_control)) {
+ DTRACE_FORMAT_COMMON_PID_AND_PORT(caller, prt);
+ DTRACE4(port_control, process_str, port_str, prt->name, command);
+ DTRACE5(driver_control, process_str, port_str, prt->name,
+ command, size);
}
- done:
- erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+#endif
+
+ prt->caller = caller;
+ cres = prt->drv_ptr->control((ErlDrvData) prt->drv_data,
+ command,
+ bufp,
+ size,
+ resp_bufp,
+ *from_size);
+ prt->caller = NIL;
+
+ if (cres < 0)
+ return ERTS_PORT_OP_BADARG;
+
+ *from_size = (ErlDrvSizeT) cres;
+
+ return ERTS_PORT_OP_DONE;
}
-/*
- * Control a port synchronously.
- * Returns either a list or a binary.
- */
-Eterm
-erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
-{
- byte* to_port = NULL; /* Buffer to write to port. */
- /* Initialization is for shutting up
- warning about use before set. */
- Uint to_len = 0; /* Length of buffer. */
- int must_free = 0; /* True if the buffer should be freed. */
- char port_result[ERL_ONHEAP_BIN_LIMIT]; /* Default buffer for result from port. */
- char* port_resp; /* Pointer to result buffer. */
- ErlDrvSSizeT n;
- ErlDrvSSizeT (*control)
- (ErlDrvData, unsigned, char*, ErlDrvSizeT, char**, ErlDrvSizeT);
- int fpe_was_unmasked;
+static void
+cleanup_scheduled_control(Binary *binp, char *bufp)
+{
+ if (binp) {
+ if (erts_refc_dectest(&binp->refc, 0) == 0)
+ erts_bin_free(binp);
+ }
+ else {
+ if (bufp)
+ erts_free(ERTS_ALC_T_DRV_CTRL_DATA, bufp);
+ }
+}
- ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- if ((control = prt->drv_ptr->control) == NULL) {
- return THE_NON_VALUE;
+static ERTS_INLINE Uint
+port_control_result_size(int control_flags,
+ char *resp_bufp,
+ ErlDrvSizeT *resp_size,
+ char *pre_alloc_buf)
+{
+ if (!resp_bufp)
+ return (Uint) 0;
+
+ if (control_flags & PORT_CONTROL_FLAG_BINARY) {
+ if (resp_bufp != pre_alloc_buf) {
+ ErlDrvBinary *dbin = (ErlDrvBinary *) resp_bufp;
+ *resp_size = dbin->orig_size;
+ if (*resp_size > ERL_ONHEAP_BIN_LIMIT)
+ return PROC_BIN_SIZE;
+ }
+ ASSERT(*resp_size <= ERL_ONHEAP_BIN_LIMIT);
+ return (Uint) heap_bin_size((*resp_size));
}
- /*
- * Convert the iolist to a buffer, pointed to by to_port,
- * and with its length in to_len.
- */
- if (is_binary(iolist) && binary_bitoffset(iolist) == 0) {
+ return (Uint) 2*(*resp_size);
+}
+
+static ERTS_INLINE Eterm
+write_port_control_result(int control_flags,
+ char *resp_bufp,
+ ErlDrvSizeT resp_size,
+ char *pre_alloc_buf,
+ Eterm **hpp,
+ ErlHeapFragment *bp,
+ ErlOffHeap *ohp)
+{
+ Eterm res;
+ if (!resp_bufp)
+ return NIL;
+ if (control_flags & PORT_CONTROL_FLAG_BINARY) {
+ /* Binary result */
+ ErlDrvBinary *dbin;
+ ErlHeapBin *hbin;
+
+ if (resp_bufp == pre_alloc_buf)
+ dbin = NULL;
+ else {
+ dbin = (ErlDrvBinary *) resp_bufp;
+ if (dbin->orig_size > ERL_ONHEAP_BIN_LIMIT) {
+ ProcBin* pb = (ProcBin *) *hpp;
+ *hpp += PROC_BIN_SIZE;
+ pb->thing_word = HEADER_PROC_BIN;
+ pb->size = dbin->orig_size;
+ pb->next = ohp->first;
+ ohp->first = (struct erl_off_heap_header *) pb;
+ pb->val = ErlDrvBinary2Binary(dbin);
+ pb->bytes = (byte*) dbin->orig_bytes;
+ pb->flags = 0;
+ OH_OVERHEAD(ohp, dbin->orig_size / sizeof(Eterm));
+ return make_binary(pb);
+ }
+ resp_bufp = dbin->orig_bytes;
+ resp_size = dbin->orig_size;
+ }
+
+ hbin = (ErlHeapBin *) *hpp;
+ *hpp += heap_bin_size(resp_size);
+ ASSERT(resp_size <= ERL_ONHEAP_BIN_LIMIT);
+ hbin->thing_word = header_heap_bin(resp_size);
+ hbin->size = resp_size;
+ sys_memcpy(hbin->data, resp_bufp, resp_size);
+ if (dbin)
+ driver_free_binary(dbin);
+ return make_binary(hbin);
+ }
+
+ /* List result */
+ res = buf_to_intlist(hpp, resp_bufp, resp_size, NIL);
+ if (resp_bufp != pre_alloc_buf)
+ driver_free(resp_bufp);
+ return res;
+}
+
+static int
+port_sig_control(Port *prt,
+ erts_aint32_t state,
+ int op,
+ ErtsProc2PortSigData *sigdp)
+{
+ ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY);
+
+ if (op == ERTS_PROC2PORT_SIG_EXEC) {
+ char resp_buf[ERL_ONHEAP_BIN_LIMIT];
+ ErlDrvSizeT resp_size = sizeof(resp_buf);
+ char *resp_bufp = &resp_buf[0];
+ ErtsPortOpResult res;
+
+ res = call_driver_control(sigdp->caller,
+ prt,
+ sigdp->u.control.command,
+ sigdp->u.control.bufp,
+ sigdp->u.control.size,
+ &resp_bufp,
+ &resp_size);
+
+ if (res == ERTS_PORT_OP_DONE) {
+ Eterm msg;
+ Eterm *hp, *hp_start;
+ ErlHeapFragment *bp;
+ ErlOffHeap *ohp;
+ Process *rp;
+ ErtsProcLocks rp_locks = 0;
+ Uint hsz;
+ int control_flags;
+
+ rp = erts_proc_lookup_raw(sigdp->caller);
+ if (!rp)
+ goto done;
+
+ control_flags = prt->control_flags;
+
+ hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+ hsz += port_control_result_size(control_flags,
+ resp_bufp,
+ &resp_size,
+ &resp_buf[0]);
+
+ hp_start = hp = erts_alloc_message_heap(hsz,
+ &bp,
+ &ohp,
+ rp,
+ &rp_locks);
+
+ msg = write_port_control_result(control_flags,
+ resp_bufp,
+ resp_size,
+ &resp_buf[0],
+ &hp,
+ bp,
+ ohp);
+
+ queue_port_sched_op_reply(rp,
+ &rp_locks,
+ hp_start,
+ hp,
+ hsz,
+ bp,
+ sigdp->ref,
+ msg);
+
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ goto done;
+ }
+ }
+
+ /* failure */
+
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+
+done:
+
+ cleanup_scheduled_control(sigdp->u.control.binp,
+ sigdp->u.control.bufp);
+
+ return ERTS_PORT_REDS_CONTROL;
+}
+
+
+ErtsPortOpResult
+erts_port_control(Process* c_p,
+ Port *prt,
+ unsigned int command,
+ Eterm data,
+ Eterm *retvalp)
+{
+ ErtsPortOpResult res;
+ char *bufp = NULL;
+ ErlDrvSizeT size = 0;
+ int try_call;
+ int tmp_alloced = 0;
+ erts_aint32_t sched_flags;
+ Binary *binp;
+ int copy;
+ ErtsProc2PortSigData *sigdp;
+
+ sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+ if (sched_flags & ERTS_PTS_FLG_EXIT)
+ return ERTS_PORT_OP_BADARG;
+
+ try_call = !(sched_flags & ERTS_PTS_FLGS_FORCE_SCHEDULE_OP);
+
+ if (is_binary(data) && binary_bitoffset(data) == 0) {
+ byte *bytep;
ERTS_DECLARE_DUMMY(Uint bitoffs);
ERTS_DECLARE_DUMMY(Uint bitsize);
- ERTS_GET_BINARY_BYTES(iolist, to_port, bitoffs, bitsize);
- to_len = binary_size(iolist);
+ ERTS_GET_BINARY_BYTES(data, bytep, bitoffs, bitsize);
+ bufp = (char *) bytep;
+ size = binary_size(data);
} else {
int r;
- /* Try with an 8KB buffer first (will often be enough I guess). */
- to_len = 8*1024;
- to_port = erts_alloc(ERTS_ALC_T_TMP, to_len);
- must_free = 1;
+ if (!try_call) {
+ if (erts_iolist_size(data, &size))
+ return ERTS_PORT_OP_BADARG;
+ bufp = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, size);
+ r = erts_iolist_to_buf(data, bufp, size);
+ ASSERT(r == 0);
+ }
+ else {
+ /* Try with an 8KB buffer first (will often be enough I guess). */
+ size = 8*1024;
+ bufp = erts_alloc(ERTS_ALC_T_TMP, size);
+ tmp_alloced = 1;
+
+ r = erts_iolist_to_buf(data, bufp, size);
+ if (ERTS_IOLIST_TO_BUF_SUCCEEDED(r)) {
+ size -= r;
+ } else {
+ if (r == ERTS_IOLIST_TO_BUF_TYPE_ERROR) { /* Type error */
+ erts_free(ERTS_ALC_T_TMP, bufp);
+ return ERTS_PORT_OP_BADARG;
+ }
+ else {
+ ASSERT(r == ERTS_IOLIST_TO_BUF_OVERFLOW); /* Overflow */
+ erts_free(ERTS_ALC_T_TMP, bufp);
+ if (erts_iolist_size(data, &size))
+ return ERTS_PORT_OP_BADARG; /* Type error */
+ }
+ bufp = erts_alloc(ERTS_ALC_T_TMP, size);
+ r = erts_iolist_to_buf(data, bufp, size);
+ ASSERT(r == 0);
+ }
+ }
+ }
- /*
- * In versions before R10B, we used to reserve random
- * amounts of extra memory. From R10B, we allocate the
- * exact amount.
- */
- r = io_list_to_buf(iolist, (char*) to_port, to_len);
- if (r >= 0) {
- to_len -= r;
- } else if (r == -2) { /* Type error */
- erts_free(ERTS_ALC_T_TMP, (void *) to_port);
- return THE_NON_VALUE;
- } else {
- ASSERT(r == -1); /* Overflow */
- erts_free(ERTS_ALC_T_TMP, (void *) to_port);
- if (erts_iolist_size(iolist, &to_len)) { /* Type error */
- return THE_NON_VALUE;
+ if (try_call) {
+ char resp_buf[ERL_ONHEAP_BIN_LIMIT];
+ char* resp_bufp = &resp_buf[0];
+ ErlDrvSizeT resp_size = sizeof(resp_buf);
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ 0,
+ am_control);
+
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK: {
+ Eterm *hp;
+ Uint hsz;
+ int control_flags;
+
+ res = call_driver_control(c_p->common.id,
+ prt,
+ command,
+ bufp,
+ size,
+ &resp_bufp,
+ &resp_size);
+ finalize_imm_drv_call(&try_call_state);
+ if (tmp_alloced)
+ erts_free(ERTS_ALC_T_TMP, bufp);
+ if (res == ERTS_PORT_OP_BADARG) {
+ return ERTS_PORT_OP_BADARG;
}
- must_free = 1;
- to_port = erts_alloc(ERTS_ALC_T_TMP, to_len);
- r = io_list_to_buf(iolist, (char*) to_port, to_len);
- ASSERT(r == 0);
+
+ control_flags = prt->control_flags;
+
+ hsz = port_control_result_size(control_flags,
+ resp_bufp,
+ &resp_size,
+ &resp_buf[0]);
+ hp = HAlloc(c_p, hsz);
+ *retvalp = write_port_control_result(control_flags,
+ resp_bufp,
+ resp_size,
+ &resp_buf[0],
+ &hp,
+ NULL,
+ &c_p->off_heap);
+ return ERTS_PORT_OP_DONE;
+ }
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ if (tmp_alloced)
+ erts_free(ERTS_ALC_T_TMP, bufp);
+ return ERTS_PORT_OP_BADARG;
+ default:
+ /* Schedule control() call instead... */
+ break;
}
}
- prt->caller = p->common.id; /* Internal pid */
+ /* Convert data into something that can be scheduled */
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
- ERTS_SMP_CHK_NO_PROC_LOCKS;
+ copy = tmp_alloced;
+
+ binp = NULL;
+
+ if (is_binary(data) && binary_bitoffset(data) == 0) {
+ Eterm *ebinp = binary_val_rel(data, NULL);
+ ASSERT(!tmp_alloced);
+ if (*ebinp == HEADER_SUB_BIN)
+ ebinp = binary_val_rel(((ErlSubBin *) ebinp)->orig, NULL);
+ if (*ebinp != HEADER_PROC_BIN)
+ copy = 1;
+ else {
+ binp = ((ProcBin *) ebinp)->val;
+ ASSERT(bufp < bufp + size);
+ ASSERT(binp->orig_bytes <= bufp
+ && bufp + size <= binp->orig_bytes + binp->orig_size);
+ erts_refc_inc(&binp->refc, 1);
+ }
+ }
+
+ if (copy) {
+ char *old_bufp = bufp;
+ bufp = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, size);
+ sys_memcpy(bufp, old_bufp, size);
+ if (tmp_alloced)
+ erts_free(ERTS_ALC_T_TMP, old_bufp);
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = 0;
+ sigdp->u.control.binp = binp;
+ sigdp->u.control.command = command;
+ sigdp->u.control.bufp = bufp;
+ sigdp->u.control.size = size;
+
+ res = erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ retvalp,
+ sigdp,
+ 0,
+ port_sig_control);
+ if (res != ERTS_PORT_OP_SCHEDULED) {
+ cleanup_scheduled_control(binp, bufp);
+ return ERTS_PORT_OP_BADARG;
+ }
+ return res;
+}
+
+static ERTS_INLINE ErtsPortOpResult
+call_driver_call(Eterm caller,
+ Port *prt,
+ unsigned int command,
+ char *bufp,
+ ErlDrvSizeT size,
+ char **resp_bufp,
+ ErlDrvSizeT *from_size,
+ unsigned *ret_flagsp)
+{
+ ErlDrvSSizeT cres;
+
+ if (!prt->drv_ptr->call)
+ return ERTS_PORT_OP_BADARG;
#ifdef USE_VM_PROBES
- if (DTRACE_ENABLED(port_control) || DTRACE_ENABLED(driver_control)) {
- DTRACE_FORMAT_COMMON_PROC_AND_PORT(p, prt);
- DTRACE4(port_control, process_str, port_str, prt->name, command);
- DTRACE5(driver_control, process_str, port_str, prt->name,
- command, to_len);
+ if (DTRACE_ENABLED(driver_call)) {
+ DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
+ DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+
+ dtrace_pid_str(caller, process_str);
+ dtrace_port_str(prt, port_str);
+ DTRACE5(driver_call, process_str, port_str, prt->name, command, size);
}
#endif
- /*
- * Call the port's control routine.
- */
+ prt->caller = caller;
+ cres = prt->drv_ptr->call((ErlDrvData) prt->drv_data,
+ command,
+ bufp,
+ size,
+ resp_bufp,
+ *from_size,
+ ret_flagsp);
+ prt->caller = NIL;
- port_resp = port_result;
- fpe_was_unmasked = erts_block_fpe();
- n = control((ErlDrvData)prt->drv_data, command, (char*)to_port, to_len,
- &port_resp, sizeof(port_result));
- erts_unblock_fpe(fpe_was_unmasked);
- if (must_free) {
- erts_free(ERTS_ALC_T_TMP, (void *) to_port);
+ if (cres <= 0
+ || ((byte) (*resp_bufp)[0]) != VERSION_MAGIC)
+ return ERTS_PORT_OP_BADARG;
+
+ *from_size = (ErlDrvSizeT) cres;
+
+ return ERTS_PORT_OP_DONE;
+}
+
+
+static
+void cleanup_scheduled_call(char *bufp)
+{
+ if (bufp)
+ erts_free(ERTS_ALC_T_DRV_CALL_DATA, bufp);
+}
+
+static int
+port_sig_call(Port *prt,
+ erts_aint32_t state,
+ int op,
+ ErtsProc2PortSigData *sigdp)
+{
+ char resp_buf[256];
+ ErlDrvSizeT resp_size = sizeof(resp_buf);
+ char *resp_bufp = &resp_buf[0];
+ unsigned ret_flags = 0U;
+
+
+ ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY);
+
+ if (op == ERTS_PROC2PORT_SIG_EXEC) {
+ ErtsPortOpResult res;
+
+ res = call_driver_call(sigdp->caller,
+ prt,
+ sigdp->u.call.command,
+ sigdp->u.call.bufp,
+ sigdp->u.call.size,
+ &resp_bufp,
+ &resp_size,
+ &ret_flags);
+
+ if (res == ERTS_PORT_OP_DONE) {
+ Eterm msg;
+ Eterm *hp;
+ ErlHeapFragment *bp;
+ ErlOffHeap *ohp;
+ Process *rp;
+ ErtsProcLocks rp_locks = 0;
+ Uint hsz;
+
+ rp = erts_proc_lookup_raw(sigdp->caller);
+ if (!rp)
+ goto done;
+
+ hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size);
+ if (hsz >= 0) {
+ Eterm *hp_start;
+ byte *endp;
+
+ hsz += 3; /* ok tuple */
+ hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+
+ hp_start = hp = erts_alloc_message_heap(hsz,
+ &bp,
+ &ohp,
+ rp,
+ &rp_locks);
+ endp = (byte *) resp_bufp;
+ msg = erts_decode_ext(&hp, ohp, &endp);
+ if (is_value(msg)) {
+ msg = TUPLE2(hp, am_ok, msg);
+ hp += 3;
+
+ queue_port_sched_op_reply(rp,
+ &rp_locks,
+ hp_start,
+ hp,
+ hsz,
+ bp,
+ sigdp->ref,
+ msg);
+
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ goto done;
+ }
+ if (bp)
+ free_message_buffer(bp);
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ }
+ }
}
- prt->caller = NIL;
-#ifdef ERTS_SMP
- if (prt->xports)
- erts_smp_xports_unlock(prt);
- ASSERT(!prt->xports);
-#endif
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
- /*
- * Handle the result.
- */
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+
+done:
- if (n < 0) {
- return THE_NON_VALUE;
+ if (resp_bufp != &resp_buf[0] && !(ret_flags & DRIVER_CALL_KEEP_BUFFER))
+ driver_free(resp_bufp);
+
+ cleanup_scheduled_call(sigdp->u.call.bufp);
+
+ return ERTS_PORT_REDS_CALL;
+}
+
+
+ErtsPortOpResult
+erts_port_call(Process* c_p,
+ Port *prt,
+ unsigned int command,
+ Eterm data,
+ Eterm *retvalp)
+{
+ ErtsPortOpResult res;
+ char input_buf[256];
+ char *bufp;
+ byte *endp;
+ ErlDrvSizeT size;
+ int try_call;
+ erts_aint32_t sched_flags;
+ ErtsProc2PortSigData *sigdp;
+
+ sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
+ if (sched_flags & ERTS_PTS_FLG_EXIT) {
+ return ERTS_PORT_OP_BADARG;
}
- if ((prt->control_flags & PORT_CONTROL_FLAG_BINARY) == 0) { /* List result */
- Eterm ret;
- Eterm* hp = HAlloc(p, 2*n);
- ret = buf_to_intlist(&hp, port_resp, n, NIL);
- if (port_resp != port_result) {
- driver_free(port_resp);
+ try_call = !(sched_flags & ERTS_PTS_FLGS_FORCE_SCHEDULE_OP);
+
+ size = erts_encode_ext_size(data);
+
+ if (!try_call)
+ bufp = erts_alloc(ERTS_ALC_T_DRV_CALL_DATA, size);
+ else if (size <= sizeof(input_buf))
+ bufp = &input_buf[0];
+ else
+ bufp = erts_alloc(ERTS_ALC_T_TMP, size);
+
+ endp = (byte *) bufp;
+ erts_encode_ext(data, &endp);
+
+ if (endp - (byte *) bufp > size)
+ ERTS_INTERNAL_ERROR("erts_internal:port_call() - Buffer overflow");
+
+ size = endp - (byte *) bufp;
+
+ if (try_call) {
+ char resp_buf[255];
+ char* resp_bufp = &resp_buf[0];
+ ErlDrvSizeT resp_size = sizeof(resp_buf);
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ 0,
+ am_call);
+
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK: {
+ Eterm *hp, *hp_end;
+ Uint hsz;
+ unsigned ret_flags = 0U;
+ Eterm term;
+
+ res = call_driver_call(c_p->common.id,
+ prt,
+ command,
+ bufp,
+ size,
+ &resp_bufp,
+ &resp_size,
+ &ret_flags);
+
+ finalize_imm_drv_call(&try_call_state);
+ if (bufp != &input_buf[0])
+ erts_free(ERTS_ALC_T_TMP, bufp);
+ if (res == ERTS_PORT_OP_BADARG)
+ return ERTS_PORT_OP_BADARG;
+ hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size);
+ if (hsz < 0)
+ return ERTS_PORT_OP_BADARG;
+ hsz += 3;
+ hp = HAlloc(c_p, hsz);
+ hp_end = hp + hsz;
+ endp = (byte *) resp_bufp;
+ term = erts_decode_ext(&hp, &MSO(c_p), &endp);
+ if (term == THE_NON_VALUE)
+ return ERTS_PORT_OP_BADARG;
+ *retvalp = TUPLE2(hp, am_ok, term);
+ hp += 3;
+ HRelease(c_p, hp_end, hp);
+ if (resp_buf != &resp_buf[0]
+ && !(ret_flags & DRIVER_CALL_KEEP_BUFFER))
+ driver_free(resp_buf);
+ return ERTS_PORT_OP_DONE;
}
- return ret;
- }
- else if (port_resp == NULL) {
- return NIL;
- }
- else { /* Binary result */
- ErlDrvBinary *dbin;
- ErlHeapBin *hbin;
- if (port_resp != port_result) {
- dbin = (ErlDrvBinary *) port_resp;
- if (dbin->orig_size > ERL_ONHEAP_BIN_LIMIT) {
- ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
- pb->thing_word = HEADER_PROC_BIN;
- pb->size = dbin->orig_size;
- pb->next = MSO(p).first;
- MSO(p).first = (struct erl_off_heap_header*)pb;
- pb->val = ErlDrvBinary2Binary(dbin);
- pb->bytes = (byte*) dbin->orig_bytes;
- pb->flags = 0;
- OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm));
- return make_binary(pb);
- }
- port_resp = dbin->orig_bytes;
- n = dbin->orig_size;
- } else {
- dbin = NULL;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ if (bufp != &input_buf[0])
+ erts_free(ERTS_ALC_T_TMP, bufp);
+ return ERTS_PORT_OP_BADARG;
+ default:
+ /* Schedule call() call instead... */
+ break;
}
- hbin = (ErlHeapBin*) HAlloc(p, heap_bin_size(n));
- ASSERT(n <= ERL_ONHEAP_BIN_LIMIT);
- hbin->thing_word = header_heap_bin(n);
- hbin->size = n;
- sys_memcpy(hbin->data, port_resp, n);
- if (dbin != NULL) {
- driver_free_binary(dbin);
+ }
+
+ /* Convert data into something that can be scheduled */
+
+ if (bufp == &input_buf[0] || try_call) {
+ char *new_bufp = erts_alloc(ERTS_ALC_T_DRV_CALL_DATA, size);
+ sys_memcpy(new_bufp, bufp, size);
+ if (bufp != &input_buf[0])
+ erts_free(ERTS_ALC_T_TMP, bufp);
+ bufp = new_bufp;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = 0;
+ sigdp->u.call.command = command;
+ sigdp->u.call.bufp = bufp;
+ sigdp->u.call.size = size;
+
+ res = erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ retvalp,
+ sigdp,
+ 0,
+ port_sig_call);
+ if (res != ERTS_PORT_OP_SCHEDULED) {
+ cleanup_scheduled_call(bufp);
+ return ERTS_PORT_OP_BADARG;
+ }
+ return res;
+}
+
+static Eterm
+make_port_info_term(Eterm **hpp_start,
+ Eterm **hpp,
+ Uint *hszp,
+ ErlHeapFragment **bpp,
+ Port *prt,
+ Eterm item)
+{
+ ErlOffHeap *ohp;
+
+ if (is_value(item)) {
+ if (erts_bld_port_info(NULL, NULL, hszp, prt, item) == am_false)
+ return THE_NON_VALUE;
+ if (*hszp) {
+ *bpp = new_message_buffer(*hszp);
+ *hpp_start = *hpp = (*bpp)->mem;
+ ohp = &(*bpp)->off_heap;
}
- return make_binary(hbin);
+ else {
+ *bpp = NULL;
+ *hpp_start = *hpp = NULL;
+ ohp = NULL;
+ }
+ return erts_bld_port_info(hpp, ohp, NULL, prt, item);
+ }
+ else {
+ int i;
+ int len;
+ int start;
+ static Eterm item[] = ERTS_PORT_INFO_1_ITEMS;
+ static Eterm value[sizeof(item)/sizeof(item[0])];
+
+ start = 0;
+ len = sizeof(item)/sizeof(item[0]);
+
+ for (i = start; i < sizeof(item)/sizeof(item[0]); i++) {
+ ASSERT(is_atom(item[i]));
+ value[i] = erts_bld_port_info(NULL, NULL, hszp, prt, item[i]);
+ }
+
+ if (value[0] == am_undefined) {
+ start++;
+ len--;
+ }
+
+ erts_bld_list(NULL, hszp, len, &value[start]);
+
+ *bpp = new_message_buffer(*hszp);
+ *hpp_start = *hpp = (*bpp)->mem;
+ ohp = &(*bpp)->off_heap;
+
+ for (i = start; i < sizeof(item)/sizeof(item[0]); i++)
+ value[i] = erts_bld_port_info(hpp, ohp, NULL, prt, item[i]);
+
+ return erts_bld_list(hpp, NULL, len, &value[start]);
+ }
+}
+
+static int
+port_sig_info(Port *prt,
+ erts_aint32_t state,
+ int op,
+ ErtsProc2PortSigData *sigdp)
+{
+ ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY);
+ if (op != ERTS_PROC2PORT_SIG_EXEC)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_undefined);
+ else {
+ Eterm *hp, *hp_start;
+ Uint hsz;
+ ErlHeapFragment *bp;
+ Eterm value;
+ Process *rp;
+ ErtsProcLocks rp_locks = 0;
+
+ rp = erts_proc_lookup_raw(sigdp->caller);
+ if (!rp)
+ return ERTS_PORT_REDS_INFO;
+
+ hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+ value = make_port_info_term(&hp_start,
+ &hp,
+ &hsz,
+ &bp,
+ prt,
+ sigdp->u.info.item);
+ if (is_value(value)) {
+ queue_port_sched_op_reply(rp,
+ &rp_locks,
+ hp_start,
+ hp,
+ hsz,
+ bp,
+ sigdp->ref,
+ value);
+ }
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ }
+ return ERTS_PORT_REDS_INFO;
+}
+
+ErtsPortOpResult
+erts_port_info(Process* c_p,
+ Port *prt,
+ Eterm item,
+ Eterm *retvalp)
+{
+ ErtsProc2PortSigData *sigdp;
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ 0,
+ am_info);
+
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK: {
+ Eterm *hp, *hp_start;
+ ErlHeapFragment *bp;
+ Uint hsz = 0;
+ Eterm value = make_port_info_term(&hp_start, &hp, &hsz, &bp, prt, item);
+ finalize_imm_drv_call(&try_call_state);
+ if (is_non_value(value))
+ return ERTS_PORT_OP_BADARG;
+ else if (is_immed(value))
+ *retvalp = value;
+ else {
+ Uint used_h_size = hp - hp_start;
+ hp = HAlloc(c_p, used_h_size);
+ *retvalp = copy_struct(value, used_h_size, &hp, &MSO(c_p));
+ free_message_buffer(bp);
+ }
+ return ERTS_PORT_OP_DONE;
}
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_DROPPED;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS:
+ case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK:
+ /* Schedule call instead... */
+ break;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = 0;
+ sigdp->u.info.item = item;
+
+ return erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ retvalp,
+ sigdp,
+ 0,
+ port_sig_info);
+}
+
+static int
+port_sig_set_data(Port *prt,
+ erts_aint32_t state,
+ int op,
+ ErtsProc2PortSigData *sigdp)
+{
+ ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY);
+
+ if (op == ERTS_PROC2PORT_SIG_EXEC) {
+ if (prt->bp)
+ free_message_buffer(prt->bp);
+ prt->bp = sigdp->u.set_data.bp;
+ prt->data = sigdp->u.set_data.data;
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_true);
+ }
+ else {
+ if (sigdp->u.set_data.bp)
+ free_message_buffer(sigdp->u.set_data.bp);
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+ }
+ return ERTS_PORT_REDS_SET_DATA;
+}
+
+ErtsPortOpResult
+erts_port_set_data(Process* c_p,
+ Port *prt,
+ Eterm data,
+ Eterm *refp)
+{
+ ErtsPortOpResult res;
+ Eterm set_data;
+ ErlHeapFragment *bp;
+ ErtsProc2PortSigData *sigdp;
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ !refp,
+ am_set_data);
+
+ if (is_immed(data)) {
+ set_data = data;
+ bp = NULL;
+ }
+ else {
+ Eterm *hp;
+ Uint sz = size_object(data);
+ bp = new_message_buffer(sz);
+ hp = bp->mem;
+ set_data = copy_struct(data, sz, &hp, &bp->off_heap);
+ }
+
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK:
+ if (prt->bp)
+ free_message_buffer(prt->bp);
+ prt->bp = bp;
+ prt->data = set_data;
+ finalize_imm_drv_call(&try_call_state);
+ return ERTS_PORT_OP_DONE;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_DROPPED;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS:
+ case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK:
+ /* Schedule call instead... */
+ break;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = 0;
+ sigdp->u.set_data.data = set_data;
+ sigdp->u.set_data.bp = bp;
+
+ res = erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ refp,
+ sigdp,
+ 0,
+ port_sig_set_data);
+ if (res != ERTS_PORT_OP_SCHEDULED && bp)
+ free_message_buffer(bp);
+ return res;
+}
+
+static int
+port_sig_get_data(Port *prt,
+ erts_aint32_t state,
+ int op,
+ ErtsProc2PortSigData *sigdp)
+{
+ ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY);
+ if (op != ERTS_PROC2PORT_SIG_EXEC)
+ port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg);
+ else {
+ Process *rp;
+ ErtsProcLocks rp_locks = 0;
+
+ rp = erts_proc_lookup_raw(sigdp->caller);
+ if (rp) {
+ Uint hsz;
+ Eterm *hp, *hp_start;
+ Eterm data, msg;
+ ErlHeapFragment *bp;
+ ErlOffHeap *ohp;
+
+ hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE;
+ hsz += 3;
+ if (prt->bp)
+ hsz += prt->bp->used_size;
+
+ hp_start = hp = erts_alloc_message_heap(hsz,
+ &bp,
+ &ohp,
+ rp,
+ &rp_locks);
+
+ if (is_immed(prt->data))
+ data = prt->data;
+ else
+ data = copy_struct(prt->data,
+ prt->bp->used_size,
+ &hp,
+ &bp->off_heap);
+
+
+
+ msg = TUPLE2(hp, am_ok, data);
+ hp += 3;
+
+ queue_port_sched_op_reply(rp,
+ &rp_locks,
+ hp_start,
+ hp,
+ hsz,
+ bp,
+ sigdp->ref,
+ msg);
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+ }
+ }
+ return ERTS_PORT_REDS_GET_DATA;
+}
+
+ErtsPortOpResult
+erts_port_get_data(Process* c_p,
+ Port *prt,
+ Eterm *retvalp)
+{
+ ErtsProc2PortSigData *sigdp;
+ ErtsTryImmDrvCallResult try_call_res;
+ ErtsTryImmDrvCallState try_call_state
+ = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(
+ c_p,
+ prt,
+ ERTS_PORT_SFLGS_INVALID_LOOKUP,
+ 0,
+ 0,
+ am_get_data);
+
+ try_call_res = try_imm_drv_call(&try_call_state);
+ switch (try_call_res) {
+ case ERTS_TRY_IMM_DRV_CALL_OK: {
+ Eterm *hp;
+ Eterm data;
+ ErlHeapFragment *bp;
+ Uint sz;
+ if (is_immed(prt->data)) {
+ bp = NULL;
+ data = prt->data;
+ }
+ else {
+ bp = new_message_buffer(prt->bp->used_size);
+ data = copy_struct(prt->data,
+ prt->bp->used_size,
+ &hp,
+ &bp->off_heap);
+ }
+ finalize_imm_drv_call(&try_call_state);
+ if (is_immed(data))
+ sz = 0;
+ else
+ sz = bp->used_size;
+
+ hp = HAlloc(c_p, sz + 3);
+ if (is_not_immed(data)) {
+ data = copy_struct(data, bp->used_size, &hp, &MSO(c_p));
+ free_message_buffer(bp);
+ }
+ *retvalp = TUPLE2(hp, am_ok, data);
+ return ERTS_PORT_OP_DONE;
+ }
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT:
+ return ERTS_PORT_OP_DROPPED;
+ case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS:
+ case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK:
+ /* Schedule call instead... */
+ break;
+ }
+
+ sigdp = erts_port_task_alloc_p2p_sig_data();
+ sigdp->flags = 0;
+
+ return erts_schedule_proc2port_signal(c_p,
+ prt,
+ c_p->common.id,
+ retvalp,
+ sigdp,
+ 0,
+ port_sig_get_data);
}
typedef struct {
@@ -2459,7 +4591,7 @@ print_port_info(Port *p, int to, void *arg)
}
void
-set_busy_port(ErlDrvPort port_num, int on)
+set_busy_port(ErlDrvPort dprt, int on)
{
Port *prt;
erts_aint32_t flags;
@@ -2470,34 +4602,38 @@ set_busy_port(ErlDrvPort port_num, int on)
ERTS_SMP_CHK_NO_PROC_LOCKS;
- prt = erts_drvport2port_raw(port_num);
+ prt = erts_drvport2port_raw(dprt);
if (!prt)
return;
if (on) {
- flags = erts_smp_atomic32_read_bor_nob(&prt->sched.flags,
- ERTS_PTS_FLG_BUSY);
+ flags = erts_smp_atomic32_read_bor_acqb(&prt->sched.flags,
+ ERTS_PTS_FLG_BUSY);
if (flags & ERTS_PTS_FLG_BUSY)
return; /* Already busy */
+
+ if (flags & ERTS_PTS_FLG_HAVE_NS_TASKS)
+ erts_port_task_abort_nosuspend_tasks(prt);
+
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(port_busy)) {
erts_snprintf(port_str, sizeof(port_str),
- "%T", prt->id);
+ "%T", prt->common.id);
DTRACE1(port_busy, port_str);
}
#endif
} else {
ErtsProcList *plp;
- flags = erts_smp_atomic32_read_band_nob(&prt->sched.flags,
- ~ERTS_PTS_FLG_BUSY);
+ flags = erts_smp_atomic32_read_band_acqb(&prt->sched.flags,
+ ~ERTS_PTS_FLG_BUSY);
if (!(flags & ERTS_PTS_FLG_BUSY))
return; /* Already non-busy */
#ifdef USE_VM_PROBES
if (DTRACE_ENABLED(port_not_busy)) {
erts_snprintf(port_str, sizeof(port_str),
- "%T", prt->id);
+ "%T", prt->common.id);
DTRACE1(port_not_busy, port_str);
}
#endif
@@ -2539,7 +4675,7 @@ set_busy_port(ErlDrvPort port_num, int on)
ErtsProcList* plp2 = plp;
erts_snprintf(port_str, sizeof(port_str),
- "%T", erts_port[port_num]);
+ "%T", prt->common.id);
while (plp2 != NULL) {
erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid);
DTRACE2(process_port_unblocked, pid_str, port_str);
@@ -3748,8 +5884,7 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size)
}
-void driver_free_binary(dbin)
-ErlDrvBinary* dbin;
+void driver_free_binary(ErlDrvBinary* dbin)
{
Binary *bin;
if (!dbin) {
@@ -4705,13 +6840,13 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof)
} else if (eof && (state & ERTS_PORT_SFLG_SOFT_EOF)) {
deliver_result(prt->common.id, ERTS_PORT_GET_CONNECTED(prt), am_eof);
} else {
- /* XXX UGLY WORK AROUND, Let do_exit_port terminate the port */
+ /* XXX UGLY WORK AROUND, Let erts_deliver_port_exit() terminate the port */
if (prt->port_data_lock)
driver_pdl_lock(prt->port_data_lock);
prt->ioq.size = 0;
if (prt->port_data_lock)
driver_pdl_unlock(prt->port_data_lock);
- erts_do_exit_port(prt, prt->common.id, eof ? am_normal : term);
+ erts_deliver_port_exit(prt, prt->common.id, eof ? am_normal : term, 0);
}
return 0;
}
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index fd6f34e96e..9c3f9551c7 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -564,7 +564,7 @@ __decl_noreturn void __noreturn erl_exit(int n, char*, ...);
#define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */
#define ERTS_INTERNAL_ERROR(What) \
- erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s", \
+ erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
__FILE__, __LINE__, __func__, What)
Eterm erts_check_io_info(void *p);
@@ -641,6 +641,7 @@ typedef struct _SysDriverOpts {
char *wd; /* Working directory. */
unsigned spawn_type; /* Bitfield of ERTS_SPAWN_DRIVER |
ERTS_SPAWN_EXTERNAL | both*/
+ int parallelism; /* Optimize for parallelism */
} SysDriverOpts;
extern char *erts_default_arg0;
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index fb203c6076..164bed51d1 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3017,12 +3017,13 @@ buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail)
** ;
**
** Return remaining bytes in buffer on success
-** -1 on overflow
-** -2 on type error (including that result would not be a whole number of bytes)
+** ERTS_IOLIST_TO_BUF_OVERFLOW on overflow
+** ERTS_IOLIST_TO_BUF_TYPE_ERROR on type error (including that result would not be a whole number of bytes)
*/
-int io_list_to_buf(Eterm obj, char* buf, int len)
+ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
{
+ ErlDrvSizeT len = (ErlDrvSizeT) alloced_len;
Eterm* objp;
DECLARE_ESTACK(s);
goto L_again;
@@ -3115,20 +3116,20 @@ int io_list_to_buf(Eterm obj, char* buf, int len)
L_type_error:
DESTROY_ESTACK(s);
- return -2;
+ return ERTS_IOLIST_TO_BUF_TYPE_ERROR;
L_overflow:
DESTROY_ESTACK(s);
- return -1;
+ return ERTS_IOLIST_TO_BUF_OVERFLOW;
}
/*
* Return 0 if successful, and non-zero if unsuccessful.
*/
-int erts_iolist_size(Eterm obj, Uint* sizep)
+int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
{
Eterm* objp;
- Uint size = 0;
+ Uint size = 0; /* Intentionally Uint due to halfword heap */
DECLARE_ESTACK(s);
goto L_again;
@@ -3180,7 +3181,7 @@ int erts_iolist_size(Eterm obj, Uint* sizep)
#undef SAFE_ADD
DESTROY_ESTACK(s);
- *sizep = size;
+ *sizep = (ErlDrvSizeT) size;
return ERTS_IOLIST_OK;
L_overflow_error: