1 files changed, 111 insertions, 28 deletions

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index cd5060ebb3..012a7d1a4b 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -914,8 +914,8 @@ int erts_port_handle_xports(Port *prt)
    (iov)->iov_base = (ptr);				\
    (iov)->iov_len = (len);				\
    if (sizeof((iov)->iov_len) < sizeof(len)				\
-       /* Check if (len) overflowed (iov)->iov_len */			\
-       && ((len) >> (sizeof((iov)->iov_len)*CHAR_BIT)) != 0) {		\
+       /* Check if (len) overflowed (iov)->iov_len */                   \
+       && (iov)->iov_len != (len)) {		                        \
        goto L_overflow;							\
    }									\
    *(bv)++ = (bin);					\
@@ -1218,9 +1218,10 @@ typedef struct {
 static ERTS_INLINE ErtsTryImmDrvCallResult
 try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 {
+    unsigned int prof_runnable_ports;
     ErtsTryImmDrvCallResult res;
     int reds_left_in;
-    erts_aint32_t invalid_state, invalid_sched_flags;
+    erts_aint32_t act, exp, invalid_state, invalid_sched_flags;
     Port *prt = sp->port;
     Process *c_p = sp->c_p;
 
@@ -1247,18 +1248,39 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	goto locked_fail;
     }
 
-    sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
-    if (sp->sched_flags & invalid_sched_flags) {
-	res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
-	goto locked_fail;
-    }
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&prt->sched);
 
+    act = erts_smp_atomic32_read_nob(&prt->sched.flags);
+
+    do {
+	erts_aint32_t new;
+	
+	if (act & invalid_sched_flags) {
+	    res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
+	    sp->sched_flags = act;
+	    goto locked_fail;
+	}
+	exp = act;
+	new = act | ERTS_PTS_FLG_EXEC_IMM;
+	act = erts_smp_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp);
+    } while (act != exp);
+    
+    sp->sched_flags = act;
 
     if (!c_p)
 	reds_left_in = CONTEXT_REDS/10;
     else {
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
 	    trace_virtual_sched(c_p, am_out);
+	/*
+	 * No status lock held while sending runnable
+	 * proc trace messages. It is however not needed
+	 * in this case, since only this thread can send
+	 * such messages for this process until the process
+	 * has been scheduled out.
+	 */
 	if (erts_system_profile_flags.runnable_procs
 	    && erts_system_profile_flags.exclusive)
 	    profile_runnable_proc(c_p, am_inactive);
@@ -1273,11 +1295,14 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-    if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
-	trace_sched_ports_where(prt, am_in, sp->port_op);
-    if (erts_system_profile_flags.runnable_ports
-	&& !erts_port_is_scheduled(prt))
-    	profile_runnable_port(prt, am_active);
+    if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
+	if (prof_runnable_ports && !(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
+	    profile_runnable_port(prt, am_active);
+	if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+	    trace_sched_ports_where(prt, am_in, sp->port_op);
+	if (prof_runnable_ports)
+	    erts_port_task_sched_unlock(&prt->sched);
+    }
 
     sp->fpe_was_unmasked = erts_block_fpe();
 
@@ -1294,17 +1319,31 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
     int reds;
     Port *prt = sp->port;
     Process *c_p = sp->c_p;
+    erts_aint32_t act;
+    unsigned int prof_runnable_ports;
 
     reds = prt->reds;
     reds += erts_port_driver_callback_epilogue(prt, NULL);
 
     erts_unblock_fpe(sp->fpe_was_unmasked);
 
-    if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
-	trace_sched_ports_where(prt, am_out, sp->port_op);
-    if (erts_system_profile_flags.runnable_ports
-	&& !erts_port_is_scheduled(prt))
-    	profile_runnable_port(prt, am_inactive);
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&prt->sched);
+
+    act = erts_smp_atomic32_read_band_mb(&prt->sched.flags,
+					 ~ERTS_PTS_FLG_EXEC_IMM);
+    ERTS_SMP_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM);
+
+    if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
+	if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+	    trace_sched_ports_where(prt, am_out, sp->port_op);
+	if (prof_runnable_ports) {
+	    if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
+		profile_runnable_port(prt, am_inactive);
+	    erts_port_task_sched_unlock(&prt->sched);
+	}
+    }
 
     erts_port_release(prt);
 
@@ -1319,6 +1358,13 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
 	    trace_virtual_sched(c_p, am_in);
+	/*
+	 * No status lock held while sending runnable
+	 * proc trace messages. It is however not needed
+	 * in this case, since only this thread can send
+	 * such messages for this process until the process
+	 * has been scheduled out.
+	 */
 	if (erts_system_profile_flags.runnable_procs
 	    && erts_system_profile_flags.exclusive)
 	    profile_runnable_proc(c_p, am_active);
@@ -2473,7 +2519,7 @@ set_port_connected(int bang_op,
 	    DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE);
 
 	    dtrace_pid_str(connect, process_str);
-	    erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
+	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id);
 	    dtrace_proc_str(rp, newprocess_str);
 	    DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str);
 	}
@@ -3591,9 +3637,9 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed)
        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
        DTRACE_CHARBUF(rreason_str, 64);
 
-       erts_snprintf(from_str, sizeof(from_str), "%T", from);
+       erts_snprintf(from_str, sizeof(DTRACE_CHARBUF_NAME(from_str)), "%T", from);
        dtrace_port_str(p, port_str);
-       erts_snprintf(rreason_str, sizeof(rreason_str), "%T", rreason);
+       erts_snprintf(rreason_str, sizeof(DTRACE_CHARBUF_NAME(rreason_str)), "%T", rreason);
        DTRACE4(port_exit, from_str, port_str, p->name, rreason_str);
    }
 #endif
@@ -4660,7 +4706,7 @@ set_busy_port(ErlDrvPort dprt, int on)
 
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(port_busy)) {
-            erts_snprintf(port_str, sizeof(port_str),
+            erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
                           "%T", prt->common.id);
             DTRACE1(port_busy, port_str);
         }
@@ -4673,7 +4719,7 @@ set_busy_port(ErlDrvPort dprt, int on)
 
 #ifdef USE_VM_PROBES
         if (DTRACE_ENABLED(port_not_busy)) {
-            erts_snprintf(port_str, sizeof(port_str),
+            erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)),
                           "%T", prt->common.id);
             DTRACE1(port_not_busy, port_str);
         }
@@ -4725,9 +4771,9 @@ erts_port_resume_procs(Port *prt)
 	    DTRACE_CHARBUF(pid_str, 16);
 	    ErtsProcList* plp2 = plp;
 
-	    erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id);
+	    erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id);
 	    while (plp2 != NULL) {
-		erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid);
+		erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid);
 		DTRACE2(process_port_unblocked, pid_str, port_str);
 	    }
 	}
@@ -6129,7 +6175,7 @@ driver_pdl_create(ErlDrvPort dp)
 	return NULL;
     pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK,
 		     sizeof(struct erl_drv_port_data_lock));
-    erts_mtx_init(&pdl->mtx, "port_data_lock");
+    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1);
     pdl_init_refc(pdl);
     erts_port_inc_refc(pp);
     pdl->prt = pp;
@@ -7166,7 +7212,7 @@ char *driver_dl_error(void)
 
 
 #define ERL_DRV_SYS_INFO_SIZE(LAST_FIELD) \
-  (((size_t) &((ErlDrvSysInfo *) 0)->LAST_FIELD) \
+  (offsetof(ErlDrvSysInfo, LAST_FIELD) \
    + sizeof(((ErlDrvSysInfo *) 0)->LAST_FIELD))
 
 void
@@ -7228,6 +7274,18 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
 	sip->nif_major_version = ERL_NIF_MAJOR_VERSION;
 	sip->nif_minor_version = ERL_NIF_MINOR_VERSION;
     }
+    /*
+     * 'dirty_scheduler_support' is the last field in the 4th version
+     * (driver version 3.1, NIF version 2.7)
+     */
+    if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) {
+#if defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) && defined(USE_THREADS)
+	sip->dirty_scheduler_support = 1;
+#else
+	sip->dirty_scheduler_support = 0;
+#endif
+    }
+
 }
 
 
@@ -7291,6 +7349,8 @@ no_stop_select_callback(ErlDrvEvent event, void* private)
     erts_send_error_to_logger_nogl(dsbufp);
 }
 
+#define IS_DRIVER_VERSION_GE(DE,MAJOR,MINOR) \
+    ((DE)->major_version >= (MAJOR) && (DE)->minor_version >= (MINOR))
 
 static int
 init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
@@ -7338,6 +7398,7 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->timeout = de->timeout ? de->timeout : no_timeout_callback;
     drv->ready_async = de->ready_async;
     drv->process_exit = de->process_exit;
+    drv->emergency_close = IS_DRIVER_VERSION_GE(de,3,2) ? de->emergency_close : NULL;
     if (de->stop_select)
 	drv->stop_select = de->stop_select;
     else
@@ -7356,6 +7417,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     }
 }
 
+#undef IS_DRIVER_VERSION_GE
+
 void
 erts_destroy_driver(erts_driver_t *drv)
 {
@@ -7499,7 +7562,7 @@ Port *erts_get_heart_port(void)
 	if (!port)
 	    continue;
 	/* only examine undead or alive ports */
-	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD)
+	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
 	    continue;
 	/* immediate atom compare */
 	reg = port->common.u.alive.reg;
@@ -7510,3 +7573,23 @@ Port *erts_get_heart_port(void)
 
     return NULL;
 }
+
+void erts_emergency_close_ports(void)
+{
+    int ix, max = erts_ptab_max(&erts_port);
+
+    for (ix = 0; ix < max; ix++) {
+	Port *port = erts_pix2port(ix);
+
+	if (!port)
+	    continue;
+	/* only examine undead or alive ports */
+	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
+	    continue;
+
+	/* emergency close socket */
+	if (port->drv_ptr->emergency_close) {
+	    port->drv_ptr->emergency_close((ErlDrvData) port->drv_data);
+	}
+    }
+}