1 files changed, 104 insertions, 28 deletions

diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index edf4a28784..dc4c6fc350 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -391,7 +391,7 @@ static Port *create_port(char *name,
     /* Set default tracing */
     erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt));
 
-    ASSERT(((char *) prt) == ((char *) &prt->common));
+    ERTS_CT_ASSERT(offsetof(Port,common) == 0);
 
 #if !ERTS_PORT_INIT_INSTR_NEED_ID
     /*
@@ -1218,9 +1218,10 @@ typedef struct {
 static ERTS_INLINE ErtsTryImmDrvCallResult
 try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 {
+    unsigned int prof_runnable_ports;
     ErtsTryImmDrvCallResult res;
     int reds_left_in;
-    erts_aint32_t invalid_state, invalid_sched_flags;
+    erts_aint32_t act, exp, invalid_state, invalid_sched_flags;
     Port *prt = sp->port;
     Process *c_p = sp->c_p;
 
@@ -1247,18 +1248,39 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 	goto locked_fail;
     }
 
-    sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags);
-    if (sp->sched_flags & invalid_sched_flags) {
-	res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
-	goto locked_fail;
-    }
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&prt->sched);
 
+    act = erts_smp_atomic32_read_nob(&prt->sched.flags);
+
+    do {
+	erts_aint32_t new;
+	
+	if (act & invalid_sched_flags) {
+	    res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS;
+	    sp->sched_flags = act;
+	    goto locked_fail;
+	}
+	exp = act;
+	new = act | ERTS_PTS_FLG_EXEC_IMM;
+	act = erts_smp_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp);
+    } while (act != exp);
+    
+    sp->sched_flags = act;
 
     if (!c_p)
 	reds_left_in = CONTEXT_REDS/10;
     else {
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
 	    trace_virtual_sched(c_p, am_out);
+	/*
+	 * No status lock held while sending runnable
+	 * proc trace messages. It is however not needed
+	 * in this case, since only this thread can send
+	 * such messages for this process until the process
+	 * has been scheduled out.
+	 */
 	if (erts_system_profile_flags.runnable_procs
 	    && erts_system_profile_flags.exclusive)
 	    profile_runnable_proc(c_p, am_inactive);
@@ -1273,11 +1295,14 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-    if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
-	trace_sched_ports_where(prt, am_in, sp->port_op);
-    if (erts_system_profile_flags.runnable_ports
-	&& !erts_port_is_scheduled(prt))
-    	profile_runnable_port(prt, am_active);
+    if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
+	if (prof_runnable_ports && !(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
+	    profile_runnable_port(prt, am_active);
+	if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+	    trace_sched_ports_where(prt, am_in, sp->port_op);
+	if (prof_runnable_ports)
+	    erts_port_task_sched_unlock(&prt->sched);
+    }
 
     sp->fpe_was_unmasked = erts_block_fpe();
 
@@ -1294,17 +1319,31 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
     int reds;
     Port *prt = sp->port;
     Process *c_p = sp->c_p;
+    erts_aint32_t act;
+    unsigned int prof_runnable_ports;
 
     reds = prt->reds;
     reds += erts_port_driver_callback_epilogue(prt, NULL);
 
     erts_unblock_fpe(sp->fpe_was_unmasked);
 
-    if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
-	trace_sched_ports_where(prt, am_out, sp->port_op);
-    if (erts_system_profile_flags.runnable_ports
-	&& !erts_port_is_scheduled(prt))
-    	profile_runnable_port(prt, am_inactive);
+    prof_runnable_ports = erts_system_profile_flags.runnable_ports;
+    if (prof_runnable_ports)
+	erts_port_task_sched_lock(&prt->sched);
+
+    act = erts_smp_atomic32_read_band_mb(&prt->sched.flags,
+					 ~ERTS_PTS_FLG_EXEC_IMM);
+    ERTS_SMP_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM);
+
+    if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) {
+	if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS))
+	    trace_sched_ports_where(prt, am_out, sp->port_op);
+	if (prof_runnable_ports) {
+	    if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC)))
+		profile_runnable_port(prt, am_inactive);
+	    erts_port_task_sched_unlock(&prt->sched);
+	}
+    }
 
     erts_port_release(prt);
 
@@ -1319,6 +1358,13 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
 
 	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS))
 	    trace_virtual_sched(c_p, am_in);
+	/*
+	 * No status lock held while sending runnable
+	 * proc trace messages. It is however not needed
+	 * in this case, since only this thread can send
+	 * such messages for this process until the process
+	 * has been scheduled out.
+	 */
 	if (erts_system_profile_flags.runnable_procs
 	    && erts_system_profile_flags.exclusive)
 	    profile_runnable_proc(c_p, am_active);
@@ -3107,8 +3153,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to,
 	Binary* bptr;
 
 	bptr = erts_bin_nrml_alloc(len);
-	bptr->flags = 0;
-	bptr->orig_size = len;
 	erts_refc_init(&bptr->refc, 1);
 	sys_memcpy(bptr->orig_bytes, buf, len);
 
@@ -5460,8 +5504,6 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 		ProcBin* pbp;
 		Binary* bp = erts_bin_nrml_alloc(size);
 		ASSERT(bufp);
-		bp->flags = 0;
-		bp->orig_size = (SWord) size;
 		erts_refc_init(&bp->refc, 1);
 		sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
 		pbp = (ProcBin *) hp;
@@ -5953,9 +5995,7 @@ driver_alloc_binary(ErlDrvSizeT size)
     bin = erts_bin_drv_alloc_fnf((Uint) size);
     if (!bin)
 	return NULL; /* The driver write must take action */
-    bin->flags = BIN_FLAG_DRV;
     erts_refc_init(&bin->refc, 1);
-    bin->orig_size = (SWord) size;
     return Binary2ErlDrvBinary(bin);
 }
 
@@ -5985,7 +6025,6 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size)
     if (!newbin)
 	return NULL;
 
-    newbin->orig_size = size;
     return Binary2ErlDrvBinary(newbin);
 }
 
@@ -6129,7 +6168,7 @@ driver_pdl_create(ErlDrvPort dp)
 	return NULL;
     pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK,
 		     sizeof(struct erl_drv_port_data_lock));
-    erts_mtx_init(&pdl->mtx, "port_data_lock");
+    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1);
     pdl_init_refc(pdl);
     erts_port_inc_refc(pp);
     pdl->prt = pp;
@@ -6659,7 +6698,7 @@ static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
 {
     RefThing *refp;
     ASSERT(is_internal_ref(ref));
-    ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor));
+    ERTS_CT_ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor));
     refp = ref_thing_ptr(ref);
     memset(mon,0,sizeof(ErlDrvMonitor));
     memcpy(mon,refp,sizeof(RefThing));
@@ -7166,7 +7205,7 @@ char *driver_dl_error(void)
 
 
 #define ERL_DRV_SYS_INFO_SIZE(LAST_FIELD) \
-  (((size_t) &((ErlDrvSysInfo *) 0)->LAST_FIELD) \
+  (offsetof(ErlDrvSysInfo, LAST_FIELD) \
    + sizeof(((ErlDrvSysInfo *) 0)->LAST_FIELD))
 
 void
@@ -7228,6 +7267,18 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size)
 	sip->nif_major_version = ERL_NIF_MAJOR_VERSION;
 	sip->nif_minor_version = ERL_NIF_MINOR_VERSION;
     }
+    /*
+     * 'dirty_scheduler_support' is the last field in the 4th version
+     * (driver version 3.1, NIF version 2.7)
+     */
+    if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) {
+#if defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) && defined(USE_THREADS)
+	sip->dirty_scheduler_support = 1;
+#else
+	sip->dirty_scheduler_support = 0;
+#endif
+    }
+
 }
 
 
@@ -7291,6 +7342,8 @@ no_stop_select_callback(ErlDrvEvent event, void* private)
     erts_send_error_to_logger_nogl(dsbufp);
 }
 
+#define IS_DRIVER_VERSION_GE(DE,MAJOR,MINOR) \
+    ((DE)->major_version >= (MAJOR) && (DE)->minor_version >= (MINOR))
 
 static int
 init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
@@ -7338,6 +7391,7 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->timeout = de->timeout ? de->timeout : no_timeout_callback;
     drv->ready_async = de->ready_async;
     drv->process_exit = de->process_exit;
+    drv->emergency_close = IS_DRIVER_VERSION_GE(de,3,2) ? de->emergency_close : NULL;
     if (de->stop_select)
 	drv->stop_select = de->stop_select;
     else
@@ -7356,6 +7410,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     }
 }
 
+#undef IS_DRIVER_VERSION_GE
+
 void
 erts_destroy_driver(erts_driver_t *drv)
 {
@@ -7499,7 +7555,7 @@ Port *erts_get_heart_port(void)
 	if (!port)
 	    continue;
 	/* only examine undead or alive ports */
-	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD)
+	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
 	    continue;
 	/* immediate atom compare */
 	reg = port->common.u.alive.reg;
@@ -7510,3 +7566,23 @@ Port *erts_get_heart_port(void)
 
     return NULL;
 }
+
+void erts_emergency_close_ports(void)
+{
+    int ix, max = erts_ptab_max(&erts_port);
+
+    for (ix = 0; ix < max; ix++) {
+	Port *port = erts_pix2port(ix);
+
+	if (!port)
+	    continue;
+	/* only examine undead or alive ports */
+	if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
+	    continue;
+
+	/* emergency close socket */
+	if (port->drv_ptr->emergency_close) {
+	    port->drv_ptr->emergency_close((ErlDrvData) port->drv_data);
+	}
+    }
+}