Merge branch 'rickard/port-optimizations/OTP-10336' into rickard/r16/port-optimizations/OTP-10336

* rickard/port-optimizations/OTP-10336:
  Change annotate level for emacs-22 in cerl
  Update etp-commands
  Add documentation on communication in Erlang
  Add support for busy port message queue
  Add driver callback epilogue
  Implement true asynchronous signaling between processes and ports
  Add erl_drv_[send|output]_term
  Move busy port flag
  Use rwlock for driver list
  Optimize management of port tasks
  Improve configuration of process and port tables
  Remove R9 compatibility features
  Use ptab functionality also for ports
  Prepare for use of ptab functionality also for ports
  Atomic port state
  Generalize process table implementation
  Implement functionality for delaying thread progress from unmanaged threads

Conflicts:
	erts/doc/src/erl_driver.xml
	erts/doc/src/erlang.xml
	erts/emulator/beam/beam_bif_load.c
	erts/emulator/beam/beam_bp.c
	erts/emulator/beam/beam_emu.c
	erts/emulator/beam/bif.c
	erts/emulator/beam/copy.c
	erts/emulator/beam/erl_alloc.c
	erts/emulator/beam/erl_alloc.types
	erts/emulator/beam/erl_bif_info.c
	erts/emulator/beam/erl_bif_port.c
	erts/emulator/beam/erl_bif_trace.c
	erts/emulator/beam/erl_init.c
	erts/emulator/beam/erl_message.c
	erts/emulator/beam/erl_port_task.c
	erts/emulator/beam/erl_process.c
	erts/emulator/beam/erl_process.h
	erts/emulator/beam/erl_process_lock.c
	erts/emulator/beam/erl_trace.c
	erts/emulator/beam/export.h
	erts/emulator/beam/global.h
	erts/emulator/beam/io.c
	erts/emulator/sys/unix/sys.c
	erts/emulator/sys/vxworks/sys.c
	erts/emulator/test/port_SUITE.erl
	erts/etc/unix/cerl.src
	erts/preloaded/ebin/erlang.beam
	erts/preloaded/ebin/prim_inet.beam
	erts/preloaded/src/prim_inet.erl
	lib/hipe/cerl/erl_bif_types.erl
	lib/kernel/doc/src/inet.xml
	lib/kernel/src/inet.erl

1 files changed, 80 insertions, 114 deletions

diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 7f7c975e78..7cbea55eac 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -104,16 +104,49 @@ static void dereference_all_processes(DE_Handle *dh);
 static void restore_process_references(DE_Handle *dh);
 static void ddll_no_more_references(void *vdh);
 
-#define lock_drv_list() erts_smp_mtx_lock(&erts_driver_list_lock)
-#define unlock_drv_list() erts_smp_mtx_unlock(&erts_driver_list_lock)
+#define lock_drv_list() erts_smp_rwmtx_rwlock(&erts_driver_list_lock)
+#define unlock_drv_list() erts_smp_rwmtx_rwunlock(&erts_driver_list_lock)
 #define assert_drv_list_locked() \
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&erts_driver_list_lock))
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \
+		       || erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
+#define assert_drv_list_rwlocked() \
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock))
+#define assert_drv_list_rlocked() \
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
 #define assert_drv_list_not_locked() \
-    ERTS_SMP_LC_ASSERT(!erts_smp_lc_mtx_is_locked(&erts_driver_list_lock))
+    ERTS_SMP_LC_ASSERT(!erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \
+		       && !erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock))
 
 
 #define FREE_PORT_FLAGS (ERTS_PORT_SFLGS_DEAD & (~ERTS_PORT_SFLG_INITIALIZING))
 
+static void
+kill_ports_driver_unloaded(DE_Handle *dh)
+{
+    int ix, max = erts_ptab_max(&erts_port);
+
+    for (ix = 0; ix < max; ix++) {
+	erts_aint32_t state;
+	Port* prt = erts_pix2port(ix);
+	if (!prt)
+	    continue;
+
+	ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
+
+	state = erts_atomic32_read_nob(&prt->state);
+	if (state & FREE_PORT_FLAGS)
+	    continue;
+
+	erts_smp_port_lock(prt);
+
+	state = erts_atomic32_read_nob(&prt->state);
+	if (!(state & ERTS_PORT_SFLGS_DEAD) && prt->drv_ptr->handle == dh)
+	    driver_failure_atom((ErlDrvPort) prt, "driver_unloaded");
+
+	erts_port_release(prt);
+    }
+}
+
 /*
  *    try_load(Path, Name, OptionList) -> {ok,Status} | 
  *                                        {ok, PendingStatus, Ref} | 
@@ -149,7 +182,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
     Eterm name_term = BIF_ARG_2;
     Eterm options = BIF_ARG_3;
     char *path = NULL;
-    Uint path_len;
+    ErlDrvSizeT path_len;
     char *name = NULL;
     DE_Handle *dh;
     erts_driver_t *drv;
@@ -228,7 +261,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
 	goto error;
     }
     path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1);
-    if (io_list_to_buf(path_term, path, path_len) != 0) {
+    if (erts_iolist_to_buf(path_term, path, path_len) != 0) {
 	goto error;
     }
     while (path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/')) {
@@ -356,40 +389,16 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
 	    ok_term = mkatom("loaded");
 	}
     }
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     if (kill_ports) {
-	int j;
-	/* Avoid closing the driver by referencing it */
+ 	/* Avoid closing the driver by referencing it */
 	erts_ddll_reference_driver(dh);
 	ASSERT(dh->status == ERL_DE_RELOAD);
 	dh->status = ERL_DE_FORCE_RELOAD;
 #if DDLL_SMP
 	unlock_drv_list();
 #endif
-	for (j = 0; j < erts_max_ports; j++) {
-	    Port* prt = &erts_port[j];
-	    erts_smp_port_state_lock(prt);
-	    if (!(prt->status & FREE_PORT_FLAGS) &&
-		prt->drv_ptr->handle == dh) {
-		erts_smp_atomic_inc_nob(&prt->refc);
-#if DDLL_SMP
-		/* Extremely rare spinlock */
-		while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
-		       erts_smp_port_state_unlock(prt);
-		       erts_smp_port_state_lock(prt);
-	        }
-		erts_smp_port_state_unlock(prt);
-		erts_smp_mtx_lock(prt->lock);
-		if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) {
-		    driver_failure_atom(j, "driver_unloaded");
-		}
-#else
-		driver_failure_atom(j, "driver_unloaded");
-#endif
-		erts_port_release(prt);
-	    }
-	    else erts_smp_port_state_unlock(prt);
-	}
+	kill_ports_driver_unloaded(dh);
 	/* Dereference, eventually causing driver destruction */
 #if DDLL_SMP
 	lock_drv_list(); 
@@ -579,45 +588,21 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2)
 	dh->reload_full_path = dh->reload_driver_name = NULL; 
 	dh->reload_flags = 0;
     } 
-    if (dh->port_count > 0) {
+    if (erts_smp_atomic32_read_nob(&dh->port_count) > 0) {
 	++kill_ports;
     }
     dh->status = ERL_DE_UNLOAD;
     ok_term = am_pending_driver;
 done:
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     if (kill_ports > 1) {
-	int j;
 	/* Avoid closing the driver by referencing it */
 	erts_ddll_reference_driver(dh);
 	dh->status = ERL_DE_FORCE_UNLOAD;
 #if DDLL_SMP
 	unlock_drv_list();
 #endif
-	for (j = 0; j < erts_max_ports; j++) {
-	    Port* prt = &erts_port[j];
-	    erts_smp_port_state_lock(prt);
-	    if (!(prt->status &  FREE_PORT_FLAGS) 
-		&& prt->drv_ptr->handle == dh) {
-		erts_smp_atomic_inc_nob(&prt->refc);
-#if DDLL_SMP
-		/* Extremely rare spinlock */
-		while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
-		       erts_smp_port_state_unlock(prt);
-		       erts_smp_port_state_lock(prt);
-	        }
-		erts_smp_port_state_unlock(prt);
-		erts_smp_mtx_lock(prt->lock);
-		if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) {
-		    driver_failure_atom(j, "driver_unloaded");
-		}
-#else
-		driver_failure_atom(j, "driver_unloaded");
-#endif
-		erts_port_release(prt);
-	    }
-	    else erts_smp_port_state_unlock(prt);
-	}
+	kill_ports_driver_unloaded(dh);
 #if DDLL_SMP
 	lock_drv_list(); 
 #endif
@@ -787,7 +772,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2)
 	} else if (drv->handle->status == ERL_DE_PERMANENT) {
 	    res = am_permanent;
 	} else {
-	    res = make_small(drv->handle->port_count);
+	    res = make_small(erts_smp_atomic32_read_nob(&drv->handle->port_count));
 	}
 	goto done;
     case am_linked_in_driver:
@@ -1045,38 +1030,16 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
 		} 
 		dh->status = ERL_DE_UNLOAD;
 	    }
-	    if (!left && drv->handle->port_count > 0) {
+	    if (!left
+		&& erts_smp_atomic32_read_nob(&drv->handle->port_count) > 0) {
 		if (kill_ports) {
-		    int j;
 		    DE_Handle *dh = drv->handle;
 		    erts_ddll_reference_driver(dh);
 		    dh->status = ERL_DE_FORCE_UNLOAD;
 #if DDLL_SMP
 		    unlock_drv_list();
 #endif
-		    for (j = 0; j < erts_max_ports; j++) {
-			Port* prt = &erts_port[j];
-			erts_smp_port_state_lock(prt);
-			if (!(prt->status & FREE_PORT_FLAGS) &&
-			    prt->drv_ptr->handle == dh) {
-			    erts_smp_atomic_inc_nob(&prt->refc);
-#if DDLL_SMP
-			    while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
-				erts_smp_port_state_unlock(prt);
-				erts_smp_port_state_lock(prt);
-			    }
-			    erts_smp_port_state_unlock(prt);
-			    erts_smp_mtx_lock(prt->lock);
-			    if (!(prt->status & ERTS_PORT_SFLGS_DEAD)) {
-				driver_failure_atom(j, "driver_unloaded");
-			    }
-#else
-			    driver_failure_atom(j, "driver_unloaded");
-#endif
-			    erts_port_release(prt);
-			}
-			else erts_smp_port_state_unlock(prt);
-		    } 
+		    kill_ports_driver_unloaded(dh);
 #if DDLL_SMP
 		    lock_drv_list(); /* Needed for future list operations */
 #endif
@@ -1098,7 +1061,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
 void erts_ddll_lock_driver(DE_Handle *dh, char *name)
 {
     DE_ProcEntry *p,*q;
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     notify_all(dh, name, 
 	       ERL_DE_PROC_AWAIT_LOAD, am_UP, am_permanent);
     notify_all(dh, name, 
@@ -1121,19 +1084,22 @@ void erts_ddll_lock_driver(DE_Handle *dh, char *name)
 void erts_ddll_increment_port_count(DE_Handle *dh) 
 {
     assert_drv_list_locked();
-    dh->port_count++;
+    erts_smp_atomic32_inc_nob(&dh->port_count);
 }
 
 void erts_ddll_decrement_port_count(DE_Handle *dh)
 {
     assert_drv_list_locked();
-    ASSERT(dh->port_count > 0);
-    dh->port_count--;
+#if DEBUG
+    ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0);
+#else
+    erts_smp_atomic32_dec_nob(&dh->port_count);
+#endif
 }
 
 static void first_ddll_reference(DE_Handle *dh) 
 {
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     erts_refc_init(&(dh->refc),1);
 }
 
@@ -1161,7 +1127,7 @@ void erts_ddll_dereference_driver(DE_Handle *dh)
 static void dereference_all_processes(DE_Handle *dh) 
 {
     DE_ProcEntry *p;
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     for(p  = dh->procs;p != NULL; p = p->next) {
 	if (p->awaiting_status == ERL_DE_PROC_LOADED) {
 	    ASSERT(!(p->flags & ERL_DE_FL_DEREFERENCED));
@@ -1174,7 +1140,7 @@ static void dereference_all_processes(DE_Handle *dh)
 static void restore_process_references(DE_Handle *dh) 
 {
     DE_ProcEntry *p;
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     ASSERT(erts_refc_read(&(dh->refc),0) == 0);
     for(p  = dh->procs;p != NULL; p = p->next) {
 	if (p->awaiting_status == ERL_DE_PROC_LOADED) {
@@ -1402,7 +1368,7 @@ static int is_last_user(DE_Handle *dh, Process *proc) {
     DE_ProcEntry *p = dh->procs;
     int found = 0;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     while (p != NULL) {
 	if (p->proc == proc && p->awaiting_status == ERL_DE_PROC_LOADED) {
@@ -1423,7 +1389,7 @@ static DE_ProcEntry *find_proc_entry(DE_Handle *dh, Process *proc, Uint status)
 {
     DE_ProcEntry *p = dh->procs;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     while (p != NULL) {
 	if (p->proc == proc && p->awaiting_status == status) {
@@ -1450,7 +1416,7 @@ static int num_procs(DE_Handle *dh, Uint status) {
     DE_ProcEntry *p = dh->procs;
     int i = 0;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     while (p != NULL) {
 	if (p->awaiting_status == status) {
@@ -1465,7 +1431,7 @@ static int num_entries(DE_Handle *dh, Process *proc, Uint status) {
     DE_ProcEntry *p = dh->procs;
     int i = 0;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     while (p != NULL) {
 	if (p->awaiting_status == status && p->proc == proc) {
 	    ++i;
@@ -1478,7 +1444,7 @@ static int num_entries(DE_Handle *dh, Process *proc, Uint status) {
 static void add_proc_loaded(DE_Handle *dh, Process *proc) 
 {
     DE_ProcEntry *p;
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry));
     p->proc = proc;
     p->flags = 0;
@@ -1490,7 +1456,7 @@ static void add_proc_loaded(DE_Handle *dh, Process *proc)
 static void add_proc_loaded_deref(DE_Handle *dh, Process *proc) 
 {
     DE_ProcEntry *p;
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry));
     p->proc = proc;
     p->awaiting_status = ERL_DE_PROC_LOADED;
@@ -1510,7 +1476,7 @@ static void add_proc_waiting(DE_Handle *dh, Process *proc,
 			     Uint status, Eterm ref) 
 {
     DE_ProcEntry *p;
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry));
     p->proc = proc;
     p->flags = 0;
@@ -1524,7 +1490,7 @@ static Eterm add_monitor(Process *p, DE_Handle *dh, Uint status)
 {
     Eterm r;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     r = erts_make_ref(p);
     add_proc_waiting(dh, p, status, r);
     return r;
@@ -1535,7 +1501,7 @@ static void set_driver_reloading(DE_Handle *dh, Process *proc, char *path, char
 {
     DE_ProcEntry *p;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     p = erts_alloc(ERTS_ALC_T_DDLL_PROCESS, sizeof(DE_ProcEntry));
     p->proc = proc;
     p->awaiting_status = ERL_DE_OK;
@@ -1556,7 +1522,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
     int res;
     ErlDrvEntry *dp;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     if ((res =  erts_sys_ddll_open(path, &(dh->handle))) != ERL_DE_NO_ERROR) {
 	return res;
@@ -1594,7 +1560,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 	goto error;
     }
     erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
-    dh->port_count = 0;
+    erts_smp_atomic32_init_nob(&dh->port_count, 0);
     dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
     sys_strcpy(dh->full_path, path);
     dh->flags = 0;
@@ -1620,7 +1586,7 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name)
 {
     erts_driver_t *q, *p = driver_list;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     while (p != NULL) {
 	if (p->handle == dh) {
@@ -1660,11 +1626,11 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name)
     int res;
     DE_Handle *dh = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sizeof(DE_Handle));
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     dh->handle = NULL;
     dh->procs = NULL;
-    dh->port_count = 0;
+    erts_smp_atomic32_init_nob(&dh->port_count, 0);
     erts_refc_init(&(dh->refc), (erts_aint_t) 0);
     dh->status = -1;
     dh->reload_full_path = NULL;
@@ -1698,7 +1664,7 @@ static int reload_driver_entry(DE_Handle *dh)
     int loadres;
     Uint flags = dh->reload_flags;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     dh->reload_full_path = NULL;
     dh->reload_driver_name = NULL;
@@ -1736,7 +1702,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
     ErtsProcLocks rp_locks = 0;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
     if (errcode != 0) {
 	int need = load_error_need(errcode);
 	Eterm e;
@@ -1769,7 +1735,7 @@ static void notify_all(DE_Handle *dh, char *name, Uint awaiting, Eterm type, Ete
 {
     DE_ProcEntry **p;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     p = &(dh->procs);
     while (*p != NULL) {
@@ -1875,7 +1841,7 @@ static Eterm mkatom(char *str)
 static char *pick_list_or_atom(Eterm name_term)
 { 
     char *name = NULL;
-    Uint name_len;
+    ErlDrvSizeT name_len;
     if (is_atom(name_term)) {
 	Atom *ap = atom_tab(atom_val(name_term));
 	if (ap->len == 0) {
@@ -1891,7 +1857,7 @@ static char *pick_list_or_atom(Eterm name_term)
 	    goto error;
 	}
 	name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1);
-	if (io_list_to_buf(name_term, name, name_len) != 0) {
+	if (erts_iolist_to_buf(name_term, name, name_len) != 0) {
 	    goto error;
 	}
 	name[name_len] = '\0';
@@ -1912,10 +1878,10 @@ static int build_proc_info(DE_Handle *dh, ProcEntryInfo **out_pei, Uint filter)
     int i;
     DE_ProcEntry *pe;
 
-    assert_drv_list_locked();
+    assert_drv_list_rwlocked();
 
     for (pe = dh->procs; pe != NULL; pe = pe->next) {
-	Eterm id = pe->proc->id;
+	Eterm id = pe->proc->common.id;
 	Uint stat = pe->awaiting_status;
 	if (stat == ERL_DE_PROC_AWAIT_UNLOAD_ONLY) {
 	    stat = ERL_DE_PROC_AWAIT_UNLOAD;