1 files changed, 74 insertions, 37 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 11ca85a41c..bae17d1569 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -38,6 +38,7 @@
 #include "erl_instrument.h"
 #include "erl_threads.h"
 #include "erl_binary.h"
+#include "beam_bp.h"
 
 #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS)
 #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \
@@ -180,6 +181,7 @@ static ErtsCpuBindData *scheduler2cpu_map;
 erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
 
 typedef enum {
+    ERTS_CPU_BIND_UNDEFINED,
     ERTS_CPU_BIND_SPREAD,
     ERTS_CPU_BIND_PROCESSOR_SPREAD,
     ERTS_CPU_BIND_THREAD_SPREAD,
@@ -190,6 +192,9 @@ typedef enum {
     ERTS_CPU_BIND_NONE
 } ErtsCpuBindOrder;
 
+#define ERTS_CPU_BIND_DEFAULT_BIND \
+  ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD
+
 ErtsCpuBindOrder cpu_bind_order;
 
 static erts_cpu_topology_t *user_cpudata;
@@ -388,7 +393,12 @@ erts_pre_init_process(void)
      erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks
 	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
      erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks
-	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
+	 = ERTS_PSD_DIST_ENTRY_SET_LOCKS;
+
+     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks
+	 = ERTS_PSD_CALL_TIME_BP_GET_LOCKS;
+     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
+	 = ERTS_PSD_CALL_TIME_BP_SET_LOCKS;
 
      /* Check that we have locks for all entries */
      for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
@@ -2078,12 +2088,14 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 
     erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS,
 					 (sizeof(ErtsAlignedRunQueue)*(n+1)));
-    if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0)
+    if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) != 0)
 	erts_aligned_run_queues = ((ErtsAlignedRunQueue *)
 				   ((((UWord) erts_aligned_run_queues)
 				     & ~ERTS_CACHE_LINE_MASK)
 				    + ERTS_CACHE_LINE_SIZE));
 
+    ASSERT((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0);
+
 #ifdef ERTS_SMP
     erts_smp_atomic_init(&no_empty_run_queues, 0);
 #endif
@@ -2175,11 +2187,14 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
     erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA,
 					     (sizeof(ErtsAlignedSchedulerData)
 					      *(n+1)));
-    if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0)
+    if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) != 0)
 	erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *)
 				       ((((UWord) erts_aligned_scheduler_data)
 					 & ~ERTS_CACHE_LINE_MASK)
 					+ ERTS_CACHE_LINE_SIZE));
+
+    ASSERT((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0);
+
     for (ix = 0; ix < n; ix++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 #ifdef ERTS_SMP
@@ -2769,7 +2784,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */
     }
     else if (on) { /* ------ BLOCK ------ */
-	if (erts_is_multi_scheduling_blocked()) {
+	if (schdlr_sspnd.msb.procs) {
 	    plp = proclist_create(p);
 	    plp->next = schdlr_sspnd.msb.procs;
 	    schdlr_sspnd.msb.procs = plp;
@@ -2975,8 +2990,11 @@ erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
 int
 erts_is_multi_scheduling_blocked(void)
 {
-    return (erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)
-	    && erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
+    int res;
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    res = schdlr_sspnd.msb.procs != NULL;
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    return res;
 }
 
 Eterm
@@ -2985,7 +3003,7 @@ erts_multi_scheduling_blockers(Process *p)
     Eterm res = NIL;
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    if (erts_is_multi_scheduling_blocked()) {
+    if (schdlr_sspnd.msb.procs) {
 	Eterm *hp, *hp_end;
 	ErtsProcList *plp1, *plp2;
 	Uint max_size;
@@ -3096,11 +3114,7 @@ erts_start_schedulers(void)
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual);
 	actual++;
 	ASSERT(actual == esdp->no);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	res = erts_lcnt_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts);
-#else
 	res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts);
-#endif
 	if (res != 0) {
 	    actual--;
 	    break;
@@ -3503,14 +3517,15 @@ erts_init_scheduler_bind_type(char *how)
     if (!system_cpudata && !user_cpudata)
 	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY;
 
-    if (sys_strcmp(how, "s") == 0)
+    if (sys_strcmp(how, "db") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+    else if (sys_strcmp(how, "s") == 0)
 	cpu_bind_order = ERTS_CPU_BIND_SPREAD;
     else if (sys_strcmp(how, "ps") == 0)
 	cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
     else if (sys_strcmp(how, "ts") == 0)
 	cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-    else if (sys_strcmp(how, "db") == 0
-	     || sys_strcmp(how, "tnnps") == 0)
+    else if (sys_strcmp(how, "tnnps") == 0)
 	cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
     else if (sys_strcmp(how, "nnps") == 0)
 	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
@@ -4153,14 +4168,15 @@ erts_bind_schedulers(Process *c_p, Eterm how)
 
 	old_cpu_bind_order = cpu_bind_order;
 
-	if (ERTS_IS_ATOM_STR("spread", how))
+	if (ERTS_IS_ATOM_STR("default_bind", how))
+	    cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+	else if (ERTS_IS_ATOM_STR("spread", how))
 	    cpu_bind_order = ERTS_CPU_BIND_SPREAD;
 	else if (ERTS_IS_ATOM_STR("processor_spread", how))
 	    cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
 	else if (ERTS_IS_ATOM_STR("thread_spread", how))
 	    cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-	else if (ERTS_IS_ATOM_STR("default_bind", how)
-		 || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
+	else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
 	    cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
 	else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
 	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
@@ -4206,14 +4222,15 @@ erts_fake_scheduler_bindings(Process *p, Eterm how)
     int cpudata_size;
     Eterm res;
 
-    if (ERTS_IS_ATOM_STR("spread", how))
+    if (ERTS_IS_ATOM_STR("default_bind", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+    else if (ERTS_IS_ATOM_STR("spread", how))
 	fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD;
     else if (ERTS_IS_ATOM_STR("processor_spread", how))
 	fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
     else if (ERTS_IS_ATOM_STR("thread_spread", how))
 	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-    else if (ERTS_IS_ATOM_STR("default_bind", how)
-	     || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
+    else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
 	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
     else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
 	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
@@ -4438,7 +4455,7 @@ early_cpu_bind_init(void)
 				(sizeof(erts_cpu_topology_t)
 				 * system_cpudata_size));
 
-    cpu_bind_order = ERTS_CPU_BIND_NONE;
+    cpu_bind_order = ERTS_CPU_BIND_UNDEFINED;
 
     if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata)
 	|| ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata,
@@ -4464,6 +4481,17 @@ late_cpu_bind_init(void)
 	scheduler2cpu_map[ix].bound_id = -1;
     }
 
+    if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) {
+	int ncpus = erts_get_cpu_configured(erts_cpuinfo);
+	if (ncpus < 1 || erts_no_schedulers < ncpus)
+	    cpu_bind_order = ERTS_CPU_BIND_NONE;
+	else
+	    cpu_bind_order = ((system_cpudata || user_cpudata)
+			      && (erts_bind_to_cpu(erts_cpuinfo, -1) != -ENOTSUP)
+			      ? ERTS_CPU_BIND_DEFAULT_BIND
+			      : ERTS_CPU_BIND_NONE);
+    }
+
     if (cpu_bind_order != ERTS_CPU_BIND_NONE) {
 	erts_cpu_topology_t *cpudata;
 	int cpudata_size;
@@ -5846,6 +5874,9 @@ Process *schedule(Process *p, int calls)
 	}
 
 	if (IS_TRACED(p)) {
+	    if (IS_TRACED_FL(p, F_TRACE_CALLS) &&  p->status != P_FREE) {
+		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
+	    }
 	    switch (p->status) {
 	    case P_EXITING:
 		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
@@ -6282,7 +6313,11 @@ Process *schedule(Process *p, int calls)
 		    trace_virtual_sched(p, am_in);
 		break;
 	    }
+	    if (IS_TRACED_FL(p, F_TRACE_CALLS)) {
+		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN);
+	    }
 	}
+
 	if (p->status != P_EXITING)
 	    p->status = P_RUNNING;
 
@@ -7187,8 +7222,6 @@ erts_debug_verify_clean_empty_process(Process* p)
 void
 erts_cleanup_empty_process(Process* p)
 {
-    ErlHeapFragment* mbufp;
-
     /* We only check fields that are known to be used... */
 
     erts_cleanup_offheap(&p->off_heap);
@@ -7199,13 +7232,10 @@ erts_cleanup_empty_process(Process* p)
     p->off_heap.externals = NULL;
     p->off_heap.overhead = 0;
 
-    mbufp = p->mbuf;
-    while (mbufp) {
-	ErlHeapFragment *next = mbufp->next;
-	free_message_buffer(mbufp);
-	mbufp = next;
+    if (p->mbuf != NULL) {
+	free_message_buffer(p->mbuf);
+	p->mbuf = NULL;
     }
-    p->mbuf = NULL;
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
     erts_lcnt_proc_lock_destroy(p);
 #endif
@@ -7221,7 +7251,6 @@ static void
 delete_process(Process* p)
 {
     ErlMessage* mp;
-    ErlHeapFragment* bp;
 
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id));
 
@@ -7271,11 +7300,8 @@ delete_process(Process* p)
     /*
      * Free all pending message buffers.
      */
-    bp = p->mbuf;
-    while (bp != NULL) {
-	ErlHeapFragment* next_bp = bp->next;
-	free_message_buffer(bp);
-	bp = next_bp;
+    if (p->mbuf != NULL) {	
+	free_message_buffer(p->mbuf);
     }
 
     erts_erase_dicts(p);
@@ -8034,8 +8060,13 @@ erts_do_exit_process(Process* p, Eterm reason)
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
 #endif
 
-    if (IS_TRACED_FL(p,F_TRACE_PROCS))
-	trace_proc(p, p, am_exit, reason);
+    if (IS_TRACED(p)) {
+	if (IS_TRACED_FL(p, F_TRACE_CALLS))
+	    erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING);
+
+	if (IS_TRACED_FL(p,F_TRACE_PROCS))
+	    trace_proc(p, p, am_exit, reason);
+    }
 
     erts_trace_check_exiting(p->id);
 
@@ -8084,6 +8115,8 @@ continue_exit_process(Process *p
     Eterm reason = p->fvalue;
     DistEntry *dep;
     struct saved_calls *scb;
+    process_breakpoint_time_t *pbt;
+
 #ifdef DEBUG
     int yield_allowed = 1;
 #endif
@@ -8223,6 +8256,7 @@ continue_exit_process(Process *p
 	   ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL)
 	   : NULL);
     scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL);
+    pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL);
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     processes_busy--;
@@ -8265,6 +8299,9 @@ continue_exit_process(Process *p
     if (scb)
         erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
 
+    if (pbt)
+        erts_free(ERTS_ALC_T_BPD, (void *) pbt);
+
     delete_process(p);
 
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);