Merge branch 'lukas/erts/gc_stat_contention/OTP-10271' into maint

* lukas/erts/gc_stat_contention/OTP-10271:
  Replace gc stat lock with sched spec data

10 files changed, 187 insertions, 45 deletions

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index ce60bb9bbc..9e12080732 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -248,6 +248,7 @@ atom get_data
 atom get_seq_token
 atom get_tcw
 atom getenv
+atom gather_gc_info_result
 atom gather_sched_wall_time_result
 atom getting_linked
 atom getting_unlinked
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 5a92ab7f24..f913525726 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -366,6 +366,7 @@ type	MONITOR_SH	STANDARD_LOW	PROCESSES	monitor_sh
 type	NLINK_SH	STANDARD_LOW	PROCESSES	nlink_sh
 type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
+type	GC_INFO_REQ	STANDARD_LOW	SYSTEM		gc_info_request
 
 +else # "fullword"
 
@@ -383,6 +384,7 @@ type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
 type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
 type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
 type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
+type	GC_INFO_REQ	SHORT_LIVED	SYSTEM		gc_info_request
 
 +endif
 
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 54eefe8d12..1744afbae1 100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -59,6 +59,7 @@ static Export* alloc_info_trap = NULL;
 static Export* alloc_sizes_trap = NULL;
 
 static Export *gather_sched_wall_time_res_trap;
+static Export *gather_gc_info_res_trap;
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
@@ -3103,18 +3104,10 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = TUPLE2(hp, cs, SMALL_ZERO);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_garbage_collection) {
-	Uint hsz = 4;
-	ErtsGCInfo gc_info;
-	Eterm gcs;
-	Eterm recl;
-	erts_gc_info(&gc_info);
-	(void) erts_bld_uint(NULL, &hsz, gc_info.garbage_collections);
-	(void) erts_bld_uint(NULL, &hsz, gc_info.reclaimed);
-	hp = HAlloc(BIF_P, hsz);
-	gcs = erts_bld_uint(&hp, NULL, gc_info.garbage_collections);
-	recl = erts_bld_uint(&hp, NULL, gc_info.reclaimed);
-	res = TUPLE3(hp, gcs, recl, SMALL_ZERO);
-	BIF_RET(res);
+	res = erts_gc_info_request(BIF_P);
+	if (is_non_value(res))
+	    BIF_RET(am_undefined);
+	BIF_TRAP1(gather_gc_info_res_trap, BIF_P, res);
     } else if (BIF_ARG_1 == am_reductions) {
 	Uint reds;
 	Uint diff;
@@ -4082,6 +4075,8 @@ erts_bif_info_init(void)
     alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1);
     gather_sched_wall_time_res_trap
 	= erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1);
+    gather_gc_info_res_trap
+	= erts_export_put(am_erlang, am_gather_gc_info_result, 1);
     process_info_init();
     os_info_init();
 }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 298909c921..0d12e658d9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -47,10 +47,6 @@
  */
 #define ALENGTH(a) (sizeof(a)/sizeof(a[0]))
 
-static erts_smp_spinlock_t info_lck;
-static Uint garbage_cols;		/* no of garbage collections */
-static Uint reclaimed;			/* no of words reclaimed in GCs */
-
 # define STACK_SZ_ON_HEAP(p) ((p)->hend - (p)->stop)
 # define OverRunCheck(P) \
     if ((P)->stop < (P)->htop) { \
@@ -120,6 +116,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
 static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
 static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size);
 
+static void init_gc_info(ErtsGCInfo *gcip);
+
 #ifdef HARDDEBUG
 static void disallow_heap_frag_ref_in_heap(Process* p);
 static void disallow_heap_frag_ref_in_old_heap(Process* p);
@@ -137,13 +135,41 @@ static int num_heap_sizes;	/* Number of heap sizes. */
 
 Uint erts_test_long_gc_sleep; /* Only used for testing... */
 
+typedef struct {
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    Uint req_sched;
+    erts_smp_atomic32_t refc;
+} ErtsGCInfoReq;
+
+#if !HALFWORD_HEAP
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
+				 ErtsGCInfoReq,
+				 5,
+				 ERTS_ALC_T_GC_INFO_REQ)
+#else
+static ERTS_INLINE ErtsGCInfoReq *
+gcireq_alloc(void)
+{
+    return erts_alloc(ERTS_ALC_T_GC_INFO_REQ,
+		      sizeof(ErtsGCInfoReq));
+}
+
+static ERTS_INLINE void
+gcireq_free(ErtsGCInfoReq *ptr)
+{
+    erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr);
+}
+#endif
+
 /*
  * Initialize GC global data.
  */
 void
 erts_init_gc(void)
 {
-    int i = 0;
+    int i = 0, ix;
     Sint max_heap_size = 0;
 
     ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
@@ -156,9 +182,6 @@ erts_init_gc(void)
     ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
     ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
 
-    erts_smp_spinlock_init(&info_lck, "gc_info");
-    garbage_cols = 0;
-    reclaimed = 0;
     erts_test_long_gc_sleep = 0;
 
     /*
@@ -199,6 +222,16 @@ erts_init_gc(void)
 	}
     }
     num_heap_sizes = i;
+    
+    for (ix = 0; ix < erts_no_schedulers; ix++) {
+      ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
+      init_gc_info(&esdp->gc_info);
+    }
+
+#if !HALFWORD_HEAP
+    init_gcireq_alloc();
+#endif
+
 }
 
 /*
@@ -287,17 +320,6 @@ erts_heap_sizes(Process* p)
     return res;
 }
 
-void
-erts_gc_info(ErtsGCInfo *gcip)
-{
-    if (gcip) {
-	erts_smp_spin_lock(&info_lck);
-	gcip->garbage_collections = garbage_cols;
-	gcip->reclaimed = reclaimed;
-	erts_smp_spin_unlock(&info_lck);
-    }
-}
-
 void 
 erts_offset_heap(Eterm* hp, Uint sz, Sint offs, Eterm* low, Eterm* high)
 {
@@ -378,6 +400,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     Uint reclaimed_now = 0;
     int done = 0;
     Uint ms1, s1, us1;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 #ifdef USE_VM_PROBES
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
@@ -455,11 +478,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 	    monitor_large_heap(p);
     }
 
-    erts_smp_spin_lock(&info_lck);
-    garbage_cols++;
-    reclaimed += reclaimed_now;
-    erts_smp_spin_unlock(&info_lck);
-
+    esdp->gc_info.garbage_cols++;
+    esdp->gc_info.reclaimed += reclaimed_now;
+    
     FLAGS(p) &= ~F_FORCE_GC;
 
 #ifdef CHECK_FOR_HOLES
@@ -2543,6 +2564,110 @@ offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
     offset_one_rootset(p, offs, area, area_size, objv, nobj);
 }
 
+static void
+init_gc_info(ErtsGCInfo *gcip)
+{
+  gcip->reclaimed = 0;
+  gcip->garbage_cols = 0;
+}
+
+static void
+reply_gc_info(void *vgcirp)
+{
+    Uint64 reclaimed = 0, garbage_cols = 0;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsGCInfoReq *gcirp = (ErtsGCInfoReq *) vgcirp;
+    ErtsProcLocks rp_locks = (gcirp->req_sched == esdp->no
+			      ? ERTS_PROC_LOCK_MAIN
+			      : 0);
+    Process *rp = gcirp->proc;
+    Eterm ref_copy = NIL, msg;
+    Eterm *hp = NULL;
+    Eterm **hpp;
+    Uint sz, *szp;
+    ErlOffHeap *ohp = NULL;
+    ErlHeapFragment *bp = NULL;
+
+    ASSERT(esdp);
+
+    reclaimed = esdp->gc_info.reclaimed;
+    garbage_cols = esdp->gc_info.garbage_cols;
+
+    sz = 0;
+    hpp = NULL;
+    szp = &sz;
+
+    while (1) {
+	if (hpp)
+	    ref_copy = STORE_NC(hpp, ohp, gcirp->ref);
+	else
+	    *szp += REF_THING_SIZE;
+
+	msg = erts_bld_tuple(hpp, szp, 3,
+			     make_small(esdp->no),
+			     erts_bld_uint64(hpp, szp, garbage_cols),
+			     erts_bld_uint64(hpp, szp, reclaimed));
+	
+	msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+	if (hpp)
+	  break;
+	
+	hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+	szp = NULL;
+	hpp = &hp;
+    }
+
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+		       );
+
+    if (gcirp->req_sched == esdp->no)
+	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+ 
+    if (rp_locks)
+	erts_smp_proc_unlock(rp, rp_locks);
+
+    erts_smp_proc_dec_refc(rp);
+
+    if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0)
+	gcireq_free(vgcirp);
+}
+
+Eterm
+erts_gc_info_request(Process *c_p)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    Eterm ref;
+    ErtsGCInfoReq *gcirp;
+    Eterm *hp;
+
+    gcirp = gcireq_alloc();
+    ref = erts_make_ref(c_p);
+    hp = &gcirp->ref_heap[0];
+
+    gcirp->proc = c_p;
+    gcirp->ref = STORE_NC(&hp, NULL, ref);
+    gcirp->req_sched = esdp->no;
+    erts_smp_atomic32_init_nob(&gcirp->refc,
+			       (erts_aint32_t) erts_no_schedulers);
+
+    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+					  erts_no_schedulers,
+					  reply_gc_info,
+					  (void *) gcirp);
+#endif
+
+    reply_gc_info((void *) gcirp);
+
+    return ref;
+}
+
 #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
 
 static int
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index b5bac2151d..b3a3c3d403 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -300,13 +300,13 @@ erl_init(int ncpu,
     init_benchmarking();
 
     erts_init_monitors();
-    erts_init_gc();
     erts_init_time();
     erts_init_sys_common_misc();
     erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab);
     erts_init_scheduling(no_schedulers,
 			 no_schedulers_online);
     erts_init_cpu_topology(); /* Must be after init_scheduling */
+    erts_init_gc(); /* Must be after init_scheduling */
     erts_alloc_late_init();
 
     H_MIN_SIZE      = erts_next_heap_size(H_MIN_SIZE, 0);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index bb3e787401..81799ddbb1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -4671,8 +4671,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online)
 	esdp->reductions = 0;
 
 	init_sched_wall_time(&esdp->sched_wall_time);
-
 	erts_port_task_handle_init(&esdp->nosuspend_port_task_handle);
+
     }
 
     init_misc_aux_work();
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 6ef1a0073d..7dd8116857 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -420,6 +420,11 @@ typedef struct {
 } ErtsSchedWallTime;
 
 typedef struct {
+  Uint64 reclaimed;
+  Uint64 garbage_cols;
+} ErtsGCInfo;
+
+typedef struct {
     int sched;
     erts_aint32_t aux_work;
 } ErtsDelayedAuxWorkWakeupJob;
@@ -507,6 +512,7 @@ struct ErtsSchedulerData_ {
 
     Uint64 reductions;
     ErtsSchedWallTime sched_wall_time;
+    ErtsGCInfo gc_info;
     ErtsPortTaskHandle nosuspend_port_task_handle;
 
 #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
@@ -1126,6 +1132,7 @@ void erts_early_init_scheduling(int);
 void erts_init_scheduling(int, int);
 
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+Eterm erts_gc_info_request(Process *c_p);
 Uint64 erts_get_proc_interval(void);
 Uint64 erts_ensure_later_proc_interval(Uint64);
 Uint64 erts_step_proc_interval(void);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index c900cdca39..26ed5f82c1 100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -688,12 +688,6 @@ void MD5Final(unsigned char [16], MD5_CTX *);
 
 /* ggc.c */
 
-
-typedef struct {
-    Uint garbage_collections;
-    Uint reclaimed;
-} ErtsGCInfo;
-
 void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
 int erts_garbage_collect(Process*, int, Eterm*, int);
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index 308cb99be5..315f9ebd47 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 7106c0a4fb..6929ca3fa5 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -45,7 +45,8 @@
 -export([alloc_info/1, alloc_sizes/1]).
 
 -export([gather_sched_wall_time_result/1,
-	 await_sched_wall_time_modifications/2]).
+	 await_sched_wall_time_modifications/2,
+	 gather_gc_info_result/1]).
 
 -deprecated([hash/2]).
 
@@ -3536,3 +3537,20 @@ sched_wall_time(Ref, N, Acc) ->
 	{Ref, undefined} -> sched_wall_time(Ref, N-1, undefined);
 	{Ref, SWT} -> sched_wall_time(Ref, N-1, [SWT|Acc])
     end.
+
+-spec erlang:gather_gc_info_result(Ref) -> [{pos_integer(),
+					     pos_integer(),
+					     0}] when
+      Ref :: reference().
+
+gather_gc_info_result(Ref) when erlang:is_reference(Ref) ->
+    gc_info(Ref, erlang:system_info(schedulers), {0,0}).
+
+gc_info(_Ref, 0, {Colls,Recl}) ->
+    {Colls,Recl,0};
+gc_info(Ref, N, {OrigColls,OrigRecl}) ->
+    receive
+	{Ref, {_,Colls, Recl}} -> 
+	    gc_info(Ref, N-1, {Colls+OrigColls,Recl+OrigRecl})
+    end.
+