aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_gc.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_gc.c')
-rw-r--r--erts/emulator/beam/erl_gc.c277
1 files changed, 208 insertions, 69 deletions
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index e2689f58c3..8ba94d89e9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2013. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -47,10 +47,6 @@
*/
#define ALENGTH(a) (sizeof(a)/sizeof(a[0]))
-static erts_smp_spinlock_t info_lck;
-static Uint garbage_cols; /* no of garbage collections */
-static Uint reclaimed; /* no of words reclaimed in GCs */
-
# define STACK_SZ_ON_HEAP(p) ((p)->hend - (p)->stop)
# define OverRunCheck(P) \
if ((P)->stop < (P)->htop) { \
@@ -59,7 +55,7 @@ static Uint reclaimed; /* no of words reclaimed in GCs */
erts_fprintf(stderr, "htop=%p\n", (p)->htop); \
erts_fprintf(stderr, "heap=%p\n", (p)->heap); \
erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \
- __FILE__,__LINE__,(P)->id); \
+ __FILE__,__LINE__,(P)->common.id); \
}
#ifdef DEBUG
@@ -120,6 +116,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size);
+static void init_gc_info(ErtsGCInfo *gcip);
+
#ifdef HARDDEBUG
static void disallow_heap_frag_ref_in_heap(Process* p);
static void disallow_heap_frag_ref_in_old_heap(Process* p);
@@ -129,7 +127,7 @@ static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int n
#if defined(ARCH_64) && !HALFWORD_HEAP
# define MAX_HEAP_SIZES 154
#else
-# define MAX_HEAP_SIZES 55
+# define MAX_HEAP_SIZES 59
#endif
static Sint heap_sizes[MAX_HEAP_SIZES]; /* Suitable heap sizes. */
@@ -137,13 +135,42 @@ static int num_heap_sizes; /* Number of heap sizes. */
Uint erts_test_long_gc_sleep; /* Only used for testing... */
+typedef struct {
+ Process *proc;
+ Eterm ref;
+ Eterm ref_heap[REF_THING_SIZE];
+ Uint req_sched;
+ erts_smp_atomic32_t refc;
+} ErtsGCInfoReq;
+
+#if !HALFWORD_HEAP
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
+ ErtsGCInfoReq,
+ 5,
+ ERTS_ALC_T_GC_INFO_REQ)
+#else
+static ERTS_INLINE ErtsGCInfoReq *
+gcireq_alloc(void)
+{
+ return erts_alloc(ERTS_ALC_T_GC_INFO_REQ,
+ sizeof(ErtsGCInfoReq));
+}
+
+static ERTS_INLINE void
+gcireq_free(ErtsGCInfoReq *ptr)
+{
+ erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr);
+}
+#endif
+
/*
* Initialize GC global data.
*/
void
erts_init_gc(void)
{
- int i = 0;
+ int i = 0, ix;
+ Sint max_heap_size = 0;
ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word));
@@ -155,9 +182,6 @@ erts_init_gc(void)
ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
- erts_smp_spinlock_init(&info_lck, "gc_info");
- garbage_cols = 0;
- reclaimed = 0;
erts_test_long_gc_sleep = 0;
/*
@@ -168,22 +192,46 @@ erts_init_gc(void)
* we really don't want that growth when the heaps are that big.
*/
- heap_sizes[0] = 34;
- heap_sizes[1] = 55;
- for (i = 2; i < 23; i++) {
- heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2];
+ /* Growth stage 1 - Fibonacci + 1*/
+ /* 12,38 will hit size 233, the old default */
+
+ heap_sizes[0] = 12;
+ heap_sizes[1] = 38;
+
+ for(i = 2; i < 23; i++) {
+ /* one extra word for block header */
+ heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-2] + 1;
}
+
+ /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] (and halfword)
+ * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words]
+ */
+
+ max_heap_size = sizeof(Eterm) < 8 ? (Sint)((~(Uint)0)/(sizeof(Eterm))) :
+ (Sint)(((Uint64)1 << 53)/sizeof(Eterm));
+
+ /* Growth stage 2 - 20% growth */
/* At 1.3 mega words heap, we start to slow down. */
for (i = 23; i < ALENGTH(heap_sizes); i++) {
- heap_sizes[i] = 5*(heap_sizes[i-1]/4);
- if (heap_sizes[i] < 0) {
+ heap_sizes[i] = heap_sizes[i-1] + heap_sizes[i-1]/5;
+ if ((heap_sizes[i] < 0) || heap_sizes[i] > max_heap_size) {
/* Size turned negative. Discard this last size. */
i--;
break;
}
}
num_heap_sizes = i;
+
+ for (ix = 0; ix < erts_no_schedulers; ix++) {
+ ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
+ init_gc_info(&esdp->gc_info);
+ }
+
+#if !HALFWORD_HEAP
+ init_gcireq_alloc();
+#endif
+
}
/*
@@ -216,7 +264,7 @@ erts_next_heap_size(Uint size, Uint offset)
low = mid + 1;
}
}
- erl_exit(1, "no next heap size found: %lu, offset %lu\n", (unsigned long)size, (unsigned long)offset);
+ erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset);
}
return 0;
}
@@ -272,17 +320,6 @@ erts_heap_sizes(Process* p)
return res;
}
-void
-erts_gc_info(ErtsGCInfo *gcip)
-{
- if (gcip) {
- erts_smp_spin_lock(&info_lck);
- gcip->garbage_collections = garbage_cols;
- gcip->reclaimed = reclaimed;
- erts_smp_spin_unlock(&info_lck);
- }
-}
-
void
erts_offset_heap(Eterm* hp, Uint sz, Sint offs, Eterm* low, Eterm* high)
{
@@ -363,6 +400,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
Uint reclaimed_now = 0;
int done = 0;
Uint ms1, s1, us1;
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef USE_VM_PROBES
DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
#endif
@@ -370,11 +408,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
trace_gc(p, am_gc_start);
}
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->gcstatus = p->status;
- p->status = P_GARBING;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-
+ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
if (erts_system_monitor_long_gc != 0) {
get_now(&ms1, &s1, &us1);
}
@@ -418,9 +452,8 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
ErtsGcQuickSanityCheck(p);
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->status = p->gcstatus;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
if (IS_TRACED_FL(p, F_TRACE_GC)) {
trace_gc(p, am_gc_end);
}
@@ -445,11 +478,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
monitor_large_heap(p);
}
- erts_smp_spin_lock(&info_lck);
- garbage_cols++;
- reclaimed += reclaimed_now;
- erts_smp_spin_unlock(&info_lck);
-
+ esdp->gc_info.garbage_cols++;
+ esdp->gc_info.reclaimed += reclaimed_now;
+
FLAGS(p) &= ~F_FORCE_GC;
#ifdef CHECK_FOR_HOLES
@@ -504,10 +535,7 @@ erts_garbage_collect_hibernate(Process* p)
/*
* Preliminaries.
*/
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->gcstatus = p->status;
- p->status = P_GARBING;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
ErtsGcQuickSanityCheck(p);
ASSERT(p->mbuf_sz == 0);
ASSERT(p->mbuf == 0);
@@ -618,9 +646,7 @@ erts_garbage_collect_hibernate(Process* p)
ErtsGcQuickSanityCheck(p);
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->status = p->gcstatus;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
}
@@ -644,10 +670,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
/*
* Set GC state.
*/
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->gcstatus = p->status;
- p->status = P_GARBING;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
/*
* We assume that the caller has already done a major collection
@@ -789,9 +812,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
/*
* Restore status.
*/
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
- p->status = p->gcstatus;
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
}
static int
@@ -883,14 +904,12 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
}
}
- if (wanted < MIN_HEAP_SIZE(p)) {
- wanted = MIN_HEAP_SIZE(p);
- } else {
- wanted = next_heap_size(p, wanted, 0);
- }
+ wanted = wanted < MIN_HEAP_SIZE(p) ? MIN_HEAP_SIZE(p)
+ : next_heap_size(p, wanted, 0);
if (wanted < HEAP_SIZE(p)) {
shrink_new_heap(p, wanted, objv, nobj);
}
+
ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
return 1; /* We are done. */
}
@@ -1449,11 +1468,10 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int
I think this is better as fullsweep is used mainly on
small memory systems, but I could be wrong... */
wanted = 2 * need_after;
- if (wanted < p->min_heap_size) {
- sz = p->min_heap_size;
- } else {
- sz = next_heap_size(p, wanted, 0);
- }
+
+ sz = wanted < p->min_heap_size ? p->min_heap_size
+ : next_heap_size(p, wanted, 0);
+
if (sz < HEAP_SIZE(p)) {
shrink_new_heap(p, sz, objv, nobj);
}
@@ -1946,6 +1964,17 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
++n;
}
+ /*
+ * A trapping BIF can add to rootset by setting the extra_root
+ * in the process_structure.
+ */
+ if (p->extra_root != NULL) {
+ roots[n].v = p->extra_root->objv;
+ roots[n].sz = p->extra_root->sz;
+ ++n;
+ }
+
+
ASSERT((is_nil(p->seq_trace_token) ||
is_tuple(follow_moved(p->seq_trace_token)) ||
is_atom(p->seq_trace_token)));
@@ -1961,9 +1990,9 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
n++;
}
#endif
- ASSERT(is_nil(p->tracer_proc) ||
- is_internal_pid(p->tracer_proc) ||
- is_internal_port(p->tracer_proc));
+ ASSERT(is_nil(ERTS_TRACER_PROC(p)) ||
+ is_internal_pid(ERTS_TRACER_PROC(p)) ||
+ is_internal_port(ERTS_TRACER_PROC(p)));
ASSERT(is_pid(follow_moved(p->group_leader)));
if (is_not_immed(p->group_leader)) {
@@ -2523,6 +2552,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
p->dictionary->used,
offs, area, area_size);
}
+ if (p->extra_root != NULL) {
+ offset_heap_ptr(p->extra_root->objv,
+ p->extra_root->sz,
+ offs, area, area_size);
+ }
+
offset_heap_ptr(&p->fvalue, 1, offs, area, area_size);
offset_heap_ptr(&p->ftrace, 1, offs, area, area_size);
offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size);
@@ -2546,6 +2581,110 @@ offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
offset_one_rootset(p, offs, area, area_size, objv, nobj);
}
+static void
+init_gc_info(ErtsGCInfo *gcip)
+{
+ gcip->reclaimed = 0;
+ gcip->garbage_cols = 0;
+}
+
+static void
+reply_gc_info(void *vgcirp)
+{
+ Uint64 reclaimed = 0, garbage_cols = 0;
+ ErtsSchedulerData *esdp = erts_get_scheduler_data();
+ ErtsGCInfoReq *gcirp = (ErtsGCInfoReq *) vgcirp;
+ ErtsProcLocks rp_locks = (gcirp->req_sched == esdp->no
+ ? ERTS_PROC_LOCK_MAIN
+ : 0);
+ Process *rp = gcirp->proc;
+ Eterm ref_copy = NIL, msg;
+ Eterm *hp = NULL;
+ Eterm **hpp;
+ Uint sz, *szp;
+ ErlOffHeap *ohp = NULL;
+ ErlHeapFragment *bp = NULL;
+
+ ASSERT(esdp);
+
+ reclaimed = esdp->gc_info.reclaimed;
+ garbage_cols = esdp->gc_info.garbage_cols;
+
+ sz = 0;
+ hpp = NULL;
+ szp = &sz;
+
+ while (1) {
+ if (hpp)
+ ref_copy = STORE_NC(hpp, ohp, gcirp->ref);
+ else
+ *szp += REF_THING_SIZE;
+
+ msg = erts_bld_tuple(hpp, szp, 3,
+ make_small(esdp->no),
+ erts_bld_uint64(hpp, szp, garbage_cols),
+ erts_bld_uint64(hpp, szp, reclaimed));
+
+ msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+ if (hpp)
+ break;
+
+ hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+ szp = NULL;
+ hpp = &hp;
+ }
+
+ erts_queue_message(rp, &rp_locks, bp, msg, NIL
+#ifdef USE_VM_PROBES
+ , NIL
+#endif
+ );
+
+ if (gcirp->req_sched == esdp->no)
+ rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+
+ if (rp_locks)
+ erts_smp_proc_unlock(rp, rp_locks);
+
+ erts_smp_proc_dec_refc(rp);
+
+ if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0)
+ gcireq_free(vgcirp);
+}
+
+Eterm
+erts_gc_info_request(Process *c_p)
+{
+ ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+ Eterm ref;
+ ErtsGCInfoReq *gcirp;
+ Eterm *hp;
+
+ gcirp = gcireq_alloc();
+ ref = erts_make_ref(c_p);
+ hp = &gcirp->ref_heap[0];
+
+ gcirp->proc = c_p;
+ gcirp->ref = STORE_NC(&hp, NULL, ref);
+ gcirp->req_sched = esdp->no;
+ erts_smp_atomic32_init_nob(&gcirp->refc,
+ (erts_aint32_t) erts_no_schedulers);
+
+ erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+ if (erts_no_schedulers > 1)
+ erts_schedule_multi_misc_aux_work(1,
+ erts_no_schedulers,
+ reply_gc_info,
+ (void *) gcirp);
+#endif
+
+ reply_gc_info((void *) gcirp);
+
+ return ref;
+}
+
#if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
static int
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-23 12:47:04 +0000