aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--erts/emulator/beam/beam_emu.c4
-rw-r--r--erts/emulator/beam/binary.c4
-rw-r--r--erts/emulator/beam/copy.c12
-rw-r--r--erts/emulator/beam/erl_bif_binary.c2
-rw-r--r--erts/emulator/beam/erl_bits.c4
-rw-r--r--erts/emulator/beam/erl_gc.c55
-rw-r--r--erts/emulator/beam/erl_message.c4
-rw-r--r--erts/emulator/beam/erl_message.h6
-rw-r--r--erts/emulator/beam/erl_nif.c2
-rw-r--r--erts/emulator/beam/erl_process.c2
-rw-r--r--erts/emulator/beam/erl_process.h8
-rw-r--r--erts/emulator/beam/io.c10
12 files changed, 60 insertions, 53 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 260f349563..0b29644f24 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3390,7 +3390,7 @@ apply_bif_or_nif_epilogue:
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
- MSO(c_p).overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
new_binary = make_binary(pb);
goto do_bits_sub_bin;
}
@@ -3492,7 +3492,7 @@ apply_bif_or_nif_epilogue:
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
- MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));
StoreBifResult(2, make_binary(pb));
}
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 3fd714f9c2..8ee8fbcb29 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len)
/*
* Miscellanous updates. Return the tagged binary.
*/
- MSO(p).overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
return make_binary(pb);
}
@@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
/*
* Miscellanous updates. Return the tagged binary.
*/
- MSO(p).overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
return make_binary(pb);
}
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 57be0169ba..8bee47232e 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -317,7 +317,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
pb->next = off_heap->first;
pb->flags = 0;
off_heap->first = (struct erl_off_heap_header*) pb;
- off_heap->overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
}
break;
case SUB_BINARY_SUBTAG:
@@ -366,7 +366,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
to->next = off_heap->first;
to->flags = 0;
off_heap->first = (struct erl_off_heap_header*) to;
- off_heap->overhead += to->size / sizeof(Eterm);
+ OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
}
*argp = make_binary(hbot);
if (extra_bytes != 0) {
@@ -652,7 +652,7 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
erts_refc_inc(&pb->val->refc, 2);
pb->next = erts_global_offheap.first;
erts_global_offheap.first = pb;
- erts_global_offheap.overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
continue;
}
@@ -777,7 +777,7 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
to_bin->bytes = from_bin->bytes + sub_offset;
to_bin->next = erts_global_offheap.first;
erts_global_offheap.first = to_bin;
- erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
+ OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm));
res_binary=make_binary(to_bin);
hp += PROC_BIN_SIZE;
}
@@ -912,7 +912,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
{
ProcBin* pb = (ProcBin *) (tp-1);
erts_refc_inc(&pb->val->refc, 2);
- off_heap->overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
}
goto off_heap_common;
@@ -977,7 +977,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
for (bp=first; bp!=NULL; bp=bp->next) {
move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
- off_heap->overhead += bp->off_heap.overhead;
+ OH_OVERHEAD(off_heap, bp->off_heap.overhead);
}
hp_end = *hpp;
for (hp=hp_start; hp<hp_end; ++hp) {
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 1d8fd11b7b..59fce04343 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2517,7 +2517,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
pb->bytes = t;
pb->flags = 0;
- MSO(BIF_P).overhead += target_size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm));
BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR);
BIF_RET(make_binary(pb));
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 96faa673f6..88d2c06246 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1340,7 +1340,7 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
- MSO(c_p).overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
/*
* Now allocate the sub binary and set its size to include the
@@ -1511,7 +1511,7 @@ erts_bs_init_writable(Process* p, Eterm sz)
pb->val = bptr;
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
- MSO(p).overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
/*
* Now allocate the sub binary.
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 38cc6195fb..0f4d2a2ef9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -743,7 +743,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
* is large enough.
*/
- if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) {
+ if (OLD_HEAP(p) &&
+ ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
+ ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
+ ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
ErlMessage *msgp;
Uint size_after;
Uint need_after;
@@ -1982,8 +1985,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
HEAP_SIZE(p) = new_sz;
}
-static Uint
-do_next_vheap_size(Uint vheap, Uint vheap_sz) {
+static Uint64
+do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) {
/* grow
*
@@ -1999,37 +2002,34 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) {
*
* ----------------------
*/
- Uint vheap_max = heap_sizes[num_heap_sizes - 1];
- if (vheap_sz == vheap_max) {
- return vheap_sz;
- }
- if ((Uint) vheap/3 > (Uint) (vheap_sz/4)) {
- Uint new_vheap_sz = vheap_sz;
+ if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+ Uint64 new_vheap_sz = vheap_sz;
- while((Uint) vheap/3 > (Uint) (vheap_sz/4)) {
- new_vheap_sz = (vheap_sz << 1);
- if (new_vheap_sz < vheap_sz || new_vheap_sz > vheap_max ) {
- return vheap_max;
+ while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+ /* the golden ratio = 1.618 */
+ new_vheap_sz = (Uint64) vheap_sz * 1.618;
+ if (new_vheap_sz < vheap_sz ) {
+ return vheap_sz;
}
vheap_sz = new_vheap_sz;
}
- return erts_next_heap_size(vheap_sz, 0);
+ return vheap_sz;
}
- if (vheap < (Uint) (vheap_sz/4)) {
- return erts_next_heap_size((Uint) (vheap_sz >> 1), 0);
+ if (vheap < (Uint64) (vheap_sz/4)) {
+ return (vheap_sz >> 1);
}
return vheap_sz;
}
-static Uint
-next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) {
- vheap_sz = do_next_vheap_size(vheap, vheap_sz);
- return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz;
+static Uint64
+next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) {
+ Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz);
+ return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz;
}
struct shrink_cand_data {
@@ -2088,7 +2088,7 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
}
-static void
+static void
sweep_off_heap(Process *p, int fullsweep)
{
struct shrink_cand_data shrink = {0};
@@ -2096,7 +2096,7 @@ sweep_off_heap(Process *p, int fullsweep)
struct erl_off_heap_header** prev;
char* oheap = NULL;
Uint oheap_sz = 0;
- Uint bin_vheap = 0;
+ Uint64 bin_vheap = 0;
#ifdef DEBUG
int seen_mature = 0;
#endif
@@ -2180,13 +2180,12 @@ sweep_off_heap(Process *p, int fullsweep)
}
}
- if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) {
- FLAGS(p) |= F_NEED_FULLSWEEP;
+ if (fullsweep) {
+ BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p));
}
-
- BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
- BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p));
- MSO(p).overhead = bin_vheap;
+ BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
+ MSO(p).overhead = bin_vheap;
+ BIN_VHEAP_MATURE(p) = bin_vheap;
/*
* If we got any shrink candidates, check them out.
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 1f61dca230..82f272d28a 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -220,7 +220,7 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
*next_p = MSO(proc).first;
MSO(proc).first = bp->off_heap.first;
bp->off_heap.first = NULL;
- MSO(proc).overhead += bp->off_heap.overhead;
+ OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
}
}
}
@@ -535,7 +535,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
goto copy_done;
}
- off_heap->overhead += bp->off_heap.overhead;
+ OH_OVERHEAD(off_heap, bp->off_heap.overhead);
sz = bp->used_size;
ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 55ea92860a..5aca0db6fe 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -37,9 +37,13 @@ struct erl_off_heap_header {
struct erl_off_heap_header* next;
};
+#define OH_OVERHEAD(oh, size) do { \
+ (oh)->overhead += size; \
+} while(0)
+
typedef struct erl_off_heap {
struct erl_off_heap_header* first;
- int overhead; /* Administrative overhead (used to force GC). */
+ Uint64 overhead; /* Administrative overhead (used to force GC). */
} ErlOffHeap;
#include "external.h"
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e95d9c4f75..eadaf137e3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -631,7 +631,7 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
pb->bytes = (byte*) bptr->orig_bytes;
pb->flags = 0;
- MSO(env->proc).overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
bin_term = make_binary(pb);
if (erts_refc_read(&bptr->refc, 1) == 1) {
/* Total ownership transfer */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a644520442..c2e62f56a5 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7799,6 +7799,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->bin_vheap_sz = p->min_vheap_size;
p->bin_old_vheap_sz = p->min_vheap_size;
p->bin_old_vheap = 0;
+ p->bin_vheap_mature = 0;
/* No need to initialize p->fcalls. */
@@ -8051,6 +8052,7 @@ void erts_init_empty_process(Process *p)
p->bin_vheap_sz = BIN_VH_MIN_SIZE;
p->bin_old_vheap_sz = BIN_VH_MIN_SIZE;
p->bin_old_vheap = 0;
+ p->bin_vheap_mature = 0;
#ifdef ERTS_SMP
p->u.ptimer = NULL;
p->bound_runq = NULL;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c3fef6d38e..08b0144c41 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -537,6 +537,7 @@ struct ErtsPendingSuspend_ {
# define MIN_VHEAP_SIZE(p) (p)->min_vheap_size
# define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz
+# define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
# define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
# define BIN_OLD_VHEAP(p) (p)->bin_old_vheap
@@ -654,9 +655,10 @@ struct process {
Uint mbuf_sz; /* Size of all message buffers */
ErtsPSD *psd; /* Rarely used process specific data */
- Uint bin_vheap_sz; /* Virtual heap block size for binaries */
- Uint bin_old_vheap_sz; /* Virtual old heap block size for binaries */
- Uint bin_old_vheap; /* Virtual old heap size for binaries */
+ Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */
+ Uint64 bin_vheap_mature; /* Virtual heap block size for binaries */
+ Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */
+ Uint64 bin_old_vheap; /* Virtual old heap size for binaries */
union {
#ifdef ERTS_SMP
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b840f65cdd..79022d5dd7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1582,7 +1582,7 @@ static void deliver_read_message(Port* prt, Eterm to,
pb->flags = 0;
hp += PROC_BIN_SIZE;
- ohp->overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
listp = make_binary(pb);
}
@@ -1732,7 +1732,7 @@ deliver_vec_message(Port* prt, /* Port */
pb->flags = 0;
hp += PROC_BIN_SIZE;
- ohp->overhead += iov->iov_len / sizeof(Eterm);
+ OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm));
if (listp == NIL) { /* compatible with deliver_bin_message */
listp = make_binary(pb);
@@ -2264,7 +2264,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
pb->val = ErlDrvBinary2Binary(dbin);
pb->bytes = (byte*) dbin->orig_bytes;
pb->flags = 0;
- MSO(p).overhead += dbin->orig_size / sizeof(Eterm);
+ OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm));
return make_binary(pb);
}
port_resp = dbin->orig_bytes;
@@ -3040,7 +3040,7 @@ driver_deliver_term(ErlDrvPort port,
pb->flags = 0;
mess = make_binary(pb);
hp += PROC_BIN_SIZE;
- ohp->overhead += pb->size / sizeof(Eterm);
+ OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
}
ptr += 3;
break;
@@ -3077,7 +3077,7 @@ driver_deliver_term(ErlDrvPort port,
pbp->val = bp;
pbp->bytes = (byte*) bp->orig_bytes;
pbp->flags = 0;
- ohp->overhead += (pbp->size / sizeof(Eterm));
+ OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
mess = make_binary(pbp);
}
ptr += 2;
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-02 20:16:50 +0000