aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/beam/copy.c4
-rw-r--r--erts/emulator/beam/erl_binary.h18
-rw-r--r--erts/emulator/beam/erl_bits.h17
-rw-r--r--erts/emulator/beam/erl_gc.c81
-rw-r--r--erts/emulator/beam/erl_gc.h95
-rw-r--r--erts/emulator/hipe/hipe_gc.c16
6 files changed, 146 insertions, 85 deletions
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 85db23393c..e567eabc82 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1990,7 +1990,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
if (is_header(val)) {
struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
ASSERT(ptr + header_arity(val) < end);
- MOVE_BOXED(ptr, val, hp, &dummy_ref);
+ move_boxed(&ptr, val, &hp, &dummy_ref);
switch (val & _HEADER_SUBTAG_MASK) {
case REF_SUBTAG:
if (is_ordinary_ref_thing(hdr))
@@ -2007,7 +2007,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite
}
else { /* must be a cons cell */
ASSERT(ptr+1 < end);
- MOVE_CONS(ptr, val, hp, &dummy_ref);
+ move_cons(&ptr, val, &hp, &dummy_ref);
ptr += 2;
}
}
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 946d3cfe03..6ff71ec6d1 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -156,6 +156,7 @@ typedef union {
#include "erl_threads.h"
#include "bif.h"
#include "erl_bif_unique.h"
+#include "erl_bits.h"
/*
* Maximum number of bytes to place in a heap binary.
@@ -163,23 +164,6 @@ typedef union {
#define ERL_ONHEAP_BIN_LIMIT 64
-/*
- * This structure represents a SUB_BINARY.
- *
- * Note: The last field (orig) is not counted in arityval in the header.
- * This simplifies garbage collection.
- */
-
-typedef struct erl_sub_bin {
- Eterm thing_word; /* Subtag SUB_BINARY_SUBTAG. */
- Uint size; /* Binary size in bytes. */
- Uint offs; /* Offset into original binary. */
- byte bitsize;
- byte bitoffs;
- byte is_writable; /* The underlying binary is writable */
- Eterm orig; /* Original binary (REFC or HEAP binary). */
-} ErlSubBin;
-
#define ERL_SUB_BIN_SIZE (sizeof(ErlSubBin)/sizeof(Eterm))
#define HEADER_SUB_BIN _make_header(ERL_SUB_BIN_SIZE-2,_TAG_HEADER_SUB_BIN)
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 4bd5b24157..1b4546722f 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -22,6 +22,23 @@
#define __ERL_BITS_H__
/*
+ * This structure represents a SUB_BINARY.
+ *
+ * Note: The last field (orig) is not counted in arityval in the header.
+ * This simplifies garbage collection.
+ */
+
+typedef struct erl_sub_bin {
+ Eterm thing_word; /* Subtag SUB_BINARY_SUBTAG. */
+ Uint size; /* Binary size in bytes. */
+ Uint offs; /* Offset into original binary. */
+ byte bitsize;
+ byte bitoffs;
+ byte is_writable; /* The underlying binary is writable */
+ Eterm orig; /* Original binary (REFC or HEAP binary). */
+} ErlSubBin;
+
+/*
* This structure represents a binary to be matched.
*/
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8cc7fc0142..50805d9cd9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1173,7 +1173,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
ASSERT(is_boxed(val));
*g_ptr++ = val;
} else if (ErtsInArea(ptr, area, area_size)) {
- MOVE_BOXED(ptr,val,old_htop,g_ptr++);
+ move_boxed(&ptr,val,&old_htop,g_ptr++);
} else {
g_ptr++;
}
@@ -1184,7 +1184,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
if (IS_MOVED_CONS(val)) { /* Moved */
*g_ptr++ = ptr[1];
} else if (ErtsInArea(ptr, area, area_size)) {
- MOVE_CONS(ptr,val,old_htop,g_ptr++);
+ move_cons(&ptr,val,&old_htop,g_ptr++);
} else {
g_ptr++;
}
@@ -1479,9 +1479,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
ASSERT(is_boxed(val));
*g_ptr++ = val;
} else if (ErtsInArea(ptr, mature, mature_size)) {
- MOVE_BOXED(ptr,val,old_htop,g_ptr++);
+ move_boxed(&ptr,val,&old_htop,g_ptr++);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
- MOVE_BOXED(ptr,val,n_htop,g_ptr++);
+ move_boxed(&ptr,val,&n_htop,g_ptr++);
} else {
g_ptr++;
}
@@ -1494,9 +1494,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
if (IS_MOVED_CONS(val)) { /* Moved */
*g_ptr++ = ptr[1];
} else if (ErtsInArea(ptr, mature, mature_size)) {
- MOVE_CONS(ptr,val,old_htop,g_ptr++);
+ move_cons(&ptr,val,&old_htop,g_ptr++);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
- MOVE_CONS(ptr,val,n_htop,g_ptr++);
+ move_cons(&ptr,val,&n_htop,g_ptr++);
} else {
g_ptr++;
}
@@ -1538,9 +1538,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
ASSERT(is_boxed(val));
*n_hp++ = val;
} else if (ErtsInArea(ptr, mature, mature_size)) {
- MOVE_BOXED(ptr,val,old_htop,n_hp++);
+ move_boxed(&ptr,val,&old_htop,n_hp++);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
- MOVE_BOXED(ptr,val,n_htop,n_hp++);
+ move_boxed(&ptr,val,&n_htop,n_hp++);
} else {
n_hp++;
}
@@ -1552,9 +1552,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
if (IS_MOVED_CONS(val)) {
*n_hp++ = ptr[1];
} else if (ErtsInArea(ptr, mature, mature_size)) {
- MOVE_CONS(ptr,val,old_htop,n_hp++);
+ move_cons(&ptr,val,&old_htop,n_hp++);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
- MOVE_CONS(ptr,val,n_htop,n_hp++);
+ move_cons(&ptr,val,&n_htop,n_hp++);
} else {
n_hp++;
}
@@ -1574,10 +1574,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
*origptr = val;
mb->base = binary_bytes(val);
} else if (ErtsInArea(ptr, mature, mature_size)) {
- MOVE_BOXED(ptr,val,old_htop,origptr);
+ move_boxed(&ptr,val,&old_htop,origptr);
mb->base = binary_bytes(mb->orig);
} else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) {
- MOVE_BOXED(ptr,val,n_htop,origptr);
+ move_boxed(&ptr,val,&n_htop,origptr);
mb->base = binary_bytes(mb->orig);
}
}
@@ -1792,7 +1792,7 @@ full_sweep_heaps(Process *p,
Eterm* ptr;
Eterm val;
Eterm gval = *g_ptr;
-
+
switch (primary_tag(gval)) {
case TAG_PRIMARY_BOXED: {
@@ -1802,7 +1802,7 @@ full_sweep_heaps(Process *p,
ASSERT(is_boxed(val));
*g_ptr++ = val;
} else if (!erts_is_literal(gval, ptr)) {
- MOVE_BOXED(ptr,val,n_htop,g_ptr++);
+ move_boxed(&ptr,val,&n_htop,g_ptr++);
} else {
g_ptr++;
}
@@ -1815,7 +1815,7 @@ full_sweep_heaps(Process *p,
if (IS_MOVED_CONS(val)) {
*g_ptr++ = ptr[1];
} else if (!erts_is_literal(gval, ptr)) {
- MOVE_CONS(ptr,val,n_htop,g_ptr++);
+ move_cons(&ptr,val,&n_htop,g_ptr++);
} else {
g_ptr++;
}
@@ -2104,7 +2104,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
ASSERT(is_boxed(val));
*n_hp++ = val;
} else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
- MOVE_BOXED(ptr,val,n_htop,n_hp++);
+ move_boxed(&ptr,val,&n_htop,n_hp++);
} else {
n_hp++;
}
@@ -2116,7 +2116,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
if (IS_MOVED_CONS(val)) {
*n_hp++ = ptr[1];
} else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
- MOVE_CONS(ptr,val,n_htop,n_hp++);
+ move_cons(&ptr,val,&n_htop,n_hp++);
} else {
n_hp++;
}
@@ -2129,7 +2129,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
if (header_is_bin_matchstate(gval)) {
ErlBinMatchState *ms = (ErlBinMatchState*) n_hp;
ErlBinMatchBuffer *mb = &(ms->mb);
- Eterm* origptr;
+ Eterm* origptr;
origptr = &(mb->orig);
ptr = boxed_val(*origptr);
val = *ptr;
@@ -2137,7 +2137,7 @@ sweep(Eterm *n_hp, Eterm *n_htop,
*origptr = val;
mb->base = binary_bytes(*origptr);
} else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) {
- MOVE_BOXED(ptr,val,n_htop,origptr);
+ move_boxed(&ptr,val,&n_htop,origptr);
mb->base = binary_bytes(*origptr);
}
}
@@ -2200,7 +2200,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
ASSERT(is_boxed(val));
*heap_ptr++ = val;
} else if (ErtsInArea(ptr, src, src_size)) {
- MOVE_BOXED(ptr,val,htop,heap_ptr++);
+ move_boxed(&ptr,val,&htop,heap_ptr++);
} else {
heap_ptr++;
}
@@ -2212,7 +2212,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
if (IS_MOVED_CONS(val)) {
*heap_ptr++ = ptr[1];
} else if (ErtsInArea(ptr, src, src_size)) {
- MOVE_CONS(ptr,val,htop,heap_ptr++);
+ move_cons(&ptr,val,&htop,heap_ptr++);
} else {
heap_ptr++;
}
@@ -2233,7 +2233,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
*origptr = val;
mb->base = binary_bytes(*origptr);
} else if (ErtsInArea(ptr, src, src_size)) {
- MOVE_BOXED(ptr,val,htop,origptr);
+ move_boxed(&ptr,val,&htop,origptr);
mb->base = binary_bytes(*origptr);
}
}
@@ -2266,11 +2266,11 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size)
ASSERT(val != ERTS_HOLE_MARKER);
if (is_header(val)) {
ASSERT(ptr + header_arity(val) < end);
- MOVE_BOXED(ptr, val, n_htop, &dummy_ref);
+ move_boxed(&ptr, val, &n_htop, &dummy_ref);
}
else { /* must be a cons cell */
ASSERT(ptr+1 < end);
- MOVE_CONS(ptr, val, n_htop, &dummy_ref);
+ move_cons(&ptr, val, &n_htop, &dummy_ref);
ptr += 2;
}
}
@@ -3256,6 +3256,39 @@ reply_gc_info(void *vgcirp)
gcireq_free(vgcirp);
}
+void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) {
+ Eterm *ptr = *pp;
+ Eterm *htop = *hpp;
+ Eterm gval;
+ ErlSubBin *sb = (ErlSubBin *)ptr;
+ ErlHeapBin *hb = (ErlHeapBin *)htop;
+ Eterm *real_bin;
+ byte *bs;
+
+ real_bin = binary_val(follow_moved(sb->orig, (Eterm)0));
+
+ if (*real_bin == HEADER_PROC_BIN) {
+ bs = ((ProcBin *) real_bin)->bytes + sb->offs;
+ } else {
+ bs = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + sb->offs;
+ }
+
+ hb->thing_word = header_heap_bin(sb->size);
+ hb->size = sb->size;
+ sys_memcpy((byte *)hb->data, bs, sb->size);
+
+ gval = make_boxed(htop);
+ *orig = gval;
+ *ptr = gval;
+
+ ptr += ERL_SUB_BIN_SIZE;
+ htop += heap_bin_size(sb->size);
+
+ *hpp = htop;
+ *pp = ptr;
+}
+
+
Eterm
erts_gc_info_request(Process *c_p)
{
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 1cce426d21..414aff1e06 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -28,44 +28,71 @@
#define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */
#include "erl_map.h"
+#include "erl_fun.h"
+#include "erl_bits.h"
#define IS_MOVED_BOXED(x) (!is_header((x)))
#define IS_MOVED_CONS(x) (is_non_value((x)))
+void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig);
-#define MOVE_CONS(PTR,CAR,HTOP,ORIG) \
-do { \
- Eterm gval; \
- \
- HTOP[0] = CAR; /* copy car */ \
- HTOP[1] = PTR[1]; /* copy cdr */ \
- gval = make_list(HTOP); /* new location */ \
- *ORIG = gval; /* redirect original reference */ \
- PTR[0] = THE_NON_VALUE; /* store forwarding indicator */ \
- PTR[1] = gval; /* store forwarding address */ \
- HTOP += 2; /* update tospace htop */ \
-} while(0)
-
-#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \
-do { \
- Eterm gval; \
- Sint nelts; \
- \
- ASSERT(is_header(HDR)); \
- nelts = header_arity(HDR); \
- switch ((HDR) & _HEADER_SUBTAG_MASK) { \
- case SUB_BINARY_SUBTAG: nelts++; break; \
- case MAP_SUBTAG: \
- if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1; \
- else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR)); \
- break; \
- case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break; \
- } \
- gval = make_boxed(HTOP); \
- *ORIG = gval; \
- *HTOP++ = HDR; \
- *PTR++ = gval; \
- while (nelts--) *HTOP++ = *PTR++; \
-} while(0)
+ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig);
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig)
+{
+ Eterm *ptr = *pp;
+ Eterm *htop = *hpp;
+ Eterm gval;
+
+ htop[0] = car; /* copy car */
+ htop[1] = ptr[1]; /* copy cdr */
+ gval = make_list(htop); /* new location */
+ *orig = gval; /* redirect original reference */
+ ptr[0] = THE_NON_VALUE; /* store forwarding indicator */
+ ptr[1] = gval; /* store forwarding address */
+ *hpp += 2; /* update tospace htop */
+}
+#endif
+
+ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig);
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig)
+{
+ Eterm gval;
+ Sint nelts;
+ Eterm *ptr = *pp;
+ Eterm *htop = *hpp;
+
+ ASSERT(is_header(hdr));
+ nelts = header_arity(hdr);
+ switch ((hdr) & _HEADER_SUBTAG_MASK) {
+ case SUB_BINARY_SUBTAG:
+ {
+ ErlSubBin *sb = (ErlSubBin *)ptr;
+ /* convert sub-binary to heap-binary if applicable */
+ if (sb->bitsize == 0 && sb->bitoffs == 0 &&
+ sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) {
+ erts_sub_binary_to_heap_binary(pp, hpp, orig);
+ return;
+ }
+ }
+ nelts++;
+ break;
+ case MAP_SUBTAG:
+ if (is_flatmap_header(hdr)) nelts+=flatmap_get_size(ptr) + 1;
+ else nelts += hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ break;
+ case FUN_SUBTAG: nelts+=((ErlFunThing*)(ptr))->num_free+1; break;
+ }
+ gval = make_boxed(htop);
+ *orig = gval;
+ *htop++ = hdr;
+ *ptr++ = gval;
+ while (nelts--) *htop++ = *ptr++;
+
+ *hpp = htop;
+ *pp = ptr;
+}
+#endif
#define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz) \
(!erts_is_literal((TPtr), (Ptr)) \
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index cf0435adc9..2311beb34a 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -91,7 +91,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
ASSERT(is_boxed(val));
*nsp_i = val;
} else if (!erts_is_literal(gval, ptr)) {
- MOVE_BOXED(ptr, val, n_htop, nsp_i);
+ move_boxed(&ptr, val, &n_htop, nsp_i);
}
} else if (is_list(gval)) {
Eterm *ptr = list_val(gval);
@@ -100,7 +100,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
*nsp_i = ptr[1];
} else if (!erts_is_literal(gval, ptr)) {
ASSERT(erts_dbg_within_proc(ptr, p, NULL));
- MOVE_CONS(ptr, val, n_htop, nsp_i);
+ move_cons(&ptr, val, &n_htop, nsp_i);
}
}
}
@@ -206,10 +206,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
ASSERT(is_boxed(val));
*nsp_i = val;
} else if (ErtsInArea(ptr, mature, mature_size)) {
- MOVE_BOXED(ptr, val, old_htop, nsp_i);
+ move_boxed(&ptr, val, &old_htop, nsp_i);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
ASSERT(erts_dbg_within_proc(ptr, p, NULL));
- MOVE_BOXED(ptr, val, n_htop, nsp_i);
+ move_boxed(&ptr, val, &n_htop, nsp_i);
}
} else if (is_list(gval)) {
Eterm *ptr = list_val(gval);
@@ -217,10 +217,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
if (IS_MOVED_CONS(val)) {
*nsp_i = ptr[1];
} else if (ErtsInArea(ptr, mature, mature_size)) {
- MOVE_CONS(ptr, val, old_htop, nsp_i);
+ move_cons(&ptr, val, &old_htop, nsp_i);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
ASSERT(erts_dbg_within_proc(ptr, p, NULL));
- MOVE_CONS(ptr, val, n_htop, nsp_i);
+ move_cons(&ptr, val, &n_htop, nsp_i);
}
}
}
@@ -278,7 +278,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
ASSERT(is_boxed(val));
*nsp_i = val;
} else if (ErtsInArea(ptr, area, area_size)) {
- MOVE_BOXED(ptr, val, old_htop, nsp_i);
+ move_boxed(&ptr, val, &old_htop, nsp_i);
}
} else if (is_list(gval)) {
Eterm *ptr = list_val(gval);
@@ -286,7 +286,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
if (IS_MOVED_CONS(val)) {
*nsp_i = ptr[1];
} else if (ErtsInArea(ptr, area, area_size)) {
- MOVE_CONS(ptr, val, old_htop, nsp_i);
+ move_cons(&ptr, val, &old_htop, nsp_i);
}
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-03 04:12:56 +0000