aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_bif_load.c24
-rw-r--r--erts/emulator/beam/beam_load.c53
-rw-r--r--erts/emulator/beam/beam_load.h8
-rw-r--r--erts/emulator/beam/dist.c2
-rw-r--r--erts/emulator/beam/erl_bif_port.c2
-rw-r--r--erts/emulator/beam/erl_gc.c47
-rw-r--r--erts/emulator/beam/erl_message.c4
-rw-r--r--erts/emulator/beam/external.c26
-rw-r--r--erts/emulator/beam/external.h4
-rw-r--r--erts/emulator/beam/global.h4
10 files changed, 141 insertions, 33 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index efb72cd3e7..bc8c001454 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -40,6 +40,7 @@ static Eterm check_process_code(Process* rp, Module* modp);
static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp);
static void delete_export_references(Eterm module);
static int purge_module(int module);
+static void decrement_refc(BeamInstr* code);
static int is_native(BeamInstr* code);
static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
@@ -553,6 +554,7 @@ check_process_code(Process* rp, Module* modp)
} else {
Eterm* literals;
Uint lit_size;
+ struct erl_off_heap_header* oh;
/*
* Try to get rid of constants by by garbage collecting.
@@ -566,7 +568,9 @@ check_process_code(Process* rp, Module* modp)
(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
literals = (Eterm *) modp->old_code[MI_LITERALS_START];
lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals;
- erts_garbage_collect_literals(rp, literals, lit_size);
+ oh = (struct erl_off_heap_header *)
+ modp->old_code[MI_LITERALS_OFF_HEAP];
+ erts_garbage_collect_literals(rp, literals, lit_size, oh);
}
}
return am_false;
@@ -664,6 +668,7 @@ purge_module(int module)
end = (BeamInstr *)((char *)code + modp->old_code_length);
erts_cleanup_funs_on_purge(code, end);
beam_catches_delmod(modp->old_catches, code, modp->old_code_length);
+ decrement_refc(code);
erts_free(ERTS_ALC_T_CODE, (void *) code);
modp->old_code = NULL;
modp->old_code_length = 0;
@@ -673,6 +678,23 @@ purge_module(int module)
}
static void
+decrement_refc(BeamInstr* code)
+{
+ struct erl_off_heap_header* oh =
+ (struct erl_off_heap_header *) code[MI_LITERALS_OFF_HEAP];
+
+ while (oh) {
+ Binary* bptr;
+ ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG);
+ bptr = ((ProcBin*)oh)->val;
+ if (erts_refc_dectest(&bptr->refc, 0) == 0) {
+ erts_bin_free(bptr);
+ }
+ oh = oh->next;
+ }
+}
+
+static void
remove_from_address_table(BeamInstr* code)
{
int i;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 10ed46417b..e43d364add 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -206,6 +206,7 @@ typedef struct {
Eterm term; /* The tagged term (in the heap). */
Uint heap_size; /* (Exact) size on the heap. */
Uint offset; /* Offset from temporary location to final. */
+ ErlOffHeap off_heap; /* Start of linked list of ProcBins. */
Eterm* heap; /* Heap for term. */
} Literal;
@@ -1419,12 +1420,14 @@ read_literal_table(LoaderState* stp)
GetInt(stp, 4, sz); /* Size of external term format. */
GetString(stp, p, sz);
- if ((heap_size = erts_decode_ext_size(p, sz, 1)) < 0) {
+ if ((heap_size = erts_decode_ext_size(p, sz)) < 0) {
LoadError1(stp, "literal %d: bad external format", i);
}
hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_LOADER_TMP,
heap_size*sizeof(Eterm));
- val = erts_decode_ext(&hp, NULL, &p);
+ stp->literals[i].off_heap.first = 0;
+ stp->literals[i].off_heap.overhead = 0;
+ val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p);
stp->literals[i].heap_size = hp - stp->literals[i].heap;
if (stp->literals[i].heap_size > heap_size) {
erl_exit(1, "overrun by %d word(s) for literal heap, term %d",
@@ -3955,6 +3958,8 @@ freeze_code(LoaderState* stp)
Uint* low;
Uint* high;
LiteralPatch* lp;
+ struct erl_off_heap_header* off_heap = 0;
+ struct erl_off_heap_header** off_heap_last = &off_heap;
low = (Uint *) (code+stp->ci);
high = low + stp->total_literal_size;
@@ -3963,6 +3968,7 @@ freeze_code(LoaderState* stp)
ptr = low;
for (i = 0; i < stp->num_literals; i++) {
Uint offset;
+ struct erl_off_heap_header* t_off_heap;
sys_memcpy(ptr, stp->literals[i].heap,
stp->literals[i].heap_size*sizeof(Eterm));
@@ -3977,9 +3983,19 @@ freeze_code(LoaderState* stp)
*ptr++ = offset_ptr(val, offset);
break;
case TAG_PRIMARY_HEADER:
- ptr++;
- if (header_is_thing(val)) {
- ptr += thing_arityval(val);
+ if (header_is_transparent(val)) {
+ ptr++;
+ } else {
+ if (thing_subtag(val) == REFC_BINARY_SUBTAG) {
+ struct erl_off_heap_header* oh;
+
+ oh = (struct erl_off_heap_header*) ptr;
+ if (oh->next) {
+ Eterm** uptr = (Eterm **) (void *) &oh->next;
+ *uptr += offset;
+ }
+ }
+ ptr += 1 + thing_arityval(val);
}
break;
default:
@@ -3988,7 +4004,23 @@ freeze_code(LoaderState* stp)
}
}
ASSERT(ptr == high);
+
+ /*
+ * Re-link the off_heap list for this term onto the
+ * off_heap list for the entire module.
+ */
+ t_off_heap = stp->literals[i].off_heap.first;
+ if (t_off_heap) {
+ t_off_heap = (struct erl_off_heap_header *)
+ offset_ptr((UWord) t_off_heap, offset);
+ while (t_off_heap) {
+ *off_heap_last = t_off_heap;
+ off_heap_last = &t_off_heap->next;
+ t_off_heap = t_off_heap->next;
+ }
+ }
}
+ code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap;
lp = stp->literal_patches;
while (lp != 0) {
BeamInstr* op_ptr;
@@ -4071,7 +4103,7 @@ freeze_code(LoaderState* stp)
sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size);
code[MI_ATTR_PTR] = (BeamInstr) attr;
code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size;
- decoded_size = erts_decode_ext_size(attr, attr_size, 0);
+ decoded_size = erts_decode_ext_size(attr, attr_size);
if (decoded_size < 0) {
LoadError0(stp, "bad external term representation of module attributes");
}
@@ -4089,7 +4121,7 @@ freeze_code(LoaderState* stp)
CHKBLK(ERTS_ALC_T_CODE,code);
code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size;
CHKBLK(ERTS_ALC_T_CODE,code);
- decoded_size = erts_decode_ext_size(compile_info, compile_size, 0);
+ decoded_size = erts_decode_ext_size(compile_info, compile_size);
CHKBLK(ERTS_ALC_T_CODE,code);
if (decoded_size < 0) {
LoadError0(stp, "bad external term representation of compilation information");
@@ -4908,6 +4940,8 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
lit->heap_size = heap_size;
lit->heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm));
lit->term = make_boxed(lit->heap);
+ lit->off_heap.first = 0;
+ lit->off_heap.overhead = 0;
*hpp = lit->heap;
return stp->num_literals++;
}
@@ -5491,7 +5525,7 @@ stub_copy_info(LoaderState* stp,
if (size != 0) {
memcpy(info, stp->chunks[chunk].start, size);
*ptr_word = (BeamInstr) info;
- decoded_size = erts_decode_ext_size(info, size, 0);
+ decoded_size = erts_decode_ext_size(info, size);
if (decoded_size < 0) {
return 0;
}
@@ -5844,6 +5878,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
code[MI_COMPILE_SIZE] = 0;
code[MI_COMPILE_SIZE_ON_HEAP] = 0;
code[MI_NUM_BREAKPOINTS] = 0;
+ code[MI_LITERALS_START] = 0;
+ code[MI_LITERALS_END] = 0;
+ code[MI_LITERALS_OFF_HEAP] = 0;
code[MI_ON_LOAD_FUNCTION_PTR] = 0;
ci = MI_FUNCTIONS + n + 1;
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 126d9a3935..4e22ee4d79 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -103,16 +103,18 @@ extern Uint erts_total_code_size;
*/
#define MI_LITERALS_START 8
#define MI_LITERALS_END 9
+#define MI_LITERALS_OFF_HEAP 10
+
/*
* Pointer to the on_load function (or NULL if none).
*/
-#define MI_ON_LOAD_FUNCTION_PTR 10
+#define MI_ON_LOAD_FUNCTION_PTR 11
/*
* Pointer to the line table (or NULL if none).
*/
-#define MI_LINE_TABLE 11
+#define MI_LINE_TABLE 12
/*
* Start of function pointer table. This table contains pointers to
@@ -123,5 +125,5 @@ extern Uint erts_total_code_size;
* this table.
*/
-#define MI_FUNCTIONS 12
+#define MI_FUNCTIONS 13
#endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 264374789c..44c5ba1e26 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -968,7 +968,7 @@ int erts_net_message(Port *prt,
res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache);
if (res >= 0)
- res = ctl_len = erts_decode_dist_ext_size(&ede, 0);
+ res = ctl_len = erts_decode_dist_ext_size(&ede);
else {
#ifdef ERTS_DIST_MSG_DBG
erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n");
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index b21cda6347..6b8f1b21fd 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -385,7 +385,7 @@ port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3)
/* Error or a binary without magic/ with wrong magic */
goto error;
}
- result_size = erts_decode_ext_size(port_resp, ret, 0);
+ result_size = erts_decode_ext_size(port_resp, ret);
if (result_size < 0) {
goto error;
}
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c29352a227..eb2b945877 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -595,7 +595,9 @@ erts_garbage_collect_hibernate(Process* p)
void
-erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
+erts_garbage_collect_literals(Process* p, Eterm* literals,
+ Uint lit_size,
+ struct erl_off_heap_header* oh)
{
Uint byte_lit_size = sizeof(Eterm)*lit_size;
Uint old_heap_size;
@@ -607,6 +609,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
Uint area_size;
Eterm* old_htop;
Uint n;
+ struct erl_off_heap_header** prev;
/*
* Set GC state.
@@ -640,6 +643,9 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
offset_heap(temp_lit, lit_size, offs, (char *) literals, byte_lit_size);
offset_heap(p->heap, p->htop - p->heap, offs, (char *) literals, byte_lit_size);
offset_rootset(p, offs, (char *) literals, byte_lit_size, p->arg_reg, p->arity);
+ if (oh) {
+ oh = (struct erl_off_heap_header *) ((Eterm *)(void *) oh + offs);
+ }
/*
* Now the literals are placed in memory that is safe to write into,
@@ -707,6 +713,45 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
p->old_htop = old_htop;
/*
+ * Prepare to sweep binaries. Since all MSOs on the new heap
+ * must be come before MSOs on the old heap, find the end of
+ * current MSO list and use that as a starting point.
+ */
+
+ if (oh) {
+ prev = &MSO(p).first;
+ while (*prev) {
+ prev = &(*prev)->next;
+ }
+ }
+
+ /*
+ * Sweep through all binaries in the temporary literal area.
+ */
+
+ while (oh) {
+ if (IS_MOVED_BOXED(oh->thing_word)) {
+ Binary* bptr;
+ struct erl_off_heap_header* ptr;
+
+ ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word);
+ ASSERT(thing_subtag(ptr->thing_word) == REFC_BINARY_SUBTAG);
+ bptr = ((ProcBin*)ptr)->val;
+
+ /*
+ * This binary has been copied to the heap.
+ * We must increment its reference count and
+ * link it into the MSO list for the process.
+ */
+
+ erts_refc_inc(&bptr->refc, 1);
+ *prev = ptr;
+ prev = &ptr->next;
+ }
+ oh = oh->next;
+ }
+
+ /*
* We no longer need this temporary area.
*/
erts_free(ERTS_ALC_T_TMP, (void *) temp_lit);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 82f272d28a..16be47d540 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -240,7 +240,7 @@ erts_msg_distext2heap(Process *pp,
Sint sz;
*bpp = NULL;
- sz = erts_decode_dist_ext_size(dist_extp, 0);
+ sz = erts_decode_dist_ext_size(dist_extp);
if (sz < 0)
goto decode_error;
if (is_not_nil(*tokenp)) {
@@ -713,7 +713,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
ASSERT(msg->data.dist_ext);
ASSERT(msg->data.dist_ext->heap_size < 0);
- sz = erts_decode_dist_ext_size(msg->data.dist_ext, 0);
+ sz = erts_decode_dist_ext_size(msg->data.dist_ext);
if (sz < 0) {
/* Bad external; remove it */
if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 80ce4b969c..4b867f2b10 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -88,7 +88,7 @@ static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
-static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins, int internal_tags);
+static Sint decoded_size(byte *ep, byte* endp, int internal_tags);
static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
@@ -810,7 +810,7 @@ bad_dist_ext(ErtsDistExternal *edep)
}
Sint
-erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
+erts_decode_dist_ext_size(ErtsDistExternal *edep)
{
Sint res;
byte *ep;
@@ -829,7 +829,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
goto fail;
ep = edep->extp+1;
}
- res = decoded_size(ep, edep->ext_endp, no_refc_bins, 0);
+ res = decoded_size(ep, edep->ext_endp, 0);
if (res >= 0)
return res;
fail:
@@ -837,16 +837,16 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
return -1;
}
-Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins)
+Sint erts_decode_ext_size(byte *ext, Uint size)
{
if (size == 0 || *ext != VERSION_MAGIC)
return -1;
- return decoded_size(ext+1, ext+size, no_refc_bins, 0);
+ return decoded_size(ext+1, ext+size, 0);
}
Sint erts_decode_ext_size_ets(byte *ext, Uint size)
{
- Sint sz = decoded_size(ext, ext+size, 0, 1);
+ Sint sz = decoded_size(ext, ext+size, 1);
ASSERT(sz >= 0);
return sz;
}
@@ -968,7 +968,7 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
ede.extp = binary_bytes(real_bin)+offset;
ede.ext_endp = ede.extp + size;
- hsz = erts_decode_dist_ext_size(&ede, 0);
+ hsz = erts_decode_dist_ext_size(&ede);
if (hsz < 0)
goto badarg;
@@ -1106,7 +1106,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
goto error;
size = (Sint) dest_len;
}
- res = decoded_size(state->extp, state->extp + size, 0, 0);
+ res = decoded_size(state->extp, state->extp + size, 0);
if (res < 0)
goto error;
return res;
@@ -2454,7 +2454,7 @@ dec_term_atom_common:
n = get_int32(ep);
ep += 4;
- if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) {
+ if (n <= ERL_ONHEAP_BIN_LIMIT) {
ErlHeapBin* hb = (ErlHeapBin *) hp;
hb->thing_word = header_heap_bin(n);
@@ -2492,7 +2492,7 @@ dec_term_atom_common:
n = get_int32(ep);
bitsize = ep[4];
ep += 5;
- if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) {
+ if (n <= ERL_ONHEAP_BIN_LIMIT) {
ErlHeapBin* hb = (ErlHeapBin *) hp;
hb->thing_word = header_heap_bin(n);
@@ -3061,7 +3061,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
}
static Sint
-decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags)
+decoded_size(byte *ep, byte* endp, int internal_tags)
{
int heap_size = 0;
int terms;
@@ -3223,7 +3223,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags)
CHKSIZE(4);
n = get_int32(ep);
SKIP2(n, 4);
- if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) {
+ if (n <= ERL_ONHEAP_BIN_LIMIT) {
heap_size += heap_bin_size(n);
} else {
heap_size += PROC_BIN_SIZE;
@@ -3234,7 +3234,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags)
CHKSIZE(5);
n = get_int32(ep);
SKIP2(n, 5);
- if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) {
+ if (n <= ERL_ONHEAP_BIN_LIMIT) {
heap_size += heap_bin_size(n) + ERL_SUB_BIN_SIZE;
} else {
heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 671b8b8781..eddd4571dd 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -175,10 +175,10 @@ void *erts_dist_ext_trailer(ErtsDistExternal *);
void erts_destroy_dist_ext_copy(ErtsDistExternal *);
int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
DistEntry *, ErtsAtomCache *);
-Sint erts_decode_dist_ext_size(ErtsDistExternal *, int);
+Sint erts_decode_dist_ext_size(ErtsDistExternal *);
Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *);
-Sint erts_decode_ext_size(byte*, Uint, int);
+Sint erts_decode_ext_size(byte*, Uint);
Sint erts_decode_ext_size_ets(byte*, Uint);
Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**);
Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 9f19172e08..b247576f1c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1126,7 +1126,9 @@ void erts_init_gc(void);
int erts_garbage_collect(Process*, int, Eterm*, int);
void erts_garbage_collect_hibernate(Process* p);
Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size);
+void erts_garbage_collect_literals(Process* p, Eterm* literals,
+ Uint lit_size,
+ struct erl_off_heap_header* oh);
Uint erts_next_heap_size(Uint, Uint);
Eterm erts_heap_sizes(Process* p);
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-14 09:30:27 +0000