Merge branch 'bjorn/erts/refc-binary-literals/OTP-9486'

* bjorn/erts/refc-binary-literals/OTP-9486:
  external.c: Remove the option to create over-sized heap binaries
  Allow refc binaries in literal pools
  code_SUITE clean-up: Remove experimental 'constant_pool' option

10 files changed, 141 insertions, 33 deletions

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index efb72cd3e7..bc8c001454 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -40,6 +40,7 @@ static Eterm check_process_code(Process* rp, Module* modp);
 static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp);
 static void delete_export_references(Eterm module);
 static int purge_module(int module);
+static void decrement_refc(BeamInstr* code);
 static int is_native(BeamInstr* code);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
@@ -553,6 +554,7 @@ check_process_code(Process* rp, Module* modp)
 	} else {
 	    Eterm* literals;
 	    Uint lit_size;
+	    struct erl_off_heap_header* oh;
 
 	    /*
 	     * Try to get rid of constants by by garbage collecting.
@@ -566,7 +568,9 @@ check_process_code(Process* rp, Module* modp)
 	    (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
 	    literals = (Eterm *) modp->old_code[MI_LITERALS_START];
 	    lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals;
-	    erts_garbage_collect_literals(rp, literals, lit_size);
+	    oh = (struct erl_off_heap_header *)
+		modp->old_code[MI_LITERALS_OFF_HEAP];
+	    erts_garbage_collect_literals(rp, literals, lit_size, oh);
 	}
     }
     return am_false;
@@ -664,6 +668,7 @@ purge_module(int module)
     end = (BeamInstr *)((char *)code + modp->old_code_length);
     erts_cleanup_funs_on_purge(code, end);
     beam_catches_delmod(modp->old_catches, code, modp->old_code_length);
+    decrement_refc(code);
     erts_free(ERTS_ALC_T_CODE, (void *) code);
     modp->old_code = NULL;
     modp->old_code_length = 0;
@@ -673,6 +678,23 @@ purge_module(int module)
 }
 
 static void
+decrement_refc(BeamInstr* code)
+{
+    struct erl_off_heap_header* oh =
+	(struct erl_off_heap_header *) code[MI_LITERALS_OFF_HEAP];
+
+    while (oh) {
+	Binary* bptr;
+	ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG);
+	bptr = ((ProcBin*)oh)->val;
+	if (erts_refc_dectest(&bptr->refc, 0) == 0) {
+	    erts_bin_free(bptr);
+	}
+	oh = oh->next;
+    }
+}
+
+static void
 remove_from_address_table(BeamInstr* code)
 {
     int i;
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 10ed46417b..e43d364add 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -206,6 +206,7 @@ typedef struct {
     Eterm term;			/* The tagged term (in the heap). */
     Uint heap_size;		/* (Exact) size on the heap. */
     Uint offset;		/* Offset from temporary location to final. */
+    ErlOffHeap off_heap;	/* Start of linked list of ProcBins. */
     Eterm* heap;		/* Heap for term. */
 } Literal;
 
@@ -1419,12 +1420,14 @@ read_literal_table(LoaderState* stp)
 
 	GetInt(stp, 4, sz);	/* Size of external term format. */
 	GetString(stp, p, sz);
-	if ((heap_size = erts_decode_ext_size(p, sz, 1)) < 0) {
+	if ((heap_size = erts_decode_ext_size(p, sz)) < 0) {
 	    LoadError1(stp, "literal %d: bad external format", i);
 	}
 	hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_LOADER_TMP,
 						heap_size*sizeof(Eterm));
-	val = erts_decode_ext(&hp, NULL, &p);
+	stp->literals[i].off_heap.first = 0;
+	stp->literals[i].off_heap.overhead = 0;
+	val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p);
 	stp->literals[i].heap_size = hp - stp->literals[i].heap;
 	if (stp->literals[i].heap_size > heap_size) {
 	    erl_exit(1, "overrun by %d word(s) for literal heap, term %d",
@@ -3955,6 +3958,8 @@ freeze_code(LoaderState* stp)
 	Uint* low;
 	Uint* high;
 	LiteralPatch* lp;
+	struct erl_off_heap_header* off_heap = 0;
+	struct erl_off_heap_header** off_heap_last = &off_heap;
 
 	low = (Uint *) (code+stp->ci);
 	high = low + stp->total_literal_size;
@@ -3963,6 +3968,7 @@ freeze_code(LoaderState* stp)
 	ptr = low;
 	for (i = 0; i < stp->num_literals; i++) {
 	    Uint offset;
+	    struct erl_off_heap_header* t_off_heap;
 
 	    sys_memcpy(ptr, stp->literals[i].heap,
 		       stp->literals[i].heap_size*sizeof(Eterm));
@@ -3977,9 +3983,19 @@ freeze_code(LoaderState* stp)
 		    *ptr++ = offset_ptr(val, offset);
 		    break;
 		case TAG_PRIMARY_HEADER:
-		    ptr++;
-		    if (header_is_thing(val)) {
-			ptr += thing_arityval(val);
+		    if (header_is_transparent(val)) {
+			ptr++;
+		    } else {
+			if (thing_subtag(val) == REFC_BINARY_SUBTAG) {
+			    struct erl_off_heap_header* oh;
+
+			    oh = (struct erl_off_heap_header*) ptr;
+			    if (oh->next) {
+				Eterm** uptr = (Eterm **) (void *) &oh->next;
+				*uptr += offset;
+			    }
+			}
+			ptr += 1 + thing_arityval(val);
 		    }
 		    break;
 		default:
@@ -3988,7 +4004,23 @@ freeze_code(LoaderState* stp)
 		}
 	    }
 	    ASSERT(ptr == high);
+
+	    /*
+	     * Re-link the off_heap list for this term onto the
+	     * off_heap list for the entire module.
+	     */
+	    t_off_heap = stp->literals[i].off_heap.first;
+	    if (t_off_heap) {
+		t_off_heap = (struct erl_off_heap_header *)
+		    offset_ptr((UWord) t_off_heap, offset);
+		while (t_off_heap) {
+		    *off_heap_last = t_off_heap;
+		    off_heap_last = &t_off_heap->next;
+		    t_off_heap = t_off_heap->next;
+		}
+	    }
 	}
+	code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap;
 	lp = stp->literal_patches;
 	while (lp != 0) {
 	    BeamInstr* op_ptr;
@@ -4071,7 +4103,7 @@ freeze_code(LoaderState* stp)
 	sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size);
 	code[MI_ATTR_PTR] = (BeamInstr) attr;
 	code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size;
-	decoded_size = erts_decode_ext_size(attr, attr_size, 0);
+	decoded_size = erts_decode_ext_size(attr, attr_size);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of module attributes");
  	}
@@ -4089,7 +4121,7 @@ freeze_code(LoaderState* stp)
 	CHKBLK(ERTS_ALC_T_CODE,code);
 	code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size;
 	CHKBLK(ERTS_ALC_T_CODE,code);
-	decoded_size = erts_decode_ext_size(compile_info, compile_size, 0);
+	decoded_size = erts_decode_ext_size(compile_info, compile_size);
 	CHKBLK(ERTS_ALC_T_CODE,code);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of compilation information");
@@ -4908,6 +4940,8 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
     lit->heap_size = heap_size;
     lit->heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm));
     lit->term = make_boxed(lit->heap);
+    lit->off_heap.first = 0;
+    lit->off_heap.overhead = 0;
     *hpp = lit->heap;
     return stp->num_literals++;
 }
@@ -5491,7 +5525,7 @@ stub_copy_info(LoaderState* stp,
     if (size != 0) {
 	memcpy(info, stp->chunks[chunk].start, size);
 	*ptr_word = (BeamInstr) info;
-	decoded_size = erts_decode_ext_size(info, size, 0);
+	decoded_size = erts_decode_ext_size(info, size);
 	if (decoded_size < 0) {
  	    return 0;
  	}
@@ -5844,6 +5878,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     code[MI_COMPILE_SIZE] = 0;
     code[MI_COMPILE_SIZE_ON_HEAP] = 0;
     code[MI_NUM_BREAKPOINTS] = 0;
+    code[MI_LITERALS_START] = 0;
+    code[MI_LITERALS_END] = 0;
+    code[MI_LITERALS_OFF_HEAP] = 0;
     code[MI_ON_LOAD_FUNCTION_PTR] = 0;
     ci = MI_FUNCTIONS + n + 1;
 
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 126d9a3935..4e22ee4d79 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -103,16 +103,18 @@ extern Uint erts_total_code_size;
  */
 #define MI_LITERALS_START	8
 #define MI_LITERALS_END		9
+#define MI_LITERALS_OFF_HEAP	10
+
 
 /*
  * Pointer to the on_load function (or NULL if none).
  */
-#define MI_ON_LOAD_FUNCTION_PTR 10
+#define MI_ON_LOAD_FUNCTION_PTR 11
 
 /*
  * Pointer to the line table (or NULL if none).
  */
-#define MI_LINE_TABLE 11
+#define MI_LINE_TABLE 12
 
 /*
  * Start of function pointer table.  This table contains pointers to
@@ -123,5 +125,5 @@ extern Uint erts_total_code_size;
  * this table.
  */
 
-#define MI_FUNCTIONS         12
+#define MI_FUNCTIONS         13
 #endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 264374789c..44c5ba1e26 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -968,7 +968,7 @@ int erts_net_message(Port *prt,
     res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache);
 
     if (res >= 0)
-	res = ctl_len = erts_decode_dist_ext_size(&ede, 0);
+	res = ctl_len = erts_decode_dist_ext_size(&ede);
     else {
 #ifdef ERTS_DIST_MSG_DBG
 	erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n");
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index b21cda6347..6b8f1b21fd 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -385,7 +385,7 @@ port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3)
 	/* Error or a binary without magic/ with wrong magic */
 	goto error;
     }
-    result_size = erts_decode_ext_size(port_resp, ret, 0);
+    result_size = erts_decode_ext_size(port_resp, ret);
     if (result_size < 0) {
 	goto error;
     }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c29352a227..eb2b945877 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -595,7 +595,9 @@ erts_garbage_collect_hibernate(Process* p)
 
 
 void
-erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
+erts_garbage_collect_literals(Process* p, Eterm* literals,
+			      Uint lit_size,
+			      struct erl_off_heap_header* oh)
 {
     Uint byte_lit_size = sizeof(Eterm)*lit_size;
     Uint old_heap_size;
@@ -607,6 +609,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     Uint area_size;
     Eterm* old_htop;
     Uint n;
+    struct erl_off_heap_header** prev;
 
     /*
      * Set GC state.
@@ -640,6 +643,9 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     offset_heap(temp_lit, lit_size, offs, (char *) literals, byte_lit_size);
     offset_heap(p->heap, p->htop - p->heap, offs, (char *) literals, byte_lit_size);
     offset_rootset(p, offs, (char *) literals, byte_lit_size, p->arg_reg, p->arity);
+    if (oh) {
+	oh = (struct erl_off_heap_header *) ((Eterm *)(void *) oh + offs);
+    }
 
     /*
      * Now the literals are placed in memory that is safe to write into,
@@ -707,6 +713,45 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     p->old_htop = old_htop;
 
     /*
+     * Prepare to sweep binaries. Since all MSOs on the new heap
+     * must be come before MSOs on the old heap, find the end of
+     * current MSO list and use that as a starting point.
+     */
+
+    if (oh) {
+	prev = &MSO(p).first;
+	while (*prev) {
+	    prev = &(*prev)->next;
+	}
+    }
+
+    /*
+     * Sweep through all binaries in the temporary literal area.
+     */
+
+    while (oh) {
+	if (IS_MOVED_BOXED(oh->thing_word)) {
+	    Binary* bptr;
+	    struct erl_off_heap_header* ptr;
+
+	    ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word);
+	    ASSERT(thing_subtag(ptr->thing_word) == REFC_BINARY_SUBTAG);
+	    bptr = ((ProcBin*)ptr)->val;
+
+	    /*
+	     * This binary has been copied to the heap.
+	     * We must increment its reference count and
+	     * link it into the MSO list for the process.
+	     */
+
+	    erts_refc_inc(&bptr->refc, 1);
+	    *prev = ptr;
+	    prev = &ptr->next;
+	}
+	oh = oh->next;
+    }
+
+    /*
      * We no longer need this temporary area.
      */
     erts_free(ERTS_ALC_T_TMP, (void *) temp_lit);
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 82f272d28a..16be47d540 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -240,7 +240,7 @@ erts_msg_distext2heap(Process *pp,
     Sint sz;
 
     *bpp = NULL;
-    sz = erts_decode_dist_ext_size(dist_extp, 0);
+    sz = erts_decode_dist_ext_size(dist_extp);
     if (sz < 0)
 	goto decode_error;
     if (is_not_nil(*tokenp)) {
@@ -713,7 +713,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
     ASSERT(msg->data.dist_ext);
     ASSERT(msg->data.dist_ext->heap_size < 0);
 
-    sz = erts_decode_dist_ext_size(msg->data.dist_ext, 0);
+    sz = erts_decode_dist_ext_size(msg->data.dist_ext);
     if (sz < 0) {
 	/* Bad external; remove it */
 	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 80ce4b969c..4b867f2b10 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -88,7 +88,7 @@ static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
 static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
-static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins, int internal_tags);
+static Sint decoded_size(byte *ep, byte* endp, int internal_tags);
 
 
 static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
@@ -810,7 +810,7 @@ bad_dist_ext(ErtsDistExternal *edep)
 }
 
 Sint
-erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
+erts_decode_dist_ext_size(ErtsDistExternal *edep)
 {
     Sint res;
     byte *ep;
@@ -829,7 +829,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
 	    goto fail;
 	ep = edep->extp+1;
     }
-    res = decoded_size(ep, edep->ext_endp, no_refc_bins, 0);
+    res = decoded_size(ep, edep->ext_endp, 0);
     if (res >= 0)
 	return res;
  fail:
@@ -837,16 +837,16 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
     return -1;
 }
 
-Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins)
+Sint erts_decode_ext_size(byte *ext, Uint size)
 {
     if (size == 0 || *ext != VERSION_MAGIC)
 	return -1;
-    return decoded_size(ext+1, ext+size, no_refc_bins, 0);
+    return decoded_size(ext+1, ext+size, 0);
 }
 
 Sint erts_decode_ext_size_ets(byte *ext, Uint size)
 {
-    Sint sz = decoded_size(ext, ext+size, 0, 1);
+    Sint sz = decoded_size(ext, ext+size, 1);
     ASSERT(sz >= 0);
     return sz;
 }
@@ -968,7 +968,7 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
     ede.extp = binary_bytes(real_bin)+offset;
     ede.ext_endp = ede.extp + size;
 
-    hsz = erts_decode_dist_ext_size(&ede, 0);
+    hsz = erts_decode_dist_ext_size(&ede);
     if (hsz < 0)
 	goto badarg;
 
@@ -1106,7 +1106,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size)
 	    goto error;
 	size = (Sint) dest_len;
     }
-    res = decoded_size(state->extp, state->extp + size, 0, 0);
+    res = decoded_size(state->extp, state->extp + size, 0);
     if (res < 0)
 	goto error;
     return res;
@@ -2454,7 +2454,7 @@ dec_term_atom_common:
 		n = get_int32(ep);
 		ep += 4;
 	    
-		if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) {
+		if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		    ErlHeapBin* hb = (ErlHeapBin *) hp;
 
 		    hb->thing_word = header_heap_bin(n);
@@ -2492,7 +2492,7 @@ dec_term_atom_common:
 		n = get_int32(ep);
 		bitsize = ep[4];
 		ep += 5;
-		if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) {
+		if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		    ErlHeapBin* hb = (ErlHeapBin *) hp;
 
 		    hb->thing_word = header_heap_bin(n);
@@ -3061,7 +3061,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 }
 
 static Sint
-decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags)
+decoded_size(byte *ep, byte* endp, int internal_tags)
 {
     int heap_size = 0;
     int terms;
@@ -3223,7 +3223,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags)
 	    CHKSIZE(4);
 	    n = get_int32(ep);
 	    SKIP2(n, 4);
-	    if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) {
+	    if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		heap_size += heap_bin_size(n);
 	    } else {
 		heap_size += PROC_BIN_SIZE;
@@ -3234,7 +3234,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags)
 		CHKSIZE(5);
 		n = get_int32(ep);
 		SKIP2(n, 5);
-		if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) {
+		if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		    heap_size += heap_bin_size(n) + ERL_SUB_BIN_SIZE;
 		} else {
 		    heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 671b8b8781..eddd4571dd 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -175,10 +175,10 @@ void *erts_dist_ext_trailer(ErtsDistExternal *);
 void erts_destroy_dist_ext_copy(ErtsDistExternal *);
 int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
 			  DistEntry *, ErtsAtomCache *);
-Sint erts_decode_dist_ext_size(ErtsDistExternal *, int);
+Sint erts_decode_dist_ext_size(ErtsDistExternal *);
 Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *);
 
-Sint erts_decode_ext_size(byte*, Uint, int);
+Sint erts_decode_ext_size(byte*, Uint);
 Sint erts_decode_ext_size_ets(byte*, Uint);
 Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**);
 Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 9f19172e08..b247576f1c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1126,7 +1126,9 @@ void erts_init_gc(void);
 int erts_garbage_collect(Process*, int, Eterm*, int);
 void erts_garbage_collect_hibernate(Process* p);
 Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size);
+void erts_garbage_collect_literals(Process* p, Eterm* literals,
+				   Uint lit_size,
+				   struct erl_off_heap_header* oh);
 Uint erts_next_heap_size(Uint, Uint);
 Eterm erts_heap_sizes(Process* p);