Simplify GC BIFs

Summary: This commit simplifies the implementation of the "GC BIFs" so
that they no longer need to do a garbage collection, removing duplicate
code for all GC BIFs in the runtime system, as well as potentially
reducing the size of the loaded BEAM code by using shorter
instructions calling those BIFs.

A GC BIF is a guard BIF that will do a garbage
collection if it needs to build anything on the heap.
For example, `abs/1` is a GC BIF because it might need to
allocate space on the heap (if the result is a floating point
number or the resulting integer is a bignum).

Before R12, a guard BIF (such as `abs/1`) that need to allocate
heap space would allocate outside of process's main heap, in
a heap fragment.

GC BIFs were introduced in R12B to support literals. During garbage
collection it become necessary to quickly test whether a term was
a literal. To make the check simple, guards BIFs were no longer
allowed to create heap fragments. Instead GC BIFs were introduced.

In OTP 19, the implementation of literals was changed to support
storing messages in heap fragments outside of the main heap for a
process. That change again made it allowed for guard BIFs to create
heap fragments when they need to build terms on the heap.

It would even be possible for the guard BIFs to build directly
on the main heap if there is room there, because the compiler
assumes that a new `test_heap/2` instruction must be emitted
when building anything after calling a GC BIF. (We don't do that
in this commit; see below.)

This commit simplifies the implementation of the GC BIFs in
the runtime system.

Each GC BIF had a dual implementation: one that was used when the GC
BIF was called directly and one used when it was called via
`apply/3`. For example, `abs/1` was implemented in `abs_1()` and
`erts_gc_abs_1()`. This commit removes the GC version of each BIF. The
other version that allocates heap space using `HAlloc()` is updated to
use the new `HeapFragOnlyAlloc()` macro that will allocate heap
space in a heap fragment outside of the main heap.

Because the BIFs will allocate outside of the main heap, the same
`bif` instructions used by nonbuilding BIFs can be used for the
(former) GC BIFs. Those instructions don't use the macros that save
and restore the heap and stack pointers (SWAPOUT/SWAPIN).  If the
former GC BIFs would build on the main heap, either new instructions
would be needed, or SWAPOUT/SWAPIN instructions would need to be added
to the `bif` instructions.

Instructions that call the former GC BIFs don't need the operand
that specifies the number of live X registers. Therefore, the
instructions that call the BIFs are usually one word shorter.

1 files changed, 15 insertions, 403 deletions

diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index 8a5c6ada6c..84783e71a0 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -19,7 +19,12 @@
  */
 
 /*
- * Numeric guard BIFs.
+ * This file implements the former GC BIFs. They used to do a GC when
+ * they needed heap space. Because of changes to the implementation of
+ * literals, those BIFs are now allowed to allocate heap fragments
+ * (using HeapFragOnlyAlloc()). Note that they must NOT call HAlloc(),
+ * because the caller does not do any SWAPIN / SWAPOUT (that is,
+ * HEAP_TOP(p) and HEAP_LIMIT(p) contain stale values).
  */
 
 #ifdef HAVE_CONFIG_H
@@ -36,15 +41,8 @@
 #include "erl_binary.h"
 #include "erl_map.h"
 
-static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live);
-
 static Eterm double_to_integer(Process* p, double x);
 
-/*
- * Guard BIFs called using apply/3 and guard BIFs that never build
- * anything on the heap.
- */
-
 BIF_RETTYPE abs_1(BIF_ALIST_1)
 {
     Eterm res;
@@ -56,7 +54,7 @@ BIF_RETTYPE abs_1(BIF_ALIST_1)
 	i0 = signed_val(BIF_ARG_1);
 	i = ERTS_SMALL_ABS(i0);
 	if (i0 == MIN_SMALL) {
-	    hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
+	    hp = HeapFragOnlyAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
 	    BIF_RET(uint_to_big(i, hp));
 	} else {
 	    BIF_RET(make_small(i));
@@ -68,7 +66,7 @@ BIF_RETTYPE abs_1(BIF_ALIST_1)
 	    int sz = big_arity(BIF_ARG_1) + 1;
 	    Uint* x;
 
-	    hp = HAlloc(BIF_P, sz);	/* See note at beginning of file */
+	    hp = HeapFragOnlyAlloc(BIF_P, sz);	/* See note at beginning of file */
 	    sz--;
 	    res = make_big(hp);
 	    x = big_val(BIF_ARG_1);
@@ -83,7 +81,7 @@ BIF_RETTYPE abs_1(BIF_ALIST_1)
 
 	GET_DOUBLE(BIF_ARG_1, f);
 	if (f.fd < 0.0) {
-	    hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT);
+	    hp = HeapFragOnlyAlloc(BIF_P, FLOAT_SIZE_OBJECT);
 	    f.fd = fabs(f.fd);
 	    res = make_float(hp);
 	    PUT_DOUBLE(f, hp);
@@ -116,7 +114,7 @@ BIF_RETTYPE float_1(BIF_ALIST_1)
     } else if (big_to_double(BIF_ARG_1, &f.fd) < 0) {
 	goto badarg;
     }
-    hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT);
+    hp = HeapFragOnlyAlloc(BIF_P, FLOAT_SIZE_OBJECT);
     res = make_float(hp);
     PUT_DOUBLE(f, hp);
     BIF_RET(res);
@@ -229,7 +227,7 @@ BIF_RETTYPE size_1(BIF_ALIST_1)
 	if (IS_USMALL(0, sz)) {
 	    return make_small(sz);
 	} else {
-	    Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
+	    Eterm* hp = HeapFragOnlyAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
 	    BIF_RET(uint_to_big(sz, hp));
 	}
     }
@@ -252,12 +250,12 @@ BIF_RETTYPE bit_size_1(BIF_ALIST_1)
 	    if (IS_USMALL(0,low_bits)) {
 		BIF_RET(make_small(low_bits));
 	    } else {
-		Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
+		Eterm* hp = HeapFragOnlyAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
 		BIF_RET(uint_to_big(low_bits, hp));
 	    }
 	} else {
 	    Uint sz = BIG_UINT_HEAP_SIZE+1;
-	    Eterm* hp = HAlloc(BIF_P, sz);
+	    Eterm* hp = HeapFragOnlyAlloc(BIF_P, sz);
 	    hp[0] = make_pos_bignum_header(sz-1);
 	    BIG_DIGIT(hp,0) = low_bits;
 	    BIG_DIGIT(hp,1) = high_bits;
@@ -281,7 +279,7 @@ BIF_RETTYPE byte_size_1(BIF_ALIST_1)
 	if (IS_USMALL(0, bytesize)) {
 	    BIF_RET(make_small(bytesize));
 	} else {
-	    Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
+	    Eterm* hp = HeapFragOnlyAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
 	    BIF_RET(uint_to_big(bytesize, hp));
 	}
     } else {
@@ -325,7 +323,7 @@ double_to_integer(Process* p, double x)
     }
     sz = BIG_NEED_SIZE(ds);          /* number of words including arity */
 
-    hp = HAlloc(p, sz);
+    hp = HeapFragOnlyAlloc(p, sz);
     res = make_big(hp);
     xp = (ErtsDigit*) (hp + 1);
 
@@ -371,389 +369,3 @@ BIF_RETTYPE binary_part_2(BIF_ALIST_2)
  badarg:
    BIF_ERROR(BIF_P,BADARG);
 }
-
-
-/*
- * The following code is used when a guard that may build on the
- * heap is called directly. They must not use HAlloc(), but must
- * do a garbage collection if there is insufficient heap space.
- *
- * Important note: All error checking MUST be done before doing
- * a garbage collection. The compiler assumes that all registers
- * are still valid if a guard BIF generates an exception.
- */
-
-#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need))
-
-Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm list = reg[live];
-    int i;
-
-    if (is_nil(list)) 
-	return SMALL_ZERO;
-    i = 0;
-    while (is_list(list)) {
-	i++;
-	list = CDR(list_val(list));
-    }
-    if (is_not_nil(list))  {
-	BIF_ERROR(p, BADARG);
-    }
-    return make_small(i);
-}
-
-Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg = reg[live];
-    if (is_tuple(arg)) {
-	Eterm* tupleptr = tuple_val(arg);
-	return make_small(arityval(*tupleptr));
-    } else if (is_binary(arg)) {
-	Uint sz = binary_size(arg);
-	if (IS_USMALL(0, sz)) {
-	    return make_small(sz);
-	} else {
-	    Eterm* hp;
-	    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
-		erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
-	    }
-	    hp = p->htop;
-	    p->htop += BIG_UINT_HEAP_SIZE;
-	    return uint_to_big(sz, hp);
-	}
-    }
-    BIF_ERROR(p, BADARG);
-}
-
-Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg = reg[live];
-    if (is_binary(arg)) {
-	Uint low_bits;
-	Uint bytesize;
-	Uint high_bits;
-	bytesize = binary_size(arg);
-	high_bits = bytesize >>  ((sizeof(Uint) * 8)-3);
-	low_bits = (bytesize << 3) + binary_bitsize(arg);
-	if (high_bits == 0) {
-	    if (IS_USMALL(0,low_bits)) {
-		return make_small(low_bits);
-	    } else {
-		Eterm* hp; 
-		if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
-		    erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
-		}
-		hp = p->htop;
-		p->htop += BIG_UINT_HEAP_SIZE;
-		return uint_to_big(low_bits, hp);
-	    }
-	} else {
-	    Uint sz = BIG_UINT_HEAP_SIZE+1;
-	    Eterm* hp;
-	    if (ERTS_NEED_GC(p, sz)) {
-		erts_garbage_collect(p, sz, reg, live);
-	    }
-	    hp = p->htop;
-	    p->htop += sz;
-	    hp[0] = make_pos_bignum_header(sz-1);
-	    BIG_DIGIT(hp,0) = low_bits;
-	    BIG_DIGIT(hp,1) = high_bits;
-	    return make_big(hp);
-	}
-    } else {
-	BIF_ERROR(p, BADARG);
-    }
-}
-
-Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg = reg[live];
-    if (is_binary(arg)) {
-	Uint bytesize = binary_size(arg);
-	if (binary_bitsize(arg) > 0) {
-	    bytesize++;
-	}
-	if (IS_USMALL(0, bytesize)) {
-	    return make_small(bytesize);
-	} else {
-	    Eterm* hp;
-	    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
-		erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
-	    }
-	    hp = p->htop;
-	    p->htop += BIG_UINT_HEAP_SIZE;
-	    return uint_to_big(bytesize, hp);
-	}
-    } else {
-	BIF_ERROR(p, BADARG);
-    }
-}
-
-Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg = reg[live];
-    if (is_flatmap(arg)) {
-	flatmap_t *mp = (flatmap_t*)flatmap_val(arg);
-        return make_small(flatmap_get_size(mp));
-    } else if (is_hashmap(arg)) {
-        Eterm* hp;
-        Uint size;
-	size = hashmap_size(arg);
-        if (IS_USMALL(0, size)) {
-            return make_small(size);
-        }
-        if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
-            erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live);
-        }
-        hp = p->htop;
-        p->htop += BIG_UINT_HEAP_SIZE;
-        return uint_to_big(size, hp);
-    }
-    p->fvalue = arg;
-    BIF_ERROR(p, BADMAP);
-}
-
-Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg;
-    Eterm res;
-    Sint i0, i;
-    Eterm* hp;
-
-    arg = reg[live];
-
-    /* integer arguments */
-    if (is_small(arg)) {
-	i0 = signed_val(arg);
-	i = ERTS_SMALL_ABS(i0);
-	if (i0 == MIN_SMALL) {
-	    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
-		erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live+1);
-		arg = reg[live];
-	    }
-	    hp = p->htop;
-	    p->htop += BIG_UINT_HEAP_SIZE;
-	    return uint_to_big(i, hp);
-	} else {
-	    return make_small(i);
-	}
-    } else if (is_big(arg)) {
-	if (!big_sign(arg)) {
-	    return arg;
-	} else {
-	    int sz = big_arity(arg) + 1;
-	    Uint* x;
-
-	    if (ERTS_NEED_GC(p, sz)) {
-		erts_garbage_collect(p, sz, reg, live+1);
-		arg = reg[live];
-	    }
-	    hp = p->htop;
-	    p->htop += sz;
-	    sz--;
-	    res = make_big(hp);
-	    x = big_val(arg);
-	    *hp++ = make_pos_bignum_header(sz);
-	    x++;                          /* skip thing */
-	    while(sz--)
-		*hp++ = *x++;
-	    return res;
-	}
-    } else if (is_float(arg)) {
-	FloatDef f;
-
-	GET_DOUBLE(arg, f);
-	if (f.fd < 0.0) {
-	    if (ERTS_NEED_GC(p, FLOAT_SIZE_OBJECT)) {
-		erts_garbage_collect(p, FLOAT_SIZE_OBJECT, reg, live+1);
-		arg = reg[live];
-	    }
-	    hp = p->htop;
-	    p->htop += FLOAT_SIZE_OBJECT;
-	    f.fd = fabs(f.fd);
-	    res = make_float(hp);
-	    PUT_DOUBLE(f, hp);
-	    return res;
-	}
-	else
-	    return arg;
-    }
-    BIF_ERROR(p, BADARG);
-}
-
-Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg;
-    Eterm res;
-    Eterm* hp;
-    FloatDef f;
-     
-    /* check args */
-    arg = reg[live];
-    if (is_not_integer(arg)) {
-	if (is_float(arg))  {
-	    return arg;
-	} else {
-	badarg:
-	    BIF_ERROR(p, BADARG);
-	}
-    }
-    if (is_small(arg)) {
-	Sint i = signed_val(arg);
-	f.fd = i;		/* use "C"'s auto casting */
-    } else if (big_to_double(arg, &f.fd) < 0) {
-	goto badarg;
-    }
-    if (ERTS_NEED_GC(p, FLOAT_SIZE_OBJECT)) {
-	erts_garbage_collect(p, FLOAT_SIZE_OBJECT, reg, live+1);
-	arg = reg[live];
-    }
-    hp = p->htop;
-    p->htop += FLOAT_SIZE_OBJECT;
-    res = make_float(hp);
-    PUT_DOUBLE(f, hp);
-    return res;
-}
-
-Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg;
-    FloatDef f;
-     
-    arg = reg[live];
-    if (is_not_float(arg)) {
-	if (is_integer(arg))  {
-	    return arg;
-	}
-	BIF_ERROR(p, BADARG);
-    }
-    GET_DOUBLE(arg, f);
-
-    return gc_double_to_integer(p, round(f.fd), reg, live);
-}
-
-Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg;
-    FloatDef f;
-     
-    arg = reg[live];
-    if (is_not_float(arg)) {
-	if (is_integer(arg))  {
-	    return arg;
-	}
-	BIF_ERROR(p, BADARG);
-    }
-    /* get the float */
-    GET_DOUBLE(arg, f);
-
-    /* truncate it and return the resultant integer */
-    return gc_double_to_integer(p, (f.fd >= 0.0) ? floor(f.fd) : ceil(f.fd),
-				reg, live);
-}
-
-Eterm erts_gc_floor_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg;
-    FloatDef f;
-
-    arg = reg[live];
-    if (is_not_float(arg)) {
-	if (is_integer(arg))  {
-	    return arg;
-	}
-	BIF_ERROR(p, BADARG);
-    }
-    GET_DOUBLE(arg, f);
-    return gc_double_to_integer(p, floor(f.fd), reg, live);
-}
-
-Eterm erts_gc_ceil_1(Process* p, Eterm* reg, Uint live)
-{
-    Eterm arg;
-    FloatDef f;
-
-    arg = reg[live];
-    if (is_not_float(arg)) {
-	if (is_integer(arg))  {
-	    return arg;
-	}
-	BIF_ERROR(p, BADARG);
-    }
-    GET_DOUBLE(arg, f);
-    return gc_double_to_integer(p, ceil(f.fd), reg, live);
-}
-
-static Eterm
-gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live)
-{
-    int is_negative;
-    int ds;
-    ErtsDigit* xp;
-    int i;
-    Eterm res;
-    size_t sz;
-    Eterm* hp;
-    double dbase;
-
-    if ((x < (double) (MAX_SMALL+1)) && (x > (double) (MIN_SMALL-1))) {
-	Sint xi = x;
-	return make_small(xi);
-    }
-
-    if (x >= 0) {
-	is_negative = 0;
-    } else {
-	is_negative = 1;
-	x = -x;
-    }
-
-    /* Unscale & (calculate exponent) */
-    ds = 0;
-    dbase = ((double)(D_MASK)+1);
-    while(x >= 1.0) {
-	x /= dbase;         /* "shift" right */
-	ds++;
-    }
-    sz = BIG_NEED_SIZE(ds);          /* number of words including arity */
-    if (ERTS_NEED_GC(p, sz)) {
-	erts_garbage_collect(p, sz, reg, live);
-    }
-    hp = p->htop;
-    p->htop += sz;
-    res = make_big(hp);
-    xp = (ErtsDigit*) (hp + 1);
-
-    for (i = ds-1; i >= 0; i--) {
-	ErtsDigit d;
-
-	x *= dbase;      /* "shift" left */
-	d = x;            /* trunc */
-	xp[i] = d;        /* store digit */
-	x -= d;           /* remove integer part */
-    }
-    while ((ds & (BIG_DIGITS_PER_WORD-1)) != 0) {
-	xp[ds++] = 0;
-    }
-
-    if (is_negative) {
-	*hp = make_neg_bignum_header(sz-1);
-    } else {
-	*hp = make_pos_bignum_header(sz-1);
-    }
-    return res;
-}
-
-/********************************************************************************
- * binary_part guards. The actual implementation is in erl_bif_binary.c
- ********************************************************************************/
-Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live)
-{
-    return erts_gc_binary_part(p,reg,live,0);
-}
-
-Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live)
-{
-    return erts_gc_binary_part(p,reg,live,1);
-}