erts: Convert small sub-binaries to heap-binaries

In many cases sub-binaries costs more memory than converting them to heap-binaries.

Sub-binaries also has a hidden cost of pinning larger binaries in memory.
By converting binaries this cost is reduced.

Byte aligned sub-binaries upto 24 bytes (64-bit) or 12 bytes (32-bit) are converted.

4 files changed, 100 insertions, 38 deletions

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 946d3cfe03..6ff71ec6d1 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -156,6 +156,7 @@ typedef union {
 #include "erl_threads.h"
 #include "bif.h"
 #include "erl_bif_unique.h"
+#include "erl_bits.h"
 
 /*
  * Maximum number of bytes to place in a heap binary.
@@ -163,23 +164,6 @@ typedef union {
 
 #define ERL_ONHEAP_BIN_LIMIT 64
 
-/*
- * This structure represents a SUB_BINARY.
- *
- * Note: The last field (orig) is not counted in arityval in the header.
- * This simplifies garbage collection.
- */
-
-typedef struct erl_sub_bin {
-    Eterm thing_word;		/* Subtag SUB_BINARY_SUBTAG. */
-    Uint size;			/* Binary size in bytes. */
-    Uint offs;			/* Offset into original binary. */
-    byte bitsize; 
-    byte bitoffs; 
-    byte is_writable;		/* The underlying binary is writable */
-    Eterm orig;			/* Original binary (REFC or HEAP binary). */
-} ErlSubBin;
-
 #define ERL_SUB_BIN_SIZE (sizeof(ErlSubBin)/sizeof(Eterm))
 #define HEADER_SUB_BIN	_make_header(ERL_SUB_BIN_SIZE-2,_TAG_HEADER_SUB_BIN)
 
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index 4bd5b24157..1b4546722f 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -22,6 +22,23 @@
 #define __ERL_BITS_H__
 
 /*
+ * This structure represents a SUB_BINARY.
+ *
+ * Note: The last field (orig) is not counted in arityval in the header.
+ * This simplifies garbage collection.
+ */
+
+typedef struct erl_sub_bin {
+    Eterm thing_word;		/* Subtag SUB_BINARY_SUBTAG. */
+    Uint size;			/* Binary size in bytes. */
+    Uint offs;			/* Offset into original binary. */
+    byte bitsize;
+    byte bitoffs;
+    byte is_writable;		/* The underlying binary is writable */
+    Eterm orig;			/* Original binary (REFC or HEAP binary). */
+} ErlSubBin;
+
+/*
  * This structure represents a binary to be matched.
  */
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 8cc7fc0142..26f6b6339d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -3256,6 +3256,39 @@ reply_gc_info(void *vgcirp)
 	gcireq_free(vgcirp);
 }
 
+void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) {
+    Eterm *ptr = *pp;
+    Eterm *htop = *hpp;
+    Eterm gval;
+    ErlSubBin *sb = (ErlSubBin *)ptr;
+    ErlHeapBin *hb = (ErlHeapBin *)htop;
+    Eterm *real_bin;
+    byte *bs;
+
+    real_bin = binary_val(follow_moved(sb->orig, (Eterm)0));
+
+    if (*real_bin == HEADER_PROC_BIN) {
+        bs = ((ProcBin *) real_bin)->bytes + sb->offs;
+    } else {
+        bs = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + sb->offs;
+    }
+
+    hb->thing_word = header_heap_bin(sb->size);
+    hb->size = sb->size;
+    sys_memcpy((byte *)hb->data, bs, sb->size);
+
+    gval  = make_boxed(htop);
+    *orig = gval;
+    *ptr  = gval;
+
+    ptr  += ERL_SUB_BIN_SIZE;
+    htop += heap_bin_size(sb->size);
+
+    *hpp = htop;
+    *pp  = ptr;
+}
+
+
 Eterm
 erts_gc_info_request(Process *c_p)
 {
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 1cce426d21..2e2be2ee47 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -28,6 +28,8 @@
 #define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */
 
 #include "erl_map.h"
+#include "erl_fun.h"
+#include "erl_bits.h"
 
 #define IS_MOVED_BOXED(x)	(!is_header((x)))
 #define IS_MOVED_CONS(x)	(is_non_value((x)))
@@ -45,27 +47,53 @@ do {									\
     HTOP += 2;			/* update tospace htop */		\
 } while(0)
 
-#define MOVE_BOXED(PTR,HDR,HTOP,ORIG)                                   \
-do {                                                                    \
-    Eterm gval;                                                         \
-    Sint nelts;                                                         \
-                                                                        \
-    ASSERT(is_header(HDR));                                             \
-    nelts = header_arity(HDR);                                          \
-    switch ((HDR) & _HEADER_SUBTAG_MASK) {                              \
-    case SUB_BINARY_SUBTAG: nelts++; break;                             \
-    case MAP_SUBTAG:                                                    \
-        if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1;   \
-        else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR));            \
-    break;                                                              \
-    case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break;   \
-    }                                                                   \
-    gval    = make_boxed(HTOP);                                         \
-    *ORIG   = gval;                                                     \
-    *HTOP++ = HDR;                                                      \
-    *PTR++  = gval;                                                     \
-    while (nelts--) *HTOP++ = *PTR++;                                   \
-} while(0)
+#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \
+    do { \
+        move_boxed(&(PTR), HDR, &(HTOP), ORIG); \
+    } while(0)
+
+void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig);
+ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig)
+{
+    Eterm gval;
+    Sint nelts;
+    Eterm *ptr = *pp;
+    Eterm *htop = *hpp;
+
+    ASSERT(is_header(hdr));
+    nelts = header_arity(hdr);
+    switch ((hdr) & _HEADER_SUBTAG_MASK) {
+    case SUB_BINARY_SUBTAG:
+        {
+            ErlSubBin *sb = (ErlSubBin *)ptr;
+            /* convert sub-binary to heap-binary if applicable */
+            if (sb->bitsize == 0 && sb->bitoffs == 0 &&
+                sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) {
+                erts_sub_binary_to_heap_binary(pp, hpp, orig);
+                return;
+            }
+        }
+        nelts++;
+        break;
+    case MAP_SUBTAG:
+        if (is_flatmap_header(hdr)) nelts+=flatmap_get_size(ptr) + 1;
+        else nelts += hashmap_bitcount(MAP_HEADER_VAL(hdr));
+    break;
+    case FUN_SUBTAG: nelts+=((ErlFunThing*)(ptr))->num_free+1; break;
+    }
+    gval    = make_boxed(htop);
+    *orig   = gval;
+    *htop++ = hdr;
+    *ptr++  = gval;
+    while (nelts--) *htop++ = *ptr++;
+
+    *hpp = htop;
+    *pp  = ptr;
+}
+#endif
 
 #define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz)			\
     (!erts_is_literal((TPtr), (Ptr))					\