Allow cross-type carrier migration

1 files changed, 54 insertions, 27 deletions

diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f26ace1534..9ab8589bf3 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -24,6 +24,7 @@
 #define ERTS_ALCU_VSN_STR "3.0"
 
 #include "erl_alloc_types.h"
+#include "erl_alloc.h"
 #define ERL_THREADS_EMU_INTERNAL__
 #include "erl_threads.h"
 
@@ -44,6 +45,7 @@ typedef struct {
 typedef struct {
     char *name_prefix;
     ErtsAlcType_t alloc_no;
+    ErtsAlcStrat_t alloc_strat;
     int force;
     int ix;
     int ts;
@@ -101,6 +103,7 @@ typedef struct {
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
     NULL,                                                                  \
     ERTS_ALC_A_INVALID,	/* (number) alloc_no: allocator number           */\
+    ERTS_ALC_S_INVALID,	/* (number) alloc_strat: allocator strategy      */\
     0,			/* (bool)   force:  force enabled                */\
     0,			/* (number) ix: instance index                   */\
     1,			/* (bool)   ts:     thread safe                  */\
@@ -138,6 +141,7 @@ typedef struct {
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
     NULL,                                                                  \
     ERTS_ALC_A_INVALID,	/* (number) alloc_no: allocator number           */\
+    ERTS_ALC_S_INVALID,	/* (number) alloc_strat: allocator strategy      */\
     0,			/* (bool)   force:  force enabled                */\
     0,			/* (number) ix: instance index                   */\
     1,			/* (bool)   ts:     thread safe                  */\
@@ -188,6 +192,7 @@ Eterm	erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *);
 void	erts_alcu_init(AlcUInit_t *);
 void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
 			       ErtsAlcUFixInfo_t *, int);
+void    erts_alcu_foreign_size(Allctr_t *, ErtsAlcType_t, AllctrSize_t *);
 void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *);
 erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
@@ -286,10 +291,18 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
 #define UNIT_FLOOR(X)	((X) & UNIT_MASK)
 #define UNIT_CEILING(X)	UNIT_FLOOR((X) + INV_UNIT_MASK)
 
-#define FLG_MASK		INV_UNIT_MASK
-#define SBC_BLK_SZ_MASK         UNIT_MASK
-#define MBC_FBLK_SZ_MASK        UNIT_MASK
-#define CARRIER_SZ_MASK         UNIT_MASK
+/* We store flags in the bits that no one will ever use. Generally these are
+ * the bits below the alignment size, but for blocks we also steal the highest
+ * bit since the header's a size and no one can expect to be able to allocate
+ * objects that large. */
+#define HIGHEST_WORD_BIT        (((UWord) 1) << (sizeof(UWord) * CHAR_BIT - 1))
+
+#define BLK_FLG_MASK            (INV_UNIT_MASK | HIGHEST_WORD_BIT)
+#define SBC_BLK_SZ_MASK         (~BLK_FLG_MASK)
+#define MBC_FBLK_SZ_MASK        (~BLK_FLG_MASK)
+
+#define CRR_FLG_MASK        INV_UNIT_MASK
+#define CRR_SZ_MASK         UNIT_MASK
 
 #if ERTS_HAVE_MSEG_SUPER_ALIGNED \
     || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC)
@@ -299,9 +312,9 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
 #    define ERTS_SUPER_ALIGN_BITS 18
 #  endif
 #  ifdef ARCH_64 
-#    define MBC_ABLK_OFFSET_BITS   24
+#    define MBC_ABLK_OFFSET_BITS   23
 #  else
-#    define MBC_ABLK_OFFSET_BITS   9
+#    define MBC_ABLK_OFFSET_BITS   8
      /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
 #  endif
 #  define ERTS_SACRR_UNIT_SHIFT		ERTS_SUPER_ALIGN_BITS
@@ -322,18 +335,17 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
 
 #if MBC_ABLK_OFFSET_BITS
 #  define MBC_ABLK_OFFSET_SHIFT  (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS)
-#  define MBC_ABLK_OFFSET_MASK   (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT)
-#  define MBC_ABLK_SZ_MASK	(~MBC_ABLK_OFFSET_MASK & ~FLG_MASK)
+#  define MBC_ABLK_OFFSET_MASK   ((~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) & ~BLK_FLG_MASK)
+#  define MBC_ABLK_SZ_MASK	(~MBC_ABLK_OFFSET_MASK & ~BLK_FLG_MASK)
 #else
-#  define MBC_ABLK_SZ_MASK	(~FLG_MASK)
+#  define MBC_ABLK_SZ_MASK	(~BLK_FLG_MASK)
 #endif
 
 #define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
 #define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
 #define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
 
-#define CARRIER_SZ(C) \
-  ((C)->chdr & CARRIER_SZ_MASK)
+#define CARRIER_SZ(C) ((C)->chdr & CRR_SZ_MASK)
 
 typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
 
@@ -351,12 +363,20 @@ typedef struct {
 #endif
 } Block_t;
 
-typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
+typedef struct ErtsAllctrDDBlock__ {
+    union  {
+        struct ErtsAllctrDDBlock__ *ptr_next;
+        erts_atomic_t atmc_next;
+    } u;
+    ErtsAlcType_t type;
+    Uint32 flags;
+} ErtsAllctrDDBlock_t;
 
-union ErtsAllctrDDBlock_t_ {
-    erts_atomic_t atmc_next;
-    ErtsAllctrDDBlock_t *ptr_next;
-};
+/* Deallocation was caused by shrinking a fix-list, so usage statistics has
+ * already been updated. */
+#define DEALLOC_FLG_FIX_SHRINK    (1 << 0)
+/* Deallocation was redirected to another instance. */
+#define DEALLOC_FLG_REDIRECTED    (1 << 1)
 
 typedef struct {
     Block_t blk;
@@ -365,11 +385,10 @@ typedef struct {
 #endif
 } ErtsFakeDDBlock_t;
 
-
-
 #define THIS_FREE_BLK_HDR_FLG 	(((UWord) 1) << 0)
 #define PREV_FREE_BLK_HDR_FLG 	(((UWord) 1) << 1)
 #define LAST_BLK_HDR_FLG 	(((UWord) 1) << 2)
+#define ATAG_BLK_HDR_FLG 	HIGHEST_WORD_BIT
 
 #define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\
     (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
@@ -381,9 +400,9 @@ typedef struct {
 #define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)
 
 #define IS_FREE_LAST_MBC_BLK(B) \
-    (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG))
+    (((B)->bhdr & BLK_FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG))
 
-#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG)
+#define IS_SBC_BLK(B) (((B)->bhdr & SBC_BLK_HDR_FLG) == SBC_BLK_HDR_FLG)
 #define IS_MBC_BLK(B) (!IS_SBC_BLK((B)))
 #define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \
 			(B)->bhdr & THIS_FREE_BLK_HDR_FLG)
@@ -394,7 +413,8 @@ typedef struct {
 #  define ABLK_TO_MBC(B) \
     (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \
      (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \
-		  (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT))))
+		  ((((B)->bhdr & ~BLK_FLG_MASK) >> MBC_ABLK_OFFSET_SHIFT) \
+                      << ERTS_SACRR_UNIT_SHIFT))))
 #  define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
 #else
 #  define FBLK_TO_MBC(B) ((B)->carrier)
@@ -433,8 +453,9 @@ typedef struct {
     ErtsThrPrgrVal thr_prgr;
     erts_atomic_t max_size;
     UWord abandon_limit;
-    UWord blocks;
-    UWord blocks_size;
+    UWord blocks[ERTS_ALC_A_MAX + 1];
+    UWord blocks_size[ERTS_ALC_A_MAX + 1];
+    UWord total_blocks_size;
     enum {
         ERTS_MBC_IS_HOME,
         ERTS_MBC_WAS_POOLED,
@@ -452,7 +473,7 @@ struct Carrier_t_ {
 };
 
 #define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
-  ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))
+  ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~CRR_FLG_MASK))
 
 typedef struct {
     Carrier_t *first;
@@ -530,7 +551,6 @@ typedef struct {
     } head;
 } ErtsAllctrDDQueue_t;
 
-
 typedef struct {
     size_t type_size;
     SWord list_size;
@@ -549,6 +569,7 @@ typedef struct {
 	    UWord used;
 	} cpool;
     } u;
+    ErtsAlcType_t type;
 } ErtsAlcFixList_t;
 
 struct Allctr_t_ {
@@ -569,6 +590,9 @@ struct Allctr_t_ {
     /* Allocator number */
     ErtsAlcType_t	alloc_no;
 
+    /* Allocator strategy */
+    ErtsAlcStrat_t	alloc_strat;
+
     /* Instance index */
     int			ix;
 
@@ -617,6 +641,9 @@ struct Allctr_t_ {
 	AOFF_RBTree_t*   pooled_tree;
 	CarrierList_t	 dc_list;
 
+        /* the sentinel of the cpool we're attached to */
+        ErtsAlcCPoolData_t  *sentinel;
+
 	UWord		abandon_limit;
 	int		disable_abandon;
 	int		check_limit_count;
@@ -624,8 +651,8 @@ struct Allctr_t_ {
         UWord           in_pool_limit;    /* acnl */
         UWord           fblk_min_limit;   /* acmfl */
 	struct {
-	    erts_atomic_t	blocks_size;
-	    erts_atomic_t	no_blocks;
+	    erts_atomic_t	blocks_size[ERTS_ALC_A_MAX + 1];
+	    erts_atomic_t	no_blocks[ERTS_ALC_A_MAX + 1];
 	    erts_atomic_t	carriers_size;
 	    erts_atomic_t	no_carriers;
             CallCounter_t       fail_pooled;