Merge branch 'sverk/egil/r16/new-alloc-header-scheme/OTP-10273'

OTP-10415

* sverk/egil/r16/new-alloc-header-scheme/OTP-10273: (42 commits)
  erts: Make ll main mbc fit into 2pow size
  erts: Clear entire mseg cache upon request
  erts: Reduce max heap sizes
  tests: Refactor away ?line macro in beam_SUITE
  tests: Fix heap_sizes check
  tests: Refactor away ?line macro in process_SUITE
  tests: Use new correct min_bin_vheap_size in test
  erts: Set super alignment (256kb) and limits for sbct (8Mb) and lmbcs (128Mb)
  erts: Do not cache segments that are misaligned
  erts: Add mseg cache for large sbc segments
  erts: Reintroduce mseg options amcbf and rmcbf
  erts: Optimize erl_alloc_util.c by substitute MBC_BLK_SZ
  erts: Fix bug when allocating size near sbc_threshold
  erts: Make gc sizes fit into MB Carrier blocks
  erts: Force sbmbc to be disabled in a crude way
  erts: Fix new header scheme for win64
  erts: Fix mseg cache. Malplaced NULL pointer
  erts: Remove unused mseg options amcbf and rmcbf
  erts: Use aligned bits as constant in mseg_alloc
  erts: Don't let zero be considered a power of two
  ...

Conflicts:
	erts/emulator/test/process_SUITE.erl

1 files changed, 46 insertions, 9 deletions

diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index cedf4ccf85..e0754e7f69 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -216,16 +216,40 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 #define UNIT_FLOOR(X)	((X) & UNIT_MASK)
 #define UNIT_CEILING(X)	UNIT_FLOOR((X) + INV_UNIT_MASK)
 
+#define FLG_MASK		INV_UNIT_MASK
+#define SBC_BLK_SZ_MASK         UNIT_MASK
+#define MBC_FBLK_SZ_MASK        UNIT_MASK
+#define CARRIER_SZ_MASK         UNIT_MASK
 
-#define SZ_MASK			(~((UWord) 0) << 3)
-#define FLG_MASK		(~(SZ_MASK))
 
+#if HAVE_ERTS_MSEG
+#  ifdef ARCH_64 
+#    define MBC_ABLK_OFFSET_BITS   24
+#  elif HAVE_SUPER_ALIGNED_MB_CARRIERS
+#    define MBC_ABLK_OFFSET_BITS   9
+     /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
+#  endif
+#endif
+#ifndef MBC_ABLK_OFFSET_BITS
+#  define MBC_ABLK_OFFSET_BITS   0 /* no carrier offset in block header */
+#endif
+
+#if MBC_ABLK_OFFSET_BITS
+#  define MBC_ABLK_OFFSET_SHIFT  (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS)
+#  define MBC_ABLK_OFFSET_MASK   (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT)
+#  define MBC_ABLK_SZ_MASK	(~MBC_ABLK_OFFSET_MASK & ~FLG_MASK)
+#  define HAVE_ERTS_SBMBC 0
+#else
+#  define MBC_ABLK_SZ_MASK	(~FLG_MASK)
+#  define HAVE_ERTS_SBMBC 1
+#endif
 
-#define BLK_SZ(B) \
-  (*((Block_t *) (B)) & SZ_MASK)
+#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
+#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
+#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
 
 #define CARRIER_SZ(C) \
-  ((C)->chdr & SZ_MASK)
+  ((C)->chdr & CARRIER_SZ_MASK)
 
 extern int erts_have_sbmbc_alloc;
 
@@ -236,6 +260,7 @@ struct Carrier_t_ {
     UWord chdr;
     Carrier_t *next;
     Carrier_t *prev;
+    Allctr_t  *allctr;
 };
 
 typedef struct {
@@ -243,8 +268,19 @@ typedef struct {
     Carrier_t *last;
 } CarrierList_t;
 
-typedef UWord Block_t;
-typedef UWord FreeBlkFtr_t;
+typedef struct {
+    UWord bhdr;
+#if !MBC_ABLK_OFFSET_BITS
+    Carrier_t *carrier;
+#else
+    union {
+	Carrier_t *carrier;     /* if free */
+	char       udata__[1];  /* if allocated */
+    }u;
+#endif
+} Block_t;
+
+typedef UWord FreeBlkFtr_t; /* Footer of a free block */
 
 typedef struct {
     UWord giga_no;
@@ -381,8 +417,6 @@ struct Allctr_t_ {
 #endif
 
     /* */
-    Uint		mbc_header_size;
-    Uint		sbc_header_size;
     Uint		min_mbc_size;
     Uint		min_mbc_first_free_size;
     Uint		min_block_size;
@@ -469,6 +503,9 @@ void	erts_alcu_verify_unused_ts(Allctr_t *allctr);
 
 unsigned long	erts_alcu_test(unsigned long, unsigned long, unsigned long);
 
+#ifdef DEBUG
+int is_sbc_blk(Block_t*);
+#endif
 
 
 #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL)