aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_alloc_util.h
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_alloc_util.h')
-rw-r--r--erts/emulator/beam/erl_alloc_util.h172
1 files changed, 129 insertions, 43 deletions
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 02cbe5c5d0..f50f09907a 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,18 +1,19 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
*
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*
* %CopyrightEnd%
*/
@@ -23,6 +24,13 @@
#define ERTS_ALCU_VSN_STR "3.0"
#include "erl_alloc_types.h"
+#ifdef USE_THREADS
+#define ERL_THREADS_EMU_INTERNAL__
+#include "erl_threads.h"
+#endif
+
+#include "erl_mseg.h"
+#include "lttng-wrapper.h"
#define ERTS_AU_PREF_ALLOC_BITS 11
#define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS)
@@ -32,6 +40,7 @@ typedef struct Allctr_t_ Allctr_t;
typedef struct {
UWord ycs;
UWord mmc;
+ int sac;
} AlcUInit_t;
typedef struct {
@@ -43,7 +52,6 @@ typedef struct {
int tspec;
int tpref;
int ramv;
- int low_mem; /* HALFWORD only */
UWord sbct;
UWord asbcst;
UWord rsbcst;
@@ -59,6 +67,16 @@ typedef struct {
void *fix;
size_t *fix_type_size;
+
+#if HAVE_ERTS_MSEG
+ void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
+ void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+ void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+ ErtsMemMapper *mseg_mmapper;
+#endif
+ void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign);
+ void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign);
+ void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
} AllctrInit_t;
typedef struct {
@@ -75,7 +93,8 @@ typedef struct {
#define ERTS_DEFAULT_ALCU_INIT { \
1024*1024, /* (bytes) ycs: sys_alloc carrier size */\
- 1024 /* (amount) mmc: max mseg carriers */\
+ ~((UWord) 0), /* (amount) mmc: max mseg carriers */\
+ 1 /* (bool) sac: sys_alloc carriers */\
}
#define ERTS_DEFAULT_ALLCTR_INIT { \
@@ -87,7 +106,6 @@ typedef struct {
0, /* (bool) tspec: thread specific */\
0, /* (bool) tpref: thread preferred */\
0, /* (bool) ramv: realloc always moves */\
- 0, /* (bool) low_mem: HALFWORD only */\
512*1024, /* (bytes) sbct: sbc threshold */\
2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\
20, /* (%) rsbcst: rel sbc shrink threshold */\
@@ -95,7 +113,7 @@ typedef struct {
50, /* (%) rmbcmt: rel mbc move threshold */\
1024*1024, /* (bytes) mmbcs: main multiblock carrier size */\
256, /* (amount) mmsbc: max mseg sbcs */\
- 10, /* (amount) mmmbc: max mseg mbcs */\
+ ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \
10*1024*1024, /* (bytes) lmbcs: largest mbc size */\
1024*1024, /* (bytes) smbcs: smallest mbc size */\
10, /* (amount) mbcgs: mbc growth stages */\
@@ -109,7 +127,8 @@ typedef struct {
#define ERTS_DEFAULT_ALCU_INIT { \
128*1024, /* (bytes) ycs: sys_alloc carrier size */\
- 1024 /* (amount) mmc: max mseg carriers */\
+ 1024, /* (amount) mmc: max mseg carriers */\
+ 1 /* (bool) sac: sys_alloc carriers */\
}
#define ERTS_DEFAULT_ALLCTR_INIT { \
@@ -121,14 +140,13 @@ typedef struct {
0, /* (bool) tspec: thread specific */\
0, /* (bool) tpref: thread preferred */\
0, /* (bool) ramv: realloc always moves */\
- 0, /* (bool) low_mem: HALFWORD only */\
64*1024, /* (bytes) sbct: sbc threshold */\
2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\
20, /* (%) rsbcst: rel sbc shrink threshold */\
80, /* (%) rsbcmt: rel sbc move threshold */\
128*1024, /* (bytes) mmbcs: main multiblock carrier size */\
256, /* (amount) mmsbc: max mseg sbcs */\
- 10, /* (amount) mmmbc: max mseg mbcs */\
+ ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \
1024*1024, /* (bytes) lmbcs: largest mbc size */\
128*1024, /* (bytes) smbcs: smallest mbc size */\
10, /* (amount) mbcgs: mbc growth stages */\
@@ -172,20 +190,44 @@ void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *
#endif
erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
+#ifdef ARCH_32
+extern UWord erts_literal_vspace_map[];
+# define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8)
#endif
+void* erts_alcu_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void erts_alcu_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+
+#if HAVE_ERTS_MSEG
+# if defined(ARCH_32)
+void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+
+# elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+# endif
+#endif /* HAVE_ERTS_MSEG */
+
+void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
+void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
+void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
+#ifdef ARCH_32
+void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
+void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign);
+void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign);
+#endif
+
+#endif /* !ERL_ALLOC_UTIL__ */
+
#if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
#define ERL_ALLOC_UTIL_IMPL__
#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0)
-#ifdef USE_THREADS
-#define ERL_THREADS_EMU_INTERNAL__
-#include "erl_threads.h"
-#endif
-
-#include "erl_mseg.h"
-
#undef ERTS_ALLOC_UTIL_HARD_DEBUG
#ifdef DEBUG
# if 0
@@ -217,25 +259,34 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
#define MBC_FBLK_SZ_MASK UNIT_MASK
#define CARRIER_SZ_MASK UNIT_MASK
-#if HAVE_ERTS_MSEG
-
-# define MSEG_UNIT_SHIFT MSEG_ALIGN_BITS
-# define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT)
-# define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT)
-
-# define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK)
-# define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK)
-
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED \
+ || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC)
+# ifdef MSEG_ALIGN_BITS
+# define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS
+# else
+# define ERTS_SUPER_ALIGN_BITS 18
+# endif
# ifdef ARCH_64
# define MBC_ABLK_OFFSET_BITS 24
-# elif HAVE_SUPER_ALIGNED_MB_CARRIERS
+# else
# define MBC_ABLK_OFFSET_BITS 9
/* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
# endif
-#endif
-#ifndef MBC_ABLK_OFFSET_BITS
+# define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS
+# define ERTS_SACRR_UNIT_SZ (1 << ERTS_SACRR_UNIT_SHIFT)
+# define ERTS_SACRR_UNIT_MASK ((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT)
+# define ERTS_SACRR_UNIT_FLOOR(X) ((X) & ERTS_SACRR_UNIT_MASK)
+# define ERTS_SACRR_UNIT_CEILING(X) ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK)
+# define ERTS_SA_MB_CARRIERS 1
+#else
+# define ERTS_SA_MB_CARRIERS 0
# define MBC_ABLK_OFFSET_BITS 0 /* no carrier offset in block header */
#endif
+#if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
+# define ERTS_SUPER_ALIGNED_MSEG_ONLY 1
+#else
+# define ERTS_SUPER_ALIGNED_MSEG_ONLY 0
+#endif
#if MBC_ABLK_OFFSET_BITS
# define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS)
@@ -245,9 +296,9 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
# define MBC_ABLK_SZ_MASK (~FLG_MASK)
#endif
-#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
-#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
-#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
+#define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
+#define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
+#define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)
#define CARRIER_SZ(C) \
((C)->chdr & CARRIER_SZ_MASK)
@@ -256,15 +307,21 @@ typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
#ifdef ERTS_SMP
+typedef struct ErtsDoubleLink_t_ {
+ struct ErtsDoubleLink_t_ *next;
+ struct ErtsDoubleLink_t_ *prev;
+}ErtsDoubleLink_t;
+
typedef struct {
erts_atomic_t next;
erts_atomic_t prev;
- Allctr_t *orig_allctr;
+ Allctr_t *orig_allctr; /* read-only while carrier is alive */
ErtsThrPrgrVal thr_prgr;
erts_atomic_t max_size;
UWord abandon_limit;
UWord blocks;
UWord blocks_size;
+ ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
} ErtsAlcCPoolData_t;
#endif
@@ -327,8 +384,8 @@ typedef struct {
(B)->u.carrier)
# define ABLK_TO_MBC(B) \
(ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \
- (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \
- (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT))))
+ (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \
+ (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT))))
# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
#else
# define FBLK_TO_MBC(B) ((B)->carrier)
@@ -362,6 +419,18 @@ typedef struct {
} blocks;
} CarriersStats_t;
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+#define LTTNG_CARRIER_STATS_TO_LTTNG_STATS(CSP, LSP) \
+ do { \
+ (LSP)->carriers.size = (CSP)->curr.norm.mseg.size \
+ + (CSP)->curr.norm.sys_alloc.size; \
+ (LSP)->carriers.no = (CSP)->curr.norm.mseg.no \
+ + (CSP)->curr.norm.sys_alloc.no; \
+ (LSP)->blocks.size = (CSP)->blocks.curr.size; \
+ (LSP)->blocks.no = (CSP)->blocks.curr.no; \
+ } while (0)
+#endif
+
#ifdef ERTS_SMP
typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
@@ -482,13 +551,20 @@ struct Allctr_t_ {
Uint min_mbc_size;
Uint min_mbc_first_free_size;
Uint min_block_size;
+ UWord crr_set_flgs;
+ UWord crr_clr_flgs;
/* Carriers */
CarrierList_t mbc_list;
CarrierList_t sbc_list;
#ifdef ERTS_SMP
struct {
- CarrierList_t dc_list;
+ /* pooled_list, traitor list and dc_list contain only
+ carriers _created_ by this allocator */
+ ErtsDoubleLink_t pooled_list;
+ ErtsDoubleLink_t traitor_list;
+ CarrierList_t dc_list;
+
UWord abandon_limit;
int disable_abandon;
int check_limit_count;
@@ -522,6 +598,16 @@ struct Allctr_t_ {
void (*remove_mbc) (Allctr_t *, Carrier_t *);
UWord (*largest_fblk_in_mbc) (Allctr_t *, Carrier_t *);
+#if HAVE_ERTS_MSEG
+ void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags);
+ void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+ void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags);
+ ErtsMemMapper *mseg_mmapper;
+#endif
+ void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign);
+ void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign);
+ void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign);
+
void (*init_atoms) (void);
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-07 02:44:25 +0000