From abb61a8b4039bb9059cb6a8cf78feaad5f2cdc28 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 8 Dec 2017 18:52:32 +0100 Subject: erts: Refactor carrier dealloc migration by adding a dedicated 'homecoming_dd_block' to use in dd-queue. + Preparation for dd-queue-migration of non-empty carriers. + Get rid of ugly hack where blk->carrier pointer is overwritten by dd-queue and then have to be restored. --- erts/emulator/beam/erl_alloc_util.h | 112 ++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 50 deletions(-) (limited to 'erts/emulator/beam/erl_alloc_util.h') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 81180382af..1291a6c10e 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -305,45 +305,7 @@ void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; -#ifdef ERTS_SMP - -typedef struct ErtsDoubleLink_t_ { - struct ErtsDoubleLink_t_ *next; - struct ErtsDoubleLink_t_ *prev; -}ErtsDoubleLink_t; - -typedef struct { - erts_atomic_t next; - erts_atomic_t prev; - Allctr_t *orig_allctr; /* read-only while carrier is alive */ - ErtsThrPrgrVal thr_prgr; - erts_atomic_t max_size; - UWord abandon_limit; - UWord blocks; - UWord blocks_size; - ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ -} ErtsAlcCPoolData_t; - -#endif - typedef struct Carrier_t_ Carrier_t; -struct Carrier_t_ { - UWord chdr; - Carrier_t *next; - Carrier_t *prev; - erts_smp_atomic_t allctr; -#ifdef ERTS_SMP - ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ -#endif -}; - -#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) - -typedef struct { - Carrier_t *first; - Carrier_t *last; -} CarrierList_t; typedef struct { UWord bhdr; @@ -357,6 +319,22 @@ typedef struct { #endif } Block_t; +typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; + +union ErtsAllctrDDBlock_t_ { + erts_atomic_t atmc_next; + ErtsAllctrDDBlock_t *ptr_next; +}; + +typedef struct { + Block_t blk; +#if !MBC_ABLK_OFFSET_BITS + ErtsAllctrDDBlock_t umem_; +#endif +} ErtsFakeDDBlock_t; + + + #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) @@ -365,14 +343,13 @@ typedef struct { (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) /* - * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for - * distinguishing empty mbc's from allocated blocks in - * handle_delayed_dealloc(). + * HOMECOMING_MBC_BLK_HDR is a special block header combo used for + * distinguishing MBC's from allocated blocks in handle_delayed_dealloc(). */ -#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) +#define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) #define IS_FREE_LAST_MBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) #define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) @@ -396,6 +373,48 @@ typedef struct { typedef UWord FreeBlkFtr_t; /* Footer of a free block */ + +#ifdef ERTS_SMP + +typedef struct ErtsDoubleLink_t_ { + struct ErtsDoubleLink_t_ *next; + struct ErtsDoubleLink_t_ *prev; +}ErtsDoubleLink_t; + +typedef struct { + ErtsFakeDDBlock_t homecoming_dd; + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; + ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ +} ErtsAlcCPoolData_t; + +#endif + +struct Carrier_t_ { + UWord chdr; + Carrier_t *next; + Carrier_t *prev; + erts_smp_atomic_t allctr; +#ifdef ERTS_SMP + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +#endif +}; + +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + +typedef struct { + Carrier_t *first; + Carrier_t *last; +} CarrierList_t; + + typedef Uint64 CallCounter_t; typedef struct { @@ -433,13 +452,6 @@ typedef struct { #ifdef ERTS_SMP -typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; - -union ErtsAllctrDDBlock_t_ { - erts_atomic_t atmc_next; - ErtsAllctrDDBlock_t *ptr_next; -}; - typedef struct { ErtsAllctrDDBlock_t marker; erts_atomic_t last; -- cgit v1.2.3 From a7f87e104e769cb7fed65076193ef0bc4c9f08fd Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 8 Dec 2017 19:02:15 +0100 Subject: erts: Improve carrier pool search * Give back carrier to owner when put in pool with use of dd-queue. * Replace pooled_list with pooled_tree for more efficient search of all owned pooled carriers. * Remove traitor_list as it does not serve much purpose anymore. * Add HOMECOMING bit flag in crr->allctr atomic to (1) avoid double enqueue into dd-enqueue. (2) trigger read barrier in get_used_allctr for newly received carriers. --- erts/emulator/beam/erl_alloc_util.h | 39 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'erts/emulator/beam/erl_alloc_util.h') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 1291a6c10e..e5bf494e97 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -307,6 +307,7 @@ typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; typedef struct Carrier_t_ Carrier_t; + typedef struct { UWord bhdr; #if !MBC_ABLK_OFFSET_BITS @@ -373,13 +374,24 @@ typedef struct { typedef UWord FreeBlkFtr_t; /* Footer of a free block */ - +/* This AOFF stuff really belong in erl_ao_firstfit_alloc.h */ +typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; +struct AOFF_RBTree_t_ { + Block_t hdr; + AOFF_RBTree_t *parent; + AOFF_RBTree_t *left; + AOFF_RBTree_t *right; + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ +}; #ifdef ERTS_SMP +void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*); +void aoff_remove_pooled_mbc(Allctr_t*, Carrier_t*); +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t*, Uint size); +void erts_aoff_larger_max_size(AOFF_RBTree_t *node); +#endif -typedef struct ErtsDoubleLink_t_ { - struct ErtsDoubleLink_t_ *next; - struct ErtsDoubleLink_t_ *prev; -}ErtsDoubleLink_t; +#ifdef ERTS_SMP typedef struct { ErtsFakeDDBlock_t homecoming_dd; @@ -391,7 +403,12 @@ typedef struct { UWord abandon_limit; UWord blocks; UWord blocks_size; - ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ + enum { + ERTS_MBC_IS_HOME, + ERTS_MBC_WAS_POOLED, + ERTS_MBC_WAS_TRAITOR + } state; + AOFF_RBTree_t pooled; /* node in pooled_tree */ } ErtsAlcCPoolData_t; #endif @@ -566,15 +583,14 @@ struct Allctr_t_ { UWord crr_set_flgs; UWord crr_clr_flgs; - /* Carriers */ + /* Carriers *employed* by this allocator */ CarrierList_t mbc_list; CarrierList_t sbc_list; #ifdef ERTS_SMP struct { - /* pooled_list, traitor list and dc_list contain only - carriers _created_ by this allocator */ - ErtsDoubleLink_t pooled_list; - ErtsDoubleLink_t traitor_list; + /* pooled_tree and dc_list contain only + carriers *created* by this allocator */ + AOFF_RBTree_t* pooled_tree; CarrierList_t dc_list; UWord abandon_limit; @@ -686,7 +702,6 @@ void erts_alcu_assert_failed(char* expr, char* file, int line, char *func); int is_sbc_blk(Block_t*); #endif - #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) */ -- cgit v1.2.3 From d74796ecb17a68d442e846c4032a57cb2c083686 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 18 Dec 2017 18:41:38 +0100 Subject: erts: Add more stats for mbcs_pool similar to the ones in OTP-19.2.3.1 --- erts/emulator/beam/erl_alloc_util.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'erts/emulator/beam/erl_alloc_util.h') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index e5bf494e97..19165d83d9 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -602,6 +602,17 @@ struct Allctr_t_ { erts_atomic_t no_blocks; erts_atomic_t carriers_size; erts_atomic_t no_carriers; + CallCounter_t fail_pooled; + CallCounter_t fail_shared; + CallCounter_t fail_pend_dealloc; + CallCounter_t fail; + CallCounter_t fetch; + CallCounter_t skip_size; + CallCounter_t skip_busy; + CallCounter_t skip_not_pooled; + CallCounter_t skip_homecoming; + CallCounter_t skip_race; + CallCounter_t entrance_removed; } stat; } cpool; #endif -- cgit v1.2.3 From 443b2e0fabe9dfbe78d7fe857b29e74f7533da39 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 23 Jan 2018 22:26:44 +0100 Subject: erts: Add migration options "acnl" and "acfml" acnl: Abandon Carrier Nr Limit acfml: Abandon Carrier Free block Min Limit --- erts/emulator/beam/erl_alloc_util.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'erts/emulator/beam/erl_alloc_util.h') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 81180382af..d2cdc3a320 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -63,7 +63,9 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - int acul; + UWord acul; + UWord acnl; + UWord acfml; void *fix; size_t *fix_type_size; @@ -118,6 +120,8 @@ typedef struct { 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -151,6 +155,8 @@ typedef struct { 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -568,7 +574,9 @@ struct Allctr_t_ { UWord abandon_limit; int disable_abandon; int check_limit_count; - int util_limit; + UWord util_limit; /* acul */ + UWord in_pool_limit; /* acnl */ + UWord fblk_min_limit; /* acmfl */ struct { erts_atomic_t blocks_size; erts_atomic_t no_blocks; -- cgit v1.2.3 From d1e89f8df4be7197fdab36a3e1662183a7dfe6ae Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 25 Jan 2018 19:05:50 +0100 Subject: erts: Add system_flags(erts_alloc,"+M?sbct *") to change sbct limit in runtime for chosen allocator type. With great power comes great responsibility. --- erts/emulator/beam/erl_alloc_util.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'erts/emulator/beam/erl_alloc_util.h') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index d2cdc3a320..6c0c5ca86a 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -227,6 +227,8 @@ void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #endif +int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); + #endif /* !ERL_ALLOC_UTIL__ */ #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) @@ -616,6 +618,8 @@ struct Allctr_t_ { void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); + int (*try_set_dyn_param)(Allctr_t*, Eterm param, Uint value); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG -- cgit v1.2.3