From 156b011958a3b80e507039ddc916db039874ada1 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 27 Aug 2013 17:23:09 +0200 Subject: erts: Refactor the ASSERT macro Introduce unconditional ERTS_ASSERT and use that for both ASSERT and ASSERT_EXPR. --- erts/emulator/beam/dist.c | 4 ++-- erts/emulator/beam/erl_alloc_util.c | 3 --- erts/emulator/beam/erl_ao_firstfit_alloc.c | 3 --- erts/emulator/beam/erl_bestfit_alloc.c | 3 --- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/beam/erl_binary.h | 4 ++-- erts/emulator/beam/erl_db.c | 4 ++-- erts/emulator/beam/io.c | 2 +- erts/emulator/beam/sys.h | 16 ++++++---------- erts/emulator/drivers/common/inet_drv.c | 2 +- erts/emulator/sys/unix/sys.c | 4 ++-- 11 files changed, 17 insertions(+), 30 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 44f4eb9d43..aabccac822 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -353,7 +353,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp) static void doit_link_net_exits(ErtsLink *lnk, void *vnecp) { LinkNetExitsContext lnec = {(NetExitsContext *) vnecp, lnk}; - ASSERT(lnk->type == LINK_PID) + ASSERT(lnk->type == LINK_PID); erts_sweep_links(ERTS_LINK_ROOT(lnk), &doit_link_net_exits_sub, (void *) &lnec); #ifdef DEBUG ERTS_LINK_ROOT(lnk) = NULL; @@ -369,7 +369,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) Process *rp; ErtsLink *rlnk; Uint i,n; - ASSERT(lnk->type == LINK_NODE) + ASSERT(lnk->type == LINK_NODE); if (is_internal_pid(lnk->pid)) { ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 825b68bb85..3ea74a12f9 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -87,9 +87,6 @@ static int initialized = 0; #define SYS_ALLOC_CARRIER_CEILING(X) \ SYS_ALLOC_CARRIER_FLOOR((X) + INV_SYS_ALLOC_CARRIER_MASK) -#undef ASSERT -#define ASSERT ASSERT_EXPR - #if 0 /* Can be useful for debugging */ #define MBC_REALLOC_ALWAYS_MOVES diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 4e6c8b317e..396aa88e0b 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -85,9 +85,6 @@ #define SET_RED(N) (((AOFF_RBTree_t *) (N))->flags |= RED_FLG) #define SET_BLACK(N) (((AOFF_RBTree_t *) (N))->flags &= ~RED_FLG) -#undef ASSERT -#define ASSERT ASSERT_EXPR - #if 1 #define RBT_ASSERT ASSERT #else diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 41f449bb28..59c14899a2 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -75,9 +75,6 @@ #define BF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) -#undef ASSERT -#define ASSERT ASSERT_EXPR - #if 1 #define RBT_ASSERT ASSERT #else diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3b25efd9af..a4f9f787cd 100755 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2091,7 +2091,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_sequential_tracer) { val = erts_get_system_seq_tracer(); - ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false) + ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 506c4813fa..331a12dc1c 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -183,7 +183,7 @@ BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); #endif #define ERTS_CHK_BIN_ALIGNMENT(B) \ - do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)) } while(0) + do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)); } while(0) ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr); ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 40b8eaf8fb..41e64fcd4f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -2236,7 +2236,7 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) CHECK_TABLES(); tptr = tuple_val(a1); - ASSERT(arityval(*tptr) >= 1) + ASSERT(arityval(*tptr) >= 1); if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) { BIF_ERROR(p, BADARG); @@ -2403,7 +2403,7 @@ static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1) CHECK_TABLES(); tptr = tuple_val(a1); - ASSERT(arityval(*tptr) >= 1) + ASSERT(arityval(*tptr) >= 1); if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) { BIF_ERROR(p, BADARG); } diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index c1e66b59af..db19f6c142 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1330,7 +1330,7 @@ force_imm_drv_call(ErtsTryImmDrvCallState *sp) erts_aint32_t invalid_state; Port *prt = sp->port; - ASSERT(ERTS_IS_CRASH_DUMPING) + ASSERT(ERTS_IS_CRASH_DUMPING); ASSERT(is_atom(sp->port_op)); invalid_state = sp->state; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 096394b878..a20106749c 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -149,20 +149,16 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_EXIT_AFTER_DUMP exit #endif -#ifdef DEBUG -# define ASSERT(e) \ - if (e) { \ - ; \ - } else { \ - erl_assert_error(#e, __FILE__, __LINE__); \ - } -# define ASSERT_EXPR(e) \ +#define ERTS_ASSERT(e) \ ((void) ((e) ? 1 : (erl_assert_error(#e, __FILE__, __LINE__), 0))) void erl_assert_error(char* expr, char* file, int line); + +#ifdef DEBUG +# define ASSERT(e) ERTS_ASSERT(e) #else -# define ASSERT(e) -# define ASSERT_EXPR(e) ((void) 1) +# define ASSERT(e) ((void) 1) #endif +#define ASSERT_EXPR ASSERT /* * Microsoft C/C++: We certainly want to use stdarg.h and prototypes. diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 60db50e80a..12f45245b5 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -4433,7 +4433,7 @@ static ErlDrvSSizeT inet_ctl_getiflist(inet_descriptor* desc, case AF_INET6: #endif case AF_INET: - ASSERT(sp+IFNAMSIZ+1 < sbuf+ifc.ifc_len+1) + ASSERT(sp+IFNAMSIZ+1 < sbuf+ifc.ifc_len+1); strncpy(sp, ifrp->ifr_name, IFNAMSIZ); sp[IFNAMSIZ] = '\0'; sp += strlen(sp), ++sp; diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index fdc3167c62..47991756df 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -2638,8 +2638,6 @@ int fd; } -#ifdef DEBUG - extern int erts_initialized; void erl_assert_error(char* expr, char* file, int line) @@ -2661,6 +2659,8 @@ erl_assert_error(char* expr, char* file, int line) abort(); } +#ifdef DEBUG + void erl_debug(char* fmt, ...) { -- cgit v1.2.3 From ca1dc60a852c7827c2934ffeacefdd0119e2d776 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 27 Aug 2013 17:36:58 +0200 Subject: erts: Remove ASSERT_EXPR macro --- erts/emulator/beam/erl_alloc_util.h | 6 +++--- erts/emulator/beam/erl_binary.h | 2 +- erts/emulator/beam/erl_db_util.h | 2 +- erts/emulator/beam/erl_node_container_utils.h | 10 +++++----- erts/emulator/beam/erl_process.c | 2 +- erts/emulator/beam/erl_process.h | 4 ++-- erts/emulator/beam/erl_vm.h | 12 ++++++------ erts/emulator/beam/external.h | 4 ++-- erts/emulator/beam/global.h | 6 +++--- erts/emulator/beam/sys.h | 1 - 10 files changed, 24 insertions(+), 25 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 70ecf28172..222f137024 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -254,9 +254,9 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); # define MBC_ABLK_SZ_MASK (~FLG_MASK) #endif -#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) -#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) -#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) +#define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) +#define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) +#define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) #define CARRIER_SZ(C) \ ((C)->chdr & CARRIER_SZ_MASK) diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 331a12dc1c..f7dc20f5e6 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -153,7 +153,7 @@ do { \ #define binary_bytes(Bin) \ (*binary_val(Bin) == HEADER_PROC_BIN ? \ ((ProcBin *) binary_val(Bin))->bytes : \ - (ASSERT_EXPR(thing_subtag(*binary_val(Bin)) == HEAP_BINARY_SUBTAG), \ + (ASSERT(thing_subtag(*binary_val(Bin)) == HEAP_BINARY_SUBTAG), \ (byte *)(&(((ErlHeapBin *) binary_val(Bin))->data)))) void erts_init_binary(void); diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 90b79e6044..328b19dfc9 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -457,7 +457,7 @@ int erts_db_is_compiled_ms(Eterm term); && ERTS_MAGIC_BIN_DESTRUCTOR((BP)) == erts_db_match_prog_destructor) #define Binary2MatchProg(BP) \ - (ASSERT_EXPR(IsMatchProgBinary((BP))), \ + (ASSERT(IsMatchProgBinary((BP))), \ ((MatchProg *) ERTS_MAGIC_BIN_DATA((BP)))) /* ** Debugging diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 0f93a3a9f0..17f6b32bb1 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -106,7 +106,7 @@ #define dist_entry_channel_no(x) \ ((x) == erts_this_dist_entry \ ? ((Uint) 0) \ - : (ASSERT_EXPR(is_atom((x)->sysname)), \ + : (ASSERT(is_atom((x)->sysname)), \ (Uint) atom_val((x)->sysname))) #define internal_channel_no(x) ((Uint) ERST_INTERNAL_CHANNEL_NO) #define external_channel_no(x) \ @@ -122,10 +122,10 @@ extern ErtsPTab erts_proc; (D), \ _TAG_IMMED1_PID) -#define internal_pid_index(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ +#define internal_pid_index(PID) (ASSERT(is_internal_pid((PID))), \ erts_ptab_id2pix(&erts_proc, (PID))) -#define internal_pid_data(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ +#define internal_pid_data(PID) (ASSERT(is_internal_pid((PID))), \ erts_ptab_id2data(&erts_proc, (PID))) #define internal_pid_number(x) _GET_PID_NUM(internal_pid_data((x))) @@ -193,10 +193,10 @@ extern ErtsPTab erts_port; (D), \ _TAG_IMMED1_PORT) -#define internal_port_index(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ +#define internal_port_index(PRT) (ASSERT(is_internal_port((PRT))), \ erts_ptab_id2pix(&erts_port, (PRT))) -#define internal_port_data(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ +#define internal_port_data(PRT) (ASSERT(is_internal_port((PRT))), \ erts_ptab_id2data(&erts_port, (PRT))) #define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x))) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 434d5ca147..79f382674a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -294,7 +294,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, ERTS_ALC_T_PROC_LIST) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ - (ASSERT_EXPR(-1 <= ((int) (IX)) \ + (ASSERT(-1 <= ((int) (IX)) \ && ((int) (IX)) < ((int) erts_no_schedulers)), \ &aligned_sched_sleep_info[(IX)].ssi) diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 8e5467f196..8d136f6e8b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1135,10 +1135,10 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; } while (0) #define ERTS_RUNQ_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues), \ &erts_aligned_run_queues[(IX)].runq) #define ERTS_SCHEDULER_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers), \ &erts_aligned_scheduler_data[(IX)].esd) void erts_pre_init_process(void); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index c962955de9..8026243555 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -98,7 +98,7 @@ * failing that, in a heap fragment. */ #define HAllocX(p, sz, xtra) \ - (ASSERT_EXPR((sz) >= 0), \ + (ASSERT((sz) >= 0), \ ErtsHAllocLockCheck(p), \ (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ ? erts_heap_alloc((p),(sz),(xtra)) \ @@ -135,14 +135,14 @@ */ #ifdef CHECK_FOR_HOLES # define HeapOnlyAlloc(p, sz) \ - (ASSERT_EXPR((sz) >= 0), \ - (ASSERT_EXPR(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ + (ASSERT((sz) >= 0), \ + (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ (erts_set_hole_marker(HEAP_TOP(p), (sz)), \ (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))) #else # define HeapOnlyAlloc(p, sz) \ - (ASSERT_EXPR((sz) >= 0), \ - (ASSERT_EXPR(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ + (ASSERT((sz) >= 0), \ + (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) #endif diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index e37d47919e..ff29e84972 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -138,8 +138,8 @@ typedef struct { #define ERTS_DIST_EXT_SIZE(EDEP) \ (sizeof(ErtsDistExternal) \ - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \ - ? (ASSERT_EXPR(0 <= (EDEP)->attab.size \ - && (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \ + ? (ASSERT(0 <= (EDEP)->attab.size \ + && (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \ sizeof(Eterm)*(ERTS_ATOM_CACHE_SIZE - (EDEP)->attab.size)) \ : sizeof(ErtsAtomTranslationTable))) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index bacd5a5752..063d16c0c7 100755 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -866,13 +866,13 @@ Eterm store_external_or_ref_in_proc_(Process *, Eterm); Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); #define NC_HEAP_SIZE(NC) \ - (ASSERT_EXPR(is_node_container((NC))), \ + (ASSERT(is_node_container((NC))), \ IS_CONST((NC)) ? 0 : (thing_arityval(*boxed_val((NC))) + 1)) #define STORE_NC(Hpp, ETpp, NC) \ - (ASSERT_EXPR(is_node_container((NC))), \ + (ASSERT(is_node_container((NC))), \ IS_CONST((NC)) ? (NC) : store_external_or_ref_((Hpp), (ETpp), (NC))) #define STORE_NC_IN_PROC(Pp, NC) \ - (ASSERT_EXPR(is_node_container((NC))), \ + (ASSERT(is_node_container((NC))), \ IS_CONST((NC)) ? (NC) : store_external_or_ref_in_proc_((Pp), (NC))) /* duplicates from big.h */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index a20106749c..97e6ed8410 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -158,7 +158,6 @@ void erl_assert_error(char* expr, char* file, int line); #else # define ASSERT(e) ((void) 1) #endif -#define ASSERT_EXPR ASSERT /* * Microsoft C/C++: We certainly want to use stdarg.h and prototypes. -- cgit v1.2.3 From c2dbcb69929ac18e7687f1df1de6613b34e2897b Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 30 Aug 2013 11:59:49 +0200 Subject: erts: Prepare erl_mmap with tree structures for free seg storage --- erts/emulator/Makefile.in | 1 + erts/emulator/beam/erl_alloc.c | 37 ++ erts/emulator/beam/sys.h | 6 + erts/emulator/sys/common/erl_mmap.c | 972 ++++++++++++++++++++++++++++++++++++ erts/emulator/sys/common/erl_mmap.h | 26 + erts/emulator/sys/common/erl_mseg.c | 6 +- erts/emulator/sys/common/erl_mseg.h | 2 + erts/etc/common/erlexec.c | 3 + 8 files changed, 1052 insertions(+), 1 deletion(-) create mode 100644 erts/emulator/sys/common/erl_mmap.c create mode 100644 erts/emulator/sys/common/erl_mmap.h (limited to 'erts') diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 9751982103..f442540f49 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -808,6 +808,7 @@ OS_OBJS += $(OBJDIR)/erl_poll.o \ endif OS_OBJS += $(OBJDIR)/erl_mseg.o \ + $(OBJDIR)/erl_mmap.o \ $(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \ $(OBJDIR)/erl_mtrace_sys_wrap.o \ $(OBJDIR)/erl_sys_common_misc.o diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 6dec383cee..2babe2f416 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1174,6 +1174,25 @@ get_kb_value(char *param_end, char** argv, int* ip) return ((Uint) tmp)*1024; } +static UWord +get_mb_value(char *param_end, char** argv, int* ip) +{ + SWord tmp; + UWord max = ((~((Uint) 0))/(1024*1024)) + 1; + char *rest; + char *param = argv[*ip]+1; + char *value = get_value(param_end, argv, ip); + errno = 0; + tmp = (SWord) ErtsStrToSint(value, &rest, 10); + if (errno != 0 || rest == value || tmp < 0 || max < ((UWord) tmp)) + bad_value(param, param_end, value); + if (max == (UWord) tmp) + return ~((UWord) 0); + else + return ((UWord) tmp)*1024*1024; +} + + #if 0 static Uint get_byte_value(char *param_end, char** argv, int* ip) @@ -1448,6 +1467,24 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) #endif get_amount_value(argv[i]+6, argv, &i); } + else if (has_prefix("scs", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.scs = +#endif + get_mb_value(argv[i]+6, argv, &i); + } + else if (has_prefix("sco", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.sco = +#endif + get_bool_value(argv[i]+6, argv, &i); + } + else if (has_prefix("scmgc", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.scmgc = +#endif + get_amount_value(argv[i]+8, argv, &i); + } else { bad_param(param, param+2); } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 97e6ed8410..dfe60d8ea0 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -277,16 +277,19 @@ typedef unsigned long UWord; typedef long SWord; #define SWORD_CONSTANT(Const) Const##L #define UWORD_CONSTANT(Const) Const##UL +#define ERTS_SWORD_MAX LONG_MAX #elif SIZEOF_VOID_P == SIZEOF_INT typedef unsigned int UWord; typedef int SWord; #define SWORD_CONSTANT(Const) Const #define UWORD_CONSTANT(Const) Const##U +#define ERTS_SWORD_MAX INT_MAX #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG typedef unsigned long long UWord; typedef long long SWord; #define SWORD_CONSTANT(Const) Const##LL #define UWORD_CONSTANT(Const) Const##ULL +#define ERTS_SWORD_MAX LLONG_MAX #else #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' #endif @@ -299,6 +302,7 @@ typedef unsigned long Uint; typedef long Sint; #define SWORD_CONSTANT(Const) Const##L #define UWORD_CONSTANT(Const) Const##UL +#define ERTS_SWORD_MAX LONG_MAX #define ERTS_SIZEOF_ETERM SIZEOF_LONG #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_INT @@ -307,6 +311,7 @@ typedef unsigned int Uint; typedef int Sint; #define SWORD_CONSTANT(Const) Const #define UWORD_CONSTANT(Const) Const##U +#define ERTS_SWORD_MAX INT_MAX #define ERTS_SIZEOF_ETERM SIZEOF_INT #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG @@ -315,6 +320,7 @@ typedef unsigned long long Uint; typedef long long Sint; #define SWORD_CONSTANT(Const) Const##LL #define UWORD_CONSTANT(Const) Const##ULL +#define ERTS_SWORD_MAX LLONG_MAX #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG #if defined(__WIN32__) #define ErtsStrToSint _strtoi64 diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c new file mode 100644 index 0000000000..a16ee7ae39 --- /dev/null +++ b/erts/emulator/sys/common/erl_mmap.c @@ -0,0 +1,972 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include + +#ifdef DEBUG +# define RBT_DEBUG +#endif +#ifdef RBT_DEBUG +# define RBT_ASSERT ERTS_ASSERT +# define IF_RBT_DEBUG(C) C +#else +# define RBT_ASSERT(x) +# define IF_RBT_DEBUG(C) +#endif + +typedef struct RBTNode_ RBTNode; +struct RBTNode_ { + RBTNode *parent; + RBTNode *left; + RBTNode *right; + int flags; +}; + +enum SortOrder { + ADDR_ORDER, + SZ_ADDR_ORDER, + SZ_REVERSE_ADDR_ORDER +}; + +typedef struct { + RBTNode* root; + enum SortOrder order; +}RBTree; + +#define RED_FLG (1) +#define IS_RED(N) ((N) && ((N)->flags & RED_FLG)) +#define IS_BLACK(N) (!IS_RED(N)) +#define SET_RED(N) ((N)->flags |= RED_FLG) +#define SET_BLACK(N) ((N)->flags &= ~RED_FLG) + +#define HARD_DEBUG /*SVERK*/ +#ifdef HARD_DEBUG +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) +# define HARD_CHECK_TREE(TREE,SZ) check_tree(TREE, SZ) +static int rbt_assert_is_member(RBTNode* root, RBTNode* node); +static RBTNode* check_tree(RBTree* tree, Uint); +#else +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) +# define HARD_CHECK_TREE(TREE,SZ) +#endif + + +typedef struct { + RBTNode snode; + RBTNode anode; + char* start; + char* end; +}ErtsFreeSegDesc; + +static ERTS_INLINE ErtsFreeSegDesc* anode_to_desc(RBTNode* anode) +{ + return (ErtsFreeSegDesc*) ((char*)anode - offsetof(ErtsFreeSegDesc, anode)); +} + +static ERTS_INLINE ErtsFreeSegDesc* snode_to_desc(RBTNode* snode) +{ + return (ErtsFreeSegDesc*) ((char*)snode - offsetof(ErtsFreeSegDesc, snode)); +} + +static ERTS_INLINE ErtsFreeSegDesc* node_to_desc(enum SortOrder order, RBTNode* node) +{ + return order==ADDR_ORDER ? anode_to_desc(node) : snode_to_desc(node); +} + +typedef struct { + RBTree stree; + RBTree atree; +}ErtsFreeSegMap; + + +#ifdef HARD_DEBUG +static ERTS_INLINE SWord cmp_blocks(enum SortOrder order, + RBTNode* lhs, RBTNode* rhs) +{ + ErtsFreeSegDesc* ldesc = node_to_desc(order, lhs); + ErtsFreeSegDesc* rdesc = node_to_desc(order, rhs); + RBT_ASSERT(lhs != rhs); + if (order != ADDR_ORDER) { + SWord lsz = ldesc->end - ldesc->start; + SWord rsz = rdesc->end - rdesc->start; + SWord diff = lsz - rsz; + if (diff) return diff; + } + if (order != SZ_REVERSE_ADDR_ORDER) { + return (char*)ldesc->start - (char*)rdesc->start; + } + else { + return (char*)rdesc->start - (char*)ldesc->start; + } +} +#endif + +static ERTS_INLINE SWord cmp_with_block(enum SortOrder order, + SWord sz, char* addr, RBTNode* rhs) +{ + ErtsFreeSegDesc* rdesc; + if (order != ADDR_ORDER) { + rdesc = snode_to_desc(rhs); + { + SWord rhs_sz = rdesc->end - rdesc->start; + SWord diff = sz - rhs_sz; + if (diff) return diff; + } + } + else + rdesc = anode_to_desc(rhs); + + if (order != SZ_REVERSE_ADDR_ORDER) + return addr - (char*)rdesc->start; + else + return (char*)rdesc->start - addr; +} + + +static ERTS_INLINE void +left_rotate(RBTNode **root, RBTNode *x) +{ + RBTNode *y = x->right; + x->right = y->left; + if (y->left) + y->left->parent = x; + y->parent = x->parent; + if (!y->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->left) + x->parent->left = y; + else { + RBT_ASSERT(x == x->parent->right); + x->parent->right = y; + } + y->left = x; + x->parent = y; + + /*SVERK y->max_sz = x->max_sz; + x->max_sz = node_max_size(x); + ASSERT(y->max_sz >= x->max_sz);*/ +} + +static ERTS_INLINE void +right_rotate(RBTNode **root, RBTNode *x) +{ + RBTNode *y = x->left; + x->left = y->right; + if (y->right) + y->right->parent = x; + y->parent = x->parent; + if (!y->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->right) + x->parent->right = y; + else { + RBT_ASSERT(x == x->parent->left); + x->parent->left = y; + } + y->right = x; + x->parent = y; + /*SVERK y->max_sz = x->max_sz; + x->max_sz = node_max_size(x); + ASSERT(y->max_sz >= x->max_sz);*/ +} + +/* + * Replace node x with node y + * NOTE: block header of y is not changed + */ +static ERTS_INLINE void +replace(RBTNode **root, RBTNode *x, RBTNode *y) +{ + + if (!x->parent) { + RBT_ASSERT(*root == x); + *root = y; + } + else if (x == x->parent->left) + x->parent->left = y; + else { + RBT_ASSERT(x == x->parent->right); + x->parent->right = y; + } + if (x->left) { + RBT_ASSERT(x->left->parent == x); + x->left->parent = y; + } + if (x->right) { + RBT_ASSERT(x->right->parent == x); + x->right->parent = y; + } + + y->flags = x->flags; + y->parent = x->parent; + y->right = x->right; + y->left = x->left; + /*SVERK y->max_sz = x->max_sz;*/ +} + +static void +tree_insert_fixup(RBTNode** root, RBTNode *blk) +{ + RBTNode *x = blk, *y; + + /* + * Rearrange the tree so that it satisfies the Red-Black Tree properties + */ + + RBT_ASSERT(x != *root && IS_RED(x->parent)); + do { + + /* + * x and its parent are both red. Move the red pair up the tree + * until we get to the root or until we can separate them. + */ + + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(x->parent->parent); + + if (x->parent == x->parent->parent->left) { + y = x->parent->parent->right; + if (IS_RED(y)) { + SET_BLACK(y); + x = x->parent; + SET_BLACK(x); + x = x->parent; + SET_RED(x); + } + else { + + if (x == x->parent->right) { + x = x->parent; + left_rotate(root, x); + } + + RBT_ASSERT(x == x->parent->parent->left->left); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_BLACK(y)); + + SET_BLACK(x->parent); + SET_RED(x->parent->parent); + right_rotate(root, x->parent->parent); + + RBT_ASSERT(x == x->parent->left); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent->right)); + RBT_ASSERT(IS_BLACK(x->parent)); + break; + } + } + else { + RBT_ASSERT(x->parent == x->parent->parent->right); + y = x->parent->parent->left; + if (IS_RED(y)) { + SET_BLACK(y); + x = x->parent; + SET_BLACK(x); + x = x->parent; + SET_RED(x); + } + else { + + if (x == x->parent->left) { + x = x->parent; + right_rotate(root, x); + } + + RBT_ASSERT(x == x->parent->parent->right->right); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent)); + RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_BLACK(y)); + + SET_BLACK(x->parent); + SET_RED(x->parent->parent); + left_rotate(root, x->parent->parent); + + RBT_ASSERT(x == x->parent->right); + RBT_ASSERT(IS_RED(x)); + RBT_ASSERT(IS_RED(x->parent->left)); + RBT_ASSERT(IS_BLACK(x->parent)); + break; + } + } + } while (x != *root && IS_RED(x->parent)); + + SET_BLACK(*root); +} + +static void +rbt_delete(RBTNode** root, RBTNode* del) +{ + Uint spliced_is_black; + RBTNode *x, *y, *z = del; + RBTNode null_x; /* null_x is used to get the fixup started when we + splice out a node without children. */ + + HARD_CHECK_IS_MEMBER(*root, del); + + null_x.parent = NULL; + + /* Remove node from tree... */ + + /* Find node to splice out */ + if (!z->left || !z->right) + y = z; + else + /* Set y to z:s successor */ + for(y = z->right; y->left; y = y->left); + /* splice out y */ + x = y->left ? y->left : y->right; + spliced_is_black = IS_BLACK(y); + if (x) { + x->parent = y->parent; + } + else if (spliced_is_black) { + x = &null_x; + x->flags = 0; + SET_BLACK(x); + x->right = x->left = NULL; + /*SVERK x->max_sz = 0;*/ + x->parent = y->parent; + y->left = x; + } + + if (!y->parent) { + RBT_ASSERT(*root == y); + *root = x; + } + else { + if (y == y->parent->left) { + y->parent->left = x; + } + else { + RBT_ASSERT(y == y->parent->right); + y->parent->right = x; + } + /*SVERK if (y->parent != z) { + lower_max_size(y->parent, (y==z ? NULL : z)); + }*/ + } + if (y != z) { + /* We spliced out the successor of z; replace z by the successor */ + RBT_ASSERT(z != &null_x); + replace(root, z, y); + /*SVERK lower_max_size(y, NULL);*/ + } + + if (spliced_is_black) { + /* We removed a black node which makes the resulting tree + violate the Red-Black Tree properties. Fixup tree... */ + + while (IS_BLACK(x) && x->parent) { + + /* + * x has an "extra black" which we move up the tree + * until we reach the root or until we can get rid of it. + * + * y is the sibbling of x + */ + + if (x == x->parent->left) { + y = x->parent->right; + RBT_ASSERT(y); + if (IS_RED(y)) { + RBT_ASSERT(y->right); + RBT_ASSERT(y->left); + SET_BLACK(y); + RBT_ASSERT(IS_BLACK(x->parent)); + SET_RED(x->parent); + left_rotate(root, x->parent); + y = x->parent->right; + } + RBT_ASSERT(y); + RBT_ASSERT(IS_BLACK(y)); + if (IS_BLACK(y->left) && IS_BLACK(y->right)) { + SET_RED(y); + x = x->parent; + } + else { + if (IS_BLACK(y->right)) { + SET_BLACK(y->left); + SET_RED(y); + right_rotate(root, y); + y = x->parent->right; + } + RBT_ASSERT(y); + if (IS_RED(x->parent)) { + + SET_BLACK(x->parent); + SET_RED(y); + } + RBT_ASSERT(y->right); + SET_BLACK(y->right); + left_rotate(root, x->parent); + x = *root; + break; + } + } + else { + RBT_ASSERT(x == x->parent->right); + y = x->parent->left; + RBT_ASSERT(y); + if (IS_RED(y)) { + RBT_ASSERT(y->right); + RBT_ASSERT(y->left); + SET_BLACK(y); + RBT_ASSERT(IS_BLACK(x->parent)); + SET_RED(x->parent); + right_rotate(root, x->parent); + y = x->parent->left; + } + RBT_ASSERT(y); + RBT_ASSERT(IS_BLACK(y)); + if (IS_BLACK(y->right) && IS_BLACK(y->left)) { + SET_RED(y); + x = x->parent; + } + else { + if (IS_BLACK(y->left)) { + SET_BLACK(y->right); + SET_RED(y); + left_rotate(root, y); + y = x->parent->left; + } + RBT_ASSERT(y); + if (IS_RED(x->parent)) { + SET_BLACK(x->parent); + SET_RED(y); + } + RBT_ASSERT(y->left); + SET_BLACK(y->left); + right_rotate(root, x->parent); + x = *root; + break; + } + } + } + SET_BLACK(x); + + if (null_x.parent) { + if (null_x.parent->left == &null_x) + null_x.parent->left = NULL; + else { + RBT_ASSERT(null_x.parent->right == &null_x); + null_x.parent->right = NULL; + } + RBT_ASSERT(!null_x.left); + RBT_ASSERT(!null_x.right); + } + else if (*root == &null_x) { + *root = NULL; + RBT_ASSERT(!null_x.left); + RBT_ASSERT(!null_x.right); + } + } +} + + +static void +rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) +{ +#ifdef RBT_DEBUG + ErtsFreeSegDesc *dbg_under=NULL, *dbg_over=NULL; +#endif + ErtsFreeSegDesc* desc = node_to_desc(order, blk); + char* blk_addr = desc->start; + SWord blk_sz = desc->end - desc->start; + /*SVERK Uint blk_sz = AOFF_BLK_SZ(blk);*/ + + blk->flags = 0; + blk->left = NULL; + blk->right = NULL; + /*SVERK blk->max_sz = blk_sz;*/ + + if (!*root) { + blk->parent = NULL; + SET_BLACK(blk); + *root = blk; + } + else { + RBTNode *x = *root; + while (1) { + SWord diff; + /*SVERK if (x->max_sz < blk_sz) { + x->max_sz = blk_sz; + }*/ + diff = cmp_with_block(order, blk_sz, blk_addr, x); + if (diff < 0) { + IF_RBT_DEBUG(dbg_over = node_to_desc(order, x)); + if (!x->left) { + blk->parent = x; + x->left = blk; + break; + } + x = x->left; + } + else { + RBT_ASSERT(diff > 0); + IF_RBT_DEBUG(dbg_under = node_to_desc(order, x)); + if (!x->right) { + blk->parent = x; + x->right = blk; + break; + } + x = x->right; + } + /*SVERK else { + ASSERT(flavor == AOFF_BF); + ASSERT(blk->flags & IS_BF_FLG); + ASSERT(x->flags & IS_BF_FLG); + SET_LIST_ELEM(blk); + LIST_NEXT(blk) = LIST_NEXT(x); + LIST_PREV(blk) = x; + if (LIST_NEXT(x)) + LIST_PREV(LIST_NEXT(x)) = blk; + LIST_NEXT(x) = blk; + return; + }*/ + } + + /* Insert block into size tree */ + RBT_ASSERT(blk->parent); +#ifdef RBT_DEBUG + if (!order) { + RBT_ASSERT(!dbg_under || dbg_under->end < desc->start); + RBT_ASSERT(!dbg_over || dbg_over->start > desc->end); + } +#endif + SET_RED(blk); + if (IS_RED(blk->parent)) + tree_insert_fixup(root, blk); + } + /*SVERK if (flavor == AOFF_BF) { + SET_TREE_NODE(blk); + LIST_NEXT(blk) = NULL; + }*/ +} + + +/* The API to keep track of a bunch of separated free segments + (non-overlapping and non-adjacent). + */ +static void init_free_seg_map(ErtsFreeSegMap*, int reverse_ao); +static void adjacent_free_seg(ErtsFreeSegMap*, char* start, char* end, + ErtsFreeSegDesc** under, ErtsFreeSegDesc** over); +static void insert_free_seg(ErtsFreeSegMap*, ErtsFreeSegDesc*, char* start, char* end); +static void resize_free_seg(ErtsFreeSegMap*, ErtsFreeSegDesc*, char* start, char* end); +static void delete_free_seg(ErtsFreeSegMap*, ErtsFreeSegDesc*); +static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap*, SWord sz); + + +static void init_free_seg_map(ErtsFreeSegMap* map, int reverse_ao) +{ + map->atree.root = NULL; + map->atree.order = ADDR_ORDER; + map->stree.root = NULL; + map->stree.order = reverse_ao ? SZ_REVERSE_ADDR_ORDER : SZ_ADDR_ORDER; +} + +static void adjacent_free_seg(ErtsFreeSegMap* map, char* start, char* end, + ErtsFreeSegDesc** under, ErtsFreeSegDesc** over) +{ + RBTNode* x = map->atree.root; + + *under = NULL; + *over = NULL; + while (x) { + if (start < anode_to_desc(x)->start) { + RBT_ASSERT(end <= anode_to_desc(x)->start); + if (end == anode_to_desc(x)->start) { + RBT_ASSERT(!*over); + *over = anode_to_desc(x); + } + x = x->left; + } + else { + RBT_ASSERT(start >= anode_to_desc(x)->end); + if (start == anode_to_desc(x)->end) { + RBT_ASSERT(!*under); + *under = anode_to_desc(x); + } + x = x->right; + } + } +} + +static void insert_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, + char* start, char* end) +{ + desc->start = start; + desc->end = end; + rbt_insert(map->atree.order, &map->atree.root, &desc->anode); + rbt_insert(map->stree.order, &map->stree.root, &desc->snode); +} + +static void resize_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, + char* start, char* end) +{ +#ifdef DEBUG + ErtsFreeSegDesc *dbg_under, *dbg_over; + rbt_delete(&map->atree.root, &desc->anode); + adjacent_free_seg(map, start, end, &dbg_under, &dbg_over); + RBT_ASSERT(dbg_under == NULL && dbg_over == NULL); + rbt_insert(map->atree.order, &map->atree.root, &desc->anode); +#endif + rbt_delete(&map->stree.root, &desc->snode); + desc->start = start; + desc->end = end; + rbt_insert(map->stree.order, &map->stree.root, &desc->snode); +} + +static void delete_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc) +{ + rbt_delete(&map->atree.root, &desc->anode); + rbt_delete(&map->stree.root, &desc->snode); +} + +static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap* map, SWord need_sz) +{ + RBTNode* x = map->stree.root; + ErtsFreeSegDesc* best_desc = NULL; + + while (x) { + ErtsFreeSegDesc* desc = snode_to_desc(x); + SWord seg_sz = desc->end - desc->start; + + if (seg_sz < need_sz) { + x = x->right; + } + else { + best_desc = desc; + x = x->left; + } + } + return best_desc; +} + + +void erts_mmap_init(ErtsMMapInit* init) +{ +#ifdef HARD_DEBUG + erts_fprintf(stderr, "SVERK: scs = %bpu\n", init->scs); + erts_fprintf(stderr, "SVERK: sco = %i\n", init->sco); + erts_fprintf(stderr, "SVERK: scmgc = %i\n", init->scmgc); + + { + void test_it(void); + test_it(); + } +#endif +} + + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Debug functions * +\* */ + + +#ifdef HARD_DEBUG + +static int rbt_assert_is_member(RBTNode* root, RBTNode* node) +{ + while (node != root) { + RBT_ASSERT(node->parent); + RBT_ASSERT(node->parent->left == node || node->parent->right == node); + node = node->parent; + } + return 1; +} + +#define LEFT_VISITED_FLG 0x1000 +#define THIS_VISITED_FLG 0x100 +#define RIGHT_VISITED_FLG 0x10 +#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) +#define IS_THIS_VISITED(FB) ((FB)->flags & THIS_VISITED_FLG) +#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) + +#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG) +#define SET_THIS_VISITED(FB) ((FB)->flags |= THIS_VISITED_FLG) +#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG) + +#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG) +#define UNSET_THIS_VISITED(FB) ((FB)->flags &= ~THIS_VISITED_FLG) +#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG) + + + +#if 1 /*SVERK*/ +# define PRINT_TREE +#else +# undef PRINT_TREE +#endif + +#ifdef PRINT_TREE +static void print_tree(enum SortOrder order, RBTNode*); +#endif + +/* + * Checks that the order between parent and children are correct, + * and that the Red-Black Tree properies are satisfied. if size > 0, + * check_tree() returns the node that satisfies "address order first fit" + * + * The Red-Black Tree properies are: + * 1. Every node is either red or black. + * 2. Every leaf (NIL) is black. + * 3. If a node is red, then both its children are black. + * 4. Every simple path from a node to a descendant leaf + * contains the same number of black nodes. + * + */ + +static RBTNode * +check_tree(RBTree* tree, Uint size) +{ + RBTNode *res = NULL; + Sint blacks; + Sint curr_blacks; + RBTNode *x; + Uint depth, max_depth, node_cnt; + ErtsFreeSegDesc* seg = NULL; + ErtsFreeSegDesc* prev_seg = NULL; + +#ifdef PRINT_TREE + print_tree(tree->order, tree->root); +#endif + + if (!tree->root) + return res; + + x = tree->root; + RBT_ASSERT(IS_BLACK(x)); + RBT_ASSERT(!x->parent); + curr_blacks = 1; + blacks = -1; + depth = 1; + max_depth = 0; + node_cnt = 0; + + /* Traverse tree in sorting order */ + while (x) { + if (!IS_LEFT_VISITED(x)) { + SET_LEFT_VISITED(x); + if (x->left) { + x = x->left; + ++depth; + if (IS_BLACK(x)) + curr_blacks++; + continue; + } + else { + if (blacks < 0) + blacks = curr_blacks; + RBT_ASSERT(blacks == curr_blacks); + } + } + + if (!IS_THIS_VISITED(x)) { + SET_THIS_VISITED(x); + ++node_cnt; + if (depth > max_depth) + max_depth = depth; + + if (IS_RED(x)) { + RBT_ASSERT(IS_BLACK(x->right)); + RBT_ASSERT(IS_BLACK(x->left)); + } + + RBT_ASSERT(x->parent || x == tree->root); + + if (x->left) { + RBT_ASSERT(x->left->parent == x); + RBT_ASSERT(cmp_blocks(tree->order, x->left, x) < 0); + } + + if (x->right) { + RBT_ASSERT(x->right->parent == x); + RBT_ASSERT(cmp_blocks(tree->order, x->right, x) > 0); + } + + seg = node_to_desc(tree->order, x); + RBT_ASSERT(seg->start < seg->end); + if (size && (seg->end - seg->start) >= size) { + if (!res || cmp_blocks(tree->order, x, res) < 0) { + res = x; + } + } + if (tree->order == ADDR_ORDER) { + RBT_ASSERT(!prev_seg || prev_seg->end < seg->start); + prev_seg = seg; + } + + } + if (!IS_RIGHT_VISITED(x)) { + SET_RIGHT_VISITED(x); + if (x->right) { + x = x->right; + ++depth; + if (IS_BLACK(x)) + curr_blacks++; + continue; + } + else { + if (blacks < 0) + blacks = curr_blacks; + RBT_ASSERT(blacks == curr_blacks); + } + } + + UNSET_LEFT_VISITED(x); + UNSET_THIS_VISITED(x); + UNSET_RIGHT_VISITED(x); + if (IS_BLACK(x)) + curr_blacks--; + x = x->parent; + --depth; + } + RBT_ASSERT(depth == 0 || (!tree->root && depth==1)); + RBT_ASSERT(curr_blacks == 0); + RBT_ASSERT((1 << (max_depth/2)) <= node_cnt); + + UNSET_LEFT_VISITED(tree->root); + UNSET_THIS_VISITED(tree->root); + UNSET_RIGHT_VISITED(tree->root); + + return res; +} + + +#ifdef PRINT_TREE +#define INDENT_STEP 2 + +#include + +static void +print_tree_aux(enum SortOrder order, RBTNode *x, int indent) +{ + int i; + + if (x) { + ErtsFreeSegDesc* desc = node_to_desc(order, x); + print_tree_aux(order, x->right, indent + INDENT_STEP); + for (i = 0; i < indent; i++) { + putc(' ', stderr); + } + fprintf(stderr, "%s: sz=%lx [%p - %p] desc=%p\r\n", + IS_BLACK(x) ? "BLACK" : "RED", + desc->end - desc->start, desc->start, desc->end, desc); + print_tree_aux(order, x->left, indent + INDENT_STEP); + } +} + + +static void +print_tree(enum SortOrder order, RBTNode* root) +{ + static const char* type[] = {"Address","Size-Address","Size-RevAddress"}; + fprintf(stderr, " --- %s ordered tree begin ---\r\n", type[order]); + print_tree_aux(order, root, 0); + fprintf(stderr, " --- %s ordered tree end ---\r\n", type[order]); +} + +#endif /* PRINT_TREE */ + + +static ErtsFreeSegDesc* new_desc(void) +{ + return (ErtsFreeSegDesc*) malloc(sizeof(ErtsFreeSegDesc)); +} + +void test_it(void) +{ + ErtsFreeSegMap map; + ErtsFreeSegDesc *desc, *under, *over, *d1, *d2; + int i; + + for (i=0; i<2; i++) { + init_free_seg_map(&map, i); + + insert_free_seg(&map, new_desc(), (char*)0x11000, (char*)0x12000); + check_tree(&map.atree, 0); check_tree(&map.stree, 0); + insert_free_seg(&map, new_desc(), (char*)0x13000, (char*)0x14000); + check_tree(&map.atree, 0); check_tree(&map.stree, 0); + insert_free_seg(&map, new_desc(), (char*)0x15000, (char*)0x17000); + check_tree(&map.atree, 0); check_tree(&map.stree, 0); + insert_free_seg(&map, new_desc(), (char*)0x8000, (char*)0x10000); + check_tree(&map.atree, 0); check_tree(&map.stree, 0); + + desc = lookup_free_seg(&map, 0x500); + ERTS_ASSERT(desc->start == (char*)(i?0x13000L:0x11000L)); + + desc = lookup_free_seg(&map, 0x1500); + ERTS_ASSERT(desc->start == (char*)0x15000); + + adjacent_free_seg(&map, (char*)0x6666, (char*)0x7777, &under, &over); + ERTS_ASSERT(!under && !over); + + adjacent_free_seg(&map, (char*)0x6666, (char*)0x8000, &under, &over); + ERTS_ASSERT(!under && over->start == (char*)0x8000); + + adjacent_free_seg(&map, (char*)0x10000, (char*)0x10500, &under, &over); + ERTS_ASSERT(under->end == (char*)0x10000 && !over); + + adjacent_free_seg(&map, (char*)0x10100, (char*)0x10500, &under, &over); + ERTS_ASSERT(!under && !over); + + adjacent_free_seg(&map, (char*)0x10100, (char*)0x11000, &under, &over); + ERTS_ASSERT(!under && over && over->start == (char*)0x11000); + + adjacent_free_seg(&map, (char*)0x12000, (char*)0x12500, &under, &over); + ERTS_ASSERT(under && under->end == (char*)0x12000 && !over); + + adjacent_free_seg(&map, (char*)0x12000, (char*)0x13000, &under, &over); + ERTS_ASSERT(under && under->end == (char*)0x12000 && + over && over->start == (char*)0x13000); + + adjacent_free_seg(&map, (char*)0x12500, (char*)0x13000, &under, &over); + ERTS_ASSERT(!under && over && over->start == (char*)0x13000); + + d1 = lookup_free_seg(&map, 0x500); + ERTS_ASSERT(d1->start == (char*)(i?0x13000L:0x11000L)); + + resize_free_seg(&map, d1, d1->start - 0x800, (char*)d1->end); + check_tree(&map.atree, 0); check_tree(&map.stree, 0); + + d2 = lookup_free_seg(&map, 0x1200); + ERTS_ASSERT(d2 == d1); + + delete_free_seg(&map, d1); + check_tree(&map.atree, 0); check_tree(&map.stree, 0); + + d1 = lookup_free_seg(&map, 0x1200); + ERTS_ASSERT(d1->start == (char*)0x15000); + } +} + +#endif /* HARD_DEBUG */ diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h new file mode 100644 index 0000000000..64baa6c493 --- /dev/null +++ b/erts/emulator/sys/common/erl_mmap.h @@ -0,0 +1,26 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +typedef struct { + SWord scs; /* super carrier size */ + int sco; /* super carrier only? */ + Uint scmgc; /* super carrier: max guaranteed (number of) carriers */ +}ErtsMMapInit; + +void erts_mmap_init(ErtsMMapInit*); diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 2748edba02..64fcb6bb40 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1668,8 +1668,12 @@ erts_mseg_init(ErtsMsegInit_t *init) erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n"); #endif -#if HAVE_MMAP && HALFWORD_HEAP +#if HAVE_MMAP +# if HALFWORD_HEAP initialize_pmmap(); +# else + erts_mmap_init(&init->mmap); +# endif #endif if (!IS_2POW(GET_PAGE_SIZE)) diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index a1b000f51c..7454e5c473 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -22,6 +22,7 @@ #include "sys.h" #include "erl_alloc_types.h" +#include "erl_mmap.h" #ifndef HAVE_MMAP # define HAVE_MMAP 0 @@ -68,6 +69,7 @@ typedef struct { Uint rmcbf; Uint mcs; Uint nos; + ErtsMMapInit mmap; } ErtsMsegInit_t; #define ERTS_MSEG_INIT_DEFAULT_INITIALIZER \ diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 30560f5a2f..1e01c5bc20 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -109,6 +109,9 @@ static char *plusM_other_switches[] = { "Mamcbf", "Mrmcbf", "Mmcs", + "Mscs", + "Mscmgc", + "Msco", "Ye", "Ym", "Ytp", -- cgit v1.2.3 From ef3da907bd566b43a4022f1cbb1ae3d103b9ec3e Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 23 Aug 2013 17:29:58 +0200 Subject: erts: erts_mmap supercarrier management and erts_mseg usage * Coalescing and trimming of free segments in supercarrier * Management of super aligned and super unaligned areas in supercarrier * Management of reservation of physical memory * erts_mseg usage of erts_mmap --- erts/emulator/beam/erl_alloc.c | 9 +- erts/emulator/beam/erl_alloc_util.c | 17 +- erts/emulator/beam/erl_lock_check.c | 3 +- erts/emulator/beam/erl_unicode.c | 3 + erts/emulator/beam/erl_vm.h | 2 +- erts/emulator/sys/common/erl_mmap.c | 1120 ++++++++++++++++++++++++++++++++++- erts/emulator/sys/common/erl_mmap.h | 89 ++- erts/emulator/sys/common/erl_mseg.c | 860 +++++---------------------- erts/emulator/sys/common/erl_mseg.h | 34 +- erts/etc/common/erlexec.c | 1 + 10 files changed, 1379 insertions(+), 759 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 2babe2f416..e30b3e7b51 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -718,6 +718,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.mseg.nos = erts_no_schedulers; erts_mseg_init(&init.mseg); #endif + erts_alcu_init(&init.alloc_util); erts_afalc_init(); erts_bfalc_init(); @@ -1178,7 +1179,7 @@ static UWord get_mb_value(char *param_end, char** argv, int* ip) { SWord tmp; - UWord max = ((~((Uint) 0))/(1024*1024)) + 1; + UWord max = ((~((UWord) 0))/(1024*1024)) + 1; char *rest; char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); @@ -1479,6 +1480,12 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) #endif get_bool_value(argv[i]+6, argv, &i); } + else if (has_prefix("scrpm", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.scrpm = +#endif + get_bool_value(argv[i]+8, argv, &i); + } else if (has_prefix("scmgc", argv[i]+3)) { #if HAVE_ERTS_MSEG init->mseg.mmap.scmgc = diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 3ea74a12f9..1fdee4db2c 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -756,8 +756,9 @@ static ERTS_INLINE void * alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; - - res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, flags, &allctr->mseg_opt); + UWord size = (UWord) *size_p; + res = erts_mseg_alloc_opt(allctr->alloc_no, &size, flags, &allctr->mseg_opt); + *size_p = (Uint) size; INC_CC(allctr->calls.mseg_alloc); return res; } @@ -766,9 +767,10 @@ static ERTS_INLINE void * alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) { void *res; - - res = erts_mseg_realloc_opt(allctr->alloc_no, seg, old_size, new_size_p, + UWord new_size = (UWord) *new_size_p; + res = erts_mseg_realloc_opt(allctr->alloc_no, seg, (UWord) old_size, &new_size, ERTS_MSEG_FLG_NONE, &allctr->mseg_opt); + *new_size_p = (Uint) new_size; INC_CC(allctr->calls.mseg_realloc); return res; } @@ -776,7 +778,7 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) static ERTS_INLINE void alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { - erts_mseg_dealloc_opt(allctr->alloc_no, seg, size, flags, &allctr->mseg_opt); + erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_dealloc); } @@ -3223,10 +3225,12 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, ASSERT(IS_MBC_BLK((B))); ASSERT(IS_MB_CARRIER((C))); ASSERT(FBLK_TO_MBC(B) == (C)); + if ((MSEGED)) { + ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ)); + } } if ((MSEGED)) { ASSERT(IS_MSEG_CARRIER((C))); - ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ)); } else { ASSERT(IS_SYS_ALLOC_CARRIER((C))); @@ -3598,7 +3602,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } else diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 2114d0c001..1e9cef3759 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -185,7 +185,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "sys_gethrtime", NULL }, #endif #endif - { "erts_alloc_hard_debug", NULL } + { "erts_alloc_hard_debug", NULL }, + { "erts_mmap", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index e00440b905..569c0a7d31 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1476,6 +1476,9 @@ static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint s Uint16 savepoints[4]; int numpoints = 0; + if (num == 0) + return NIL; + ASSERT(num > 0); hp = HAlloc(p,num * 2); /* May be to much */ diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 8026243555..337422eead 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -80,7 +80,7 @@ # ifdef CHECK_FOR_HOLES # define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz)) # else -# define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),DEBUG_BAD_BYTE,(sz)*sizeof(Eterm*)) +# define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*)) # endif #else # define INIT_HEAP_MEM(p,sz) ((void)0) diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index a16ee7ae39..aac01bf93c 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -20,11 +20,57 @@ # include "config.h" #endif -#include -#include +#include "sys.h" #include +#include "erl_smp.h" +#include "erl_mmap.h" -#ifdef DEBUG +#if defined(DEBUG) || 0 +# undef ERTS_MMAP_DEBUG +# define ERTS_MMAP_DEBUG +#endif + +/* #define ERTS_MMAP_DEBUG_FILL_AREAS */ + +#ifdef ERTS_MMAP_DEBUG +# define ERTS_MMAP_ASSERT(A) \ + ((void) (!(A) \ + ? erts_mmap_assert_failed(#A, __func__, __FILE__, __LINE__)\ + : 1)) +static int +erts_mmap_assert_failed(const char *a, const char *func, const char *file, int line) +{ + erts_fprintf(stderr, "%s:%d:%s() Assertion failed: %s\n", + (char *) file, line, (char *) func, (char *) a); + abort(); + return 0; +} +#else +# define ERTS_MMAP_ASSERT(A) ((void) 1) +#endif + +/* + * `mmap_state.sa.bot` and `mmap_state.sua.top` are read only after + * initialization, but the other pointers are not; i.e., only + * ERTS_MMAP_IN_SUPERCARRIER() is allowed without the mutex held. + */ +#define ERTS_MMAP_IN_SUPERCARRIER(PTR) \ + (((UWord) (PTR)) - ((UWord) mmap_state.sa.bot) \ + < ((UWord) mmap_state.sua.top) - ((UWord) mmap_state.sa.bot)) +#define ERTS_MMAP_IN_SUPERALIGNED_AREA(PTR) \ + (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mmap_state.mtx)), \ + (((UWord) (PTR)) - ((UWord) mmap_state.sa.bot) \ + < ((UWord) mmap_state.sa.top) - ((UWord) mmap_state.sa.bot))) +#define ERTS_MMAP_IN_SUPERUNALIGNED_AREA(PTR) \ + (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mmap_state.mtx)), \ + (((UWord) (PTR)) - ((UWord) mmap_state.sua.bot) \ + < ((UWord) mmap_state.sua.top) - ((UWord) mmap_state.sua.bot))) + +int erts_have_erts_mmap; +UWord erts_page_inv_mask; + +#if defined(DEBUG) || defined(ERTS_MMAP_DEBUG) +# undef RBT_DEBUG # define RBT_DEBUG #endif #ifdef RBT_DEBUG @@ -60,7 +106,7 @@ typedef struct { #define SET_RED(N) ((N)->flags |= RED_FLG) #define SET_BLACK(N) ((N)->flags &= ~RED_FLG) -#define HARD_DEBUG /*SVERK*/ +/* #define HARD_DEBUG */ #ifdef HARD_DEBUG # define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) # define HARD_CHECK_TREE(TREE,SZ) check_tree(TREE, SZ) @@ -79,6 +125,134 @@ typedef struct { char* end; }ErtsFreeSegDesc; +typedef struct { + RBTree stree; + RBTree atree; +}ErtsFreeSegMap; + +static struct { + int (*reserve_physical)(char *, UWord); + void (*unreserve_physical)(char *, UWord); + int supercarrier; + int no_os_mmap; + /* + * Super unaligend area is located above super aligned + * area. That is, `sa.bot` is beginning of the super + * carrier, `sua.top` is the end of the super carrier, + * and sa.top and sua.bot moves towards eachother. + */ + struct { + char *top; + char *bot; + ErtsFreeSegMap map; + } sua; + struct { + char *top; + char *bot; + ErtsFreeSegMap map; + } sa; +#if HAVE_MMAP && (!defined(MAP_ANON) && !defined(MAP_ANONYMOUS)) + int mmap_fd; +#endif + erts_smp_mtx_t mtx; + char *desc_free_list; + struct { + struct { + UWord total; + struct { + UWord total; + UWord sa; + UWord sua; + } used; + } supercarrier; + struct { + UWord used; + } os; + } size; +} mmap_state; + +#define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ + do { \ + mmap_state.size.supercarrier.used.total += (SZ); \ + mmap_state.size.supercarrier.used.sa += (SZ); \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.total \ + <= mmap_state.size.supercarrier.total); \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.sa \ + <= mmap_state.size.supercarrier.used.total); \ + } while (0) +#define ERTS_MMAP_SIZE_SC_SA_DEC(SZ) \ + do { \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.total >= (SZ)); \ + mmap_state.size.supercarrier.used.total -= (SZ); \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.sa >= (SZ)); \ + mmap_state.size.supercarrier.used.sa -= (SZ); \ + } while (0) +#define ERTS_MMAP_SIZE_SC_SUA_INC(SZ) \ + do { \ + mmap_state.size.supercarrier.used.total += (SZ); \ + mmap_state.size.supercarrier.used.sua += (SZ); \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.total \ + <= mmap_state.size.supercarrier.total); \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.sua \ + <= mmap_state.size.supercarrier.used.total); \ + } while (0) +#define ERTS_MMAP_SIZE_SC_SUA_DEC(SZ) \ + do { \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.total >= (SZ)); \ + mmap_state.size.supercarrier.used.total -= (SZ); \ + ERTS_MMAP_ASSERT(mmap_state.size.supercarrier.used.sua >= (SZ)); \ + mmap_state.size.supercarrier.used.sua -= (SZ); \ + } while (0) +#define ERTS_MMAP_SIZE_OS_INC(SZ) \ + do { \ + ERTS_MMAP_ASSERT(mmap_state.size.os.used + (SZ) >= (SZ)); \ + mmap_state.size.os.used += (SZ); \ + } while (0) +#define ERTS_MMAP_SIZE_OS_DEC(SZ) \ + do { \ + ERTS_MMAP_ASSERT(mmap_state.size.os.used >= (SZ)); \ + mmap_state.size.os.used -= (SZ); \ + } while (0) + +static void +add_free_desc_area(char *start, char *end) +{ + if (end > start && sizeof(ErtsFreeSegDesc) <= end - start) { + ErtsFreeSegDesc *prev_desc, *desc; + char *desc_end; + + prev_desc = (ErtsFreeSegDesc *) start; + prev_desc->start = mmap_state.desc_free_list; + desc = (ErtsFreeSegDesc *) (start + sizeof(ErtsFreeSegDesc)); + desc_end = start + 2*sizeof(ErtsFreeSegDesc); + + while (desc_end <= end) { + desc->start = (char *) prev_desc; + prev_desc = desc; + desc = (ErtsFreeSegDesc *) desc_end; + desc_end += sizeof(ErtsFreeSegDesc); + } + mmap_state.desc_free_list = (char *) prev_desc; + } +} + +static ERTS_INLINE ErtsFreeSegDesc * +alloc_desc(void) +{ + ErtsFreeSegDesc *res; + res = (ErtsFreeSegDesc *) mmap_state.desc_free_list; + if (res) + mmap_state.desc_free_list = res->start; + return res; +} + +static ERTS_INLINE void +free_desc(ErtsFreeSegDesc *desc) +{ + desc->start = mmap_state.desc_free_list; + mmap_state.desc_free_list = (char *) desc; +} + static ERTS_INLINE ErtsFreeSegDesc* anode_to_desc(RBTNode* anode) { return (ErtsFreeSegDesc*) ((char*)anode - offsetof(ErtsFreeSegDesc, anode)); @@ -94,12 +268,6 @@ static ERTS_INLINE ErtsFreeSegDesc* node_to_desc(enum SortOrder order, RBTNode* return order==ADDR_ORDER ? anode_to_desc(node) : snode_to_desc(node); } -typedef struct { - RBTree stree; - RBTree atree; -}ErtsFreeSegMap; - - #ifdef HARD_DEBUG static ERTS_INLINE SWord cmp_blocks(enum SortOrder order, RBTNode* lhs, RBTNode* rhs) @@ -671,19 +839,927 @@ static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap* map, SWord need_sz) return best_desc; } +#if ERTS_HAVE_OS_MMAP +/* Implementation of os_mmap()/os_munmap()/os_mremap()... */ + +#if HAVE_MMAP +# define ERTS_MMAP_PROT (PROT_READ|PROT_WRITE) +# if defined(MAP_ANONYMOUS) +# define ERTS_MMAP_FLAGS (MAP_ANON|MAP_PRIVATE) +# define ERTS_MMAP_FD (-1) +# elif defined(MAP_ANON) +# define ERTS_MMAP_FLAGS (MAP_ANON|MAP_PRIVATE) +# define ERTS_MMAP_FD (-1) +# else +# define ERTS_MMAP_FLAGS (MAP_PRIVATE) +# define ERTS_MMAP_FD mmap_state.mmap_fd +# endif +#endif -void erts_mmap_init(ErtsMMapInit* init) +static ERTS_INLINE void * +os_mmap(UWord size, int try_superalign) { -#ifdef HARD_DEBUG - erts_fprintf(stderr, "SVERK: scs = %bpu\n", init->scs); - erts_fprintf(stderr, "SVERK: sco = %i\n", init->sco); - erts_fprintf(stderr, "SVERK: scmgc = %i\n", init->scmgc); +#if HAVE_MMAP + void *res; +#ifdef MAP_ALIGN + if (try_superalign) + res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, ERTS_MMAP_PROT, + ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0); + else +#endif + res = mmap((void *) 0, size, ERTS_MMAP_PROT, + ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0); + if (res == MAP_FAILED) + return NULL; + return res; +#elif HAVE_VIRTUALALLOC + return (void *) VirtualAlloc(NULL, (SIZE_T) size, + MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE); +#else +# error "missing mmap() or similar" +#endif +} + +static ERTS_INLINE void +os_munmap(void *ptr, UWord size) +{ +#if HAVE_MMAP +#ifdef ERTS_MMAP_DEBUG + int res = +#endif + munmap(ptr, size); + ERTS_MMAP_ASSERT(res == 0); +#elif HAVE_VIRTUALALLOC +#ifdef DEBUG + BOOL res = +#endif + VirtualFree((LPVOID) ptr, (SIZE_T) 0, MEM_RELEASE); + ERTS_MMAP_ASSERT(res != 0); +#else +# error "missing munmap() or similar" +#endif +} + +#ifdef ERTS_HAVE_OS_MREMAP +# if HAVE_MREMAP +# if defined(__NetBSD__) +# define ERTS_MREMAP_FLAGS (0) +# else +# define ERTS_MREMAP_FLAGS (MREMAP_MAYMOVE) +# endif +# endif +static ERTS_INLINE void * +os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign) +{ + void *new_seg; +#if HAVE_MREMAP + new_seg = mremap(ptr, (size_t) old_size, +# if defined(__NetBSD__) + NULL, +# endif + (size_t) new_size, ERTS_MREMAP_FLAGS); + if (new_seg == (void *) MAP_FAILED) + return NULL; + return new_seg; +#else +# error "missing mremap() or similar" +#endif +} +#endif + +#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION +#if HAVE_MMAP + +#define ERTS_MMAP_RESERVE_PROT (ERTS_MMAP_PROT) +#define ERTS_MMAP_RESERVE_FLAGS (ERTS_MMAP_FLAGS|MAP_FIXED) +#define ERTS_MMAP_UNRESERVE_PROT (PROT_NONE) +#define ERTS_MMAP_UNRESERVE_FLAGS (ERTS_MMAP_FLAGS|MAP_NORESERVE|MAP_FIXED) +#define ERTS_MMAP_VIRTUAL_PROT (PROT_NONE) +#define ERTS_MMAP_VIRTUAL_FLAGS (ERTS_MMAP_FLAGS|MAP_NORESERVE) + +static int +os_reserve_physical(char *ptr, UWord size) +{ + void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_RESERVE_PROT, + ERTS_MMAP_RESERVE_FLAGS, ERTS_MMAP_FD, 0); + if (res == (void *) MAP_FAILED) + return 0; + return 1; +} + +static void +os_unreserve_physical(char *ptr, UWord size) +{ + void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_UNRESERVE_PROT, + ERTS_MMAP_UNRESERVE_FLAGS, ERTS_MMAP_FD, 0); + if (res == (void *) MAP_FAILED) + erl_exit(ERTS_ABORT_EXIT, "Failed to unreserve memory"); +} + +static void * +os_mmap_virtual(char *ptr, UWord size) +{ + void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_VIRTUAL_PROT, + ERTS_MMAP_VIRTUAL_FLAGS, ERTS_MMAP_FD, 0); + if (res == (void *) MAP_FAILED) + return NULL; + return res; +} + +#else +#error "Missing reserve/unreserve physical memory implementation" +#endif +#endif /* ERTS_HAVE_OS_RESERVE_PHYSICAL_MEMORY */ + +#endif /* ERTS_HAVE_OS_MMAP */ + +static int reserve_noop(char *ptr, UWord size) +{ +#ifdef ERTS_MMAP_DEBUG_FILL_AREAS + Uint32 *uip, *end = (Uint32 *) (ptr + size); + + for (uip = (Uint32 *) ptr; uip < end; uip++) + ERTS_MMAP_ASSERT(*uip == (Uint32) 0xdeadbeef); + for (uip = (Uint32 *) ptr; uip < end; uip++) + *uip = (Uint32) 0xfeedfeed; +#endif + return 1; +} + +static void unreserve_noop(char *ptr, UWord size) +{ +#ifdef ERTS_MMAP_DEBUG_FILL_AREAS + Uint32 *uip, *end = (Uint32 *) (ptr + size); + + for (uip = (Uint32 *) ptr; uip < end; uip++) + *uip = (Uint32) 0xdeadbeef; +#endif +} + +void * +erts_mmap(Uint32 flags, UWord *sizep) +{ + char *seg; + UWord asize = ERTS_PAGEALIGNED_CEILING(*sizep); + + /* Map in premapped supercarrier */ + if (mmap_state.supercarrier && !(ERTS_MMAPFLG_OS_ONLY & flags)) { + char *end; + ErtsFreeSegDesc *desc; + Uint32 superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); + + erts_smp_mtx_lock(&mmap_state.mtx); + + if (!superaligned) { + desc = lookup_free_seg(&mmap_state.sua.map, asize); + if (desc) { + seg = desc->start; + end = seg+asize; + if (!mmap_state.reserve_physical(seg, asize)) + goto supercarrier_reserve_failure; + if (desc->end == end) { + delete_free_seg(&mmap_state.sua.map, desc); + free_desc(desc); + } + else { + ERTS_MMAP_ASSERT(end < desc->end); + resize_free_seg(&mmap_state.sua.map, desc, end, desc->end); + } + ERTS_MMAP_SIZE_SC_SUA_INC(asize); + goto supercarrier_success; + } + + if (asize <= mmap_state.sua.bot - mmap_state.sa.top) { + if (!mmap_state.reserve_physical(mmap_state.sua.bot - asize, + asize)) + goto supercarrier_reserve_failure; + mmap_state.sua.bot -= asize; + seg = mmap_state.sua.bot; + ERTS_MMAP_SIZE_SC_SUA_INC(asize); + goto supercarrier_success; + } + } + + asize = ERTS_SUPERALIGNED_CEILING(asize); + + desc = lookup_free_seg(&mmap_state.sa.map, asize); + if (desc) { + seg = desc->start; + end = seg+asize; + if (!mmap_state.reserve_physical(seg, asize)) + goto supercarrier_reserve_failure; + if (desc->end == end) { + delete_free_seg(&mmap_state.sa.map, desc); + free_desc(desc); + } + else { + ERTS_MMAP_ASSERT(end < desc->end); + resize_free_seg(&mmap_state.sa.map, desc, end, desc->end); + } + ERTS_MMAP_SIZE_SC_SA_INC(asize); + goto supercarrier_success; + } + + if (superaligned) { + + if (asize <= mmap_state.sua.bot - mmap_state.sa.top) { + seg = (void *) mmap_state.sa.top; + if (!mmap_state.reserve_physical(seg, asize)) + goto supercarrier_reserve_failure; + mmap_state.sa.top += asize; + ERTS_MMAP_SIZE_SC_SA_INC(asize); + goto supercarrier_success; + } + + desc = lookup_free_seg(&mmap_state.sua.map, asize + ERTS_SUPERALIGNED_SIZE); + if (desc) { + char *org_start = desc->start; + char *org_end = desc->end; + + seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start); + end = seg + asize; + if (!mmap_state.reserve_physical(seg, asize)) + goto supercarrier_reserve_failure; + if (org_start != seg) { + ERTS_MMAP_ASSERT(org_start < seg); + resize_free_seg(&mmap_state.sua.map, desc, org_start, seg); + desc = NULL; + } + if (end != org_end) { + ERTS_MMAP_ASSERT(end < org_end); + if (desc) + resize_free_seg(&mmap_state.sua.map, desc, end, org_end); + else { + desc = alloc_desc(); + if (!desc) + add_free_desc_area(end, org_end); + else + insert_free_seg(&mmap_state.sua.map, desc, end, org_end); + } + } + ERTS_MMAP_SIZE_SC_SA_INC(asize); + goto supercarrier_success; + } + } + + erts_smp_mtx_unlock(&mmap_state.mtx); + } + +#if ERTS_HAVE_OS_MMAP + /* Map using OS primitives */ + if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mmap_state.no_os_mmap) { + if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) { + seg = os_mmap(asize, 0); + if (!seg) + return NULL; + } + else { + asize = ERTS_SUPERALIGNED_CEILING(*sizep); + seg = os_mmap(asize, 1); + if (!seg) + return NULL; + + if (!ERTS_IS_SUPERALIGNED(seg)) { + char *ptr; + UWord sz; + + os_munmap(seg, asize); + + ptr = os_mmap(asize + ERTS_SUPERALIGNED_SIZE, 1); + if (!ptr) + return NULL; + + seg = (char *) ERTS_SUPERALIGNED_CEILING(ptr); + sz = (UWord) (seg - ptr); + ERTS_MMAP_ASSERT(sz <= ERTS_SUPERALIGNED_SIZE); + if (sz) + os_munmap(ptr, sz); + sz = ERTS_SUPERALIGNED_SIZE - sz; + if (sz) + os_munmap(seg+asize, sz); + } + } + + ERTS_MMAP_SIZE_OS_INC(asize); + *sizep = asize; + return (void *) seg; + } +#endif + *sizep = 0; + return NULL; + +supercarrier_success: + +#ifdef ERTS_MMAP_DEBUG + if ((ERTS_MMAPFLG_SUPERALIGNED & flags) + || ERTS_MMAP_IN_SUPERALIGNED_AREA(seg)) { + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(seg)); + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); + } + else { + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(seg)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); + } +#endif + + erts_smp_mtx_unlock(&mmap_state.mtx); + + *sizep = asize; + return (void *) seg; + +supercarrier_reserve_failure: + erts_smp_mtx_unlock(&mmap_state.mtx); + + *sizep = 0; + return NULL; + +} + +void +erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) +{ + void *ptr = *ptrp; + UWord size = *sizep; + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size)); + if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) { + ERTS_MMAP_ASSERT(!mmap_state.no_os_mmap); +#if ERTS_HAVE_OS_MMAP + ERTS_MMAP_SIZE_OS_DEC(size); + os_munmap(ptr, size); +#endif + } + else { + char *start, *end; + ErtsFreeSegMap *map; + ErtsFreeSegDesc *prev, *next, *desc; + + ERTS_MMAP_ASSERT(mmap_state.supercarrier); + + start = (char *) ptr; + end = start + size; + + erts_smp_mtx_lock(&mmap_state.mtx); + + if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { + + start = (char *) ERTS_SUPERALIGNED_CEILING(start); + end = (char *) ERTS_SUPERALIGNED_FLOOR(end); + + size = (UWord) (end - start); + *ptrp = start; + *sizep = size; + + map = &mmap_state.sa.map; + adjacent_free_seg(map, start, end, &prev, &next); + + ERTS_MMAP_SIZE_SC_SA_DEC(size); + if (end == mmap_state.sa.top) { + ERTS_MMAP_ASSERT(!next); + if (prev) { + start = prev->start; + delete_free_seg(map, prev); + free_desc(prev); + } + mmap_state.sa.top = start; + goto supercarrier_success; + } + } + else { + ERTS_MMAP_ASSERT(ERTS_MMAP_IN_SUPERUNALIGNED_AREA(ptr)); + + map = &mmap_state.sua.map; + adjacent_free_seg(map, start, end, &prev, &next); + + ERTS_MMAP_SIZE_SC_SUA_DEC(size); + if (start == mmap_state.sua.bot) { + ERTS_MMAP_ASSERT(!prev); + if (next) { + end = next->end; + delete_free_seg(map, next); + free_desc(next); + } + mmap_state.sua.bot = end; + goto supercarrier_success; + } + } + + desc = NULL; + + if (next) { + ERTS_MMAP_ASSERT(end < next->end); + end = next->end; + if (prev) { + delete_free_seg(map, next); + free_desc(next); + goto save_prev; + } + desc = next; + } else if (prev) { + save_prev: + ERTS_MMAP_ASSERT(prev->start < start); + start = prev->start; + desc = prev; + } + + if (desc) + resize_free_seg(map, desc, start, end); + else { + desc = alloc_desc(); + if (desc) + insert_free_seg(map, desc, start, end); + else { + if (map == &mmap_state.sa.map) + ERTS_MMAP_SIZE_SC_SA_INC(size); + else + ERTS_MMAP_SIZE_SC_SUA_INC(size); + add_free_desc_area(start, end); + } + } + + supercarrier_success: + erts_smp_mtx_unlock(&mmap_state.mtx); + + mmap_state.unreserve_physical((char *) ptr, size); + } +} + +static void * +remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) +{ + UWord size = *sizep; + UWord um_size = old_size; + void *um_ptr = ptr; + void *new_ptr = erts_mmap(flags, &size); + if (!new_ptr) + return NULL; + *sizep = size; + if (old_size < size) + size = old_size; + sys_memcpy(new_ptr, ptr, (size_t) size); + erts_munmap(flags, &um_ptr, &um_size); + ERTS_MMAP_ASSERT(um_ptr == ptr); + ERTS_MMAP_ASSERT(um_size == old_size); + return new_ptr; +} + +void * +erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) +{ + void *new_ptr; + Uint32 superaligned; + UWord asize; + + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size)); + ERTS_MMAP_ASSERT(sizep && ERTS_IS_PAGEALIGNED(*sizep)); + + if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) { + + ERTS_MMAP_ASSERT(!mmap_state.no_os_mmap); + + if (!(ERTS_MMAPFLG_OS_ONLY & flags) && mmap_state.supercarrier) { + new_ptr = remap_move(ERTS_MMAPFLG_SUPERCARRIER_ONLY|flags, ptr, + old_size, sizep); + if (new_ptr) + return new_ptr; + } + + if (ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) + return NULL; + +#if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP + superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); + + if (superaligned) { + asize = ERTS_SUPERALIGNED_CEILING(*sizep); + if (asize == old_size && ERTS_IS_SUPERALIGNED(ptr)) { + *sizep = asize; + return ptr; + } + } + else { + asize = ERTS_PAGEALIGNED_CEILING(*sizep); + if (asize == old_size) { + *sizep = asize; + return ptr; + } + } + +#if ERTS_HAVE_GENUINE_OS_MMAP + if (asize < old_size + && (!superaligned + || ERTS_IS_SUPERALIGNED(ptr))) { + UWord um_sz; + new_ptr = ((char *) ptr) + asize; + ERTS_MMAP_ASSERT((((char *)ptr) + old_size) > (char *) new_ptr); + um_sz = (UWord) ((((char *) ptr) + old_size) - (char *) new_ptr); + ERTS_MMAP_SIZE_OS_DEC(um_sz); + os_munmap(new_ptr, um_sz); + *sizep = asize; + return ptr; + } +#endif +#if ERTS_HAVE_OS_MREMAP + if (superaligned) + return remap_move(flags, new_ptr, old_size, sizep); + else { + new_ptr = os_mremap(ptr, old_size, asize, 0); + if (!new_ptr) + return NULL; + if (asize > old_size) + ERTS_MMAP_SIZE_OS_INC(asize - old_size); + else + ERTS_MMAP_SIZE_OS_DEC(old_size - asize); + *sizep = asize; + return new_ptr; + } +#endif +#endif + } + else { /* In super carrier */ + char *start, *end, *new_end; + ErtsFreeSegMap *map; + ErtsFreeSegDesc *prev, *next, *desc; + + ERTS_MMAP_ASSERT(mmap_state.supercarrier); + + if (ERTS_MMAPFLG_OS_ONLY & flags) + return remap_move(flags, ptr, old_size, sizep); + + superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); + + asize = (superaligned + ? ERTS_SUPERALIGNED_CEILING(*sizep) + : ERTS_PAGEALIGNED_CEILING(*sizep)); + + erts_smp_mtx_lock(&mmap_state.mtx); + + if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr) + ? (!superaligned && lookup_free_seg(&mmap_state.sua.map, asize)) + : (superaligned && lookup_free_seg(&mmap_state.sa.map, asize))) { + erts_smp_mtx_unlock(&mmap_state.mtx); + /* + * Segment currently in wrong area (due to a previous memory + * shortage), move it to the right area. + * (remap_move() will succeed) + */ + return remap_move(ERTS_MMAPFLG_SUPERCARRIER_ONLY|flags, ptr, + old_size, sizep); + } + + if (asize == old_size) { + new_ptr = ptr; + goto supercarrier_resize_success; + } + + start = (char *) ptr; + end = start + old_size; + new_end = start+asize; + + if (asize < old_size) { + new_ptr = ptr; + if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); + map = &mmap_state.sua.map; + ERTS_MMAP_SIZE_SC_SUA_DEC(old_size - asize); + } + else { + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size)); + if (!superaligned) { + /* must be a superaligned size in this area */ + asize = ERTS_SUPERALIGNED_CEILING(asize); + ERTS_MMAP_ASSERT(asize <= old_size); + if (asize == old_size) + goto supercarrier_resize_success; + new_end = start+asize; + } + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); + if (end == mmap_state.sa.top) { + mmap_state.sa.top = new_end; + mmap_state.unreserve_physical(((char *) ptr) + asize, + old_size - asize); + goto supercarrier_resize_success; + } + ERTS_MMAP_SIZE_SC_SA_DEC(old_size - asize); + map = &mmap_state.sa.map; + } + + adjacent_free_seg(map, start, end, &prev, &next); + + if (next) + resize_free_seg(map, next, new_end, next->end); + else { + desc = alloc_desc(); + if (desc) + insert_free_seg(map, desc, new_end, end); + else { + if (map == &mmap_state.sa.map) + ERTS_MMAP_SIZE_SC_SA_INC(old_size - asize); + else + ERTS_MMAP_SIZE_SC_SUA_INC(old_size - asize); + add_free_desc_area(new_end, end); + goto supercarrier_resize_success; + } + } + mmap_state.unreserve_physical(((char *) ptr) + asize, + old_size - asize); + goto supercarrier_resize_success; + } + + if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); + + adjacent_free_seg(&mmap_state.sua.map, start, end, &prev, &next); + + if (next && new_end <= next->end) { + if (!mmap_state.reserve_physical(((char *) ptr) + old_size, + asize - old_size)) + goto supercarrier_reserve_failure; + if (new_end < next->end) + resize_free_seg(&mmap_state.sua.map, next, new_end, next->end); + else { + delete_free_seg(&mmap_state.sua.map, next); + free_desc(next); + } + new_ptr = ptr; + ERTS_MMAP_SIZE_SC_SUA_INC(asize - old_size); + goto supercarrier_resize_success; + } + } + else { /* Superaligned area */ + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size)); + + if (!superaligned) { + /* must be a superaligned size in this area */ + asize = ERTS_PAGEALIGNED_CEILING(asize); + new_end = start+asize; + } + + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); + + if (end == mmap_state.sa.top) { + if (new_end <= mmap_state.sua.bot) { + if (!mmap_state.reserve_physical(((char *) ptr) + old_size, + asize - old_size)) + goto supercarrier_reserve_failure; + mmap_state.sa.top = new_end; + new_ptr = ptr; + ERTS_MMAP_SIZE_SC_SA_INC(asize - old_size); + goto supercarrier_resize_success; + } + } + else { + adjacent_free_seg(&mmap_state.sa.map, start, end, &prev, &next); + if (next && new_end <= next->end) { + if (!mmap_state.reserve_physical(((char *) ptr) + old_size, + asize - old_size)) + goto supercarrier_reserve_failure; + if (new_end < next->end) + resize_free_seg(&mmap_state.sa.map, next, new_end, next->end); + else { + delete_free_seg(&mmap_state.sa.map, next); + free_desc(next); + } + new_ptr = ptr; + ERTS_MMAP_SIZE_SC_SA_INC(asize - old_size); + goto supercarrier_resize_success; + } + } + } + erts_smp_mtx_unlock(&mmap_state.mtx); + + /* Failed to resize... */ + } + + return remap_move(flags, ptr, old_size, sizep); + +supercarrier_resize_success: + +#ifdef ERTS_MMAP_DEBUG + if ((ERTS_MMAPFLG_SUPERALIGNED & flags) + || ERTS_MMAP_IN_SUPERALIGNED_AREA(new_ptr)) { + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(new_ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); + } + else { + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(new_ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); + } +#endif + + erts_smp_mtx_unlock(&mmap_state.mtx); + + *sizep = asize; + return new_ptr; + +supercarrier_reserve_failure: + + erts_smp_mtx_unlock(&mmap_state.mtx); + *sizep = 0; + return NULL; + +} + +int erts_mmap_in_supercarrier(void *ptr) +{ + return ERTS_MMAP_IN_SUPERCARRIER(ptr); +} + +void +erts_mmap_init(ErtsMMapInit *init) +{ + int virtual_map = 0; + char *start = NULL, *end = NULL; + UWord pagesize; +#if defined(__WIN32__) + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + pagesize = (UWord) sysinfo.dwPageSize; +#elif defined(_SC_PAGESIZE) + pagesize = (UWord) sysconf(_SC_PAGESIZE); +#elif defined(HAVE_GETPAGESIZE) + pagesize = (UWord) getpagesize(); +#else +# error "Do not know how to get page size" +#endif +#if defined(HARD_DEBUG) || 0 + erts_fprintf(stderr, "erts_mmap: scs = %bpu\n", init->scs); + erts_fprintf(stderr, "erts_mmap: sco = %i\n", init->sco); + erts_fprintf(stderr, "erts_mmap: scmgc = %i\n", init->scmgc); +#endif + erts_page_inv_mask = pagesize - 1; + if (pagesize & erts_page_inv_mask) + erl_exit(-1, "erts_mmap: Invalid pagesize: %bpu\n", + pagesize); + + erts_have_erts_mmap = 0; + + mmap_state.reserve_physical = reserve_noop; + mmap_state.unreserve_physical = unreserve_noop; + +#if HAVE_MMAP && !defined(MAP_ANON) + mmap_state.mmap_fd = open("/dev/zero", O_RDWR); + if (mmap_state.mmap_fd < 0) + erl_exit(-1, "erts_mmap: Failed to open /dev/zero\n"); +#endif + + erts_smp_mtx_init(&mmap_state.mtx, "erts_mmap"); + +#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + if (init->virtual_range.start) { + char *ptr; + UWord sz; + ptr = (char *) ERTS_PAGEALIGNED_CEILING(init->virtual_range.start); + end = (char *) ERTS_PAGEALIGNED_FLOOR(init->virtual_range.end); + sz = end - ptr; + start = os_mmap_virtual(ptr, sz); + if (!start || start > ptr || start >= end) + erl_exit(-1, + "erts_mmap: Failed to create virtual range for super carrier\n"); + sz = start - ptr; + if (sz) + os_munmap(end, sz); + mmap_state.reserve_physical = os_reserve_physical; + mmap_state.unreserve_physical = os_unreserve_physical; + virtual_map = 1; + } + else +#endif + if (init->predefined_area.start) { + start = init->predefined_area.start; + end = init->predefined_area.end; + if (end != (void *) 0 && end < start) + end = start; + } +#if ERTS_HAVE_OS_MMAP + else if (init->scs) { + UWord sz; + sz = ERTS_PAGEALIGNED_CEILING(init->scs); +#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + if (!init->scrpm) { + start = os_mmap_virtual(NULL, sz); + mmap_state.reserve_physical = os_reserve_physical; + mmap_state.unreserve_physical = os_unreserve_physical; + virtual_map = 1; + } + else +#endif + { + /* + * The whole supercarrier will by physically + * reserved all the time. + */ + start = os_mmap(sz, 1); + } + if (!start) + erl_exit(-1, + "erts_mmap: Failed to create super carrier of size %bpu MB\n", + init->scs/1024/1024); + end = start + sz; +#ifdef ERTS_MMAP_DEBUG_FILL_AREAS + if (!virtual_map) { + Uint32 *uip; + + for (uip = (Uint32 *) start; uip < (Uint32 *) end; uip++) + *uip = (Uint32) 0xdeadbeef; + } +#endif + } + if (!mmap_state.no_os_mmap) + erts_have_erts_mmap |= ERTS_HAVE_ERTS_OS_MMAP; +#endif + + mmap_state.size.supercarrier.total = 0; + mmap_state.size.supercarrier.used.total = 0; + mmap_state.size.supercarrier.used.sa = 0; + mmap_state.size.supercarrier.used.sua = 0; + mmap_state.size.os.used = 0; + + if (!start) { + mmap_state.sa.bot = NULL; + mmap_state.sua.top = NULL; + mmap_state.sa.bot = NULL; + mmap_state.sua.top = NULL; + mmap_state.no_os_mmap = 0; + } + else { + size_t desc_size; + + mmap_state.no_os_mmap = init->sco; + + desc_size = init->scmgc; + if (desc_size < 100) + desc_size = 100; + desc_size *= sizeof(ErtsFreeSegDesc); + if ((desc_size + + ERTS_SUPERALIGNED_SIZE + + ERTS_PAGEALIGNED_SIZE) > end - start) + erl_exit(-1, "erts_mmap: No space for segments in super carrier\n"); + + mmap_state.sa.bot = start; + mmap_state.sa.bot += desc_size; + mmap_state.sa.bot = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.bot); + mmap_state.sa.top = mmap_state.sa.bot; + mmap_state.sua.top = (char *) ERTS_SUPERALIGNED_FLOOR(end); + mmap_state.sua.bot = mmap_state.sua.top; + + mmap_state.size.os.used += (UWord) (mmap_state.sa.bot - start); + + if (end == (void *) 0) { + /* + * Very unlikely, but we need a guarantee + * that `mmap_state.sua.top` always will + * compare as larger than all segment pointers + * into the super carrier... + */ + mmap_state.sua.top -= ERTS_PAGEALIGNED_SIZE; + mmap_state.size.os.used += ERTS_PAGEALIGNED_SIZE; + } + + mmap_state.size.supercarrier.total = (UWord) (mmap_state.sua.top - mmap_state.sa.bot); + + /* + * Area before (and after) super carrier + * will be used for free segment descritors. + */ + mmap_state.desc_free_list = NULL; +#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + if (virtual_map && mmap_state.sa.bot - start > 0) + os_reserve_physical(start, mmap_state.sa.bot - start); +#endif + add_free_desc_area(start, mmap_state.sa.bot); +#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + if (virtual_map && end - mmap_state.sua.top > 0) + os_reserve_physical(mmap_state.sua.top, end - mmap_state.sua.top); +#endif + add_free_desc_area(mmap_state.sua.top, end); + + init_free_seg_map(&mmap_state.sa.map, 0); + init_free_seg_map(&mmap_state.sua.map, 1); + + mmap_state.supercarrier = 1; + erts_have_erts_mmap |= ERTS_HAVE_ERTS_SUPERCARRIER_MMAP; + +#ifdef HARD_DEBUG { void test_it(void); test_it(); } #endif + + } +#if !ERTS_HAVE_OS_MMAP + mmap_state.no_os_mmap = 1; +#endif + } @@ -897,12 +1973,6 @@ print_tree(enum SortOrder order, RBTNode* root) #endif /* PRINT_TREE */ - -static ErtsFreeSegDesc* new_desc(void) -{ - return (ErtsFreeSegDesc*) malloc(sizeof(ErtsFreeSegDesc)); -} - void test_it(void) { ErtsFreeSegMap map; @@ -912,13 +1982,13 @@ void test_it(void) for (i=0; i<2; i++) { init_free_seg_map(&map, i); - insert_free_seg(&map, new_desc(), (char*)0x11000, (char*)0x12000); + insert_free_seg(&map, alloc_desc(), (char*)0x11000, (char*)0x12000); check_tree(&map.atree, 0); check_tree(&map.stree, 0); - insert_free_seg(&map, new_desc(), (char*)0x13000, (char*)0x14000); + insert_free_seg(&map, alloc_desc(), (char*)0x13000, (char*)0x14000); check_tree(&map.atree, 0); check_tree(&map.stree, 0); - insert_free_seg(&map, new_desc(), (char*)0x15000, (char*)0x17000); + insert_free_seg(&map, alloc_desc(), (char*)0x15000, (char*)0x17000); check_tree(&map.atree, 0); check_tree(&map.stree, 0); - insert_free_seg(&map, new_desc(), (char*)0x8000, (char*)0x10000); + insert_free_seg(&map, alloc_desc(), (char*)0x8000, (char*)0x10000); check_tree(&map.atree, 0); check_tree(&map.stree, 0); desc = lookup_free_seg(&map, 0x500); diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 64baa6c493..143f1aff3e 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * Copyright Ericsson AB 2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -17,10 +17,95 @@ * %CopyrightEnd% */ +#ifndef ERL_MMAP_H__ +#define ERL_MMAP_H__ + +#include "sys.h" + +#define ERTS_MMAP_SUPERALIGNED_BITS (18) +/* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ + +#define ERTS_MMAPFLG_OS_ONLY (((Uint32) 1) << 0) +#define ERTS_MMAPFLG_SUPERCARRIER_ONLY (((Uint32) 1) << 1) +#define ERTS_MMAPFLG_SUPERALIGNED (((Uint32) 1) << 2) + +#define ERTS_HAVE_ERTS_OS_MMAP (1 << 0) +#define ERTS_HAVE_ERTS_SUPERCARRIER_MMAP (1 << 1) +extern int erts_have_erts_mmap; +extern UWord erts_page_inv_mask; + typedef struct { - SWord scs; /* super carrier size */ + struct { + char *start; + char *end; + } virtual_range; + struct { + char *start; + char *end; + } predefined_area; + UWord scs; /* super carrier size */ int sco; /* super carrier only? */ Uint scmgc; /* super carrier: max guaranteed (number of) carriers */ + int scrpm; }ErtsMMapInit; +#define ERTS_MMAP_INIT_DEFAULT_INITER \ + {{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1} + +void *erts_mmap(Uint32 flags, UWord *sizep); +void erts_munmap(Uint32 flags, void **ptrp, UWord *sizep); +void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep); +int erts_mmap_in_supercarrier(void *ptr); void erts_mmap_init(ErtsMMapInit*); + +#define ERTS_SUPERALIGNED_SIZE \ + (1 << ERTS_MMAP_SUPERALIGNED_BITS) +#define ERTS_INV_SUPERALIGNED_MASK \ + ((UWord) (ERTS_SUPERALIGNED_SIZE - 1)) +#define ERTS_SUPERALIGNED_MASK \ + (~ERTS_INV_SUPERALIGNED_MASK) +#define ERTS_SUPERALIGNED_FLOOR(X) \ + (((UWord) (X)) & ERTS_SUPERALIGNED_MASK) +#define ERTS_SUPERALIGNED_CEILING(X) \ + ERTS_SUPERALIGNED_FLOOR((X) + ERTS_INV_SUPERALIGNED_MASK) +#define ERTS_IS_SUPERALIGNED(X) \ + (((UWord) (X) & ERTS_INV_SUPERALIGNED_MASK) == 0) + +#define ERTS_INV_PAGEALIGNED_MASK \ + (erts_page_inv_mask) +#define ERTS_PAGEALIGNED_MASK \ + (~ERTS_INV_PAGEALIGNED_MASK) +#define ERTS_PAGEALIGNED_FLOOR(X) \ + (((UWord) (X)) & ERTS_PAGEALIGNED_MASK) +#define ERTS_PAGEALIGNED_CEILING(X) \ + ERTS_PAGEALIGNED_FLOOR((X) + ERTS_INV_PAGEALIGNED_MASK) +#define ERTS_IS_PAGEALIGNED(X) \ + (((UWord) (X) & ERTS_INV_PAGEALIGNED_MASK) == 0) +#define ERTS_PAGEALIGNED_SIZE \ + (ERTS_INV_PAGEALIGNED_MASK + 1) + +#ifndef HAVE_MMAP +# define HAVE_MMAP 0 +#endif +#ifndef HAVE_MREMAP +# define HAVE_MREMAP 0 +#endif +#if HAVE_MMAP +# define ERTS_HAVE_OS_MMAP 1 +# define ERTS_HAVE_GENUINE_OS_MMAP 1 +# if HAVE_MREMAP +# define ERTS_HAVE_OS_MREMAP 1 +# endif +# if defined(MAP_FIXED) && defined(MAP_NORESERVE) +# define ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION 1 +# endif +#endif + +#ifndef HAVE_VIRTUALALLOC +# define HAVE_VIRTUALALLOC 0 +#endif +#if HAVE_VIRTUALALLOC +# define ERTS_HAVE_OS_MMAP 1 +#endif + +#endif /* ERL_MMAP_H__ */ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 64fcb6bb40..b21d6ca393 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -100,45 +100,6 @@ static int atoms_initialized; typedef struct mem_kind_t MemKind; -#if HALFWORD_HEAP -static int initialize_pmmap(void); -static void *pmmap(size_t size); -static int pmunmap(void *p, size_t size); -static void *pmremap(void *old_address, size_t old_size, - size_t new_size); -#endif - -#if HAVE_MMAP -/* Mmap ... */ - -#define MMAP_PROT (PROT_READ|PROT_WRITE) - - -#ifdef MAP_ANON -# define MMAP_FLAGS (MAP_ANON|MAP_PRIVATE) -# define MMAP_FD (-1) -#else -# define MMAP_FLAGS (MAP_PRIVATE) -# define MMAP_FD mmap_fd -static int mmap_fd; -#endif - -#if HAVE_MREMAP -# define HAVE_MSEG_RECREATE 1 -#else -# define HAVE_MSEG_RECREATE 0 -#endif - -#if HALFWORD_HEAP -#define CAN_PARTLY_DESTROY 0 -#else -#define CAN_PARTLY_DESTROY 1 -#endif -#else /* #if HAVE_MMAP */ -#define CAN_PARTLY_DESTROY 0 -#error "Not supported" -#endif /* #if HAVE_MMAP */ - const ErtsMsegOpt_t erts_mseg_default_opt = { 1, /* Use cache */ 1, /* Preserv data */ @@ -163,9 +124,7 @@ typedef struct { CallCounter create; CallCounter create_resize; CallCounter destroy; -#if HAVE_MSEG_RECREATE CallCounter recreate; -#endif CallCounter clear_cache; CallCounter check_cache; } ErtsMsegCalls; @@ -236,11 +195,6 @@ struct ErtsMsegAllctr_t_ { Uint rel_max_cache_bad_fit; ErtsMsegCalls calls; - -#if CAN_PARTLY_DESTROY - Uint min_seg_size; -#endif - }; typedef union { @@ -344,69 +298,31 @@ schedule_cache_check(ErtsMsegAllctr_t *ma) { } } -/* remove ErtsMsegAllctr_t from arguments? - * only used for statistics - */ -static ERTS_INLINE void * -mmap_align(ErtsMsegAllctr_t *ma, void *addr, size_t length, int prot, int flags, int fd, off_t offset) { - - char *p, *q; - UWord d; - - p = mmap(addr, length, prot, flags, fd, offset); - - if (MAP_IS_ALIGNED(p) || p == MAP_FAILED) - return p; - - if (ma) - INC_CC(ma, create_resize); - - munmap(p, length); - - if ((p = mmap(addr, length + MSEG_ALIGNED_SIZE, prot, flags, fd, offset)) == MAP_FAILED) - return MAP_FAILED; - - q = (void *)ALIGNED_CEILING((char *)p); - d = (UWord)(q - p); - - if (d > 0) - munmap(p, d); - - if (MSEG_ALIGNED_SIZE - d > 0) - munmap((void *)(q + length), MSEG_ALIGNED_SIZE - d); - - return q; -} +/* #define ERTS_PRINT_ERTS_MMAP */ static ERTS_INLINE void * -mseg_create(ErtsMsegAllctr_t *ma, MemKind* mk, Uint size) +mseg_create(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, UWord *sizep) { +#ifdef ERTS_PRINT_ERTS_MMAP + UWord req_size = *sizep; +#endif void *seg; - ASSERT(size % MSEG_ALIGNED_SIZE == 0); - + Uint32 mmap_flags = 0; #if HALFWORD_HEAP - if (mk == &ma->low_mem) { - seg = pmmap(size); - if ((unsigned long) seg & CHECK_POINTER_MASK) { - erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg); - return NULL; - } - } else + mmap_flags |= ((mk == &ma->low_mem) + ? ERTS_MMAPFLG_SUPERCARRIER_ONLY + : ERTS_MMAPFLG_OS_ONLY); #endif - { -#if HAVE_MMAP - { - seg = (void *) mmap_align(ma, (void *) 0, (size_t) size, - MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0); - if (seg == (void *) MAP_FAILED) - seg = NULL; - - ASSERT(MAP_IS_ALIGNED(seg) || !seg); - } -#else -# error "Missing mseg_create() implementation" + if (MSEG_FLG_IS_2POW(flags)) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + seg = erts_mmap(mmap_flags, sizep); + +#ifdef ERTS_PRINT_ERTS_MMAP + erts_fprintf(stderr, "%p = erts_mmap(%s, {%bpu, %bpu});\n", seg, + (mmap_flags & ERTS_MMAPFLG_SUPERALIGNED) ? "sa" : "sua", + req_size, *sizep); #endif - } INC_CC(ma, create); @@ -414,91 +330,55 @@ mseg_create(ErtsMsegAllctr_t *ma, MemKind* mk, Uint size) } static ERTS_INLINE void -mseg_destroy(ErtsMsegAllctr_t *ma, MemKind* mk, void *seg, Uint size) { - ERTS_DECLARE_DUMMY(int res); - +mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord *size_p) { + + Uint32 mmap_flags = 0; #if HALFWORD_HEAP - if (mk == &ma->low_mem) { - res = pmunmap((void *) seg, size); - } - else + mmap_flags |= ((mk == &ma->low_mem) + ? ERTS_MMAPFLG_SUPERCARRIER_ONLY + : ERTS_MMAPFLG_OS_ONLY); #endif - { -#ifdef HAVE_MMAP - res = munmap((void *) seg, size); -#else -# error "Missing mseg_destroy() implementation" + if (MSEG_FLG_IS_2POW(flags)) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + erts_munmap(mmap_flags, seg_pp, size_p); +#ifdef ERTS_PRINT_ERTS_MMAP + erts_fprintf(stderr, "erts_munmap(%s, %p, %bpu);\n", + (mmap_flags & ERTS_MMAPFLG_SUPERALIGNED) ? "sa" : "sua", + *seg_pp, *size_p); #endif - } - - ASSERT(size % MSEG_ALIGNED_SIZE == 0); - ASSERT(res == 0); - INC_CC(ma, destroy); } -#if HAVE_MSEG_RECREATE -#if defined(__NetBSD__) -#define MREMAP_FLAGS (0) -#else -#define MREMAP_FLAGS (MREMAP_MAYMOVE) -#endif - - -/* mseg_recreate - * May return *unaligned* segments as in address not aligned to MSEG_ALIGNMENT - * it is still page aligned - * - * This is fine for single block carriers as long as we don't cache misaligned - * segments (since multiblock carriers may use them) - * - * For multiblock carriers we *need* MSEG_ALIGNMENT but mbc's will never be - * reallocated. - * - * This should probably be fixed the following way: - * 1) Use an option to segment allocation - NEED_ALIGNMENT - * 2) Add mremap_align which takes care of aligning a new a mremaped area - * 3) Fix the cache to handle of aligned and unaligned segments - */ - static ERTS_INLINE void * -mseg_recreate(ErtsMsegAllctr_t *ma, MemKind* mk, void *old_seg, Uint old_size, Uint new_size) +mseg_recreate(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void *old_seg, UWord old_size, UWord *sizep) { +#ifdef ERTS_PRINT_ERTS_MMAP + UWord req_size = *sizep; +#endif void *new_seg; - - ASSERT(old_size % MSEG_ALIGNED_SIZE == 0); - ASSERT(new_size % MSEG_ALIGNED_SIZE == 0); - + Uint32 mmap_flags = 0; #if HALFWORD_HEAP - if (mk == &ma->low_mem) { - new_seg = (void *) pmremap((void *) old_seg, - (size_t) old_size, - (size_t) new_size); - } - else -#endif - { -#if HAVE_MREMAP -#if defined(__NetBSD__) - new_seg = mremap(old_seg, (size_t)old_size, NULL, new_size, MREMAP_FLAGS); -#else - new_seg = mremap(old_seg, (size_t)old_size, (size_t)new_size, MREMAP_FLAGS); + mmap_flags |= ((mk == &ma->low_mem) + ? ERTS_MMAPFLG_SUPERCARRIER_ONLY + : ERTS_MMAPFLG_OS_ONLY); #endif - if (new_seg == (void *) MAP_FAILED) - new_seg = NULL; -#else -#error "Missing mseg_recreate() implementation" -#endif - } + if (MSEG_FLG_IS_2POW(flags)) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + new_seg = erts_mremap(mmap_flags, old_seg, old_size, sizep); +#ifdef ERTS_PRINT_ERTS_MMAP + erts_fprintf(stderr, "%p = erts_mremap(%s, %p, %bpu, {%bpu, %bpu});\n", + new_seg, (mmap_flags & ERTS_MMAPFLG_SUPERALIGNED) ? "sa" : "sua", + old_seg, old_size, req_size, *sizep); +#endif INC_CC(ma, recreate); return new_seg; } -#endif /* #if HAVE_MSEG_RECREATE */ - #ifdef DEBUG #define ERTS_DBG_MA_CHK_THR_ACCESS(MA) \ do { \ @@ -566,14 +446,19 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui return 1; } else if (!MSEG_FLG_IS_2POW(flags) && !erts_circleq_is_empty(&(mk->cache_unpowered_node))) { - + void *destr_seg; + UWord destr_size; /* No free slots. * Evict oldest slot from unpowered cache so we can cache an unpowered (sbc) segment */ c = erts_circleq_tail(&(mk->cache_unpowered_node)); erts_circleq_remove(c); - mseg_destroy(mk->ma, mk, c->seg, c->size); + destr_seg = c->seg; + destr_size = c->size; + mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, &destr_seg, &destr_size); + ASSERT(destr_seg == c->seg); + ASSERT(destr_size == c->size); mseg_cache_clear_node(c); c->seg = seg; @@ -593,13 +478,20 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui int i; for( i = 0; i < CACHE_AREAS; i++) { + void *destr_seg; + UWord destr_size; if (erts_circleq_is_empty(&(mk->cache_powered_node[i]))) continue; c = erts_circleq_tail(&(mk->cache_powered_node[i])); erts_circleq_remove(c); - mseg_destroy(mk->ma, mk, c->seg, c->size); + destr_seg = seg; + destr_size = c->size; + mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, &destr_seg, &destr_size); + ASSERT(destr_seg == c->seg); + ASSERT(destr_size == c->size); + mseg_cache_clear_node(c); c->seg = seg; @@ -614,9 +506,9 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui return 0; } -static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p, Uint flags) { +static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flags) { - Uint size = *size_p; + Uint size = (UWord) *size_p; ERTS_DBG_MK_CHK_THR_ACCESS(mk); @@ -653,7 +545,11 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p, Uint flags ASSERT(!(mk->cache_size < 0)); if (csize != size) { - mseg_destroy(mk->ma, mk, (char *)seg + size, csize - size); + void *destr_seg = ((char *) seg) + size; + UWord destr_size = csize - size; + mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, &destr_seg, &destr_size); + *size_p = (UWord) (destr_seg - seg); + ASSERT(c->seg + c->size == destr_seg + destr_size); } return seg; @@ -683,7 +579,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p, Uint flags mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); - *size_p = csize; + *size_p = (UWord) csize; return seg; @@ -714,7 +610,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p, Uint flags ASSERT((size % GET_PAGE_SIZE) == 0); ASSERT((best->size % GET_PAGE_SIZE) == 0); - *size_p = size; + *size_p = (UWord) size; return seg; @@ -728,7 +624,9 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p, Uint flags * using callbacks from aux-work in the scheduler. */ -static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t *head) { +static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags, cache_t *head) { + void *destr_seg; + UWord destr_size; cache_t *c = NULL; c = erts_circleq_tail(head); @@ -737,7 +635,11 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t *h if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg); - mseg_destroy(mk->ma, mk, c->seg, c->size); + destr_seg = c->seg; + destr_size = c->size; + mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size); + ASSERT(destr_seg == c->seg); + ASSERT(destr_size == c->size); mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); @@ -749,10 +651,12 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t *h return mk->cache_size; } -static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, cache_t *head) { +static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, cache_t *head) { cache_t *c = NULL; while (!erts_circleq_is_empty(head)) { + void *destr_seg; + UWord destr_size; c = erts_circleq_tail(head); erts_circleq_remove(c); @@ -760,7 +664,11 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, cache_t *head) if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg); - mseg_destroy(mk->ma, mk, c->seg, c->size); + destr_seg = c->seg; + destr_size = c->size; + mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size); + ASSERT(destr_seg == c->seg); + ASSERT(destr_size == c->size); mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); @@ -788,11 +696,11 @@ static Uint mseg_check_memkind_cache(MemKind *mk) { for (i = 0; i < CACHE_AREAS; i++) { if (!erts_circleq_is_empty(&(mk->cache_powered_node[i]))) - return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_powered_node[i])); + return mseg_drop_one_memkind_cache_size(mk, ERTS_MSEG_FLG_2POW, &(mk->cache_powered_node[i])); } if (!erts_circleq_is_empty(&(mk->cache_unpowered_node))) - return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_unpowered_node)); + return mseg_drop_one_memkind_cache_size(mk, ERTS_MSEG_FLG_NONE, &(mk->cache_unpowered_node)); return 0; } @@ -851,12 +759,12 @@ static void mseg_clear_memkind_cache(MemKind *mk) { if (erts_circleq_is_empty(&(mk->cache_powered_node[i]))) continue; - mseg_drop_memkind_cache_size(mk, &(mk->cache_powered_node[i])); + mseg_drop_memkind_cache_size(mk, ERTS_MSEG_FLG_2POW, &(mk->cache_powered_node[i])); ASSERT(erts_circleq_is_empty(&(mk->cache_powered_node[i]))); } /* drop varied caches */ if (!erts_circleq_is_empty(&(mk->cache_unpowered_node))) - mseg_drop_memkind_cache_size(mk, &(mk->cache_unpowered_node)); + mseg_drop_memkind_cache_size(mk, ERTS_MSEG_FLG_NONE, &(mk->cache_unpowered_node)); ASSERT(erts_circleq_is_empty(&(mk->cache_unpowered_node))); ASSERT(mk->cache_size == 0); @@ -896,36 +804,33 @@ static ERTS_INLINE MemKind* memkind(ErtsMsegAllctr_t *ma, } static void * -mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, Uint *size_p, +mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, UWord *size_p, Uint flags, const ErtsMsegOpt_t *opt) { - Uint size; + UWord size; void *seg; MemKind* mk = memkind(ma, opt); INC_CC(ma, alloc); - /* Carrier align */ - size = ALIGNED_CEILING(*size_p); - - /* Cache optim (if applicable) */ - if (MSEG_FLG_IS_2POW(flags) && !IS_2POW(size)) + if (!MSEG_FLG_IS_2POW(flags) && !IS_2POW(size)) + size = ERTS_PAGEALIGNED_CEILING(*size_p); + else { + size = ALIGNED_CEILING(*size_p); + /* Cache optim (if applicable) */ size = ceil_2pow(size); + } -#if CAN_PARTLY_DESTROY - if (size < ma->min_seg_size) - ma->min_seg_size = size; -#endif - if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size, flags)) != NULL) goto done; - if ((seg = mseg_create(ma, mk, size)) == NULL) - size = 0; + seg = mseg_create(ma, flags, mk, &size); + if (!seg) + *size_p = 0; + else { done: - *size_p = size; - if (seg) { + *size_p = size; if (erts_mtrace_enabled) erts_mtrace_crr_alloc(seg, atype, ERTS_MTRACE_SEGMENT_ID, size); @@ -937,14 +842,16 @@ done: static void -mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, Uint size, +mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size, Uint flags, const ErtsMsegOpt_t *opt) { + void *destr_seg; + UWord destr_size; MemKind* mk = memkind(ma, opt); ERTS_MSEG_DEALLOC_STAT(mk,size); - if (opt->cache && cache_bless_segment(mk, seg, size, flags)) { + if (opt->cache && cache_bless_segment(mk, seg, (Uint) size, flags)) { schedule_cache_check(ma); goto done; } @@ -952,7 +859,11 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, Uint size, if (erts_mtrace_enabled) erts_mtrace_crr_free(atype, SEGTYPE, seg); - mseg_destroy(ma, mk, seg, size); + destr_seg = seg; + destr_size = size; + mseg_destroy(ma, flags, mk, &destr_seg, &destr_size); + ASSERT(destr_seg == seg); + ASSERT(destr_size == size); done: @@ -961,11 +872,11 @@ done: static void * mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, - Uint old_size, Uint *new_size_p, Uint flags, const ErtsMsegOpt_t *opt) + UWord old_size, UWord *new_size_p, Uint flags, const ErtsMsegOpt_t *opt) { MemKind* mk; void *new_seg; - Uint new_size; + UWord new_size; /* Just allocate a new segment if we didn't have one before */ if (!seg || !old_size) { @@ -985,90 +896,46 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, mk = memkind(ma, opt); new_seg = seg; - /* Carrier align */ - new_size = ALIGNED_CEILING(*new_size_p); - /* Cache optim (if applicable) */ - if (MSEG_FLG_IS_2POW(flags) && !IS_2POW(new_size)) + if (!MSEG_FLG_IS_2POW(flags)) + new_size = ERTS_PAGEALIGNED_CEILING(*new_size_p); + else { + new_size = ALIGNED_CEILING(*new_size_p); + /* Cache optim (if applicable) */ new_size = ceil_2pow(new_size); + } - if (new_size == old_size) - ; - else if (new_size < old_size) { - Uint shrink_sz = old_size - new_size; - -#if CAN_PARTLY_DESTROY - if (new_size < ma->min_seg_size) - ma->min_seg_size = new_size; -#endif - /* +Mrsbcst */ - if (shrink_sz < opt->abs_shrink_th - && 100*shrink_sz < opt->rel_shrink_th*old_size) { - new_size = old_size; + if (new_size > old_size) { + if (opt->preserv) { + new_seg = mseg_recreate(ma, flags, mk, (void *) seg, old_size, &new_size); + if (!new_seg) + new_size = old_size; } else { - -#if CAN_PARTLY_DESTROY - - if (erts_mtrace_enabled) - erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size); - - mseg_destroy(ma, mk, ((char *) seg) + new_size, shrink_sz); - -#elif HAVE_MSEG_RECREATE - goto do_recreate; -#else + mseg_dealloc(ma, atype, seg, old_size, flags, opt); new_seg = mseg_alloc(ma, atype, &new_size, flags, opt); - - ASSERT(MAP_IS_ALIGNED(new_seg) || !new_seg); - if (!new_seg) - new_size = old_size; - else { - sys_memcpy(((char *) new_seg), - ((char *) seg), - MIN(new_size, old_size)); - mseg_dealloc(ma, atype, seg, old_size, flags, opt); - } -#endif + new_size = 0; } } - else { + else if (new_size < old_size) { + UWord shrink_sz = old_size - new_size; - if (!opt->preserv) { - mseg_dealloc(ma, atype, seg, old_size, flags, opt); - new_seg = mseg_alloc(ma, atype, &new_size, flags, opt); - ASSERT(MAP_IS_ALIGNED(new_seg) || !new_seg); + /* +Mrsbcst */ + if (shrink_sz < opt->abs_shrink_th + && 100*shrink_sz < opt->rel_shrink_th*old_size) { + new_size = old_size; } else { -#if HAVE_MSEG_RECREATE -#if !CAN_PARTLY_DESTROY - do_recreate: -#endif - new_seg = mseg_recreate(ma, mk, (void *) seg, old_size, new_size); - /* ASSERT(MAP_IS_ALIGNED(new_seg) || !new_seg); - * will not always be aligned and it ok for now - */ - - if (erts_mtrace_enabled) - erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size); + new_seg = mseg_recreate(ma, flags, mk, (void *) seg, old_size, &new_size); if (!new_seg) new_size = old_size; -#else - new_seg = mseg_alloc(ma, atype, &new_size, flags, opt); - - ASSERT(MAP_IS_ALIGNED(new_seg) || !new_seg); - - if (!new_seg) - new_size = old_size; - else { - sys_memcpy(((char *) new_seg), ((char *) seg), MIN(new_size, old_size)); - mseg_dealloc(ma, atype, seg, old_size, flags, opt); - } -#endif } } + if (erts_mtrace_enabled) + erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size); + INC_CC(ma, realloc); ASSERT(!MSEG_FLG_IS_2POW(flags) || IS_2POW(new_size)); @@ -1106,9 +973,7 @@ static struct { Eterm mseg_create; Eterm mseg_create_resize; Eterm mseg_destroy; -#if HAVE_MSEG_RECREATE Eterm mseg_recreate; -#endif Eterm mseg_clear_cache; Eterm mseg_check_cache; @@ -1163,9 +1028,7 @@ init_atoms(ErtsMsegAllctr_t *ma) AM_INIT(mseg_create); AM_INIT(mseg_create_resize); AM_INIT(mseg_destroy); -#if HAVE_MSEG_RECREATE AM_INIT(mseg_recreate); -#endif AM_INIT(mseg_clear_cache); AM_INIT(mseg_check_cache); @@ -1293,9 +1156,7 @@ info_calls(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp PRINT_CC(to, arg, create); PRINT_CC(to, arg, create_resize); PRINT_CC(to, arg, destroy); -#if HAVE_MSEG_RECREATE PRINT_CC(to, arg, recreate); -#endif PRINT_CC(to, arg, clear_cache); PRINT_CC(to, arg, check_cache); @@ -1316,12 +1177,10 @@ info_calls(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp bld_unstable_uint(hpp, szp, ma->calls.clear_cache.giga_no), bld_unstable_uint(hpp, szp, ma->calls.clear_cache.no)); -#if HAVE_MSEG_RECREATE add_3tup(hpp, szp, &res, am.mseg_recreate, bld_unstable_uint(hpp, szp, ma->calls.recreate.giga_no), bld_unstable_uint(hpp, szp, ma->calls.recreate.no)); -#endif add_3tup(hpp, szp, &res, am.mseg_destroy, bld_unstable_uint(hpp, szp, ma->calls.destroy.giga_no), @@ -1521,7 +1380,7 @@ erts_mseg_info(int ix, } void * -erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, Uint flags, const ErtsMsegOpt_t *opt) +erts_mseg_alloc_opt(ErtsAlcType_t atype, UWord *size_p, Uint flags, const ErtsMsegOpt_t *opt) { ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); void *seg; @@ -1533,14 +1392,14 @@ erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, Uint flags, const ErtsMse } void * -erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p, Uint flags) +erts_mseg_alloc(ErtsAlcType_t atype, UWord *size_p, Uint flags) { return erts_mseg_alloc_opt(atype, size_p, flags, &erts_mseg_default_opt); } void erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, - Uint size, Uint flags, const ErtsMsegOpt_t *opt) + UWord size, Uint flags, const ErtsMsegOpt_t *opt) { ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); ERTS_MSEG_LOCK(ma); @@ -1550,14 +1409,14 @@ erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, } void -erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, Uint flags) +erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, UWord size, Uint flags) { erts_mseg_dealloc_opt(atype, seg, size, flags, &erts_mseg_default_opt); } void * erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, - Uint old_size, Uint *new_size_p, + UWord old_size, UWord *new_size_p, Uint flags, const ErtsMsegOpt_t *opt) { @@ -1572,7 +1431,7 @@ erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, void * erts_mseg_realloc(ErtsAlcType_t atype, void *seg, - Uint old_size, Uint *new_size_p, Uint flags) + UWord old_size, UWord *new_size_p, Uint flags) { return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, flags, &erts_mseg_default_opt); @@ -1662,19 +1521,16 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); -#if HAVE_MMAP && !defined(MAP_ANON) - mmap_fd = open("/dev/zero", O_RDWR); - if (mmap_fd < 0) - erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n"); +#if HALFWORD_HEAP + if (sizeof(void *) != 8) + erl_exit(-1,"Halfword emulator cannot be run in 32bit mode"); + + init->mmap.virtual_range.start = (char *) sbrk(0); + init->mmap.virtual_range.end = (char *) 0x100000000UL; + init->mmap.sco = 0; #endif -#if HAVE_MMAP -# if HALFWORD_HEAP - initialize_pmmap(); -# else erts_mmap_init(&init->mmap); -# endif -#endif if (!IS_2POW(GET_PAGE_SIZE)) erl_exit(ERTS_ABORT_EXIT, "erts_mseg: Unexpected page_size %beu\n", GET_PAGE_SIZE); @@ -1716,10 +1572,6 @@ erts_mseg_init(ErtsMsegInit_t *init) #endif sys_memzero((void *) &ma->calls, sizeof(ErtsMsegCalls)); - -#if CAN_PARTLY_DESTROY - ma->min_seg_size = ~((Uint) 0); -#endif } } @@ -1761,7 +1613,7 @@ erts_mseg_test(UWord op, UWord a1, UWord a2, UWord a3) case 0x400: /* Have erts_mseg */ return (UWord) 1; case 0x401: - return (UWord) erts_mseg_alloc(ERTS_ALC_A_INVALID, (Uint *) a1, (Uint) 0); + return (UWord) erts_mseg_alloc(ERTS_ALC_A_INVALID, (UWord *) a1, (Uint) 0); case 0x402: erts_mseg_dealloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2, (Uint) 0); return (UWord) 0; @@ -1769,7 +1621,7 @@ erts_mseg_test(UWord op, UWord a1, UWord a2, UWord a3) return (UWord) erts_mseg_realloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2, - (Uint *) a3, + (UWord *) a3, (Uint) 0); case 0x404: erts_mseg_clear_cache(); @@ -1792,405 +1644,3 @@ erts_mseg_test(UWord op, UWord a1, UWord a2, UWord a3) } } - - -#if HALFWORD_HEAP -/* - * Very simple page oriented mmap replacer. Works in the lower - * 32 bit address range of a 64bit program. - * Implements anonymous mmap mremap and munmap with address order first fit. - * The free list is expected to be very short... - * To be used for compressed pointers in Erlang halfword emulator - * implementation. The MacOS X version is more of a toy, it's not really - * for production as the halfword erlang VM relies on Linux specific memory - * mapping tricks. - */ - -/* #define HARDDEBUG 1 */ - -#ifdef HARDDEBUG -static void dump_freelist(void) -{ - FreeBlock *p = first; - - while (p) { - fprintf(stderr, "p = %p\r\np->num = %ld\r\np->next = %p\r\n\r\n", - (void *) p, (unsigned long) p->num, (void *) p->next); - p = p->next; - } -} - -#define HARDDEBUG_HW_INCOMPLETE_ALIGNMENT(PTR, SZ) \ - fprintf(stderr,"Mapping of address %p with size %ld " \ - "does not map complete pages (%s:%d)\r\n", \ - (void *) (PTR), (unsigned long) (SZ),__FILE__, __LINE__) - -#define HARDDEBUG_HW_UNALIGNED_ALIGNMENT(PTR, SZ) \ - fprintf(stderr,"Mapping of address %p with size %ld " \ - "is not page aligned (%s:%d)\r\n", \ - (void *) (PTR), (unsigned long) (SZ),__FILE__, __LINE__) - -#define HARDDEBUG_MAP_FAILED(PTR, SZ) \ - fprintf(stderr, "Could not actually map memory " \ - "at address %p with size %ld (%s:%d) ..\r\n", \ - (void *) (PTR), (unsigned long) (SZ),__FILE__, __LINE__) -#else -#define HARDDEBUG_HW_INCOMPLETE_ALIGNMENT(PTR, SZ) do{}while(0) -#define HARDDEBUG_HW_UNALIGNED_ALIGNMENT(PTR, SZ) do{}while(0) -#define HARDDEBUG_MAP_FAILED(PTR, SZ) do{}while(0) -#endif - - -#ifdef __APPLE__ -#define MAP_ANONYMOUS MAP_ANON -#endif - -#define INIT_LOCK() do {erts_mtx_init(&pmmap_mutex, "pmmap");} while(0) - -#define TAKE_LOCK() do {erts_mtx_lock(&pmmap_mutex);} while(0) - -#define RELEASE_LOCK() do {erts_mtx_unlock(&pmmap_mutex);} while(0) - -static erts_mtx_t pmmap_mutex; /* Also needed when !USE_THREADS */ - -typedef struct _free_block { - unsigned long num; /*pages*/ - struct _free_block *next; -} FreeBlock; - -/* Protect with lock */ -static FreeBlock *first; - -static void *do_map(void *ptr, size_t sz) -{ - void *res; - - if (ALIGNED_CEILING(sz) != sz) { - HARDDEBUG_HW_INCOMPLETE_ALIGNMENT(ptr, sz); - return NULL; - } - - if (((unsigned long) ptr) % MSEG_ALIGNED_SIZE) { - HARDDEBUG_HW_UNALIGNED_ALIGNMENT(ptr, sz); - return NULL; - } - -#if HAVE_MMAP - res = mmap(ptr, sz, - PROT_READ | PROT_WRITE, MAP_PRIVATE | - MAP_ANONYMOUS | MAP_FIXED, - -1 , 0); -#else -# error "Missing mmap support" -#endif - - if (res == MAP_FAILED) { - HARDDEBUG_MAP_FAILED(ptr, sz); - return NULL; - } - - return res; -} - -static int do_unmap(void *ptr, size_t sz) -{ - void *res; - - if (ALIGNED_CEILING(sz) != sz) { - HARDDEBUG_HW_INCOMPLETE_ALIGNMENT(ptr, sz); - return 1; - } - - if (((unsigned long) ptr) % MSEG_ALIGNED_SIZE) { - HARDDEBUG_HW_UNALIGNED_ALIGNMENT(ptr, sz); - return 1; - } - - res = mmap(ptr, sz, - PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED, - -1 , 0); - - if (res == MAP_FAILED) { - HARDDEBUG_MAP_FAILED(ptr, sz); - return 1; - } - - return 0; -} - -#ifdef __APPLE__ -/* - * The first 4 gig's are protected on Macos X for 64bit processes :( - * The range 0x1000000000 - 0x10FFFFFFFF is selected as an arbitrary - * value of a normally unused range... Real MMAP's will avoid - * it and all 32bit compressed pointers can be in that range... - * More expensive than on Linux where expansion of compressed - * poiters involves no masking (as they are in the first 4 gig's). - * It's also very uncertain if the MAP_NORESERVE flag really has - * any effect in MacOS X. Swap space may always be allocated... - */ -#define SET_RANGE_MIN() /* nothing */ -#define RANGE_MIN 0x1000000000UL -#define RANGE_MAX 0x1100000000UL -#define RANGE_MASK (RANGE_MIN) -#define EXTRA_MAP_FLAGS (MAP_FIXED) -#else -static size_t range_min; -#define SET_RANGE_MIN() do { range_min = (size_t) sbrk(0); } while (0) -#define RANGE_MIN range_min -#define RANGE_MAX 0x100000000UL -#define RANGE_MASK 0UL -#define EXTRA_MAP_FLAGS (0) -#endif - -static int initialize_pmmap(void) -{ - char *p,*q,*rptr; - size_t rsz; - FreeBlock *initial; - - SET_RANGE_MIN(); - if (sizeof(void *) != 8) { - erl_exit(1,"Halfword emulator cannot be run in 32bit mode"); - } - - p = (char *) RANGE_MIN; - q = (char *) RANGE_MAX; - - rsz = ALIGNED_FLOOR(q - p); - - rptr = mmap_align(NULL, (void *) p, rsz, - PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | - MAP_NORESERVE | EXTRA_MAP_FLAGS, - -1 , 0); -#ifdef HARDDEBUG - printf("p=%p, rsz = %ld, pages = %ld, got range = %p -> %p\r\n", - p, (unsigned long) rsz, (unsigned long) (rsz / MSEG_ALIGNED_SIZE), - (void *) rptr, (void*)(rptr + rsz)); -#endif - if ((UWord)(rptr + rsz) > RANGE_MAX) { - size_t rsz_trunc = RANGE_MAX - (UWord)rptr; -#ifdef HARDDEBUG - printf("Reducing mmap'ed memory from %lu to %lu Mb, reduced range = %p -> %p\r\n", - rsz/(1024*1024), rsz_trunc/(1024*1024), rptr, rptr+rsz_trunc); -#endif - munmap((void*)RANGE_MAX, rsz - rsz_trunc); - rsz = rsz_trunc; - } - if (!do_map(rptr, MSEG_ALIGNED_SIZE)) { - erl_exit(1,"Could not actually mmap first page for halfword emulator...\n"); - } - initial = (FreeBlock *) rptr; - initial->num = (rsz / MSEG_ALIGNED_SIZE); - initial->next = NULL; - first = initial; - INIT_LOCK(); - return 0; -} - -static void *pmmap(size_t size) -{ - size_t real_size = ALIGNED_CEILING(size); - size_t num_pages = real_size / MSEG_ALIGNED_SIZE; - FreeBlock **block; - FreeBlock *tail; - FreeBlock *res; - - TAKE_LOCK(); - - for (block = &first; - *block != NULL && (*block)->num < num_pages; - block = &((*block)->next)) - ; - if (!(*block)) { - RELEASE_LOCK(); - return NULL; - } - if ((*block)->num == num_pages) { - /* nice, perfect fit */ - res = *block; - *block = (*block)->next; - } else { - tail = (FreeBlock *) (((char *) ((void *) (*block))) + real_size); - if (!do_map(tail, MSEG_ALIGNED_SIZE)) { - HARDDEBUG_MAP_FAILED(tail, MSEG_ALIGNED_SIZE); - RELEASE_LOCK(); - return NULL; - } - tail->num = (*block)->num - num_pages; - tail->next = (*block)->next; - res = *block; - *block = tail; - } - - RELEASE_LOCK(); - - if (!do_map(res, real_size)) { - HARDDEBUG_MAP_FAILED(res, real_size); - return NULL; - } - - return (void *) res; -} - -static int pmunmap(void *p, size_t size) -{ - size_t real_size = ALIGNED_CEILING(size); - size_t num_pages = real_size / MSEG_ALIGNED_SIZE; - - FreeBlock *block; - FreeBlock *last; - FreeBlock *nb = (FreeBlock *) p; - - ASSERT(((unsigned long)p & CHECK_POINTER_MASK)==0); - - if (real_size > MSEG_ALIGNED_SIZE) { - if (do_unmap(((char *) p) + MSEG_ALIGNED_SIZE, real_size - MSEG_ALIGNED_SIZE)) { - return 1; - } - } - - TAKE_LOCK(); - - last = NULL; - block = first; - while(block != NULL && ((void *) block) < p) { - last = block; - block = block->next; - } - - if (block != NULL && - ((void *) block) == ((void *) (((char *) p) + real_size))) { - /* Merge new free block with following */ - nb->num = block->num + num_pages; - nb->next = block->next; - if (do_unmap(block, MSEG_ALIGNED_SIZE)) { - RELEASE_LOCK(); - return 1; - } - } else { - /* just link in */ - nb->num = num_pages; - nb->next = block; - } - if (last != NULL) { - if (p == ((void *) (((char *) last) + (last->num * MSEG_ALIGNED_SIZE)))) { - /* Merge with previous */ - last->num += nb->num; - last->next = nb->next; - if (do_unmap(nb, MSEG_ALIGNED_SIZE)) { - RELEASE_LOCK(); - return 1; - } - } else { - last->next = nb; - } - } else { - first = nb; - } - RELEASE_LOCK(); - return 0; -} - -static void *pmremap(void *old_address, size_t old_size, - size_t new_size) -{ - size_t new_real_size = ALIGNED_CEILING(new_size); - size_t new_num_pages = new_real_size / MSEG_ALIGNED_SIZE; - size_t old_real_size = ALIGNED_CEILING(old_size); - size_t old_num_pages = old_real_size / MSEG_ALIGNED_SIZE; - if (new_num_pages == old_num_pages) { - return old_address; - } else if (new_num_pages < old_num_pages) { /* Shrink */ - size_t nfb_pages = old_num_pages - new_num_pages; - size_t nfb_real_size = old_real_size - new_real_size; - void *vnfb = (void *) (((char *)old_address) + new_real_size); - FreeBlock *nfb = (FreeBlock *) vnfb; - FreeBlock **block; - TAKE_LOCK(); - for (block = &first; - *block != NULL && (*block) < nfb; - block = &((*block)->next)) - ; - if (!(*block) || - (*block) > ((FreeBlock *)(((char *) vnfb) + nfb_real_size))) { - /* Normal link in */ - if (nfb_pages > 1) { - if (do_unmap((void *)(((char *) vnfb) + MSEG_ALIGNED_SIZE), - (nfb_pages - 1)*MSEG_ALIGNED_SIZE)) { - return NULL; - } - } - nfb->next = (*block); - nfb->num = nfb_pages; - (*block) = nfb; - } else { /* block merge */ - nfb->next = (*block)->next; - nfb->num = nfb_pages + (*block)->num; - /* unmap also the first page of the next freeblock */ - (*block) = nfb; - if (do_unmap((void *)(((char *) vnfb) + MSEG_ALIGNED_SIZE), - nfb_pages*MSEG_ALIGNED_SIZE)) { - return NULL; - } - } - RELEASE_LOCK(); - return old_address; - } else { /* Enlarge */ - FreeBlock **block; - void *old_end = (void *) (((char *)old_address) + old_real_size); - TAKE_LOCK(); - for (block = &first; - *block != NULL && (*block) < (FreeBlock *) old_address; - block = &((*block)->next)) - ; - if ((*block) == NULL || old_end > ((void *) RANGE_MAX) || - (*block) != old_end || - (*block)->num < (new_num_pages - old_num_pages)) { - /* cannot extend */ - void *result; - RELEASE_LOCK(); - result = pmmap(new_size); - if (result == NULL) { - return NULL; - } - memcpy(result,old_address,old_size); - if (pmunmap(old_address,old_size)) { - /* Oups... */ - pmunmap(result,new_size); - return NULL; - } - return result; - } else { /* extend */ - size_t remaining_pages = (*block)->num - - (new_num_pages - old_num_pages); - if (!remaining_pages) { - void *p = (void *) (((char *) (*block)) + MSEG_ALIGNED_SIZE); - void *n = (*block)->next; - size_t x = ((*block)->num - 1) * MSEG_ALIGNED_SIZE; - if (x > 0) { - if (do_map(p,x) == NULL) { - RELEASE_LOCK(); - return NULL; - } - } - (*block) = n; - } else { - FreeBlock *nfb = (FreeBlock *) ((void *) - (((char *) old_address) + - new_real_size)); - void *p = (void *) (((char *) (*block)) + MSEG_ALIGNED_SIZE); - if (do_map(p,new_real_size - old_real_size) == NULL) { - RELEASE_LOCK(); - return NULL; - } - nfb->num = remaining_pages; - nfb->next = (*block)->next; - (*block) = nfb; - } - RELEASE_LOCK(); - return old_address; - } - } -} -#endif /* HALFWORD_HEAP */ diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index 7454e5c473..2284b3f8f1 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -24,14 +24,14 @@ #include "erl_alloc_types.h" #include "erl_mmap.h" -#ifndef HAVE_MMAP -# define HAVE_MMAP 0 -#endif -#ifndef HAVE_MREMAP -# define HAVE_MREMAP 0 -#endif - -#if HAVE_MMAP +/* + * We currently only enable mseg_alloc if we got + * a genuine mmap()/munmap() primitive. It is possible + * to utilize erts_mmap() withiout a mmap support but + * alloc_util needs to be prepared before we can do + * that. + */ +#ifdef ERTS_HAVE_GENUINE_OS_MMAP # define HAVE_ERTS_MSEG 1 # define ERTS_HAVE_MSEG_SUPER_ALIGNED 1 #else @@ -40,8 +40,7 @@ #endif #if ERTS_HAVE_MSEG_SUPER_ALIGNED -# define MSEG_ALIGN_BITS (18) - /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ +# define MSEG_ALIGN_BITS ERTS_MMAP_SUPERALIGNED_BITS #else /* If we don't use super aligned multiblock carriers * we will mmap with page size alignment (and thus use corresponding @@ -77,7 +76,8 @@ typedef struct { 4*1024*1024, /* amcbf: Absolute max cache bad fit */ \ 20, /* rmcbf: Relative max cache bad fit */ \ 10, /* mcs: Max cache size */ \ - 1000 /* cci: Cache check interval */ \ + 1000, /* cci: Cache check interval */ \ + ERTS_MMAP_INIT_DEFAULT_INITER \ } typedef struct { @@ -93,12 +93,12 @@ typedef struct { extern const ErtsMsegOpt_t erts_mseg_default_opt; -void *erts_mseg_alloc(ErtsAlcType_t, Uint *, Uint); -void *erts_mseg_alloc_opt(ErtsAlcType_t, Uint *, Uint, const ErtsMsegOpt_t *); -void erts_mseg_dealloc(ErtsAlcType_t, void *, Uint, Uint); -void erts_mseg_dealloc_opt(ErtsAlcType_t, void *, Uint, Uint, const ErtsMsegOpt_t *); -void *erts_mseg_realloc(ErtsAlcType_t, void *, Uint, Uint *, Uint); -void *erts_mseg_realloc_opt(ErtsAlcType_t, void *, Uint, Uint *, Uint, const ErtsMsegOpt_t *); +void *erts_mseg_alloc(ErtsAlcType_t, UWord *, Uint); +void *erts_mseg_alloc_opt(ErtsAlcType_t, UWord *, Uint, const ErtsMsegOpt_t *); +void erts_mseg_dealloc(ErtsAlcType_t, void *, UWord, Uint); +void erts_mseg_dealloc_opt(ErtsAlcType_t, void *, UWord, Uint, const ErtsMsegOpt_t *); +void *erts_mseg_realloc(ErtsAlcType_t, void *, UWord, UWord *, Uint); +void *erts_mseg_realloc_opt(ErtsAlcType_t, void *, UWord, UWord *, Uint, const ErtsMsegOpt_t *); void erts_mseg_clear_cache(void); void erts_mseg_cache_check(void); Uint erts_mseg_no( const ErtsMsegOpt_t *); diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 1e01c5bc20..c9908caf20 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -112,6 +112,7 @@ static char *plusM_other_switches[] = { "Mscs", "Mscmgc", "Msco", + "Mscrpm", "Ye", "Ym", "Ytp", -- cgit v1.2.3 From 5e4a2c0cd69736ee1b1f54f8bb63a68688bc84e8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 2 Sep 2013 15:30:02 +0200 Subject: erts: Save one word in ErtsFreeSegDesc by putting red/black color bit in 'parent' pointer --- erts/emulator/sys/common/erl_mmap.c | 298 +++++++++++++++++++----------------- 1 file changed, 155 insertions(+), 143 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index aac01bf93c..cd71127ce5 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -83,16 +83,43 @@ UWord erts_page_inv_mask; typedef struct RBTNode_ RBTNode; struct RBTNode_ { - RBTNode *parent; + UWord parent_and_color; /* color in bit 0 of parent ptr */ RBTNode *left; RBTNode *right; +#ifdef HARD_DEBUG int flags; +#endif }; +#define RED_FLG (1) +#define IS_RED(N) ((N) && ((N)->parent_and_color & RED_FLG)) +#define IS_BLACK(N) (!IS_RED(N)) +#define SET_RED(N) ((N)->parent_and_color |= RED_FLG) +#define SET_BLACK(N) ((N)->parent_and_color &= ~RED_FLG) + +static ERTS_INLINE RBTNode* parent(RBTNode* node) +{ + return (RBTNode*) (node->parent_and_color & ~RED_FLG); +} + +static ERTS_INLINE void set_parent(RBTNode* node, RBTNode* parent) +{ + RBT_ASSERT(!((UWord)parent & RED_FLG)); + node->parent_and_color = ((UWord)parent) | (node->parent_and_color & RED_FLG); +} + +static ERTS_INLINE UWord parent_and_color(RBTNode* parent, int color) +{ + RBT_ASSERT(!((UWord)parent & RED_FLG)); + RBT_ASSERT(!(color & ~RED_FLG)); + return ((UWord)parent) | color; +} + + enum SortOrder { - ADDR_ORDER, - SZ_ADDR_ORDER, - SZ_REVERSE_ADDR_ORDER + ADDR_ORDER, /* only address order */ + SZ_ADDR_ORDER, /* first size then address order as tiebreaker */ + SZ_REVERSE_ADDR_ORDER /* first size then reverse address order */ }; typedef struct { @@ -100,13 +127,6 @@ typedef struct { enum SortOrder order; }RBTree; -#define RED_FLG (1) -#define IS_RED(N) ((N) && ((N)->flags & RED_FLG)) -#define IS_BLACK(N) (!IS_RED(N)) -#define SET_RED(N) ((N)->flags |= RED_FLG) -#define SET_BLACK(N) ((N)->flags &= ~RED_FLG) - -/* #define HARD_DEBUG */ #ifdef HARD_DEBUG # define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) # define HARD_CHECK_TREE(TREE,SZ) check_tree(TREE, SZ) @@ -318,20 +338,20 @@ left_rotate(RBTNode **root, RBTNode *x) RBTNode *y = x->right; x->right = y->left; if (y->left) - y->left->parent = x; - y->parent = x->parent; - if (!y->parent) { + set_parent(y->left, x); + set_parent(y, parent(x)); + if (!parent(y)) { RBT_ASSERT(*root == x); *root = y; } - else if (x == x->parent->left) - x->parent->left = y; + else if (x == parent(x)->left) + parent(x)->left = y; else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; + RBT_ASSERT(x == parent(x)->right); + parent(x)->right = y; } y->left = x; - x->parent = y; + set_parent(x, y); /*SVERK y->max_sz = x->max_sz; x->max_sz = node_max_size(x); @@ -344,20 +364,20 @@ right_rotate(RBTNode **root, RBTNode *x) RBTNode *y = x->left; x->left = y->right; if (y->right) - y->right->parent = x; - y->parent = x->parent; - if (!y->parent) { + set_parent(y->right, x); + set_parent(y, parent(x)); + if (!parent(y)) { RBT_ASSERT(*root == x); *root = y; } - else if (x == x->parent->right) - x->parent->right = y; + else if (x == parent(x)->right) + parent(x)->right = y; else { - RBT_ASSERT(x == x->parent->left); - x->parent->left = y; + RBT_ASSERT(x == parent(x)->left); + parent(x)->left = y; } y->right = x; - x->parent = y; + set_parent(x, y); /*SVERK y->max_sz = x->max_sz; x->max_sz = node_max_size(x); ASSERT(y->max_sz >= x->max_sz);*/ @@ -371,27 +391,27 @@ static ERTS_INLINE void replace(RBTNode **root, RBTNode *x, RBTNode *y) { - if (!x->parent) { + if (!parent(x)) { RBT_ASSERT(*root == x); *root = y; } - else if (x == x->parent->left) - x->parent->left = y; + else if (x == parent(x)->left) + parent(x)->left = y; else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; + RBT_ASSERT(x == parent(x)->right); + parent(x)->right = y; } if (x->left) { - RBT_ASSERT(x->left->parent == x); - x->left->parent = y; + RBT_ASSERT(parent(x->left) == x); + set_parent(x->left, y); } if (x->right) { - RBT_ASSERT(x->right->parent == x); - x->right->parent = y; + RBT_ASSERT(parent(x->right) == x); + set_parent(x->right, y); } - y->flags = x->flags; - y->parent = x->parent; + /*y->flags = x->flags;*/ + y->parent_and_color = x->parent_and_color; y->right = x->right; y->left = x->left; /*SVERK y->max_sz = x->max_sz;*/ @@ -406,7 +426,7 @@ tree_insert_fixup(RBTNode** root, RBTNode *blk) * Rearrange the tree so that it satisfies the Red-Black Tree properties */ - RBT_ASSERT(x != *root && IS_RED(x->parent)); + RBT_ASSERT(x != *root && IS_RED(parent(x))); do { /* @@ -415,77 +435,77 @@ tree_insert_fixup(RBTNode** root, RBTNode *blk) */ RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(x->parent->parent); + RBT_ASSERT(IS_BLACK(parent(parent(x)))); + RBT_ASSERT(parent(parent(x))); - if (x->parent == x->parent->parent->left) { - y = x->parent->parent->right; + if (parent(x) == parent(parent(x))->left) { + y = parent(parent(x))->right; if (IS_RED(y)) { SET_BLACK(y); - x = x->parent; + x = parent(x); SET_BLACK(x); - x = x->parent; + x = parent(x); SET_RED(x); } else { - if (x == x->parent->right) { - x = x->parent; + if (x == parent(x)->right) { + x = parent(x); left_rotate(root, x); } - RBT_ASSERT(x == x->parent->parent->left->left); + RBT_ASSERT(x == parent(parent(x))->left->left); RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_RED(parent(x))); + RBT_ASSERT(IS_BLACK(parent(parent(x)))); RBT_ASSERT(IS_BLACK(y)); - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - right_rotate(root, x->parent->parent); + SET_BLACK(parent(x)); + SET_RED(parent(parent(x))); + right_rotate(root, parent(parent(x))); - RBT_ASSERT(x == x->parent->left); + RBT_ASSERT(x == parent(x)->left); RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->right)); - RBT_ASSERT(IS_BLACK(x->parent)); + RBT_ASSERT(IS_RED(parent(x)->right)); + RBT_ASSERT(IS_BLACK(parent(x))); break; } } else { - RBT_ASSERT(x->parent == x->parent->parent->right); - y = x->parent->parent->left; + RBT_ASSERT(parent(x) == parent(parent(x))->right); + y = parent(parent(x))->left; if (IS_RED(y)) { SET_BLACK(y); - x = x->parent; + x = parent(x); SET_BLACK(x); - x = x->parent; + x = parent(x); SET_RED(x); } else { - if (x == x->parent->left) { - x = x->parent; + if (x == parent(x)->left) { + x = parent(x); right_rotate(root, x); } - RBT_ASSERT(x == x->parent->parent->right->right); + RBT_ASSERT(x == parent(parent(x))->right->right); RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); + RBT_ASSERT(IS_RED(parent(x))); + RBT_ASSERT(IS_BLACK(parent(parent(x)))); RBT_ASSERT(IS_BLACK(y)); - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - left_rotate(root, x->parent->parent); + SET_BLACK(parent(x)); + SET_RED(parent(parent(x))); + left_rotate(root, parent(parent(x))); - RBT_ASSERT(x == x->parent->right); + RBT_ASSERT(x == parent(x)->right); RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->left)); - RBT_ASSERT(IS_BLACK(x->parent)); + RBT_ASSERT(IS_RED(parent(x)->left)); + RBT_ASSERT(IS_BLACK(parent(x))); break; } } - } while (x != *root && IS_RED(x->parent)); + } while (x != *root && IS_RED(parent(x))); SET_BLACK(*root); } @@ -500,7 +520,7 @@ rbt_delete(RBTNode** root, RBTNode* del) HARD_CHECK_IS_MEMBER(*root, del); - null_x.parent = NULL; + null_x.parent_and_color = parent_and_color(NULL, !RED_FLG); /* Remove node from tree... */ @@ -514,29 +534,29 @@ rbt_delete(RBTNode** root, RBTNode* del) x = y->left ? y->left : y->right; spliced_is_black = IS_BLACK(y); if (x) { - x->parent = y->parent; + set_parent(x, parent(y)); } else if (spliced_is_black) { x = &null_x; - x->flags = 0; - SET_BLACK(x); + /*x->flags = 0; + SET_BLACK(x);*/ x->right = x->left = NULL; /*SVERK x->max_sz = 0;*/ - x->parent = y->parent; + x->parent_and_color = parent_and_color(parent(y), !RED_FLG); y->left = x; } - if (!y->parent) { + if (!parent(y)) { RBT_ASSERT(*root == y); *root = x; } else { - if (y == y->parent->left) { - y->parent->left = x; + if (y == parent(y)->left) { + parent(y)->left = x; } else { - RBT_ASSERT(y == y->parent->right); - y->parent->right = x; + RBT_ASSERT(y == parent(y)->right); + parent(y)->right = x; } /*SVERK if (y->parent != z) { lower_max_size(y->parent, (y==z ? NULL : z)); @@ -553,7 +573,7 @@ rbt_delete(RBTNode** root, RBTNode* del) /* We removed a black node which makes the resulting tree violate the Red-Black Tree properties. Fixup tree... */ - while (IS_BLACK(x) && x->parent) { + while (IS_BLACK(x) && parent(x)) { /* * x has an "extra black" which we move up the tree @@ -562,78 +582,78 @@ rbt_delete(RBTNode** root, RBTNode* del) * y is the sibbling of x */ - if (x == x->parent->left) { - y = x->parent->right; + if (x == parent(x)->left) { + y = parent(x)->right; RBT_ASSERT(y); if (IS_RED(y)) { RBT_ASSERT(y->right); RBT_ASSERT(y->left); SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - left_rotate(root, x->parent); - y = x->parent->right; + RBT_ASSERT(IS_BLACK(parent(x))); + SET_RED(parent(x)); + left_rotate(root, parent(x)); + y = parent(x)->right; } RBT_ASSERT(y); RBT_ASSERT(IS_BLACK(y)); if (IS_BLACK(y->left) && IS_BLACK(y->right)) { SET_RED(y); - x = x->parent; + x = parent(x); } else { if (IS_BLACK(y->right)) { SET_BLACK(y->left); SET_RED(y); right_rotate(root, y); - y = x->parent->right; + y = parent(x)->right; } RBT_ASSERT(y); - if (IS_RED(x->parent)) { + if (IS_RED(parent(x))) { - SET_BLACK(x->parent); + SET_BLACK(parent(x)); SET_RED(y); } RBT_ASSERT(y->right); SET_BLACK(y->right); - left_rotate(root, x->parent); + left_rotate(root, parent(x)); x = *root; break; } } else { - RBT_ASSERT(x == x->parent->right); - y = x->parent->left; + RBT_ASSERT(x == parent(x)->right); + y = parent(x)->left; RBT_ASSERT(y); if (IS_RED(y)) { RBT_ASSERT(y->right); RBT_ASSERT(y->left); SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - right_rotate(root, x->parent); - y = x->parent->left; + RBT_ASSERT(IS_BLACK(parent(x))); + SET_RED(parent(x)); + right_rotate(root, parent(x)); + y = parent(x)->left; } RBT_ASSERT(y); RBT_ASSERT(IS_BLACK(y)); if (IS_BLACK(y->right) && IS_BLACK(y->left)) { SET_RED(y); - x = x->parent; + x = parent(x); } else { if (IS_BLACK(y->left)) { SET_BLACK(y->right); SET_RED(y); left_rotate(root, y); - y = x->parent->left; + y = parent(x)->left; } RBT_ASSERT(y); - if (IS_RED(x->parent)) { - SET_BLACK(x->parent); + if (IS_RED(parent(x))) { + SET_BLACK(parent(x)); SET_RED(y); } RBT_ASSERT(y->left); SET_BLACK(y->left); - right_rotate(root, x->parent); + right_rotate(root, parent(x)); x = *root; break; } @@ -641,12 +661,12 @@ rbt_delete(RBTNode** root, RBTNode* del) } SET_BLACK(x); - if (null_x.parent) { - if (null_x.parent->left == &null_x) - null_x.parent->left = NULL; + if (parent(&null_x)) { + if (parent(&null_x)->left == &null_x) + parent(&null_x)->left = NULL; else { - RBT_ASSERT(null_x.parent->right == &null_x); - null_x.parent->right = NULL; + RBT_ASSERT(parent(&null_x)->right == &null_x); + parent(&null_x)->right = NULL; } RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); @@ -671,14 +691,14 @@ rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) SWord blk_sz = desc->end - desc->start; /*SVERK Uint blk_sz = AOFF_BLK_SZ(blk);*/ - blk->flags = 0; + /*blk->flags = 0;*/ blk->left = NULL; blk->right = NULL; /*SVERK blk->max_sz = blk_sz;*/ if (!*root) { - blk->parent = NULL; - SET_BLACK(blk); + blk->parent_and_color = parent_and_color(NULL, !RED_FLG); + /*SET_BLACK(blk);*/ *root = blk; } else { @@ -692,7 +712,7 @@ rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) if (diff < 0) { IF_RBT_DEBUG(dbg_over = node_to_desc(order, x)); if (!x->left) { - blk->parent = x; + blk->parent_and_color = parent_and_color(x, RED_FLG); x->left = blk; break; } @@ -702,7 +722,7 @@ rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) RBT_ASSERT(diff > 0); IF_RBT_DEBUG(dbg_under = node_to_desc(order, x)); if (!x->right) { - blk->parent = x; + blk->parent_and_color = parent_and_color(x, RED_FLG); x->right = blk; break; } @@ -723,15 +743,16 @@ rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) } /* Insert block into size tree */ - RBT_ASSERT(blk->parent); + RBT_ASSERT(parent(blk)); #ifdef RBT_DEBUG if (!order) { RBT_ASSERT(!dbg_under || dbg_under->end < desc->start); RBT_ASSERT(!dbg_over || dbg_over->start > desc->end); } #endif - SET_RED(blk); - if (IS_RED(blk->parent)) + /*SET_RED(blk);*/ + RBT_ASSERT(IS_RED(blk)); + if (IS_RED(parent(blk))) tree_insert_fixup(root, blk); } /*SVERK if (flavor == AOFF_BF) { @@ -1746,20 +1767,11 @@ erts_mmap_init(ErtsMMapInit *init) mmap_state.supercarrier = 1; erts_have_erts_mmap |= ERTS_HAVE_ERTS_SUPERCARRIER_MMAP; - - -#ifdef HARD_DEBUG - { - void test_it(void); - test_it(); } -#endif - } #if !ERTS_HAVE_OS_MMAP mmap_state.no_os_mmap = 1; #endif - } @@ -1773,9 +1785,9 @@ erts_mmap_init(ErtsMMapInit *init) static int rbt_assert_is_member(RBTNode* root, RBTNode* node) { while (node != root) { - RBT_ASSERT(node->parent); - RBT_ASSERT(node->parent->left == node || node->parent->right == node); - node = node->parent; + RBT_ASSERT(parent(node)); + RBT_ASSERT(parent(node)->left == node || parent(node)->right == node); + node = parent(node); } return 1; } @@ -1841,7 +1853,7 @@ check_tree(RBTree* tree, Uint size) x = tree->root; RBT_ASSERT(IS_BLACK(x)); - RBT_ASSERT(!x->parent); + RBT_ASSERT(!parent(x)); curr_blacks = 1; blacks = -1; depth = 1; @@ -1877,15 +1889,15 @@ check_tree(RBTree* tree, Uint size) RBT_ASSERT(IS_BLACK(x->left)); } - RBT_ASSERT(x->parent || x == tree->root); + RBT_ASSERT(parent(x) || x == tree->root); if (x->left) { - RBT_ASSERT(x->left->parent == x); + RBT_ASSERT(parent(x->left) == x); RBT_ASSERT(cmp_blocks(tree->order, x->left, x) < 0); } if (x->right) { - RBT_ASSERT(x->right->parent == x); + RBT_ASSERT(parent(x->right) == x); RBT_ASSERT(cmp_blocks(tree->order, x->right, x) > 0); } @@ -1923,7 +1935,7 @@ check_tree(RBTree* tree, Uint size) UNSET_RIGHT_VISITED(x); if (IS_BLACK(x)) curr_blacks--; - x = x->parent; + x = parent(x); --depth; } RBT_ASSERT(depth == 0 || (!tree->root && depth==1)); @@ -1937,6 +1949,8 @@ check_tree(RBTree* tree, Uint size) return res; } +#endif /* HARD_DEBUG */ + #ifdef PRINT_TREE #define INDENT_STEP 2 @@ -1983,13 +1997,13 @@ void test_it(void) init_free_seg_map(&map, i); insert_free_seg(&map, alloc_desc(), (char*)0x11000, (char*)0x12000); - check_tree(&map.atree, 0); check_tree(&map.stree, 0); + HARD_CHECK_TREE(&map.atree, 0); HARD_CHECK_TREE(&map.stree, 0); insert_free_seg(&map, alloc_desc(), (char*)0x13000, (char*)0x14000); - check_tree(&map.atree, 0); check_tree(&map.stree, 0); + HARD_CHECK_TREE(&map.atree, 0); HARD_CHECK_TREE(&map.stree, 0); insert_free_seg(&map, alloc_desc(), (char*)0x15000, (char*)0x17000); - check_tree(&map.atree, 0); check_tree(&map.stree, 0); + HARD_CHECK_TREE(&map.atree, 0); HARD_CHECK_TREE(&map.stree, 0); insert_free_seg(&map, alloc_desc(), (char*)0x8000, (char*)0x10000); - check_tree(&map.atree, 0); check_tree(&map.stree, 0); + HARD_CHECK_TREE(&map.atree, 0); HARD_CHECK_TREE(&map.stree, 0); desc = lookup_free_seg(&map, 0x500); ERTS_ASSERT(desc->start == (char*)(i?0x13000L:0x11000L)); @@ -2026,17 +2040,15 @@ void test_it(void) ERTS_ASSERT(d1->start == (char*)(i?0x13000L:0x11000L)); resize_free_seg(&map, d1, d1->start - 0x800, (char*)d1->end); - check_tree(&map.atree, 0); check_tree(&map.stree, 0); + HARD_CHECK_TREE(&map.atree, 0); HARD_CHECK_TREE(&map.stree, 0); d2 = lookup_free_seg(&map, 0x1200); ERTS_ASSERT(d2 == d1); delete_free_seg(&map, d1); - check_tree(&map.atree, 0); check_tree(&map.stree, 0); + HARD_CHECK_TREE(&map.atree, 0); HARD_CHECK_TREE(&map.stree, 0); d1 = lookup_free_seg(&map, 0x1200); ERTS_ASSERT(d1->start == (char*)0x15000); } } - -#endif /* HARD_DEBUG */ -- cgit v1.2.3 From 97f11582fba9e8b141764273b94bd1ccee3d9b08 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 2 Sep 2013 16:22:32 +0200 Subject: erts: Remove HARD_DEBUG flags for tree traversal --- erts/emulator/sys/common/erl_mmap.c | 120 ++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 66 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index cd71127ce5..77c6abadf9 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -86,9 +86,6 @@ struct RBTNode_ { UWord parent_and_color; /* color in bit 0 of parent ptr */ RBTNode *left; RBTNode *right; -#ifdef HARD_DEBUG - int flags; -#endif }; #define RED_FLG (1) @@ -1792,22 +1789,6 @@ static int rbt_assert_is_member(RBTNode* root, RBTNode* node) return 1; } -#define LEFT_VISITED_FLG 0x1000 -#define THIS_VISITED_FLG 0x100 -#define RIGHT_VISITED_FLG 0x10 -#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) -#define IS_THIS_VISITED(FB) ((FB)->flags & THIS_VISITED_FLG) -#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) - -#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG) -#define SET_THIS_VISITED(FB) ((FB)->flags |= THIS_VISITED_FLG) -#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG) - -#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG) -#define UNSET_THIS_VISITED(FB) ((FB)->flags &= ~THIS_VISITED_FLG) -#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG) - - #if 1 /*SVERK*/ # define PRINT_TREE @@ -1843,6 +1824,7 @@ check_tree(RBTree* tree, Uint size) Uint depth, max_depth, node_cnt; ErtsFreeSegDesc* seg = NULL; ErtsFreeSegDesc* prev_seg = NULL; + enum { RECURSE_LEFT, CHECK_NODE, RECURSE_RIGHT, RETURN_TO_PARENT }state; #ifdef PRINT_TREE print_tree(tree->order, tree->root); @@ -1859,27 +1841,29 @@ check_tree(RBTree* tree, Uint size) depth = 1; max_depth = 0; node_cnt = 0; + state = RECURSE_LEFT; /* Traverse tree in sorting order */ while (x) { - if (!IS_LEFT_VISITED(x)) { - SET_LEFT_VISITED(x); - if (x->left) { - x = x->left; - ++depth; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - RBT_ASSERT(blacks == curr_blacks); - } - } + switch (state) { + case RECURSE_LEFT: + if (x->left) { + RBT_ASSERT(parent(x->left) == x); + x = x->left; + ++depth; + if (IS_BLACK(x)) + curr_blacks++; + state = RECURSE_LEFT; + } + else { + if (blacks < 0) + blacks = curr_blacks; + RBT_ASSERT(blacks == curr_blacks); + state = CHECK_NODE; + } + break; - if (!IS_THIS_VISITED(x)) { - SET_THIS_VISITED(x); + case CHECK_NODE: ++node_cnt; if (depth > max_depth) max_depth = depth; @@ -1892,12 +1876,9 @@ check_tree(RBTree* tree, Uint size) RBT_ASSERT(parent(x) || x == tree->root); if (x->left) { - RBT_ASSERT(parent(x->left) == x); RBT_ASSERT(cmp_blocks(tree->order, x->left, x) < 0); } - if (x->right) { - RBT_ASSERT(parent(x->right) == x); RBT_ASSERT(cmp_blocks(tree->order, x->right, x) > 0); } @@ -1912,40 +1893,47 @@ check_tree(RBTree* tree, Uint size) RBT_ASSERT(!prev_seg || prev_seg->end < seg->start); prev_seg = seg; } + state = RECURSE_RIGHT; + break; - } - if (!IS_RIGHT_VISITED(x)) { - SET_RIGHT_VISITED(x); - if (x->right) { - x = x->right; - ++depth; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - RBT_ASSERT(blacks == curr_blacks); - } - } + case RECURSE_RIGHT: + if (x->right) { + RBT_ASSERT(parent(x->right) == x); + x = x->right; + ++depth; + if (IS_BLACK(x)) + curr_blacks++; + state = RECURSE_LEFT; + } + else { + if (blacks < 0) + blacks = curr_blacks; + RBT_ASSERT(blacks == curr_blacks); + state = RETURN_TO_PARENT; + } + break; - UNSET_LEFT_VISITED(x); - UNSET_THIS_VISITED(x); - UNSET_RIGHT_VISITED(x); - if (IS_BLACK(x)) - curr_blacks--; - x = parent(x); - --depth; + case RETURN_TO_PARENT: + if (parent(x)) { + if (x == parent(x)->left) { + state = CHECK_NODE; + } + else { + RBT_ASSERT(x == parent(x)->right); + state = RETURN_TO_PARENT; + } + } + if (IS_BLACK(x)) + curr_blacks--; + x = parent(x); + --depth; + break; + } } RBT_ASSERT(depth == 0 || (!tree->root && depth==1)); RBT_ASSERT(curr_blacks == 0); RBT_ASSERT((1 << (max_depth/2)) <= node_cnt); - UNSET_LEFT_VISITED(tree->root); - UNSET_THIS_VISITED(tree->root); - UNSET_RIGHT_VISITED(tree->root); - return res; } -- cgit v1.2.3 From 9fe8915784a23622577a8bc8604e809d34623c94 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 3 Sep 2013 11:25:36 +0200 Subject: erts: Add build_free_seg_list --- erts/emulator/sys/common/erl_mmap.c | 136 +++++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 77c6abadf9..5624ec9813 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -21,9 +21,10 @@ #endif #include "sys.h" -#include +#include "erl_process.h" #include "erl_smp.h" #include "erl_mmap.h" +#include #if defined(DEBUG) || 0 # undef ERTS_MMAP_DEBUG @@ -145,6 +146,7 @@ typedef struct { typedef struct { RBTree stree; RBTree atree; + Uint nseg; }ErtsFreeSegMap; static struct { @@ -758,6 +760,96 @@ rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) }*/ } +/* + * Traverse tree in (reverse) sorting order + */ +static void +rbt_foreach_node(RBTree* tree, + void (*fn)(RBTNode*,void*), + void* arg, int reverse) +{ +#ifdef HARD_DEBUG + Sint blacks = -1; + Sint curr_blacks = 1; + Uint depth = 1; +#endif + enum { RECURSE_LEFT, DO_NODE, RECURSE_RIGHT, RETURN_TO_PARENT }state; + RBTNode *x = tree->root; + + RBT_ASSERT(!parent(x)); + + state = reverse ? RECURSE_RIGHT : RECURSE_LEFT; + while (x) { + switch (state) { + case RECURSE_LEFT: + if (x->left) { + RBT_ASSERT(parent(x->left) == x); + #ifdef HARD_DEBUG + ++depth; + if (IS_BLACK(x->left)) + curr_blacks++; + #endif + x = x->left; + state = reverse ? RECURSE_RIGHT : RECURSE_LEFT; + } + else { + #ifdef HARD_DEBUG + if (blacks < 0) + blacks = curr_blacks; + RBT_ASSERT(blacks == curr_blacks); + #endif + state = reverse ? RETURN_TO_PARENT : DO_NODE; + } + break; + + case DO_NODE: + (*fn) (x, arg); + state = reverse ? RECURSE_LEFT : RECURSE_RIGHT; + break; + + case RECURSE_RIGHT: + if (x->right) { + RBT_ASSERT(parent(x->right) == x); + #ifdef HARD_DEBUG + ++depth; + if (IS_BLACK(x->right)) + curr_blacks++; + #endif + x = x->right; + state = reverse ? RECURSE_RIGHT : RECURSE_LEFT; + } + else { + #ifdef HARD_DEBUG + if (blacks < 0) + blacks = curr_blacks; + RBT_ASSERT(blacks == curr_blacks); + #endif + state = reverse ? DO_NODE : RETURN_TO_PARENT; + } + break; + + case RETURN_TO_PARENT: + #ifdef HARD_DEBUG + if (IS_BLACK(x)) + curr_blacks--; + --depth; + #endif + if (parent(x)) { + if (x == parent(x)->left) { + state = reverse ? RETURN_TO_PARENT : DO_NODE; + } + else { + RBT_ASSERT(x == parent(x)->right); + state = reverse ? DO_NODE : RETURN_TO_PARENT; + } + } + x = parent(x); + break; + } + } +} + + /* The API to keep track of a bunch of separated free segments (non-overlapping and non-adjacent). @@ -777,6 +869,7 @@ static void init_free_seg_map(ErtsFreeSegMap* map, int reverse_ao) map->atree.order = ADDR_ORDER; map->stree.root = NULL; map->stree.order = reverse_ao ? SZ_REVERSE_ADDR_ORDER : SZ_ADDR_ORDER; + map->nseg = 0; } static void adjacent_free_seg(ErtsFreeSegMap* map, char* start, char* end, @@ -813,6 +906,7 @@ static void insert_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, desc->end = end; rbt_insert(map->atree.order, &map->atree.root, &desc->anode); rbt_insert(map->stree.order, &map->stree.root, &desc->snode); + map->nseg++; } static void resize_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, @@ -835,6 +929,7 @@ static void delete_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc) { rbt_delete(&map->atree.root, &desc->anode); rbt_delete(&map->stree.root, &desc->snode); + map->nseg--; } static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap* map, SWord need_sz) @@ -857,6 +952,44 @@ static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap* map, SWord need_sz) return best_desc; } +struct build_arg_t +{ + Process* p; + Eterm* hp; + Eterm acc; +}; + +static void build_free_seg_tuple(RBTNode* node, void* arg) +{ + struct build_arg_t* a = (struct build_arg_t*)arg; + ErtsFreeSegDesc* desc = anode_to_desc(node); + Eterm start= erts_bld_uword(&a->hp, NULL, (UWord)desc->start); + Eterm end = erts_bld_uword(&a->hp, NULL, (UWord)desc->end); + Eterm tpl = TUPLE2(a->hp, start, end); + + a->hp += 3; + a->acc = CONS(a->hp, tpl, a->acc); + a->hp += 2; +} + +static +Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map) +{ + struct build_arg_t barg; + Eterm* hp_end; + const Uint may_need = map->nseg * (2 + 3 + 2*2); /* cons + tuple + bigs */ + + barg.p = p; + barg.hp = HAlloc(p, may_need); + hp_end = barg.hp + may_need; + barg.acc = NIL; + rbt_foreach_node(&map->atree, build_free_seg_tuple, &barg, 1); + + RBT_ASSERT(barg.hp <= hp_end); + HRelease(p, hp_end, barg.hp); + return barg.acc; +} + #if ERTS_HAVE_OS_MMAP /* Implementation of os_mmap()/os_munmap()/os_mremap()... */ @@ -1975,6 +2108,7 @@ print_tree(enum SortOrder order, RBTNode* root) #endif /* PRINT_TREE */ + void test_it(void) { ErtsFreeSegMap map; -- cgit v1.2.3 From 0d9c5c44a46810f0d9f45a533ca8a3754c11c643 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 3 Sep 2013 17:18:34 +0200 Subject: erts: Use rbt_foreach_node to check_tree --- erts/emulator/sys/common/erl_mmap.c | 161 +++++++++++++----------------------- 1 file changed, 58 insertions(+), 103 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 5624ec9813..2d749b2402 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -772,6 +772,8 @@ rbt_foreach_node(RBTree* tree, Sint blacks = -1; Sint curr_blacks = 1; Uint depth = 1; + Uint max_depth = 0; + Uint node_cnt = 0; #endif enum { RECURSE_LEFT, DO_NODE, RECURSE_RIGHT, RETURN_TO_PARENT }state; RBTNode *x = tree->root; @@ -803,7 +805,12 @@ rbt_foreach_node(RBTree* tree, break; case DO_NODE: - (*fn) (x, arg); + #ifdef HARD_DEBUG + ++node_cnt; + if (depth > max_depth) + max_depth = depth; + #endif + (*fn) (x, arg); /* Do it! */ state = reverse ? RECURSE_LEFT : RECURSE_RIGHT; break; @@ -847,6 +854,11 @@ rbt_foreach_node(RBTree* tree, break; } } +#ifdef HARD_DEBUG + RBT_ASSERT(depth == 0 || (!tree->root && depth==1)); + RBT_ASSERT(curr_blacks == 0); + RBT_ASSERT((1 << (max_depth/2)) <= node_cnt); +#endif } @@ -1947,127 +1959,70 @@ static void print_tree(enum SortOrder order, RBTNode*); * */ +struct check_arg_t { + RBTree* tree; + ErtsFreeSegDesc* prev_seg; + Uint size; + RBTNode *res; +}; +static void check_node(RBTNode* x, void* arg); + + static RBTNode * check_tree(RBTree* tree, Uint size) { - RBTNode *res = NULL; - Sint blacks; - Sint curr_blacks; - RBTNode *x; - Uint depth, max_depth, node_cnt; - ErtsFreeSegDesc* seg = NULL; - ErtsFreeSegDesc* prev_seg = NULL; - enum { RECURSE_LEFT, CHECK_NODE, RECURSE_RIGHT, RETURN_TO_PARENT }state; + struct check_arg_t carg; + carg.tree = tree; + carg.prev_seg = NULL; + carg.size = size; + carg.res = NULL; #ifdef PRINT_TREE print_tree(tree->order, tree->root); #endif if (!tree->root) - return res; + return NULL; - x = tree->root; - RBT_ASSERT(IS_BLACK(x)); - RBT_ASSERT(!parent(x)); - curr_blacks = 1; - blacks = -1; - depth = 1; - max_depth = 0; - node_cnt = 0; - state = RECURSE_LEFT; - - /* Traverse tree in sorting order */ - while (x) { - switch (state) { - case RECURSE_LEFT: - if (x->left) { - RBT_ASSERT(parent(x->left) == x); - x = x->left; - ++depth; - if (IS_BLACK(x)) - curr_blacks++; - state = RECURSE_LEFT; - } - else { - if (blacks < 0) - blacks = curr_blacks; - RBT_ASSERT(blacks == curr_blacks); - state = CHECK_NODE; - } - break; + RBT_ASSERT(IS_BLACK(tree->root)); + RBT_ASSERT(!parent(tree->root)); - case CHECK_NODE: - ++node_cnt; - if (depth > max_depth) - max_depth = depth; + rbt_foreach_node(tree, check_node, &carg, 0); - if (IS_RED(x)) { - RBT_ASSERT(IS_BLACK(x->right)); - RBT_ASSERT(IS_BLACK(x->left)); - } + return carg.res; +} - RBT_ASSERT(parent(x) || x == tree->root); +/* callback */ +static void check_node(RBTNode* x, void* arg) +{ + struct check_arg_t* a = (struct check_arg_t*) arg; + ErtsFreeSegDesc* seg; - if (x->left) { - RBT_ASSERT(cmp_blocks(tree->order, x->left, x) < 0); - } - if (x->right) { - RBT_ASSERT(cmp_blocks(tree->order, x->right, x) > 0); - } + if (IS_RED(x)) { + RBT_ASSERT(IS_BLACK(x->right)); + RBT_ASSERT(IS_BLACK(x->left)); + } - seg = node_to_desc(tree->order, x); - RBT_ASSERT(seg->start < seg->end); - if (size && (seg->end - seg->start) >= size) { - if (!res || cmp_blocks(tree->order, x, res) < 0) { - res = x; - } - } - if (tree->order == ADDR_ORDER) { - RBT_ASSERT(!prev_seg || prev_seg->end < seg->start); - prev_seg = seg; - } - state = RECURSE_RIGHT; - break; + RBT_ASSERT(parent(x) || x == a->tree->root); - case RECURSE_RIGHT: - if (x->right) { - RBT_ASSERT(parent(x->right) == x); - x = x->right; - ++depth; - if (IS_BLACK(x)) - curr_blacks++; - state = RECURSE_LEFT; - } - else { - if (blacks < 0) - blacks = curr_blacks; - RBT_ASSERT(blacks == curr_blacks); - state = RETURN_TO_PARENT; - } - break; + if (x->left) { + RBT_ASSERT(cmp_blocks(a->tree->order, x->left, x) < 0); + } + if (x->right) { + RBT_ASSERT(cmp_blocks(a->tree->order, x->right, x) > 0); + } - case RETURN_TO_PARENT: - if (parent(x)) { - if (x == parent(x)->left) { - state = CHECK_NODE; - } - else { - RBT_ASSERT(x == parent(x)->right); - state = RETURN_TO_PARENT; - } - } - if (IS_BLACK(x)) - curr_blacks--; - x = parent(x); - --depth; - break; + seg = node_to_desc(a->tree->order, x); + RBT_ASSERT(seg->start < seg->end); + if (a->size && (seg->end - seg->start) >= a->size) { + if (!a->res || cmp_blocks(a->tree->order, x, a->res) < 0) { + a->res = x; } } - RBT_ASSERT(depth == 0 || (!tree->root && depth==1)); - RBT_ASSERT(curr_blacks == 0); - RBT_ASSERT((1 << (max_depth/2)) <= node_cnt); - - return res; + if (a->tree->order == ADDR_ORDER) { + RBT_ASSERT(!a->prev_seg || a->prev_seg->end < seg->start); + a->prev_seg = seg; + } } #endif /* HARD_DEBUG */ -- cgit v1.2.3 From 984f0e42665e9fa09467145976183bf88f8f3da8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 3 Sep 2013 19:54:38 +0200 Subject: erts: Optimize rb-tree operations by "caching" parent ptr --- erts/emulator/sys/common/erl_mmap.c | 152 +++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 71 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 2d749b2402..ede7c66fdc 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -419,13 +419,14 @@ replace(RBTNode **root, RBTNode *x, RBTNode *y) static void tree_insert_fixup(RBTNode** root, RBTNode *blk) { - RBTNode *x = blk, *y; + RBTNode *x = blk, *y, *papa_x, *granpa_x; /* * Rearrange the tree so that it satisfies the Red-Black Tree properties */ - RBT_ASSERT(x != *root && IS_RED(parent(x))); + papa_x = parent(x); + RBT_ASSERT(x != *root && IS_RED(papa_x)); do { /* @@ -433,35 +434,36 @@ tree_insert_fixup(RBTNode** root, RBTNode *blk) * until we get to the root or until we can separate them. */ + granpa_x = parent(papa_x); RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_BLACK(parent(parent(x)))); - RBT_ASSERT(parent(parent(x))); + RBT_ASSERT(IS_BLACK(granpa_x)); + RBT_ASSERT(granpa_x); - if (parent(x) == parent(parent(x))->left) { - y = parent(parent(x))->right; + if (papa_x == granpa_x->left) { + y = granpa_x->right; if (IS_RED(y)) { SET_BLACK(y); - x = parent(x); - SET_BLACK(x); - x = parent(x); - SET_RED(x); + SET_BLACK(papa_x); + SET_RED(granpa_x); + x = granpa_x; } else { - if (x == parent(x)->right) { - x = parent(x); - left_rotate(root, x); + if (x == papa_x->right) { + left_rotate(root, papa_x); + papa_x = x; + x = papa_x->left; } - RBT_ASSERT(x == parent(parent(x))->left->left); + RBT_ASSERT(x == granpa_x->left->left); RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(parent(x))); - RBT_ASSERT(IS_BLACK(parent(parent(x)))); + RBT_ASSERT(IS_RED(papa_x)); + RBT_ASSERT(IS_BLACK(granpa_x)); RBT_ASSERT(IS_BLACK(y)); - SET_BLACK(parent(x)); - SET_RED(parent(parent(x))); - right_rotate(root, parent(parent(x))); + SET_BLACK(papa_x); + SET_RED(granpa_x); + right_rotate(root, granpa_x); RBT_ASSERT(x == parent(x)->left); RBT_ASSERT(IS_RED(x)); @@ -471,31 +473,31 @@ tree_insert_fixup(RBTNode** root, RBTNode *blk) } } else { - RBT_ASSERT(parent(x) == parent(parent(x))->right); - y = parent(parent(x))->left; + RBT_ASSERT(papa_x == granpa_x->right); + y = granpa_x->left; if (IS_RED(y)) { SET_BLACK(y); - x = parent(x); - SET_BLACK(x); - x = parent(x); - SET_RED(x); + SET_BLACK(papa_x); + SET_RED(granpa_x); + x = granpa_x; } else { - if (x == parent(x)->left) { - x = parent(x); - right_rotate(root, x); + if (x == papa_x->left) { + right_rotate(root, papa_x); + papa_x = x; + x = papa_x->right; } - RBT_ASSERT(x == parent(parent(x))->right->right); + RBT_ASSERT(x == granpa_x->right->right); RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(parent(x))); - RBT_ASSERT(IS_BLACK(parent(parent(x)))); + RBT_ASSERT(IS_RED(papa_x)); + RBT_ASSERT(IS_BLACK(granpa_x)); RBT_ASSERT(IS_BLACK(y)); - SET_BLACK(parent(x)); - SET_RED(parent(parent(x))); - left_rotate(root, parent(parent(x))); + SET_BLACK(papa_x); + SET_RED(granpa_x); + left_rotate(root, granpa_x); RBT_ASSERT(x == parent(x)->right); RBT_ASSERT(IS_RED(x)); @@ -504,7 +506,7 @@ tree_insert_fixup(RBTNode** root, RBTNode *blk) break; } } - } while (x != *root && IS_RED(parent(x))); + } while (x != *root && (papa_x=parent(x), IS_RED(papa_x))); SET_BLACK(*root); } @@ -513,7 +515,7 @@ static void rbt_delete(RBTNode** root, RBTNode* del) { Uint spliced_is_black; - RBTNode *x, *y, *z = del; + RBTNode *x, *y, *z = del, *papa_y; RBTNode null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ @@ -532,8 +534,9 @@ rbt_delete(RBTNode** root, RBTNode* del) /* splice out y */ x = y->left ? y->left : y->right; spliced_is_black = IS_BLACK(y); + papa_y = parent(y); if (x) { - set_parent(x, parent(y)); + set_parent(x, papa_y); } else if (spliced_is_black) { x = &null_x; @@ -541,21 +544,21 @@ rbt_delete(RBTNode** root, RBTNode* del) SET_BLACK(x);*/ x->right = x->left = NULL; /*SVERK x->max_sz = 0;*/ - x->parent_and_color = parent_and_color(parent(y), !RED_FLG); + x->parent_and_color = parent_and_color(papa_y, !RED_FLG); y->left = x; } - if (!parent(y)) { + if (!papa_y) { RBT_ASSERT(*root == y); *root = x; } else { - if (y == parent(y)->left) { - parent(y)->left = x; + if (y == papa_y->left) { + papa_y->left = x; } else { - RBT_ASSERT(y == parent(y)->right); - parent(y)->right = x; + RBT_ASSERT(y == papa_y->right); + papa_y->right = x; } /*SVERK if (y->parent != z) { lower_max_size(y->parent, (y==z ? NULL : z)); @@ -569,10 +572,12 @@ rbt_delete(RBTNode** root, RBTNode* del) } if (spliced_is_black) { + RBTNode* papa_x; /* We removed a black node which makes the resulting tree violate the Red-Black Tree properties. Fixup tree... */ - while (IS_BLACK(x) && parent(x)) { + papa_x = parent(x); + while (IS_BLACK(x) && papa_x) { /* * x has an "extra black" which we move up the tree @@ -581,91 +586,96 @@ rbt_delete(RBTNode** root, RBTNode* del) * y is the sibbling of x */ - if (x == parent(x)->left) { - y = parent(x)->right; + if (x == papa_x->left) { + y = papa_x->right; RBT_ASSERT(y); if (IS_RED(y)) { RBT_ASSERT(y->right); RBT_ASSERT(y->left); SET_BLACK(y); - RBT_ASSERT(IS_BLACK(parent(x))); - SET_RED(parent(x)); - left_rotate(root, parent(x)); - y = parent(x)->right; + RBT_ASSERT(IS_BLACK(papa_x)); + SET_RED(papa_x); + left_rotate(root, papa_x); + RBT_ASSERT(papa_x == parent(x)); + y = papa_x->right; } RBT_ASSERT(y); RBT_ASSERT(IS_BLACK(y)); if (IS_BLACK(y->left) && IS_BLACK(y->right)) { SET_RED(y); - x = parent(x); } else { if (IS_BLACK(y->right)) { SET_BLACK(y->left); SET_RED(y); right_rotate(root, y); - y = parent(x)->right; + RBT_ASSERT(papa_x == parent(x)); + y = papa_x->right; } RBT_ASSERT(y); - if (IS_RED(parent(x))) { + if (IS_RED(papa_x)) { - SET_BLACK(parent(x)); + SET_BLACK(papa_x); SET_RED(y); } RBT_ASSERT(y->right); SET_BLACK(y->right); - left_rotate(root, parent(x)); + left_rotate(root, papa_x); x = *root; break; } } else { - RBT_ASSERT(x == parent(x)->right); - y = parent(x)->left; + RBT_ASSERT(x == papa_x->right); + y = papa_x->left; RBT_ASSERT(y); if (IS_RED(y)) { RBT_ASSERT(y->right); RBT_ASSERT(y->left); SET_BLACK(y); - RBT_ASSERT(IS_BLACK(parent(x))); - SET_RED(parent(x)); - right_rotate(root, parent(x)); - y = parent(x)->left; + RBT_ASSERT(IS_BLACK(papa_x)); + SET_RED(papa_x); + right_rotate(root, papa_x); + RBT_ASSERT(papa_x == parent(x)); + y = papa_x->left; } RBT_ASSERT(y); RBT_ASSERT(IS_BLACK(y)); if (IS_BLACK(y->right) && IS_BLACK(y->left)) { SET_RED(y); - x = parent(x); } else { if (IS_BLACK(y->left)) { SET_BLACK(y->right); SET_RED(y); left_rotate(root, y); - y = parent(x)->left; + RBT_ASSERT(papa_x == parent(x)); + y = papa_x->left; } RBT_ASSERT(y); - if (IS_RED(parent(x))) { - SET_BLACK(parent(x)); + if (IS_RED(papa_x)) { + SET_BLACK(papa_x); SET_RED(y); } RBT_ASSERT(y->left); SET_BLACK(y->left); - right_rotate(root, parent(x)); + right_rotate(root, papa_x); x = *root; break; } } + x = papa_x; + papa_x = parent(x); } SET_BLACK(x); - if (parent(&null_x)) { - if (parent(&null_x)->left == &null_x) - parent(&null_x)->left = NULL; + papa_x = parent(&null_x); + if (papa_x) { + if (papa_x->left == &null_x) + papa_x->left = NULL; else { - RBT_ASSERT(parent(&null_x)->right == &null_x); - parent(&null_x)->right = NULL; + RBT_ASSERT(papa_x->right == &null_x); + papa_x->right = NULL; } RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); -- cgit v1.2.3 From e0e17136506f9c8363b46a991012422980925dd1 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 4 Sep 2013 14:56:49 +0200 Subject: erts: Add __func__ to ERTS_ASSERT macro --- erts/emulator/beam/sys.h | 4 ++-- erts/emulator/sys/common/erl_mmap.c | 13 +------------ erts/emulator/sys/unix/sys.c | 6 +++--- erts/emulator/sys/win32/sys.c | 2 +- 4 files changed, 7 insertions(+), 18 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index dfe60d8ea0..9561c0be96 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -150,8 +150,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif #define ERTS_ASSERT(e) \ - ((void) ((e) ? 1 : (erl_assert_error(#e, __FILE__, __LINE__), 0))) -void erl_assert_error(char* expr, char* file, int line); + ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0))) +void erl_assert_error(const char* expr, const char *func, const char* file, int line); #ifdef DEBUG # define ASSERT(e) ERTS_ASSERT(e) diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index ede7c66fdc..58795bd3a6 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -34,18 +34,7 @@ /* #define ERTS_MMAP_DEBUG_FILL_AREAS */ #ifdef ERTS_MMAP_DEBUG -# define ERTS_MMAP_ASSERT(A) \ - ((void) (!(A) \ - ? erts_mmap_assert_failed(#A, __func__, __FILE__, __LINE__)\ - : 1)) -static int -erts_mmap_assert_failed(const char *a, const char *func, const char *file, int line) -{ - erts_fprintf(stderr, "%s:%d:%s() Assertion failed: %s\n", - (char *) file, line, (char *) func, (char *) a); - abort(); - return 0; -} +# define ERTS_MMAP_ASSERT ERTS_ASSERT #else # define ERTS_MMAP_ASSERT(A) ((void) 1) #endif diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 47991756df..401b37b9d2 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -2640,11 +2640,11 @@ int fd; extern int erts_initialized; void -erl_assert_error(char* expr, char* file, int line) +erl_assert_error(const char* expr, const char* func, const char* file, int line) { fflush(stdout); - fprintf(stderr, "Assertion failed: %s in %s, line %d\n", - expr, file, line); + fprintf(stderr, "%s:%d:%s() Assertion failed: %s\n", + file, line, func, expr); fflush(stderr); #if !defined(ERTS_SMP) && 0 /* Writing a crashdump from a failed assertion when smp support diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 6a1d6b08f4..99951bb45e 100755 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -3224,7 +3224,7 @@ erl_bin_write(buf, sz, max) } void -erl_assert_error(char* expr, char* file, int line) +erl_assert_error(const char* expr, const char* func, const char* file, int line) { char message[1024]; -- cgit v1.2.3 From 5295a138518df6c9f4907a2fb9afb1282b96ffb7 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 4 Sep 2013 15:00:30 +0200 Subject: erts: Cleanup erl_mmap --- erts/emulator/sys/common/erl_mmap.c | 233 +++++++++++++++++++----------------- 1 file changed, 121 insertions(+), 112 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 58795bd3a6..723efa772f 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -126,15 +126,15 @@ static RBTNode* check_tree(RBTree* tree, Uint); typedef struct { - RBTNode snode; - RBTNode anode; + RBTNode snode; /* node in 'stree' */ + RBTNode anode; /* node in 'atree' */ char* start; char* end; }ErtsFreeSegDesc; typedef struct { - RBTree stree; - RBTree atree; + RBTree stree; /* size ordered tree */ + RBTree atree; /* address ordered tree */ Uint nseg; }ErtsFreeSegMap; @@ -277,8 +277,8 @@ static ERTS_INLINE ErtsFreeSegDesc* node_to_desc(enum SortOrder order, RBTNode* } #ifdef HARD_DEBUG -static ERTS_INLINE SWord cmp_blocks(enum SortOrder order, - RBTNode* lhs, RBTNode* rhs) +static ERTS_INLINE SWord cmp_nodes(enum SortOrder order, + RBTNode* lhs, RBTNode* rhs) { ErtsFreeSegDesc* ldesc = node_to_desc(order, lhs); ErtsFreeSegDesc* rdesc = node_to_desc(order, rhs); @@ -296,10 +296,10 @@ static ERTS_INLINE SWord cmp_blocks(enum SortOrder order, return (char*)rdesc->start - (char*)ldesc->start; } } -#endif +#endif /* HARD_DEBUG */ -static ERTS_INLINE SWord cmp_with_block(enum SortOrder order, - SWord sz, char* addr, RBTNode* rhs) +static ERTS_INLINE SWord cmp_with_node(enum SortOrder order, + SWord sz, char* addr, RBTNode* rhs) { ErtsFreeSegDesc* rdesc; if (order != ADDR_ORDER) { @@ -340,10 +340,6 @@ left_rotate(RBTNode **root, RBTNode *x) } y->left = x; set_parent(x, y); - - /*SVERK y->max_sz = x->max_sz; - x->max_sz = node_max_size(x); - ASSERT(y->max_sz >= x->max_sz);*/ } static ERTS_INLINE void @@ -366,14 +362,11 @@ right_rotate(RBTNode **root, RBTNode *x) } y->right = x; set_parent(x, y); - /*SVERK y->max_sz = x->max_sz; - x->max_sz = node_max_size(x); - ASSERT(y->max_sz >= x->max_sz);*/ } /* * Replace node x with node y - * NOTE: block header of y is not changed + * NOTE: segment descriptor of y is not changed */ static ERTS_INLINE void replace(RBTNode **root, RBTNode *x, RBTNode *y) @@ -398,17 +391,15 @@ replace(RBTNode **root, RBTNode *x, RBTNode *y) set_parent(x->right, y); } - /*y->flags = x->flags;*/ y->parent_and_color = x->parent_and_color; y->right = x->right; y->left = x->left; - /*SVERK y->max_sz = x->max_sz;*/ } static void -tree_insert_fixup(RBTNode** root, RBTNode *blk) +tree_insert_fixup(RBTNode** root, RBTNode *node) { - RBTNode *x = blk, *y, *papa_x, *granpa_x; + RBTNode *x = node, *y, *papa_x, *granpa_x; /* * Rearrange the tree so that it satisfies the Red-Black Tree properties @@ -501,14 +492,15 @@ tree_insert_fixup(RBTNode** root, RBTNode *blk) } static void -rbt_delete(RBTNode** root, RBTNode* del) +rbt_delete(RBTree* tree, RBTNode* del) { Uint spliced_is_black; RBTNode *x, *y, *z = del, *papa_y; RBTNode null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ - HARD_CHECK_IS_MEMBER(*root, del); + HARD_CHECK_IS_MEMBER(tree->root, del); + HARD_CHECK_TREE(tree, 0); null_x.parent_and_color = parent_and_color(NULL, !RED_FLG); @@ -529,17 +521,14 @@ rbt_delete(RBTNode** root, RBTNode* del) } else if (spliced_is_black) { x = &null_x; - /*x->flags = 0; - SET_BLACK(x);*/ x->right = x->left = NULL; - /*SVERK x->max_sz = 0;*/ x->parent_and_color = parent_and_color(papa_y, !RED_FLG); y->left = x; } if (!papa_y) { - RBT_ASSERT(*root == y); - *root = x; + RBT_ASSERT(tree->root == y); + tree->root = x; } else { if (y == papa_y->left) { @@ -549,15 +538,11 @@ rbt_delete(RBTNode** root, RBTNode* del) RBT_ASSERT(y == papa_y->right); papa_y->right = x; } - /*SVERK if (y->parent != z) { - lower_max_size(y->parent, (y==z ? NULL : z)); - }*/ } if (y != z) { /* We spliced out the successor of z; replace z by the successor */ RBT_ASSERT(z != &null_x); - replace(root, z, y); - /*SVERK lower_max_size(y, NULL);*/ + replace(&tree->root, z, y); } if (spliced_is_black) { @@ -584,7 +569,7 @@ rbt_delete(RBTNode** root, RBTNode* del) SET_BLACK(y); RBT_ASSERT(IS_BLACK(papa_x)); SET_RED(papa_x); - left_rotate(root, papa_x); + left_rotate(&tree->root, papa_x); RBT_ASSERT(papa_x == parent(x)); y = papa_x->right; } @@ -597,7 +582,7 @@ rbt_delete(RBTNode** root, RBTNode* del) if (IS_BLACK(y->right)) { SET_BLACK(y->left); SET_RED(y); - right_rotate(root, y); + right_rotate(&tree->root, y); RBT_ASSERT(papa_x == parent(x)); y = papa_x->right; } @@ -609,8 +594,8 @@ rbt_delete(RBTNode** root, RBTNode* del) } RBT_ASSERT(y->right); SET_BLACK(y->right); - left_rotate(root, papa_x); - x = *root; + left_rotate(&tree->root, papa_x); + x = tree->root; break; } } @@ -624,7 +609,7 @@ rbt_delete(RBTNode** root, RBTNode* del) SET_BLACK(y); RBT_ASSERT(IS_BLACK(papa_x)); SET_RED(papa_x); - right_rotate(root, papa_x); + right_rotate(&tree->root, papa_x); RBT_ASSERT(papa_x == parent(x)); y = papa_x->left; } @@ -637,7 +622,7 @@ rbt_delete(RBTNode** root, RBTNode* del) if (IS_BLACK(y->left)) { SET_BLACK(y->right); SET_RED(y); - left_rotate(root, y); + left_rotate(&tree->root, y); RBT_ASSERT(papa_x == parent(x)); y = papa_x->left; } @@ -648,8 +633,8 @@ rbt_delete(RBTNode** root, RBTNode* del) } RBT_ASSERT(y->left); SET_BLACK(y->left); - right_rotate(root, papa_x); - x = *root; + right_rotate(&tree->root, papa_x); + x = tree->root; break; } } @@ -669,49 +654,44 @@ rbt_delete(RBTNode** root, RBTNode* del) RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); } - else if (*root == &null_x) { - *root = NULL; + else if (tree->root == &null_x) { + tree->root = NULL; RBT_ASSERT(!null_x.left); RBT_ASSERT(!null_x.right); } } + HARD_CHECK_TREE(tree, 0); } static void -rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) +rbt_insert(enum SortOrder order, RBTree* tree, RBTNode* node) { #ifdef RBT_DEBUG ErtsFreeSegDesc *dbg_under=NULL, *dbg_over=NULL; #endif - ErtsFreeSegDesc* desc = node_to_desc(order, blk); - char* blk_addr = desc->start; - SWord blk_sz = desc->end - desc->start; - /*SVERK Uint blk_sz = AOFF_BLK_SZ(blk);*/ - - /*blk->flags = 0;*/ - blk->left = NULL; - blk->right = NULL; - /*SVERK blk->max_sz = blk_sz;*/ - - if (!*root) { - blk->parent_and_color = parent_and_color(NULL, !RED_FLG); - /*SET_BLACK(blk);*/ - *root = blk; + ErtsFreeSegDesc* desc = node_to_desc(order, node); + char* seg_addr = desc->start; + SWord seg_sz = desc->end - desc->start; + + HARD_CHECK_TREE(tree, 0); + + node->left = NULL; + node->right = NULL; + + if (!tree->root) { + node->parent_and_color = parent_and_color(NULL, !RED_FLG); + tree->root = node; } else { - RBTNode *x = *root; + RBTNode *x = tree->root; while (1) { - SWord diff; - /*SVERK if (x->max_sz < blk_sz) { - x->max_sz = blk_sz; - }*/ - diff = cmp_with_block(order, blk_sz, blk_addr, x); + SWord diff = cmp_with_node(order, seg_sz, seg_addr, x); if (diff < 0) { IF_RBT_DEBUG(dbg_over = node_to_desc(order, x)); if (!x->left) { - blk->parent_and_color = parent_and_color(x, RED_FLG); - x->left = blk; + node->parent_and_color = parent_and_color(x, RED_FLG); + x->left = node; break; } x = x->left; @@ -720,43 +700,26 @@ rbt_insert(enum SortOrder order, RBTNode** root, RBTNode* blk) RBT_ASSERT(diff > 0); IF_RBT_DEBUG(dbg_under = node_to_desc(order, x)); if (!x->right) { - blk->parent_and_color = parent_and_color(x, RED_FLG); - x->right = blk; + node->parent_and_color = parent_and_color(x, RED_FLG); + x->right = node; break; } x = x->right; } - /*SVERK else { - ASSERT(flavor == AOFF_BF); - ASSERT(blk->flags & IS_BF_FLG); - ASSERT(x->flags & IS_BF_FLG); - SET_LIST_ELEM(blk); - LIST_NEXT(blk) = LIST_NEXT(x); - LIST_PREV(blk) = x; - if (LIST_NEXT(x)) - LIST_PREV(LIST_NEXT(x)) = blk; - LIST_NEXT(x) = blk; - return; - }*/ } - /* Insert block into size tree */ - RBT_ASSERT(parent(blk)); + RBT_ASSERT(parent(node)); #ifdef RBT_DEBUG - if (!order) { + if (order == ADDR_ORDER) { RBT_ASSERT(!dbg_under || dbg_under->end < desc->start); RBT_ASSERT(!dbg_over || dbg_over->start > desc->end); } #endif - /*SET_RED(blk);*/ - RBT_ASSERT(IS_RED(blk)); - if (IS_RED(parent(blk))) - tree_insert_fixup(root, blk); + RBT_ASSERT(IS_RED(node)); + if (IS_RED(parent(node))) + tree_insert_fixup(&tree->root, node); } - /*SVERK if (flavor == AOFF_BF) { - SET_TREE_NODE(blk); - LIST_NEXT(blk) = NULL; - }*/ + HARD_CHECK_TREE(tree, 0); } /* @@ -860,9 +823,39 @@ rbt_foreach_node(RBTree* tree, #endif } +#ifdef RBT_DEBUG +static RBTNode* rbt_prev_node(RBTNode* node) +{ + RBTNode* x; + if (node->left) { + for (x=node->left; x->right; x=x->right) + ; + return x; + } + for (x=node; parent(x); x=parent(x)) { + if (parent(x)->right == x) + return parent(x); + } + return NULL; +} +static RBTNode* rbt_next_node(RBTNode* node) +{ + RBTNode* x; + if (node->right) { + for (x=node->right; x->left; x=x->left) + ; + return x; + } + for (x=node; parent(x); x=parent(x)) { + if (parent(x)->left == x) + return parent(x); + } + return NULL; +} +#endif /* RBT_DEBUG */ -/* The API to keep track of a bunch of separated free segments +/* The API to keep track of a bunch of separated (free) segments (non-overlapping and non-adjacent). */ static void init_free_seg_map(ErtsFreeSegMap*, int reverse_ao); @@ -883,6 +876,9 @@ static void init_free_seg_map(ErtsFreeSegMap* map, int reverse_ao) map->nseg = 0; } +/* Lookup directly adjacent free segments to the given area [start->end]. + * The given area must not contain any free segments. + */ static void adjacent_free_seg(ErtsFreeSegMap* map, char* start, char* end, ErtsFreeSegDesc** under, ErtsFreeSegDesc** over) { @@ -910,39 +906,49 @@ static void adjacent_free_seg(ErtsFreeSegMap* map, char* start, char* end, } } +/* Initialize 'desc' and insert as new free segment [start->end]. + * The new segment must not contain or be adjacent to any free segment in 'map'. + */ static void insert_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, char* start, char* end) { desc->start = start; desc->end = end; - rbt_insert(map->atree.order, &map->atree.root, &desc->anode); - rbt_insert(map->stree.order, &map->stree.root, &desc->snode); + rbt_insert(map->atree.order, &map->atree, &desc->anode); + rbt_insert(map->stree.order, &map->stree, &desc->snode); map->nseg++; } +/* Resize existing free segment 'desc' to [start->end]. + * The new segment location must overlap the old location and + * it must not contain or be adjacent to any other free segment in 'map'. + */ static void resize_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, char* start, char* end) { -#ifdef DEBUG - ErtsFreeSegDesc *dbg_under, *dbg_over; - rbt_delete(&map->atree.root, &desc->anode); - adjacent_free_seg(map, start, end, &dbg_under, &dbg_over); - RBT_ASSERT(dbg_under == NULL && dbg_over == NULL); - rbt_insert(map->atree.order, &map->atree.root, &desc->anode); +#ifdef RBT_DEBUG + RBTNode* prev = rbt_prev_node(&desc->anode); + RBTNode* next = rbt_next_node(&desc->anode); + RBT_ASSERT(!prev || anode_to_desc(prev)->end < start); + RBT_ASSERT(!next || anode_to_desc(next)->start > end); #endif - rbt_delete(&map->stree.root, &desc->snode); + rbt_delete(&map->stree, &desc->snode); desc->start = start; desc->end = end; - rbt_insert(map->stree.order, &map->stree.root, &desc->snode); + rbt_insert(map->stree.order, &map->stree, &desc->snode); } +/* Delete existing free segment 'desc' from 'map'. + */ static void delete_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc) { - rbt_delete(&map->atree.root, &desc->anode); - rbt_delete(&map->stree.root, &desc->snode); + rbt_delete(&map->atree, &desc->anode); + rbt_delete(&map->stree, &desc->snode); map->nseg--; } +/* Lookup a free segment in 'map' with a size of at least 'need_sz' bytes. + */ static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap* map, SWord need_sz) { RBTNode* x = map->stree.root; @@ -1934,7 +1940,7 @@ static int rbt_assert_is_member(RBTNode* root, RBTNode* node) } -#if 1 /*SVERK*/ +#if 0 # define PRINT_TREE #else # undef PRINT_TREE @@ -1964,7 +1970,7 @@ struct check_arg_t { Uint size; RBTNode *res; }; -static void check_node(RBTNode* x, void* arg); +static void check_node_callback(RBTNode* x, void* arg); static RBTNode * @@ -1986,13 +1992,12 @@ check_tree(RBTree* tree, Uint size) RBT_ASSERT(IS_BLACK(tree->root)); RBT_ASSERT(!parent(tree->root)); - rbt_foreach_node(tree, check_node, &carg, 0); + rbt_foreach_node(tree, check_node_callback, &carg, 0); return carg.res; } -/* callback */ -static void check_node(RBTNode* x, void* arg) +static void check_node_callback(RBTNode* x, void* arg) { struct check_arg_t* a = (struct check_arg_t*) arg; ErtsFreeSegDesc* seg; @@ -2005,16 +2010,16 @@ static void check_node(RBTNode* x, void* arg) RBT_ASSERT(parent(x) || x == a->tree->root); if (x->left) { - RBT_ASSERT(cmp_blocks(a->tree->order, x->left, x) < 0); + RBT_ASSERT(cmp_nodes(a->tree->order, x->left, x) < 0); } if (x->right) { - RBT_ASSERT(cmp_blocks(a->tree->order, x->right, x) > 0); + RBT_ASSERT(cmp_nodes(a->tree->order, x->right, x) > 0); } seg = node_to_desc(a->tree->order, x); RBT_ASSERT(seg->start < seg->end); if (a->size && (seg->end - seg->start) >= a->size) { - if (!a->res || cmp_blocks(a->tree->order, x, a->res) < 0) { + if (!a->res || cmp_nodes(a->tree->order, x, a->res) < 0) { a->res = x; } } @@ -2063,6 +2068,8 @@ print_tree(enum SortOrder order, RBTNode* root) #endif /* PRINT_TREE */ +#ifdef FREE_SEG_API_SMOKE_TEST + void test_it(void) { ErtsFreeSegMap map; @@ -2128,3 +2135,5 @@ void test_it(void) ERTS_ASSERT(d1->start == (char*)0x15000); } } + +#endif /* FREE_SEG_API_SMOKE_TEST */ -- cgit v1.2.3 From d04e3d43f3a96030db0c9b8da8f88cb78f8ec8dc Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 5 Sep 2013 14:45:26 +0200 Subject: erts: Improve erts_mmap out of free descriptor management --- erts/emulator/sys/common/erl_mmap.c | 100 +++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 40 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 723efa772f..0ac08a0004 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -222,6 +222,7 @@ static struct { mmap_state.size.os.used -= (SZ); \ } while (0) + static void add_free_desc_area(char *start, char *end) { @@ -1164,6 +1165,39 @@ static void unreserve_noop(char *ptr, UWord size) #endif } +static ERTS_INLINE UWord +alloc_desc_insert_free_seg(ErtsFreeSegMap *map, char* start, char* end) +{ + UWord ad_sz; + char *new_end; + ErtsFreeSegDesc *desc = alloc_desc(); + if (desc) { + insert_free_seg(map, desc, start, end); + return 0; + } + + /* Use part of the free segment for descriptors */ + + if (map == &mmap_state.sa.map) { + ERTS_MMAP_SIZE_SC_SA_INC(ERTS_SUPERALIGNED_SIZE); + ad_sz = ERTS_SUPERALIGNED_SIZE; + } + else { + ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE); + ad_sz = ERTS_PAGEALIGNED_SIZE; + } + + new_end = end - ad_sz; + ERTS_MMAP_ASSERT(start <= new_end); + if (start != new_end) { + desc = alloc_desc(); + ERTS_MMAP_ASSERT(desc); + insert_free_seg(map, desc, start, new_end); + } + + return ad_sz; +} + void * erts_mmap(Uint32 flags, UWord *sizep) { @@ -1248,6 +1282,7 @@ erts_mmap(Uint32 flags, UWord *sizep) end = seg + asize; if (!mmap_state.reserve_physical(seg, asize)) goto supercarrier_reserve_failure; + ERTS_MMAP_SIZE_SC_SUA_INC(asize); if (org_start != seg) { ERTS_MMAP_ASSERT(org_start < seg); resize_free_seg(&mmap_state.sua.map, desc, org_start, seg); @@ -1257,15 +1292,10 @@ erts_mmap(Uint32 flags, UWord *sizep) ERTS_MMAP_ASSERT(end < org_end); if (desc) resize_free_seg(&mmap_state.sua.map, desc, end, org_end); - else { - desc = alloc_desc(); - if (!desc) - add_free_desc_area(end, org_end); - else - insert_free_seg(&mmap_state.sua.map, desc, end, org_end); - } + else + alloc_desc_insert_free_seg(&mmap_state.sua.map, + end, org_end); } - ERTS_MMAP_SIZE_SC_SA_INC(asize); goto supercarrier_success; } } @@ -1361,6 +1391,7 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) char *start, *end; ErtsFreeSegMap *map; ErtsFreeSegDesc *prev, *next, *desc; + UWord ad_sz = 0; ERTS_MMAP_ASSERT(mmap_state.supercarrier); @@ -1432,23 +1463,19 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) if (desc) resize_free_seg(map, desc, start, end); - else { - desc = alloc_desc(); - if (desc) - insert_free_seg(map, desc, start, end); - else { - if (map == &mmap_state.sa.map) - ERTS_MMAP_SIZE_SC_SA_INC(size); - else - ERTS_MMAP_SIZE_SC_SUA_INC(size); - add_free_desc_area(start, end); - } - } + else + ad_sz = alloc_desc_insert_free_seg(map, start, end); + + supercarrier_success: { + UWord unres_sz; - supercarrier_success: - erts_smp_mtx_unlock(&mmap_state.mtx); + erts_smp_mtx_unlock(&mmap_state.mtx); - mmap_state.unreserve_physical((char *) ptr, size); + ERTS_MMAP_ASSERT(size >= ad_sz); + unres_sz = size - ad_sz; + if (unres_sz) + mmap_state.unreserve_physical((char *) ptr, unres_sz); + } } } @@ -1548,7 +1575,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) else { /* In super carrier */ char *start, *end, *new_end; ErtsFreeSegMap *map; - ErtsFreeSegDesc *prev, *next, *desc; + ErtsFreeSegDesc *prev, *next; + UWord ad_sz = 0; ERTS_MMAP_ASSERT(mmap_state.supercarrier); @@ -1586,6 +1614,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) new_end = start+asize; if (asize < old_size) { + UWord unres_sz; new_ptr = ptr; if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); @@ -1620,21 +1649,13 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) if (next) resize_free_seg(map, next, new_end, next->end); - else { - desc = alloc_desc(); - if (desc) - insert_free_seg(map, desc, new_end, end); - else { - if (map == &mmap_state.sa.map) - ERTS_MMAP_SIZE_SC_SA_INC(old_size - asize); - else - ERTS_MMAP_SIZE_SC_SUA_INC(old_size - asize); - add_free_desc_area(new_end, end); - goto supercarrier_resize_success; - } - } - mmap_state.unreserve_physical(((char *) ptr) + asize, - old_size - asize); + else + ad_sz = alloc_desc_insert_free_seg(map, new_end, end); + ERTS_MMAP_ASSERT(old_size - asize >= ad_sz); + unres_sz = old_size - asize - ad_sz; + if (unres_sz) + mmap_state.unreserve_physical(((char *) ptr) + asize, + unres_sz); goto supercarrier_resize_success; } @@ -1921,7 +1942,6 @@ erts_mmap_init(ErtsMMapInit *init) #endif } - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Debug functions * \* */ -- cgit v1.2.3 From 0820017c421bfab27d23aff4da474974f988006c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 5 Sep 2013 19:23:07 +0200 Subject: erts: Add mmap argument to erts_debug:get_internal_state --- erts/emulator/beam/erl_bif_info.c | 3 +++ erts/emulator/sys/common/erl_mmap.c | 48 ++++++++++++++++++++++++++++++++++++- erts/emulator/sys/common/erl_mmap.h | 2 ++ 3 files changed, 52 insertions(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index a4f9f787cd..7aa439f2e6 100755 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3289,6 +3289,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) erts_smp_thr_progress_unblock(); BIF_RET(res); } + else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { + BIF_RET(erts_mmap_info(BIF_P)); + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 0ac08a0004..1d18c1fcc9 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -23,6 +23,7 @@ #include "sys.h" #include "erl_process.h" #include "erl_smp.h" +#include "atom.h" #include "erl_mmap.h" #include @@ -741,7 +742,7 @@ rbt_foreach_node(RBTree* tree, enum { RECURSE_LEFT, DO_NODE, RECURSE_RIGHT, RETURN_TO_PARENT }state; RBTNode *x = tree->root; - RBT_ASSERT(!parent(x)); + RBT_ASSERT(!x || !parent(x)); state = reverse ? RECURSE_RIGHT : RECURSE_LEFT; while (x) { @@ -1942,6 +1943,51 @@ erts_mmap_init(ErtsMMapInit *init) #endif } +Eterm erts_mmap_info(Process* p) +{ + if (mmap_state.supercarrier) { + ERTS_DECL_AM(sabot); + ERTS_DECL_AM(satop); + ERTS_DECL_AM(suabot); + ERTS_DECL_AM(suatop); + Eterm sa_list, sua_list, list; + Eterm tags[] = { AM_sabot, AM_satop, AM_suabot, AM_suatop }; + UWord values[4]; + Eterm *hp, *hp_end; + Uint may_need; + const Uint PTR_BIG_SZ = HALFWORD_HEAP ? 3 : 2; + + erts_smp_mtx_lock(&mmap_state.mtx); + values[0] = (UWord)mmap_state.sa.bot; + values[1] = (UWord)mmap_state.sa.top; + values[2] = (UWord)mmap_state.sua.bot; + values[3] = (UWord)mmap_state.sua.top; + sa_list = build_free_seg_list(p, &mmap_state.sa.map); + sua_list = build_free_seg_list(p, &mmap_state.sua.map); + erts_smp_mtx_unlock(&mmap_state.mtx); + + may_need = 4*(2+3+PTR_BIG_SZ) + 2*(2+3); + hp = HAlloc(p, may_need); + hp_end = hp + may_need; + + list = erts_bld_atom_uint_2tup_list(&hp, NULL, + sizeof(values)/sizeof(*values), + tags, values); + + sa_list = TUPLE2(hp, am_atom_put("sa_free_segs",12), sa_list); hp+=3; + sua_list = TUPLE2(hp, am_atom_put("sua_free_segs",13), sua_list); hp+=3; + list = CONS(hp, sua_list, list); hp+=2; + list = CONS(hp, sa_list, list); hp+=2; + + ASSERT(hp <= hp_end); + HRelease(p, hp_end, hp); + return list; + } + else { + return am_undefined; + } +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Debug functions * \* */ diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 143f1aff3e..b75200f4e9 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -57,6 +57,8 @@ void erts_munmap(Uint32 flags, void **ptrp, UWord *sizep); void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep); int erts_mmap_in_supercarrier(void *ptr); void erts_mmap_init(ErtsMMapInit*); +struct process; +Eterm erts_mmap_info(struct process*); #define ERTS_SUPERALIGNED_SIZE \ (1 << ERTS_MMAP_SUPERALIGNED_BITS) -- cgit v1.2.3 From 2d64c6e31966d9e63d1aa1835d41ded22f799175 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 6 Sep 2013 19:34:02 +0200 Subject: erts: Fix ASSERT bug and void* arithmetics --- erts/emulator/sys/common/erl_mseg.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index b21d6ca393..09035ca73e 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -515,7 +515,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag if (MSEG_FLG_IS_2POW(flags)) { int i, ix = SIZE_TO_CACHE_AREA_IDX(size); - void *seg; + char *seg; cache_t *c; Uint csize; @@ -533,7 +533,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag ASSERT(MAP_IS_ALIGNED(c->seg)); csize = c->size; - seg = c->seg; + seg = (char*) c->seg; mk->cache_size--; mk->cache_hits++; @@ -545,11 +545,11 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag ASSERT(!(mk->cache_size < 0)); if (csize != size) { - void *destr_seg = ((char *) seg) + size; + char *destr_seg = seg + size; UWord destr_size = csize - size; - mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, &destr_seg, &destr_size); + mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, (void**)&destr_seg, &destr_size); *size_p = (UWord) (destr_seg - seg); - ASSERT(c->seg + c->size == destr_seg + destr_size); + ASSERT(seg + csize == destr_seg + destr_size); } return seg; -- cgit v1.2.3 From ac32bccf21354d7e6896d8b83b6e9a45bb1bccd7 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 9 Sep 2013 15:49:21 +0200 Subject: erts: Sort tree in super aligned sizes (SA_SZ_ADDR_ORDER) --- erts/emulator/sys/common/erl_mmap.c | 52 +++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 20 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 1d18c1fcc9..eae1a1a410 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -106,9 +106,12 @@ static ERTS_INLINE UWord parent_and_color(RBTNode* parent, int color) enum SortOrder { ADDR_ORDER, /* only address order */ - SZ_ADDR_ORDER, /* first size then address order as tiebreaker */ + SA_SZ_ADDR_ORDER, /* first super-aligned size then address order */ SZ_REVERSE_ADDR_ORDER /* first size then reverse address order */ }; +#ifdef HARD_DEBUG +static const char* sort_order_names[] = {"Address","SuperAlignedSize-Address","Size-RevAddress"}; +#endif typedef struct { RBTNode* root; @@ -286,9 +289,17 @@ static ERTS_INLINE SWord cmp_nodes(enum SortOrder order, ErtsFreeSegDesc* rdesc = node_to_desc(order, rhs); RBT_ASSERT(lhs != rhs); if (order != ADDR_ORDER) { - SWord lsz = ldesc->end - ldesc->start; - SWord rsz = rdesc->end - rdesc->start; - SWord diff = lsz - rsz; + SWord lsz, rsz, diff; + if (order == SA_SZ_ADDR_ORDER) { + lsz = ERTS_SUPERALIGNED_FLOOR(ldesc->end) - ERTS_SUPERALIGNED_CEILING(ldesc->start); + rsz = ERTS_SUPERALIGNED_FLOOR(rdesc->end) - ERTS_SUPERALIGNED_CEILING(rdesc->start); + } + else { + RBT_ASSERT(order == SZ_REVERSE_ADDR_ORDER); + lsz = ldesc->end - ldesc->start; + rsz = rdesc->end - rdesc->start; + } + diff = lsz - rsz; if (diff) return diff; } if (order != SZ_REVERSE_ADDR_ORDER) { @@ -305,12 +316,14 @@ static ERTS_INLINE SWord cmp_with_node(enum SortOrder order, { ErtsFreeSegDesc* rdesc; if (order != ADDR_ORDER) { + SWord rhs_sz, diff; rdesc = snode_to_desc(rhs); - { - SWord rhs_sz = rdesc->end - rdesc->start; - SWord diff = sz - rhs_sz; - if (diff) return diff; - } + if (order == SA_SZ_ADDR_ORDER) + rhs_sz = ERTS_SUPERALIGNED_FLOOR(rdesc->end) - ERTS_SUPERALIGNED_CEILING(rdesc->start); + else + rhs_sz = rdesc->end - rdesc->start; + diff = sz - rhs_sz; + if (diff) return diff; } else rdesc = anode_to_desc(rhs); @@ -860,7 +873,7 @@ static RBTNode* rbt_next_node(RBTNode* node) /* The API to keep track of a bunch of separated (free) segments (non-overlapping and non-adjacent). */ -static void init_free_seg_map(ErtsFreeSegMap*, int reverse_ao); +static void init_free_seg_map(ErtsFreeSegMap*, enum SortOrder); static void adjacent_free_seg(ErtsFreeSegMap*, char* start, char* end, ErtsFreeSegDesc** under, ErtsFreeSegDesc** over); static void insert_free_seg(ErtsFreeSegMap*, ErtsFreeSegDesc*, char* start, char* end); @@ -869,12 +882,12 @@ static void delete_free_seg(ErtsFreeSegMap*, ErtsFreeSegDesc*); static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap*, SWord sz); -static void init_free_seg_map(ErtsFreeSegMap* map, int reverse_ao) +static void init_free_seg_map(ErtsFreeSegMap* map, enum SortOrder order) { map->atree.root = NULL; map->atree.order = ADDR_ORDER; map->stree.root = NULL; - map->stree.order = reverse_ao ? SZ_REVERSE_ADDR_ORDER : SZ_ADDR_ORDER; + map->stree.order = order; map->nseg = 0; } @@ -1931,8 +1944,8 @@ erts_mmap_init(ErtsMMapInit *init) #endif add_free_desc_area(mmap_state.sua.top, end); - init_free_seg_map(&mmap_state.sa.map, 0); - init_free_seg_map(&mmap_state.sua.map, 1); + init_free_seg_map(&mmap_state.sa.map, SA_SZ_ADDR_ORDER); + init_free_seg_map(&mmap_state.sua.map, SZ_REVERSE_ADDR_ORDER); mmap_state.supercarrier = 1; erts_have_erts_mmap |= ERTS_HAVE_ERTS_SUPERCARRIER_MMAP; @@ -2125,10 +2138,9 @@ print_tree_aux(enum SortOrder order, RBTNode *x, int indent) static void print_tree(enum SortOrder order, RBTNode* root) { - static const char* type[] = {"Address","Size-Address","Size-RevAddress"}; - fprintf(stderr, " --- %s ordered tree begin ---\r\n", type[order]); + fprintf(stderr, " --- %s ordered tree begin ---\r\n", sort_order_names[order]); print_tree_aux(order, root, 0); - fprintf(stderr, " --- %s ordered tree end ---\r\n", type[order]); + fprintf(stderr, " --- %s ordered tree end ---\r\n", sort_order_names[order]); } #endif /* PRINT_TREE */ @@ -2140,10 +2152,10 @@ void test_it(void) { ErtsFreeSegMap map; ErtsFreeSegDesc *desc, *under, *over, *d1, *d2; - int i; + const int i = 1; /* reverse addr order */ - for (i=0; i<2; i++) { - init_free_seg_map(&map, i); + { + init_free_seg_map(&map, SZ_REVERSE_ADDR_ORDER); insert_free_seg(&map, alloc_desc(), (char*)0x11000, (char*)0x12000); HARD_CHECK_TREE(&map.atree, 0); HARD_CHECK_TREE(&map.stree, 0); -- cgit v1.2.3 From c8f87b4ec91b67c9d3373c8466a07db638e32cc2 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 9 Sep 2013 16:07:29 +0200 Subject: erts: Allow page aligned erts_munmap() --- erts/emulator/sys/common/erl_mmap.c | 102 +++++++++++++++++------------------- erts/emulator/sys/common/erl_mmap.h | 2 +- erts/emulator/sys/common/erl_mseg.c | 70 +++++++------------------ 3 files changed, 67 insertions(+), 107 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index eae1a1a410..73874759f5 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -1260,30 +1260,53 @@ erts_mmap(Uint32 flags, UWord *sizep) desc = lookup_free_seg(&mmap_state.sa.map, asize); if (desc) { - seg = desc->start; + char *start = seg = desc->start; + seg = (char *) ERTS_SUPERALIGNED_CEILING(seg); end = seg+asize; - if (!mmap_state.reserve_physical(seg, asize)) + if (!mmap_state.reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; + ERTS_MMAP_SIZE_SC_SA_INC(asize); if (desc->end == end) { - delete_free_seg(&mmap_state.sa.map, desc); - free_desc(desc); + if (start != seg) + resize_free_seg(&mmap_state.sa.map, desc, start, seg); + else { + delete_free_seg(&mmap_state.sa.map, desc); + free_desc(desc); + } } else { ERTS_MMAP_ASSERT(end < desc->end); resize_free_seg(&mmap_state.sa.map, desc, end, desc->end); + if (start != seg) { + UWord ad_sz; + ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map, + start, seg); + start += ad_sz; + if (start != seg) + mmap_state.unreserve_physical(start, (UWord) (seg - start)); + } } - ERTS_MMAP_SIZE_SC_SA_INC(asize); goto supercarrier_success; } if (superaligned) { + seg = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.top); - if (asize <= mmap_state.sua.bot - mmap_state.sa.top) { - seg = (void *) mmap_state.sa.top; - if (!mmap_state.reserve_physical(seg, asize)) + if (asize <= mmap_state.sua.bot - seg) { + char *start = mmap_state.sa.top; + end = seg + asize; + if (!mmap_state.reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; - mmap_state.sa.top += asize; + mmap_state.sa.top = end; ERTS_MMAP_SIZE_SC_SA_INC(asize); + if (start != seg) { + UWord ad_sz; + ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map, + start, seg); + start += ad_sz; + if (start != seg) + mmap_state.unreserve_physical(start, (UWord) (seg - start)); + } goto supercarrier_success; } @@ -1294,7 +1317,7 @@ erts_mmap(Uint32 flags, UWord *sizep) seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start); end = seg + asize; - if (!mmap_state.reserve_physical(seg, asize)) + if (!mmap_state.reserve_physical(seg, (UWord) (org_end - seg))) goto supercarrier_reserve_failure; ERTS_MMAP_SIZE_SC_SUA_INC(asize); if (org_start != seg) { @@ -1303,12 +1326,17 @@ erts_mmap(Uint32 flags, UWord *sizep) desc = NULL; } if (end != org_end) { + UWord ad_sz = 0; ERTS_MMAP_ASSERT(end < org_end); if (desc) resize_free_seg(&mmap_state.sua.map, desc, end, org_end); else - alloc_desc_insert_free_seg(&mmap_state.sua.map, - end, org_end); + ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map, + end, org_end); + end += ad_sz; + if (end != org_end) + mmap_state.unreserve_physical(end, + (UWord) (org_end - end)); } goto supercarrier_success; } @@ -1363,8 +1391,7 @@ erts_mmap(Uint32 flags, UWord *sizep) supercarrier_success: #ifdef ERTS_MMAP_DEBUG - if ((ERTS_MMAPFLG_SUPERALIGNED & flags) - || ERTS_MMAP_IN_SUPERALIGNED_AREA(seg)) { + if (ERTS_MMAPFLG_SUPERALIGNED & flags) { ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(seg)); ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); } @@ -1388,10 +1415,8 @@ supercarrier_reserve_failure: } void -erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) +erts_munmap(Uint32 flags, void *ptr, UWord size) { - void *ptr = *ptrp; - UWord size = *sizep; ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size)); if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) { @@ -1416,13 +1441,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { - start = (char *) ERTS_SUPERALIGNED_CEILING(start); - end = (char *) ERTS_SUPERALIGNED_FLOOR(end); - - size = (UWord) (end - start); - *ptrp = start; - *sizep = size; - map = &mmap_state.sa.map; adjacent_free_seg(map, start, end, &prev, &next); @@ -1439,8 +1457,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep) } } else { - ERTS_MMAP_ASSERT(ERTS_MMAP_IN_SUPERUNALIGNED_AREA(ptr)); - map = &mmap_state.sua.map; adjacent_free_seg(map, start, end, &prev, &next); @@ -1497,8 +1513,6 @@ static void * remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) { UWord size = *sizep; - UWord um_size = old_size; - void *um_ptr = ptr; void *new_ptr = erts_mmap(flags, &size); if (!new_ptr) return NULL; @@ -1506,9 +1520,7 @@ remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) if (old_size < size) size = old_size; sys_memcpy(new_ptr, ptr, (size_t) size); - erts_munmap(flags, &um_ptr, &um_size); - ERTS_MMAP_ASSERT(um_ptr == ptr); - ERTS_MMAP_ASSERT(um_size == old_size); + erts_munmap(flags, ptr, old_size); return new_ptr; } @@ -1627,28 +1639,18 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) end = start + old_size; new_end = start+asize; + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size)); + ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); + if (asize < old_size) { UWord unres_sz; new_ptr = ptr; if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { - ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); - ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size)); - ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize)); map = &mmap_state.sua.map; ERTS_MMAP_SIZE_SC_SUA_DEC(old_size - asize); } else { - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr)); - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size)); - if (!superaligned) { - /* must be a superaligned size in this area */ - asize = ERTS_SUPERALIGNED_CEILING(asize); - ERTS_MMAP_ASSERT(asize <= old_size); - if (asize == old_size) - goto supercarrier_resize_success; - new_end = start+asize; - } - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); if (end == mmap_state.sa.top) { mmap_state.sa.top = new_end; mmap_state.unreserve_physical(((char *) ptr) + asize, @@ -1696,16 +1698,6 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) } } else { /* Superaligned area */ - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr)); - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size)); - - if (!superaligned) { - /* must be a superaligned size in this area */ - asize = ERTS_PAGEALIGNED_CEILING(asize); - new_end = start+asize; - } - - ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize)); if (end == mmap_state.sa.top) { if (new_end <= mmap_state.sua.bot) { diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index b75200f4e9..6cb51fb0b4 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -53,7 +53,7 @@ typedef struct { {{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1} void *erts_mmap(Uint32 flags, UWord *sizep); -void erts_munmap(Uint32 flags, void **ptrp, UWord *sizep); +void erts_munmap(Uint32 flags, void *ptr, UWord size); void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep); int erts_mmap_in_supercarrier(void *ptr); void erts_mmap_init(ErtsMMapInit*); diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 09035ca73e..2474015bcb 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -330,7 +330,7 @@ mseg_create(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, UWord *sizep) } static ERTS_INLINE void -mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord *size_p) { +mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void *seg_p, UWord size) { Uint32 mmap_flags = 0; #if HALFWORD_HEAP @@ -341,11 +341,11 @@ mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord if (MSEG_FLG_IS_2POW(flags)) mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; - erts_munmap(mmap_flags, seg_pp, size_p); + erts_munmap(mmap_flags, seg_p, size); #ifdef ERTS_PRINT_ERTS_MMAP erts_fprintf(stderr, "erts_munmap(%s, %p, %bpu);\n", (mmap_flags & ERTS_MMAPFLG_SUPERALIGNED) ? "sa" : "sua", - *seg_pp, *size_p); + seg_p, *size); #endif INC_CC(ma, destroy); @@ -446,19 +446,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui return 1; } else if (!MSEG_FLG_IS_2POW(flags) && !erts_circleq_is_empty(&(mk->cache_unpowered_node))) { - void *destr_seg; - UWord destr_size; /* No free slots. * Evict oldest slot from unpowered cache so we can cache an unpowered (sbc) segment */ c = erts_circleq_tail(&(mk->cache_unpowered_node)); erts_circleq_remove(c); - destr_seg = c->seg; - destr_size = c->size; - mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, c->seg, c->size); mseg_cache_clear_node(c); c->seg = seg; @@ -478,19 +472,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui int i; for( i = 0; i < CACHE_AREAS; i++) { - void *destr_seg; - UWord destr_size; if (erts_circleq_is_empty(&(mk->cache_powered_node[i]))) continue; c = erts_circleq_tail(&(mk->cache_powered_node[i])); erts_circleq_remove(c); - destr_seg = seg; - destr_size = c->size; - mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, c->seg, c->size); mseg_cache_clear_node(c); @@ -544,13 +532,8 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag ASSERT(!(mk->cache_size < 0)); - if (csize != size) { - char *destr_seg = seg + size; - UWord destr_size = csize - size; - mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, (void**)&destr_seg, &destr_size); - *size_p = (UWord) (destr_seg - seg); - ASSERT(seg + csize == destr_seg + destr_size); - } + if (csize != size) + mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, seg + size, csize - size); return seg; } @@ -625,8 +608,6 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag */ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags, cache_t *head) { - void *destr_seg; - UWord destr_size; cache_t *c = NULL; c = erts_circleq_tail(head); @@ -635,11 +616,7 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg); - destr_seg = c->seg; - destr_size = c->size; - mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, flags, mk, c->seg, c->size); mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); @@ -655,8 +632,6 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca cache_t *c = NULL; while (!erts_circleq_is_empty(head)) { - void *destr_seg; - UWord destr_size; c = erts_circleq_tail(head); erts_circleq_remove(c); @@ -664,11 +639,7 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg); - destr_seg = c->seg; - destr_size = c->size; - mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == c->seg); - ASSERT(destr_size == c->size); + mseg_destroy(mk->ma, flags, mk, c->seg, c->size); mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); @@ -813,12 +784,14 @@ mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, UWord *size_p, INC_CC(ma, alloc); - if (!MSEG_FLG_IS_2POW(flags) && !IS_2POW(size)) + if (!MSEG_FLG_IS_2POW(flags)) size = ERTS_PAGEALIGNED_CEILING(*size_p); else { size = ALIGNED_CEILING(*size_p); - /* Cache optim (if applicable) */ - size = ceil_2pow(size); + if (!IS_2POW(size)) { + /* Cache optim (if applicable) */ + size = ceil_2pow(size); + } } if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size, flags)) != NULL) @@ -845,8 +818,6 @@ static void mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size, Uint flags, const ErtsMsegOpt_t *opt) { - void *destr_seg; - UWord destr_size; MemKind* mk = memkind(ma, opt); ERTS_MSEG_DEALLOC_STAT(mk,size); @@ -859,11 +830,7 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size, if (erts_mtrace_enabled) erts_mtrace_crr_free(atype, SEGTYPE, seg); - destr_seg = seg; - destr_size = size; - mseg_destroy(ma, flags, mk, &destr_seg, &destr_size); - ASSERT(destr_seg == seg); - ASSERT(destr_size == size); + mseg_destroy(ma, flags, mk, seg, size); done: @@ -896,13 +863,14 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, mk = memkind(ma, opt); new_seg = seg; - if (!MSEG_FLG_IS_2POW(flags)) new_size = ERTS_PAGEALIGNED_CEILING(*new_size_p); else { new_size = ALIGNED_CEILING(*new_size_p); - /* Cache optim (if applicable) */ - new_size = ceil_2pow(new_size); + if (!IS_2POW(new_size)) { + /* Cache optim (if applicable) */ + new_size = ceil_2pow(new_size); + } } if (new_size > old_size) { -- cgit v1.2.3 From ffbd1fe3fe4fdd1657f98d650eb3b40139e4b115 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 10 Sep 2013 15:33:33 +0200 Subject: erts: Add documentation for +MMsc* system flags --- erts/doc/src/erts_alloc.xml | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) (limited to 'erts') diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index 6ce2261430..9dab5b3876 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -271,6 +271,65 @@ memory segment cache is not reused if its size exceeds the requested size with more than relative max cache bad fit percent of the requested size. Default value is 20. + ]]> + + Set super carrier max guaranteed + no of carriers. This parameter defaults to 65536. This + parameter determines an amount of pre-allocated structures that is + needed in order to keep track of different areas in the super carrier. + When the system runs out of such structures it may crash due to an + out of memory condition. + + + + Set super carrier only flag. This + flag defaults to true. When a super carrier is used and this + flag is true, the system will crash when a carrier request + cannot be satisfied by the super carrier. When the flag is false + the system will try to create requested carrier by other means. +

+ NOTE: Setting this flag to false may not be supported + on all systems. This flag will in that case be ignored. +

+ NOTE: The super carrier cannot be enabled nor + disabled on halfword heap systems. This flag will be + ignored on halfword heap systems. + + + + Set super carrier reserve physical + memory flag. This flag defaults to true. When this flag is + true, physical memory will be reserved for the whole super + carrier at once when it is created. The reservation will after that + be left unchanged. When this flag is set to false only virtual + address space will be reserved for the super carrier upon creation. + The system will attempt to reserve physical memory upon carrier + creations in the super carrier, and attempt to unreserve physical + memory upon carrier destructions in the super carrier. +

+ NOTE: What reservation of physical memory actually means + highly depends on the operating system, and how it is configured. For + example, different memory overcommit settings on Linux drastically + change the behaviour. Also note, setting this flag to false + may not be supported on all systems. This flag will in that case + be ignored. +

+ NOTE: The super carrier cannot be enabled nor + disabled on halfword heap systems. This flag will be + ignored on halfword heap systems. + + ]]> + + Set super carrier size (in MB). The super carrier size defaults to + zero; i.e, the super carrier is by default disabled. The super + carrier is a large continuous area in the virtual address space. + The system will always try to create new carriers in the super + carrier. +

+ NOTE: The super carrier cannot be enabled nor + disabled on halfword heap systems. This flag will be + ignored on halfword heap systems. + ]]> Max cached segments. The maximum number of memory segments -- cgit v1.2.3 From e107965576ccd0cfd4f235e463cd6cc8da11a259 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 13 Sep 2013 02:07:03 +0200 Subject: erts: erts_mmap improved free seg desc management --- erts/emulator/sys/common/erl_mmap.c | 389 +++++++++++++++++++++++++++++++----- erts/emulator/sys/common/erl_mseg.c | 22 +- 2 files changed, 349 insertions(+), 62 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 73874759f5..1f9168df28 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -27,9 +27,18 @@ #include "erl_mmap.h" #include +/* #define ERTS_MMAP_OP_RINGBUF_SZ 100 */ + #if defined(DEBUG) || 0 # undef ERTS_MMAP_DEBUG # define ERTS_MMAP_DEBUG +# ifndef ERTS_MMAP_OP_RINGBUF_SZ +# define ERTS_MMAP_OP_RINGBUF_SZ 100 +# endif +#endif + +#ifndef ERTS_MMAP_OP_RINGBUF_SZ +# define ERTS_MMAP_OP_RINGBUF_SZ 0 #endif /* #define ERTS_MMAP_DEBUG_FILL_AREAS */ @@ -128,6 +137,148 @@ static RBTNode* check_tree(RBTree* tree, Uint); # define HARD_CHECK_TREE(TREE,SZ) #endif +#if ERTS_MMAP_OP_RINGBUF_SZ + +static int mmap_op_ix; + +typedef enum { + ERTS_OP_TYPE_NONE, + ERTS_OP_TYPE_MMAP, + ERTS_OP_TYPE_MUNMAP, + ERTS_OP_TYPE_MREMAP +} ErtsMMapOpType; + +typedef struct { + ErtsMMapOpType type; + void *result; + UWord in_size; + UWord out_size; + void *old_ptr; + UWord old_size; +} ErtsMMapOp; + +static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; + +#define ERTS_MMAP_OP_RINGBUF_INIT() \ + do { \ + int ix__; \ + for (ix__ = 0; ix__ < ERTS_MMAP_OP_RINGBUF_SZ; ix__++) {\ + mmap_ops[ix__].type = ERTS_OP_TYPE_NONE; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = 0; \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = NULL; \ + mmap_ops[ix__].old_size = 0; \ + } \ + mmap_op_ix = ERTS_MMAP_OP_RINGBUF_SZ-1; \ + } while (0) + +#define ERTS_MMAP_OP_START(SZ) \ + do { \ + int ix__; \ + if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \ + mmap_op_ix = 0; \ + ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_MMAP; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = (SZ); \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = NULL; \ + mmap_ops[ix__].old_size = 0; \ + } while (0) + +#define ERTS_MMAP_OP_END(PTR, SZ) \ + do { \ + int ix__ = mmap_op_ix; \ + mmap_ops[ix__].result = (PTR); \ + mmap_ops[ix__].out_size = (SZ); \ + } while (0) + +#define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ) \ + do { \ + erts_smp_mtx_lock(&mmap_state.mtx); \ + ERTS_MMAP_OP_START((IN_SZ)); \ + ERTS_MMAP_OP_END((RES), (OUT_SZ)); \ + erts_smp_mtx_unlock(&mmap_state.mtx); \ + } while (0) + +#define ERTS_MUNMAP_OP(PTR, SZ) \ + do { \ + int ix__; \ + if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \ + mmap_op_ix = 0; \ + ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_MUNMAP; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = 0; \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = (PTR); \ + mmap_ops[ix__].old_size = (SZ); \ + } while (0) + +#define ERTS_MUNMAP_OP_LCK(PTR, SZ) \ + do { \ + erts_smp_mtx_lock(&mmap_state.mtx); \ + ERTS_MUNMAP_OP((PTR), (SZ)); \ + erts_smp_mtx_unlock(&mmap_state.mtx); \ + } while (0) + +#define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ) \ + do { \ + int ix__; \ + if (++mmap_op_ix >= ERTS_MMAP_OP_RINGBUF_SZ) \ + mmap_op_ix = 0; \ + ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_MREMAP; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = (IN_SZ); \ + mmap_ops[ix__].out_size = (OLD_SZ); \ + mmap_ops[ix__].old_ptr = (OLD_PTR); \ + mmap_ops[ix__].old_size = (OLD_SZ); \ + } while (0) + +#define ERTS_MREMAP_OP_END(PTR, SZ) \ + do { \ + int ix__ = mmap_op_ix; \ + mmap_ops[ix__].result = (PTR); \ + mmap_ops[mmap_op_ix].out_size = (SZ); \ + } while (0) + +#define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ) \ + do { \ + erts_smp_mtx_lock(&mmap_state.mtx); \ + ERTS_MREMAP_OP_START((OLD_PTR), (OLD_SZ), (IN_SZ)); \ + ERTS_MREMAP_OP_END((RES), (OUT_SZ)); \ + erts_smp_mtx_unlock(&mmap_state.mtx); \ + } while (0) + +#define ERTS_MMAP_OP_ABORT() \ + do { \ + int ix__ = mmap_op_ix; \ + mmap_ops[ix__].type = ERTS_OP_TYPE_NONE; \ + mmap_ops[ix__].result = NULL; \ + mmap_ops[ix__].in_size = 0; \ + mmap_ops[ix__].out_size = 0; \ + mmap_ops[ix__].old_ptr = NULL; \ + mmap_ops[ix__].old_size = 0; \ + if (--mmap_op_ix < 0) \ + mmap_op_ix = ERTS_MMAP_OP_RINGBUF_SZ-1; \ + } while (0) + +#else + +#define ERTS_MMAP_OP_RINGBUF_INIT() +#define ERTS_MMAP_OP_START(SZ) +#define ERTS_MMAP_OP_END(PTR, SZ) +#define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ) +#define ERTS_MUNMAP_OP(PTR, SZ) +#define ERTS_MUNMAP_OP_LCK(PTR, SZ) +#define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ) +#define ERTS_MREMAP_OP_END(PTR, SZ) +#define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ) +#define ERTS_MMAP_OP_ABORT() + +#endif typedef struct { RBTNode snode; /* node in 'stree' */ @@ -167,7 +318,19 @@ static struct { int mmap_fd; #endif erts_smp_mtx_t mtx; - char *desc_free_list; + struct { + char *free_list; + char *unused_start; + char *unused_end; + char *new_area_hint; + } desc; + struct { + UWord free_seg_descs; + struct { + UWord curr; + UWord max; + } free_segs; + } no; struct { struct { UWord total; @@ -230,12 +393,15 @@ static struct { static void add_free_desc_area(char *start, char *end) { - if (end > start && sizeof(ErtsFreeSegDesc) <= end - start) { + ERTS_MMAP_ASSERT(end == (void *) 0 || end > start); + if (sizeof(ErtsFreeSegDesc) <= ((UWord) end) - ((UWord) start)) { + UWord no; ErtsFreeSegDesc *prev_desc, *desc; char *desc_end; + no = 1; prev_desc = (ErtsFreeSegDesc *) start; - prev_desc->start = mmap_state.desc_free_list; + prev_desc->start = mmap_state.desc.free_list; desc = (ErtsFreeSegDesc *) (start + sizeof(ErtsFreeSegDesc)); desc_end = start + 2*sizeof(ErtsFreeSegDesc); @@ -244,26 +410,61 @@ add_free_desc_area(char *start, char *end) prev_desc = desc; desc = (ErtsFreeSegDesc *) desc_end; desc_end += sizeof(ErtsFreeSegDesc); + no++; } - mmap_state.desc_free_list = (char *) prev_desc; + mmap_state.desc.free_list = (char *) prev_desc; + mmap_state.no.free_seg_descs += no; } } +static ErtsFreeSegDesc * +add_unused_free_desc_area(void) +{ + char *ptr; + if (!mmap_state.desc.unused_start) + return NULL; + + ERTS_MMAP_ASSERT(mmap_state.desc.unused_end); + ERTS_MMAP_ASSERT(ERTS_PAGEALIGNED_SIZE + <= mmap_state.desc.unused_end - mmap_state.desc.unused_start); + + ptr = mmap_state.desc.unused_start + ERTS_PAGEALIGNED_SIZE; + add_free_desc_area(mmap_state.desc.unused_start, ptr); + + if ((mmap_state.desc.unused_end - ptr) >= ERTS_PAGEALIGNED_SIZE) + mmap_state.desc.unused_start = ptr; + else + mmap_state.desc.unused_end = mmap_state.desc.unused_start = NULL; + + ERTS_MMAP_ASSERT(mmap_state.desc.free_list); + return (ErtsFreeSegDesc *) mmap_state.desc.free_list; +} + static ERTS_INLINE ErtsFreeSegDesc * alloc_desc(void) { ErtsFreeSegDesc *res; - res = (ErtsFreeSegDesc *) mmap_state.desc_free_list; - if (res) - mmap_state.desc_free_list = res->start; + res = (ErtsFreeSegDesc *) mmap_state.desc.free_list; + if (!res) { + res = add_unused_free_desc_area(); + if (!res) + return NULL; + } + mmap_state.desc.free_list = res->start; + ASSERT(mmap_state.no.free_segs.curr < mmap_state.no.free_seg_descs); + mmap_state.no.free_segs.curr++; + if (mmap_state.no.free_segs.max < mmap_state.no.free_segs.curr) + mmap_state.no.free_segs.max = mmap_state.no.free_segs.curr; return res; } static ERTS_INLINE void free_desc(ErtsFreeSegDesc *desc) { - desc->start = mmap_state.desc_free_list; - mmap_state.desc_free_list = (char *) desc; + desc->start = mmap_state.desc.free_list; + mmap_state.desc.free_list = (char *) desc; + ERTS_MMAP_ASSERT(mmap_state.no.free_segs.curr > 0); + mmap_state.no.free_segs.curr--; } static ERTS_INLINE ErtsFreeSegDesc* anode_to_desc(RBTNode* anode) @@ -1040,7 +1241,7 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map) #endif static ERTS_INLINE void * -os_mmap(UWord size, int try_superalign) +os_mmap(void *hint_ptr, UWord size, int try_superalign) { #if HAVE_MMAP void *res; @@ -1050,7 +1251,7 @@ os_mmap(UWord size, int try_superalign) ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0); else #endif - res = mmap((void *) 0, size, ERTS_MMAP_PROT, + res = mmap((void *) hint_ptr, size, ERTS_MMAP_PROT, ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0); if (res == MAP_FAILED) return NULL; @@ -1179,37 +1380,90 @@ static void unreserve_noop(char *ptr, UWord size) #endif } -static ERTS_INLINE UWord +static UWord alloc_desc_insert_free_seg(ErtsFreeSegMap *map, char* start, char* end) { - UWord ad_sz; - char *new_end; + char *ptr; + ErtsFreeSegMap *da_map; ErtsFreeSegDesc *desc = alloc_desc(); if (desc) { insert_free_seg(map, desc, start, end); return 0; } - /* Use part of the free segment for descriptors */ + /* + * Ahh; ran out of free segment descriptors. + * + * First try to map a new page... + */ - if (map == &mmap_state.sa.map) { - ERTS_MMAP_SIZE_SC_SA_INC(ERTS_SUPERALIGNED_SIZE); - ad_sz = ERTS_SUPERALIGNED_SIZE; +#if ERTS_HAVE_OS_MMAP + ptr = os_mmap(mmap_state.desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0); + if (ptr) { + mmap_state.desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE; + ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE); + add_free_desc_area(ptr, ptr+ERTS_PAGEALIGNED_SIZE); + desc = alloc_desc(); + ERTS_MMAP_ASSERT(desc); + insert_free_seg(map, desc, start, end); + return 0; + } +#endif + + /* + * ...then try to find a good place in the supercarrier... + */ + da_map = &mmap_state.sua.map; + desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); + if (desc) { + if (mmap_state.reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) + ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE); + else + desc = NULL; + } else { - ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE); - ad_sz = ERTS_PAGEALIGNED_SIZE; + da_map = &mmap_state.sa.map; + desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); + if (desc) { + if (mmap_state.reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) + ERTS_MMAP_SIZE_SC_SA_INC(ERTS_PAGEALIGNED_SIZE); + else + desc = NULL; + } } + if (desc) { + char *da_end = desc->start + ERTS_PAGEALIGNED_SIZE; + add_free_desc_area(desc->start, da_end); + if (da_end != desc->end) + resize_free_seg(da_map, desc, da_end, desc->end); + else { + delete_free_seg(da_map, desc); + free_desc(desc); + } - new_end = end - ad_sz; - ERTS_MMAP_ASSERT(start <= new_end); - if (start != new_end) { desc = alloc_desc(); ERTS_MMAP_ASSERT(desc); - insert_free_seg(map, desc, start, new_end); + insert_free_seg(map, desc, start, end); + return 0; } - return ad_sz; + /* + * ... and then as last resort use the first page of the + * free segment we are trying to insert for free descriptors. + */ + ptr = start + ERTS_PAGEALIGNED_SIZE; + ERTS_MMAP_ASSERT(ptr <= end); + + add_free_desc_area(start, ptr); + + if (ptr != end) { + desc = alloc_desc(); + ERTS_MMAP_ASSERT(desc); + insert_free_seg(map, desc, ptr, end); + } + + return ERTS_PAGEALIGNED_SIZE; } void * @@ -1226,6 +1480,8 @@ erts_mmap(Uint32 flags, UWord *sizep) erts_smp_mtx_lock(&mmap_state.mtx); + ERTS_MMAP_OP_START(*sizep); + if (!superaligned) { desc = lookup_free_seg(&mmap_state.sua.map, asize); if (desc) { @@ -1290,10 +1546,10 @@ erts_mmap(Uint32 flags, UWord *sizep) } if (superaligned) { - seg = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.top); + char *start = mmap_state.sa.top; + seg = (char *) ERTS_SUPERALIGNED_CEILING(start); - if (asize <= mmap_state.sua.bot - seg) { - char *start = mmap_state.sa.top; + if (asize + (seg - start) <= mmap_state.sua.bot - start) { end = seg + asize; if (!mmap_state.reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; @@ -1342,6 +1598,7 @@ erts_mmap(Uint32 flags, UWord *sizep) } } + ERTS_MMAP_OP_ABORT(); erts_smp_mtx_unlock(&mmap_state.mtx); } @@ -1349,15 +1606,15 @@ erts_mmap(Uint32 flags, UWord *sizep) /* Map using OS primitives */ if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mmap_state.no_os_mmap) { if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) { - seg = os_mmap(asize, 0); + seg = os_mmap(NULL, asize, 0); if (!seg) - return NULL; + goto failure; } else { asize = ERTS_SUPERALIGNED_CEILING(*sizep); - seg = os_mmap(asize, 1); + seg = os_mmap(NULL, asize, 1); if (!seg) - return NULL; + goto failure; if (!ERTS_IS_SUPERALIGNED(seg)) { char *ptr; @@ -1365,9 +1622,9 @@ erts_mmap(Uint32 flags, UWord *sizep) os_munmap(seg, asize); - ptr = os_mmap(asize + ERTS_SUPERALIGNED_SIZE, 1); + ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1); if (!ptr) - return NULL; + goto failure; seg = (char *) ERTS_SUPERALIGNED_CEILING(ptr); sz = (UWord) (seg - ptr); @@ -1380,11 +1637,14 @@ erts_mmap(Uint32 flags, UWord *sizep) } } + ERTS_MMAP_OP_LCK(seg, *sizep, asize); ERTS_MMAP_SIZE_OS_INC(asize); *sizep = asize; return (void *) seg; } +failure: #endif + ERTS_MMAP_OP_LCK(NULL, *sizep, 0); *sizep = 0; return NULL; @@ -1401,6 +1661,7 @@ supercarrier_success: } #endif + ERTS_MMAP_OP_END(seg, asize); erts_smp_mtx_unlock(&mmap_state.mtx); *sizep = asize; @@ -1408,10 +1669,8 @@ supercarrier_success: supercarrier_reserve_failure: erts_smp_mtx_unlock(&mmap_state.mtx); - *sizep = 0; return NULL; - } void @@ -1419,9 +1678,11 @@ erts_munmap(Uint32 flags, void *ptr, UWord size) { ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr)); ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size)); + if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) { ERTS_MMAP_ASSERT(!mmap_state.no_os_mmap); #if ERTS_HAVE_OS_MMAP + ERTS_MUNMAP_OP_LCK(ptr, size); ERTS_MMAP_SIZE_OS_DEC(size); os_munmap(ptr, size); #endif @@ -1439,6 +1700,8 @@ erts_munmap(Uint32 flags, void *ptr, UWord size) erts_smp_mtx_lock(&mmap_state.mtx); + ERTS_MUNMAP_OP(ptr, size); + if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) { map = &mmap_state.sa.map; @@ -1504,7 +1767,7 @@ erts_munmap(Uint32 flags, void *ptr, UWord size) ERTS_MMAP_ASSERT(size >= ad_sz); unres_sz = size - ad_sz; if (unres_sz) - mmap_state.unreserve_physical((char *) ptr, unres_sz); + mmap_state.unreserve_physical(((char *) ptr) + ad_sz, unres_sz); } } } @@ -1546,8 +1809,10 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) return new_ptr; } - if (ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) + if (ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) { + ERTS_MREMAP_OP_LCK(NULL, ptr, old_size, *sizep, old_size); return NULL; + } #if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); @@ -1555,6 +1820,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) if (superaligned) { asize = ERTS_SUPERALIGNED_CEILING(*sizep); if (asize == old_size && ERTS_IS_SUPERALIGNED(ptr)) { + ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize); *sizep = asize; return ptr; } @@ -1562,6 +1828,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) else { asize = ERTS_PAGEALIGNED_CEILING(*sizep); if (asize == old_size) { + ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize); *sizep = asize; return ptr; } @@ -1577,6 +1844,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) um_sz = (UWord) ((((char *) ptr) + old_size) - (char *) new_ptr); ERTS_MMAP_SIZE_OS_DEC(um_sz); os_munmap(new_ptr, um_sz); + ERTS_MREMAP_OP_LCK(ptr, ptr, old_size, *sizep, asize); *sizep = asize; return ptr; } @@ -1592,6 +1860,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) ERTS_MMAP_SIZE_OS_INC(asize - old_size); else ERTS_MMAP_SIZE_OS_DEC(old_size - asize); + ERTS_MREMAP_OP_LCK(new_ptr, ptr, old_size, *sizep, asize); *sizep = asize; return new_ptr; } @@ -1630,6 +1899,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) old_size, sizep); } + ERTS_MREMAP_OP_START(ptr, old_size, *sizep); + if (asize == old_size) { new_ptr = ptr; goto supercarrier_resize_success; @@ -1670,7 +1941,7 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) ERTS_MMAP_ASSERT(old_size - asize >= ad_sz); unres_sz = old_size - asize - ad_sz; if (unres_sz) - mmap_state.unreserve_physical(((char *) ptr) + asize, + mmap_state.unreserve_physical(((char *) ptr) + asize + ad_sz, unres_sz); goto supercarrier_resize_success; } @@ -1728,6 +1999,8 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep) } } } + + ERTS_MMAP_OP_ABORT(); erts_smp_mtx_unlock(&mmap_state.mtx); /* Failed to resize... */ @@ -1749,15 +2022,16 @@ supercarrier_resize_success: } #endif + ERTS_MREMAP_OP_END(new_ptr, asize); erts_smp_mtx_unlock(&mmap_state.mtx); *sizep = asize; return new_ptr; supercarrier_reserve_failure: - + ERTS_MREMAP_OP_END(NULL, old_size); erts_smp_mtx_unlock(&mmap_state.mtx); - *sizep = 0; + *sizep = old_size; return NULL; } @@ -1794,8 +2068,11 @@ erts_mmap_init(ErtsMMapInit *init) erl_exit(-1, "erts_mmap: Invalid pagesize: %bpu\n", pagesize); + ERTS_MMAP_OP_RINGBUF_INIT(); + erts_have_erts_mmap = 0; + mmap_state.supercarrier = 0; mmap_state.reserve_physical = reserve_noop; mmap_state.unreserve_physical = unreserve_noop; @@ -1851,7 +2128,7 @@ erts_mmap_init(ErtsMMapInit *init) * The whole supercarrier will by physically * reserved all the time. */ - start = os_mmap(sz, 1); + start = os_mmap(NULL, sz, 1); } if (!start) erl_exit(-1, @@ -1871,12 +2148,18 @@ erts_mmap_init(ErtsMMapInit *init) erts_have_erts_mmap |= ERTS_HAVE_ERTS_OS_MMAP; #endif + mmap_state.no.free_seg_descs = 0; + mmap_state.no.free_segs.curr = 0; + mmap_state.no.free_segs.max = 0; + mmap_state.size.supercarrier.total = 0; mmap_state.size.supercarrier.used.total = 0; mmap_state.size.supercarrier.used.sa = 0; mmap_state.size.supercarrier.used.sua = 0; mmap_state.size.os.used = 0; + mmap_state.desc.new_area_hint = NULL; + if (!start) { mmap_state.sa.bot = NULL; mmap_state.sua.top = NULL; @@ -1907,6 +2190,8 @@ erts_mmap_init(ErtsMMapInit *init) mmap_state.size.os.used += (UWord) (mmap_state.sa.bot - start); + mmap_state.desc.free_list = NULL; + if (end == (void *) 0) { /* * Very unlikely, but we need a guarantee @@ -1916,6 +2201,10 @@ erts_mmap_init(ErtsMMapInit *init) */ mmap_state.sua.top -= ERTS_PAGEALIGNED_SIZE; mmap_state.size.os.used += ERTS_PAGEALIGNED_SIZE; +#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + if (!virtual_map || os_reserve_physical(mmap_state.sua.top, ERTS_PAGEALIGNED_SIZE)) +#endif + add_free_desc_area(mmap_state.sua.top, end); } mmap_state.size.supercarrier.total = (UWord) (mmap_state.sua.top - mmap_state.sa.bot); @@ -1924,23 +2213,21 @@ erts_mmap_init(ErtsMMapInit *init) * Area before (and after) super carrier * will be used for free segment descritors. */ - mmap_state.desc_free_list = NULL; -#ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (virtual_map && mmap_state.sa.bot - start > 0) - os_reserve_physical(start, mmap_state.sa.bot - start); -#endif - add_free_desc_area(start, mmap_state.sa.bot); #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (virtual_map && end - mmap_state.sua.top > 0) - os_reserve_physical(mmap_state.sua.top, end - mmap_state.sua.top); + if (virtual_map && !os_reserve_physical(start, mmap_state.sa.bot - start)) + erl_exit(-1, "erts_mmap: Failed to reserve physical memory for descriptors\n"); #endif - add_free_desc_area(mmap_state.sua.top, end); + mmap_state.desc.unused_start = start; + mmap_state.desc.unused_end = mmap_state.sa.bot; init_free_seg_map(&mmap_state.sa.map, SA_SZ_ADDR_ORDER); init_free_seg_map(&mmap_state.sua.map, SZ_REVERSE_ADDR_ORDER); mmap_state.supercarrier = 1; erts_have_erts_mmap |= ERTS_HAVE_ERTS_SUPERCARRIER_MMAP; + + mmap_state.desc.new_area_hint = end; + } #if !ERTS_HAVE_OS_MMAP diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 2474015bcb..c65973b3be 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -132,7 +132,7 @@ typedef struct { typedef struct cache_t_ cache_t; struct cache_t_ { - Uint size; + UWord size; void *seg; cache_t *next; cache_t *prev; @@ -408,7 +408,7 @@ static ERTS_INLINE void mseg_cache_clear_node(cache_t *c) { c->prev = c; } -static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Uint flags) { +static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, UWord size, Uint flags) { cache_t *c; ERTS_DBG_MK_CHK_THR_ACCESS(mk); @@ -496,7 +496,7 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flags) { - Uint size = (UWord) *size_p; + UWord size = *size_p; ERTS_DBG_MK_CHK_THR_ACCESS(mk); @@ -505,7 +505,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag int i, ix = SIZE_TO_CACHE_AREA_IDX(size); char *seg; cache_t *c; - Uint csize; + UWord csize; ASSERT(IS_2POW(size)); @@ -542,10 +542,10 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag void *seg; cache_t *c; cache_t *best = NULL; - Uint bdiff = 0; - Uint csize; - Uint bad_max_abs = mk->ma->abs_max_cache_bad_fit; - Uint bad_max_rel = mk->ma->rel_max_cache_bad_fit; + UWord bdiff = 0; + UWord csize; + UWord bad_max_abs = mk->ma->abs_max_cache_bad_fit; + UWord bad_max_rel = mk->ma->rel_max_cache_bad_fit; erts_circleq_foreach(c, &(mk->cache_unpowered_node)) { csize = c->size; @@ -562,7 +562,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag mseg_cache_clear_node(c); erts_circleq_push_head(&(mk->cache_free), c); - *size_p = (UWord) csize; + *size_p = csize; return seg; @@ -593,7 +593,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag ASSERT((size % GET_PAGE_SIZE) == 0); ASSERT((best->size % GET_PAGE_SIZE) == 0); - *size_p = (UWord) size; + *size_p = size; return seg; @@ -822,7 +822,7 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size, ERTS_MSEG_DEALLOC_STAT(mk,size); - if (opt->cache && cache_bless_segment(mk, seg, (Uint) size, flags)) { + if (opt->cache && cache_bless_segment(mk, seg, size, flags)) { schedule_cache_check(ma); goto done; } -- cgit v1.2.3 From 19e47f24c1fe3dc996e836da0e5f99cea86acdbd Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 12 Sep 2013 17:52:33 +0200 Subject: erts: Refactor rbt_insert in erl_mmap --- erts/emulator/sys/common/erl_mmap.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 1f9168df28..1876398ec4 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -881,12 +881,12 @@ rbt_delete(RBTree* tree, RBTNode* del) static void -rbt_insert(enum SortOrder order, RBTree* tree, RBTNode* node) +rbt_insert(RBTree* tree, RBTNode* node) { #ifdef RBT_DEBUG ErtsFreeSegDesc *dbg_under=NULL, *dbg_over=NULL; #endif - ErtsFreeSegDesc* desc = node_to_desc(order, node); + ErtsFreeSegDesc* desc = node_to_desc(tree->order, node); char* seg_addr = desc->start; SWord seg_sz = desc->end - desc->start; @@ -902,9 +902,9 @@ rbt_insert(enum SortOrder order, RBTree* tree, RBTNode* node) else { RBTNode *x = tree->root; while (1) { - SWord diff = cmp_with_node(order, seg_sz, seg_addr, x); + SWord diff = cmp_with_node(tree->order, seg_sz, seg_addr, x); if (diff < 0) { - IF_RBT_DEBUG(dbg_over = node_to_desc(order, x)); + IF_RBT_DEBUG(dbg_over = node_to_desc(tree->order, x)); if (!x->left) { node->parent_and_color = parent_and_color(x, RED_FLG); x->left = node; @@ -914,7 +914,7 @@ rbt_insert(enum SortOrder order, RBTree* tree, RBTNode* node) } else { RBT_ASSERT(diff > 0); - IF_RBT_DEBUG(dbg_under = node_to_desc(order, x)); + IF_RBT_DEBUG(dbg_under = node_to_desc(tree->order, x)); if (!x->right) { node->parent_and_color = parent_and_color(x, RED_FLG); x->right = node; @@ -926,7 +926,7 @@ rbt_insert(enum SortOrder order, RBTree* tree, RBTNode* node) RBT_ASSERT(parent(node)); #ifdef RBT_DEBUG - if (order == ADDR_ORDER) { + if (tree->order == ADDR_ORDER) { RBT_ASSERT(!dbg_under || dbg_under->end < desc->start); RBT_ASSERT(!dbg_over || dbg_over->start > desc->end); } @@ -1130,8 +1130,8 @@ static void insert_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, { desc->start = start; desc->end = end; - rbt_insert(map->atree.order, &map->atree, &desc->anode); - rbt_insert(map->stree.order, &map->stree, &desc->snode); + rbt_insert(&map->atree, &desc->anode); + rbt_insert(&map->stree, &desc->snode); map->nseg++; } @@ -1151,7 +1151,7 @@ static void resize_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc, rbt_delete(&map->stree, &desc->snode); desc->start = start; desc->end = end; - rbt_insert(map->stree.order, &map->stree, &desc->snode); + rbt_insert(&map->stree, &desc->snode); } /* Delete existing free segment 'desc' from 'map'. -- cgit v1.2.3 From b9e82ba0be1364c64e90274d5e9bf37f78b676ee Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 13 Sep 2013 13:31:05 +0200 Subject: erts: Add HARD_DBG_MSEG --- erts/emulator/beam/erl_lock_check.c | 1 + erts/emulator/sys/common/erl_mmap.c | 142 +++++++++++++++++++++++++++++++++++- erts/emulator/sys/common/erl_mmap.h | 11 +++ erts/emulator/sys/common/erl_mseg.c | 6 ++ 4 files changed, 158 insertions(+), 2 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 1e9cef3759..87efbdbc3e 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -186,6 +186,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { #endif #endif { "erts_alloc_hard_debug", NULL }, + { "hard_dbg_mseg", NULL }, { "erts_mmap", NULL } }; diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 1876398ec4..93a95e5eef 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -1039,7 +1039,7 @@ rbt_foreach_node(RBTree* tree, #endif } -#ifdef RBT_DEBUG +#if defined(RBT_DEBUG) || defined(HARD_DEBUG_MSEG) static RBTNode* rbt_prev_node(RBTNode* node) { RBTNode* x; @@ -1068,7 +1068,7 @@ static RBTNode* rbt_next_node(RBTNode* node) } return NULL; } -#endif /* RBT_DEBUG */ +#endif /* RBT_DEBUG || HARD_DEBUG_MSEG */ /* The API to keep track of a bunch of separated (free) segments @@ -2041,6 +2041,10 @@ int erts_mmap_in_supercarrier(void *ptr) return ERTS_MMAP_IN_SUPERCARRIER(ptr); } +#ifdef HARD_DEBUG_MSEG +static void hard_dbg_mseg_init(void); +#endif + void erts_mmap_init(ErtsMMapInit *init) { @@ -2233,6 +2237,10 @@ erts_mmap_init(ErtsMMapInit *init) #if !ERTS_HAVE_OS_MMAP mmap_state.no_os_mmap = 1; #endif + +#ifdef HARD_DEBUG_MSEG + hard_dbg_mseg_init(); +#endif } Eterm erts_mmap_info(Process* p) @@ -2494,3 +2502,133 @@ void test_it(void) } #endif /* FREE_SEG_API_SMOKE_TEST */ + + +#ifdef HARD_DEBUG_MSEG + +/* + * Debug stuff used by erl_mseg to check that it does the right thing. + * The reason for keeping it here is that we (ab)use the rb-tree code + * for keeping track of *allocated* segments. + */ + +typedef struct ErtsFreeSegDesc_fake_ { + /*RBTNode snode; Save memory by skipping unused size tree node */ + RBTNode anode; /* node in 'atree' */ + union { + char* start; + struct ErtsFreeSegDesc_fake_* next_free; + }u; + char* end; +}ErtsFreeSegDesc_fake; + +static ErtsFreeSegDesc_fake hard_dbg_mseg_desc_pool[10000]; +static ErtsFreeSegDesc_fake* hard_dbg_mseg_desc_first; +RBTree hard_dbg_mseg_tree; + +static erts_mtx_t hard_dbg_mseg_mtx; + +static void hard_dbg_mseg_init(void) +{ + ErtsFreeSegDesc_fake* p; + + erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg"); + hard_dbg_mseg_tree.root = NULL; + hard_dbg_mseg_tree.order = ADDR_ORDER; + + p = &hard_dbg_mseg_desc_pool[(sizeof(hard_dbg_mseg_desc_pool) / + sizeof(*hard_dbg_mseg_desc_pool)) - 1]; + p->u.next_free = NULL; + while (--p >= hard_dbg_mseg_desc_pool) { + p->u.next_free = (p+1); + } + hard_dbg_mseg_desc_first = &hard_dbg_mseg_desc_pool[0]; +} + +static ErtsFreeSegDesc* hard_dbg_alloc_desc(void) +{ + ErtsFreeSegDesc_fake* p = hard_dbg_mseg_desc_first; + ERTS_ASSERT(p || !"HARD_DEBUG_MSEG: Out of mseg descriptors"); + hard_dbg_mseg_desc_first = p->u.next_free; + + /* Creative pointer arithmetic to return something that looks like + * a ErtsFreeSegDesc as long as we don't use the absent 'snode'. + */ + return (ErtsFreeSegDesc*) ((char*)p - offsetof(ErtsFreeSegDesc,anode)); +} + +static void hard_dbg_free_desc(ErtsFreeSegDesc* desc) +{ + ErtsFreeSegDesc_fake* p = (ErtsFreeSegDesc_fake*) &desc->anode; + memset(p, 0xfe, sizeof(*p)); + p->u.next_free = hard_dbg_mseg_desc_first; + hard_dbg_mseg_desc_first = p; +} + +static void check_seg_writable(void* seg, UWord sz) +{ + UWord* seg_end = (UWord*)((char*)seg + sz); + volatile UWord* p; + ERTS_ASSERT(ERTS_IS_PAGEALIGNED(seg)); + ERTS_ASSERT(ERTS_IS_PAGEALIGNED(sz)); + for (p=(UWord*)seg; pstart = (char*)seg; + desc->end = desc->start + sz - 1; /* -1 to allow adjacent segments in tree */ + rbt_insert(&hard_dbg_mseg_tree, &desc->anode); + prev = rbt_prev_node(&desc->anode); + next = rbt_next_node(&desc->anode); + ERTS_ASSERT(!prev || anode_to_desc(prev)->end < desc->start); + ERTS_ASSERT(!next || anode_to_desc(next)->start > desc->end); + } + erts_mtx_unlock(&hard_dbg_mseg_mtx); +} + +static ErtsFreeSegDesc* hard_dbg_lookup_seg_at(RBTree* tree, char* start) +{ + RBTNode* x = tree->root; + + while (x) { + ErtsFreeSegDesc* desc = anode_to_desc(x); + if (start < desc->start) { + x = x->left; + } + else if (start > desc->start) { + ERTS_ASSERT(start > desc->end); + x = x->right; + } + else + return desc; + } + return NULL; +} + +void hard_dbg_remove_mseg(void* seg, UWord sz) +{ + check_seg_writable(seg, sz); + erts_mtx_lock(&hard_dbg_mseg_mtx); + { + ErtsFreeSegDesc* desc = hard_dbg_lookup_seg_at(&hard_dbg_mseg_tree, (char*)seg); + ERTS_ASSERT(desc); + ERTS_ASSERT(desc->start == (char*)seg); + ERTS_ASSERT(desc->end == (char*)seg + sz - 1); + + rbt_delete(&hard_dbg_mseg_tree, &desc->anode); + hard_dbg_free_desc(desc); + } + erts_mtx_unlock(&hard_dbg_mseg_mtx); +} + +#endif /* HARD_DEBUG_MSEG */ diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 6cb51fb0b4..106459f872 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -110,4 +110,15 @@ Eterm erts_mmap_info(struct process*); # define ERTS_HAVE_OS_MMAP 1 #endif +/*#define HARD_DEBUG_MSEG*/ +#ifdef HARD_DEBUG_MSEG +# define HARD_DBG_INSERT_MSEG hard_dbg_insert_mseg +# define HARD_DBG_REMOVE_MSEG hard_dbg_remove_mseg +void hard_dbg_insert_mseg(void* seg, UWord sz); +void hard_dbg_remove_mseg(void* seg, UWord sz); +#else +# define HARD_DBG_INSERT_MSEG(SEG,SZ) +# define HARD_DBG_REMOVE_MSEG(SEG,SZ) +#endif + #endif /* ERL_MMAP_H__ */ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index c65973b3be..13f8069cf5 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1356,6 +1356,7 @@ erts_mseg_alloc_opt(ErtsAlcType_t atype, UWord *size_p, Uint flags, const ErtsMs ERTS_DBG_MA_CHK_THR_ACCESS(ma); seg = mseg_alloc(ma, atype, size_p, flags, opt); ERTS_MSEG_UNLOCK(ma); + HARD_DBG_INSERT_MSEG(seg, *size_p); return seg; } @@ -1370,6 +1371,8 @@ erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, UWord size, Uint flags, const ErtsMsegOpt_t *opt) { ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); + + HARD_DBG_REMOVE_MSEG(seg, size); ERTS_MSEG_LOCK(ma); ERTS_DBG_MA_CHK_THR_ACCESS(ma); mseg_dealloc(ma, atype, seg, size, flags, opt); @@ -1390,10 +1393,13 @@ erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, { ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt); void *new_seg; + + HARD_DBG_REMOVE_MSEG(seg, old_size); ERTS_MSEG_LOCK(ma); ERTS_DBG_MA_CHK_THR_ACCESS(ma); new_seg = mseg_realloc(ma, atype, seg, old_size, new_size_p, flags, opt); ERTS_MSEG_UNLOCK(ma); + HARD_DBG_INSERT_MSEG(new_seg, *new_size_p); return new_seg; } -- cgit v1.2.3 From 4ba6824a90943e74e8fdd02f3cb695931093bcca Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 13 Sep 2013 17:37:44 +0200 Subject: erts: Fix race bug in erts_munmap Must keep mutex to serialize (un)reserve ops. --- erts/emulator/sys/common/erl_mmap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 93a95e5eef..317e3ec391 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -1762,12 +1762,12 @@ erts_munmap(Uint32 flags, void *ptr, UWord size) supercarrier_success: { UWord unres_sz; - erts_smp_mtx_unlock(&mmap_state.mtx); - ERTS_MMAP_ASSERT(size >= ad_sz); unres_sz = size - ad_sz; if (unres_sz) mmap_state.unreserve_physical(((char *) ptr) + ad_sz, unres_sz); + + erts_smp_mtx_unlock(&mmap_state.mtx); } } } -- cgit v1.2.3 From 98c01224ff2c0dacc97613c080cc0f7268c6b2f1 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 24 Sep 2013 22:26:31 +0200 Subject: erts: Fix bug in lookup_free_seg that could return segments that are too small after being super aligned. --- erts/emulator/sys/common/erl_mmap.c | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 317e3ec391..237820d3ec 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -482,6 +482,14 @@ static ERTS_INLINE ErtsFreeSegDesc* node_to_desc(enum SortOrder order, RBTNode* return order==ADDR_ORDER ? anode_to_desc(node) : snode_to_desc(node); } +static ERTS_INLINE SWord usable_size(enum SortOrder order, + ErtsFreeSegDesc* desc) +{ + return ((order == SA_SZ_ADDR_ORDER) ? + ERTS_SUPERALIGNED_FLOOR(desc->end) - ERTS_SUPERALIGNED_CEILING(desc->start) + : desc->end - desc->start); +} + #ifdef HARD_DEBUG static ERTS_INLINE SWord cmp_nodes(enum SortOrder order, RBTNode* lhs, RBTNode* rhs) @@ -490,17 +498,7 @@ static ERTS_INLINE SWord cmp_nodes(enum SortOrder order, ErtsFreeSegDesc* rdesc = node_to_desc(order, rhs); RBT_ASSERT(lhs != rhs); if (order != ADDR_ORDER) { - SWord lsz, rsz, diff; - if (order == SA_SZ_ADDR_ORDER) { - lsz = ERTS_SUPERALIGNED_FLOOR(ldesc->end) - ERTS_SUPERALIGNED_CEILING(ldesc->start); - rsz = ERTS_SUPERALIGNED_FLOOR(rdesc->end) - ERTS_SUPERALIGNED_CEILING(rdesc->start); - } - else { - RBT_ASSERT(order == SZ_REVERSE_ADDR_ORDER); - lsz = ldesc->end - ldesc->start; - rsz = rdesc->end - rdesc->start; - } - diff = lsz - rsz; + SWord diff = usable_size(order, ldesc) - usable_size(order, rdesc); if (diff) return diff; } if (order != SZ_REVERSE_ADDR_ORDER) { @@ -517,13 +515,9 @@ static ERTS_INLINE SWord cmp_with_node(enum SortOrder order, { ErtsFreeSegDesc* rdesc; if (order != ADDR_ORDER) { - SWord rhs_sz, diff; + SWord diff; rdesc = snode_to_desc(rhs); - if (order == SA_SZ_ADDR_ORDER) - rhs_sz = ERTS_SUPERALIGNED_FLOOR(rdesc->end) - ERTS_SUPERALIGNED_CEILING(rdesc->start); - else - rhs_sz = rdesc->end - rdesc->start; - diff = sz - rhs_sz; + diff = sz - usable_size(order, rdesc); if (diff) return diff; } else @@ -1163,16 +1157,17 @@ static void delete_free_seg(ErtsFreeSegMap* map, ErtsFreeSegDesc* desc) map->nseg--; } -/* Lookup a free segment in 'map' with a size of at least 'need_sz' bytes. +/* Lookup a free segment in 'map' with a size of at least 'need_sz' usable bytes. */ static ErtsFreeSegDesc* lookup_free_seg(ErtsFreeSegMap* map, SWord need_sz) { RBTNode* x = map->stree.root; ErtsFreeSegDesc* best_desc = NULL; + const enum SortOrder order = map->stree.order; while (x) { ErtsFreeSegDesc* desc = snode_to_desc(x); - SWord seg_sz = desc->end - desc->start; + SWord seg_sz = usable_size(order, desc); if (seg_sz < need_sz) { x = x->right; -- cgit v1.2.3 From d6d531012957f8e3315d44c2bcb10938ab5d6e72 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 25 Sep 2013 17:23:55 +0200 Subject: erts: Rename erts_bld_atom_uint_2tup_list to *_uword_* and change from Uint to UWord values --- erts/emulator/beam/erl_trace.c | 38 ++++++++++++++++++------------------- erts/emulator/beam/erl_utils.h | 6 +++--- erts/emulator/beam/utils.c | 4 ++-- erts/emulator/sys/common/erl_mmap.c | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index fa015ee4b9..ff7fdfcfca 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2184,7 +2184,7 @@ trace_gc(Process *p, Eterm what) AM_bin_old_vheap_block_size }; - Uint values[] = { + UWord values[] = { OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, HEAP_SIZE(p), MBUF_SIZE(p), @@ -2198,7 +2198,7 @@ trace_gc(Process *p, Eterm what) BIN_OLD_VHEAP_SZ(p) }; #define LOCAL_HEAP_SIZE \ - (sizeof(values)/sizeof(Eterm)) * \ + (sizeof(values)/sizeof(*values)) * \ (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \ 5/*4-tuple */ + TS_HEAP_WORDS DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); @@ -2206,7 +2206,7 @@ trace_gc(Process *p, Eterm what) Eterm* limit; #endif - ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm)); + ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm)); UseTmpHeap(LOCAL_HEAP_SIZE,p); @@ -2214,9 +2214,9 @@ trace_gc(Process *p, Eterm what) hp = local_heap; #ifdef DEBUG size = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &size, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); size += 5/*4-tuple*/ + TS_SIZE(p); @@ -2229,9 +2229,9 @@ trace_gc(Process *p, Eterm what) ERTS_TRACE_FLAGS(p)); size = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &size, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); size += 5/*4-tuple*/ + TS_SIZE(p); @@ -2244,9 +2244,9 @@ trace_gc(Process *p, Eterm what) ASSERT(size <= LOCAL_HEAP_SIZE); #endif - msg = erts_bld_atom_uint_2tup_list(&hp, + msg = erts_bld_atom_uword_2tup_list(&hp, NULL, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); @@ -2415,7 +2415,7 @@ monitor_long_gc(Process *p, Uint time) { am_old_heap_size, am_heap_size }; - Eterm values[] = { + UWord values[] = { time, OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, HEAP_SIZE(p), @@ -2436,9 +2436,9 @@ monitor_long_gc(Process *p, Uint time) { #endif hsz = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &hsz, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); hsz += 5 /* 4-tuple */; @@ -2449,9 +2449,9 @@ monitor_long_gc(Process *p, Uint time) { hp_end = hp + hsz; #endif - list = erts_bld_atom_uint_2tup_list(&hp, + list = erts_bld_atom_uword_2tup_list(&hp, NULL, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); msg = TUPLE4(hp, am_monitor, p->common.id, am_long_gc, list); @@ -2489,7 +2489,7 @@ monitor_large_heap(Process *p) { am_old_heap_size, am_heap_size }; - Uint values[] = { + UWord values[] = { OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, HEAP_SIZE(p), MBUF_SIZE(p), @@ -2511,9 +2511,9 @@ monitor_large_heap(Process *p) { #endif hsz = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &hsz, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); hsz += 5 /* 4-tuple */; @@ -2524,9 +2524,9 @@ monitor_large_heap(Process *p) { hp_end = hp + hsz; #endif - list = erts_bld_atom_uint_2tup_list(&hp, + list = erts_bld_atom_uword_2tup_list(&hp, NULL, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); msg = TUPLE4(hp, am_monitor, p->common.id, am_large_heap, list); diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 80d29d554a..f85c7410c4 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -175,8 +175,8 @@ Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]); Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp, Sint length, Eterm terms1[], Uint terms2[]); Eterm -erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp, - Sint length, Eterm atoms[], Uint uints[]); +erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp, + Sint length, Eterm atoms[], UWord uints[]); Eterm erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, Eterm atoms[], Uint uints1[], Uint uints2[]); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index bd2be7afca..0d75bbcc77 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -576,8 +576,8 @@ erts_bld_2tup_list(Uint **hpp, Uint *szp, } Eterm -erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp, - Sint length, Eterm atoms[], Uint uints[]) +erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp, + Sint length, Eterm atoms[], UWord uints[]) { Sint i; Eterm res = THE_NON_VALUE; diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 237820d3ec..4392ec4f9c 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -2265,7 +2265,7 @@ Eterm erts_mmap_info(Process* p) hp = HAlloc(p, may_need); hp_end = hp + may_need; - list = erts_bld_atom_uint_2tup_list(&hp, NULL, + list = erts_bld_atom_uword_2tup_list(&hp, NULL, sizeof(values)/sizeof(*values), tags, values); -- cgit v1.2.3 From 8d5b9a53a1fd5e2264d705911af23cd484ccead0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 26 Sep 2013 22:30:26 +0200 Subject: erts: Add erts_bld_tupleX macros for compile time argument checking --- erts/emulator/beam/erl_alloc.c | 12 ++++++------ erts/emulator/beam/erl_utils.h | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index e30b3e7b51..ad1020d7d6 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -3060,13 +3060,13 @@ reply_alloc_info(void *vair) ? NIL : erts_mseg_info(0, NULL, NULL, hpp != NULL, hpp, szp)); - ainfo = erts_bld_tuple(hpp, szp, 3, - alloc_atom, - make_small(0), - ainfo); + ainfo = erts_bld_tuple3(hpp, szp, + alloc_atom, + make_small(0), + ainfo); #else - ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom, - am_false); + ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, + am_false); #endif break; default: diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index f85c7410c4..292d135946 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -168,6 +168,10 @@ Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64); Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64); Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr); Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...); +#define erts_bld_tuple2(H,S,E1,E2) erts_bld_tuple(H,S,2,E1,E2) +#define erts_bld_tuple3(H,S,E1,E2,E3) erts_bld_tuple(H,S,3,E1,E2,E3) +#define erts_bld_tuple4(H,S,E1,E2,E3,E4) erts_bld_tuple(H,S,4,E1,E2,E3,E4) +#define erts_bld_tuple5(H,S,E1,E2,E3,E4,E5) erts_bld_tuple(H,S,5,E1,E2,E3,E4,E5) Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]); Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len); #define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str)) -- cgit v1.2.3 From 059d8b76011f960cc5938501a33002b051b0bca2 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 26 Sep 2013 22:36:34 +0200 Subject: erts: Add erts_mmap stats As part of erlang:system_info({allocator,mseg_alloc}) and erl_crash.dump --- erts/emulator/beam/erl_alloc.c | 14 ++- erts/emulator/beam/erl_bif_info.c | 2 +- erts/emulator/sys/common/erl_mmap.c | 193 +++++++++++++++++++++++++++++++++++- erts/emulator/sys/common/erl_mmap.h | 12 ++- erts/emulator/sys/common/erl_mseg.c | 5 +- erts/preloaded/ebin/erlang.beam | Bin 94156 -> 94244 bytes erts/preloaded/src/erlang.erl | 2 + 7 files changed, 222 insertions(+), 6 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index ad1020d7d6..d8da616d05 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -2712,6 +2712,7 @@ erts_allocator_info(int to, void *arg) #if HAVE_ERTS_MSEG { + struct erts_mmap_info_struct emis; #ifdef ERTS_SMP int max = (int) erts_no_schedulers; #else @@ -2722,6 +2723,8 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i); erts_mseg_info(i, &to, arg, 0, NULL, NULL); } + erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n"); + erts_mmap_info(&to, arg, NULL, NULL, &emis); } #endif @@ -2948,6 +2951,7 @@ reply_alloc_info(void *vair) Uint sz, *szp; ErlOffHeap *ohp = NULL; ErlHeapFragment *bp = NULL; + struct erts_mmap_info_struct emis; int i; Eterm (*info_func)(Allctr_t *, int, @@ -3064,11 +3068,19 @@ reply_alloc_info(void *vair) alloc_atom, make_small(0), ainfo); + + ai_list = erts_bld_cons(hpp, szp, + ainfo, ai_list); + ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis)); + ainfo = erts_bld_tuple3(hpp, szp, + alloc_atom, + erts_bld_atom(hpp,szp,"erts_mmap"), + ainfo); #else ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, am_false); #endif - break; + break; default: alloc_atom = erts_bld_atom(hpp, szp, (char *) ERTS_ALC_A2AD(ai)); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 7aa439f2e6..5fbcbbe250 100755 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3290,7 +3290,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(res); } else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { - BIF_RET(erts_mmap_info(BIF_P)); + BIF_RET(erts_mmap_debug_info(BIF_P)); } } else if (is_tuple(BIF_ARG_1)) { diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 4392ec4f9c..60b32e8115 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -323,6 +323,7 @@ static struct { char *unused_start; char *unused_end; char *new_area_hint; + Uint reserved; } desc; struct { UWord free_seg_descs; @@ -2190,6 +2191,7 @@ erts_mmap_init(ErtsMMapInit *init) mmap_state.size.os.used += (UWord) (mmap_state.sa.bot - start); mmap_state.desc.free_list = NULL; + mmap_state.desc.reserved = 0; if (end == (void *) 0) { /* @@ -2204,6 +2206,7 @@ erts_mmap_init(ErtsMMapInit *init) if (!virtual_map || os_reserve_physical(mmap_state.sua.top, ERTS_PAGEALIGNED_SIZE)) #endif add_free_desc_area(mmap_state.sua.top, end); + mmap_state.desc.reserved += (end - mmap_state.sua.top) / sizeof(ErtsFreeSegDesc); } mmap_state.size.supercarrier.total = (UWord) (mmap_state.sua.top - mmap_state.sa.bot); @@ -2218,6 +2221,8 @@ erts_mmap_init(ErtsMMapInit *init) #endif mmap_state.desc.unused_start = start; mmap_state.desc.unused_end = mmap_state.sa.bot; + mmap_state.desc.reserved += ((mmap_state.desc.unused_end - start) + / sizeof(ErtsFreeSegDesc)); init_free_seg_map(&mmap_state.sa.map, SA_SZ_ADDR_ORDER); init_free_seg_map(&mmap_state.sua.map, SZ_REVERSE_ADDR_ORDER); @@ -2238,7 +2243,192 @@ erts_mmap_init(ErtsMMapInit *init) #endif } -Eterm erts_mmap_info(Process* p) +static struct { + Eterm total; + Eterm total_sa; + Eterm total_sua; + Eterm used; + Eterm used_sa; + Eterm used_sua; + Eterm max; + Eterm allocated; + Eterm reserved; + Eterm sizes; + Eterm free_segs; + Eterm supercarrier; + Eterm os; + Eterm scs; + Eterm sco; + Eterm scrpm; + Eterm scmgc; + + int is_initialized; +}am; + +static void ERTS_INLINE atom_init(Eterm *atom, char *name) +{ + *atom = am_atom_put(name, strlen(name)); +} +#define AM_INIT(AM) atom_init(&am.AM, #AM) + +static void init_atoms(void) +{ + AM_INIT(total); + AM_INIT(total_sa); + AM_INIT(total_sua); + AM_INIT(used); + AM_INIT(used_sa); + AM_INIT(used_sua); + AM_INIT(max); + AM_INIT(allocated); + AM_INIT(reserved); + AM_INIT(sizes); + AM_INIT(free_segs); + AM_INIT(supercarrier); + AM_INIT(os); + AM_INIT(scs); + AM_INIT(sco); + AM_INIT(scrpm); + AM_INIT(scmgc); + am.is_initialized = 1; +}; + + +static ERTS_INLINE void +add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) +{ + *lp = erts_bld_cons(hpp, szp, erts_bld_tuple(hpp, szp, 2, el1, el2), *lp); +} + +Eterm erts_mmap_info(int *print_to_p, + void *print_to_arg, + Eterm** hpp, Uint* szp, + struct erts_mmap_info_struct* emis) +{ + Eterm size_tags[] = { am.total, am.total_sa, am.total_sua, am.used, am.used_sa, am.used_sua }; + Eterm seg_tags[] = { am.used, am.max, am.allocated, am.reserved, am.used_sa, am.used_sua }; + Eterm group[2]; + Eterm group_tags[] = { am.sizes, am.free_segs }; + Eterm list[2]; + Eterm list_tags[2]; /* { am.supercarrier, am.os } */ + int lix; + Eterm res = THE_NON_VALUE; + + if (!hpp) { + erts_smp_mtx_lock(&mmap_state.mtx); + emis->sizes[0] = mmap_state.size.supercarrier.total; + emis->sizes[1] = mmap_state.sa.top - mmap_state.sa.bot; + emis->sizes[2] = mmap_state.sua.top - mmap_state.sua.bot; + emis->sizes[3] = mmap_state.size.supercarrier.used.total; + emis->sizes[4] = mmap_state.size.supercarrier.used.sa; + emis->sizes[5] = mmap_state.size.supercarrier.used.sua; + + emis->segs[0] = mmap_state.no.free_segs.curr; + emis->segs[1] = mmap_state.no.free_segs.max; + emis->segs[2] = mmap_state.no.free_seg_descs; + emis->segs[3] = mmap_state.desc.reserved; + emis->segs[4] = mmap_state.sa.map.nseg; + emis->segs[5] = mmap_state.sua.map.nseg; + + emis->os_used = mmap_state.size.os.used; + erts_smp_mtx_unlock(&mmap_state.mtx); + } + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + if (mmap_state.supercarrier) { + const char* prefix = "supercarrier "; + erts_print(to, arg, "%stotal size: %bpu\n", prefix, emis->sizes[0]); + erts_print(to, arg, "%stotal sa size: %bpu\n", prefix, emis->sizes[1]); + erts_print(to, arg, "%stotal sua size: %bpu\n", prefix, emis->sizes[2]); + erts_print(to, arg, "%sused size: %bpu\n", prefix, emis->sizes[3]); + erts_print(to, arg, "%sused sa size: %bpu\n", prefix, emis->sizes[4]); + erts_print(to, arg, "%sused sua size: %bpu\n", prefix, emis->sizes[5]); + erts_print(to, arg, "%sused free segs: %bpu\n", prefix, emis->segs[0]); + erts_print(to, arg, "%smax free segs: %bpu\n", prefix, emis->segs[1]); + erts_print(to, arg, "%sallocated free segs: %bpu\n", prefix, emis->segs[2]); + erts_print(to, arg, "%sreserved free segs: %bpu\n", prefix, emis->segs[3]); + erts_print(to, arg, "%ssa free segs: %bpu\n", prefix, emis->segs[4]); + erts_print(to, arg, "%ssua free segs: %bpu\n", prefix, emis->segs[5]); + } + if (!mmap_state.no_os_mmap) { + erts_print(to, arg, "os mmap size used: %bpu\n", emis->os_used); + } + } + + + if (hpp || szp) { + if (!am.is_initialized) { + init_atoms(); + } + + lix = 0; + if (mmap_state.supercarrier) { + group[0] = erts_bld_atom_uword_2tup_list(hpp, szp, + sizeof(size_tags)/sizeof(Eterm), + size_tags, emis->sizes); + group[1] = erts_bld_atom_uword_2tup_list(hpp, szp, + sizeof(seg_tags)/sizeof(Eterm), + seg_tags, emis->segs); + list[lix] = erts_bld_2tup_list(hpp, szp, 2, group_tags, group); + list_tags[lix] = am.supercarrier; + lix++; + } + + if (!mmap_state.no_os_mmap) { + group[0] = erts_bld_atom_uword_2tup_list(hpp, szp, + 1, &am.used, &emis->os_used); + list[lix] = erts_bld_2tup_list(hpp, szp, 1, group_tags, group); + list_tags[lix] = am.os; + lix++; + } + res = erts_bld_2tup_list(hpp, szp, lix, list_tags, list); + } + return res; +} + +Eterm erts_mmap_info_options(char *prefix, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + const UWord scs = mmap_state.sua.top - mmap_state.sa.bot; + const Eterm sco = mmap_state.no_os_mmap ? am_true : am_false; + const Eterm scrpm = (mmap_state.reserve_physical == reserve_noop) ? am_true : am_false; + Eterm res = THE_NON_VALUE; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, arg, "%sscs: %bpu\n", prefix, scs); + if (mmap_state.supercarrier) { + erts_print(to, arg, "%ssco: %T\n", prefix, sco); + erts_print(to, arg, "%sscrpm: %T\n", prefix, scrpm); + erts_print(to, arg, "%sscmgc: %beu\n", prefix, mmap_state.desc.reserved); + } + } + + if (hpp || szp) { + if (!am.is_initialized) { + init_atoms(); + } + + res = NIL; + if (mmap_state.supercarrier) { + add_2tup(hpp, szp, &res, am.scmgc, + erts_bld_uint(hpp,szp, mmap_state.desc.reserved)); + add_2tup(hpp, szp, &res, am.scrpm, scrpm); + add_2tup(hpp, szp, &res, am.sco, sco); + } + add_2tup(hpp, szp, &res, am.scs, erts_bld_uword(hpp, szp, scs)); + } + return res; +} + + +Eterm erts_mmap_debug_info(Process* p) { if (mmap_state.supercarrier) { ERTS_DECL_AM(sabot); @@ -2283,6 +2473,7 @@ Eterm erts_mmap_info(Process* p) } } + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Debug functions * \* */ diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 106459f872..e6934dbb26 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -57,8 +57,18 @@ void erts_munmap(Uint32 flags, void *ptr, UWord size); void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep); int erts_mmap_in_supercarrier(void *ptr); void erts_mmap_init(ErtsMMapInit*); +struct erts_mmap_info_struct +{ + UWord sizes[6]; + UWord segs[6]; + UWord os_used; +}; +Eterm erts_mmap_info(int *print_to_p, void *print_to_arg, + Eterm** hpp, Uint* szp, struct erts_mmap_info_struct*); +Eterm erts_mmap_info_options(char *prefix, int *print_to_p, void *print_to_arg, + Uint **hpp, Uint *szp); struct process; -Eterm erts_mmap_info(struct process*); +Eterm erts_mmap_debug_info(struct process*); #define ERTS_SUPERALIGNED_SIZE \ (1 << ERTS_MMAP_SUPERALIGNED_BITS) diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 13f8069cf5..b165f76c96 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1070,7 +1070,9 @@ info_options(ErtsMsegAllctr_t *ma, Uint **hpp, Uint *szp) { - Eterm res = THE_NON_VALUE; + Eterm res; + + res = erts_mmap_info_options(prefix, print_to_p, print_to_arg, hpp, szp); if (print_to_p) { int to = *print_to_p; @@ -1085,7 +1087,6 @@ info_options(ErtsMsegAllctr_t *ma, if (!atoms_initialized) init_atoms(ma); - res = NIL; add_2tup(hpp, szp, &res, am.mcs, bld_uint(hpp, szp, ma->max_cache_size)); diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam index f1791200e0..5a9ffa89d1 100644 Binary files a/erts/preloaded/ebin/erlang.beam and b/erts/preloaded/ebin/erlang.beam differ diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index d40ee7c59a..2b6760c675 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -3499,6 +3499,8 @@ mk_res_list([]) -> mk_res_list([Alloc | Rest]) -> [{Alloc, []} | mk_res_list(Rest)]. +insert_instance(I, N, Rest) when erlang:is_atom(N) -> + [{N, I} | Rest]; insert_instance(I, N, []) -> [{instance, N, I}]; insert_instance(I, N, [{instance, M, _}|_] = Rest) when N < M -> -- cgit v1.2.3 From ee39709b4ae375277c6380033c9c9f04b0242b43 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 26 Sep 2013 22:37:43 +0200 Subject: erts: Fix misc minor bugs in supercarrier initialization --- erts/emulator/sys/common/erl_mmap.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 60b32e8115..52041fd03e 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -1394,15 +1394,17 @@ alloc_desc_insert_free_seg(ErtsFreeSegMap *map, char* start, char* end) */ #if ERTS_HAVE_OS_MMAP - ptr = os_mmap(mmap_state.desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0); - if (ptr) { - mmap_state.desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE; - ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE); - add_free_desc_area(ptr, ptr+ERTS_PAGEALIGNED_SIZE); - desc = alloc_desc(); - ERTS_MMAP_ASSERT(desc); - insert_free_seg(map, desc, start, end); - return 0; + if (!mmap_state.no_os_mmap) { + ptr = os_mmap(mmap_state.desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0); + if (ptr) { + mmap_state.desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE; + ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE); + add_free_desc_area(ptr, ptr+ERTS_PAGEALIGNED_SIZE); + desc = alloc_desc(); + ERTS_MMAP_ASSERT(desc); + insert_free_seg(map, desc, start, end); + return 0; + } } #endif @@ -2166,6 +2168,7 @@ erts_mmap_init(ErtsMMapInit *init) mmap_state.sa.bot = NULL; mmap_state.sua.top = NULL; mmap_state.no_os_mmap = 0; + mmap_state.supercarrier = 0; } else { size_t desc_size; @@ -2185,10 +2188,10 @@ erts_mmap_init(ErtsMMapInit *init) mmap_state.sa.bot += desc_size; mmap_state.sa.bot = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.bot); mmap_state.sa.top = mmap_state.sa.bot; - mmap_state.sua.top = (char *) ERTS_SUPERALIGNED_FLOOR(end); + mmap_state.sua.top = end; mmap_state.sua.bot = mmap_state.sua.top; - mmap_state.size.os.used += (UWord) (mmap_state.sa.bot - start); + mmap_state.size.supercarrier.used.total += (UWord) (mmap_state.sa.bot - start); mmap_state.desc.free_list = NULL; mmap_state.desc.reserved = 0; @@ -2201,7 +2204,7 @@ erts_mmap_init(ErtsMMapInit *init) * into the super carrier... */ mmap_state.sua.top -= ERTS_PAGEALIGNED_SIZE; - mmap_state.size.os.used += ERTS_PAGEALIGNED_SIZE; + mmap_state.size.supercarrier.used.total += ERTS_PAGEALIGNED_SIZE; #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION if (!virtual_map || os_reserve_physical(mmap_state.sua.top, ERTS_PAGEALIGNED_SIZE)) #endif @@ -2209,7 +2212,7 @@ erts_mmap_init(ErtsMMapInit *init) mmap_state.desc.reserved += (end - mmap_state.sua.top) / sizeof(ErtsFreeSegDesc); } - mmap_state.size.supercarrier.total = (UWord) (mmap_state.sua.top - mmap_state.sa.bot); + mmap_state.size.supercarrier.total = (UWord) (mmap_state.sua.top - start); /* * Area before (and after) super carrier -- cgit v1.2.3 From e9f670e542dd2ea2dc29dff66d3516a3fd0d2d21 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 1 Oct 2013 17:11:27 +0200 Subject: erts: Fix lock violation for init_atoms in erl_mmap.c by not holding the mseg lock while reading version and option info which is unnecessary anyway. --- erts/emulator/sys/common/erl_mseg.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) (limited to 'erts') diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index b165f76c96..94a381e168 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -962,8 +962,6 @@ init_atoms(ErtsMsegAllctr_t *ma) #ifdef DEBUG Eterm *atom; #endif - - ERTS_MSEG_UNLOCK(ma); erts_mtx_lock(&init_atoms_mutex); if (!atoms_initialized) { @@ -1007,7 +1005,6 @@ init_atoms(ErtsMsegAllctr_t *ma) #endif } - ERTS_MSEG_LOCK(ma); atoms_initialized = 1; erts_mtx_unlock(&init_atoms_mutex); } @@ -1293,14 +1290,8 @@ erts_mseg_info_options(int ix, ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(ix); Eterm res; - ERTS_MSEG_LOCK(ma); - - ERTS_DBG_MA_CHK_THR_ACCESS(ma); - res = info_options(ma, "option ", print_to_p, print_to_arg, hpp, szp); - ERTS_MSEG_UNLOCK(ma); - return res; } @@ -1318,10 +1309,6 @@ erts_mseg_info(int ix, Eterm values[4]; Uint n = 0; - ERTS_MSEG_LOCK(ma); - - ERTS_DBG_MA_CHK_THR_ACCESS(ma); - if (hpp || szp) { if (!atoms_initialized) @@ -1334,6 +1321,10 @@ erts_mseg_info(int ix, } values[n++] = info_version(ma, print_to_p, print_to_arg, hpp, szp); values[n++] = info_options(ma, "option ", print_to_p, print_to_arg, hpp, szp); + + ERTS_MSEG_LOCK(ma); + ERTS_DBG_MA_CHK_THR_ACCESS(ma); + #if HALFWORD_HEAP values[n++] = info_memkind(ma, &ma->low_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); values[n++] = info_memkind(ma, &ma->hi_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); -- cgit v1.2.3 From ff95e85937007a7952477c7acc7619791405ab1c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 1 Oct 2013 18:12:43 +0200 Subject: erts: Add mutex to init_atoms in erts_mmap.c --- erts/emulator/beam/erl_lock_check.c | 1 + erts/emulator/sys/common/erl_mmap.c | 109 +++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 50 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 87efbdbc3e..0dd83fa6ed 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -132,6 +132,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { #endif /* __WIN32__ */ { "alcu_init_atoms", NULL }, { "mseg_init_atoms", NULL }, + { "mmap_init_atoms", NULL }, { "drv_tsd", NULL }, { "async_enq_mtx", NULL }, #ifdef ERTS_SMP diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 52041fd03e..a9da7430fb 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -2039,6 +2039,64 @@ int erts_mmap_in_supercarrier(void *ptr) return ERTS_MMAP_IN_SUPERCARRIER(ptr); } + +static struct { + Eterm total; + Eterm total_sa; + Eterm total_sua; + Eterm used; + Eterm used_sa; + Eterm used_sua; + Eterm max; + Eterm allocated; + Eterm reserved; + Eterm sizes; + Eterm free_segs; + Eterm supercarrier; + Eterm os; + Eterm scs; + Eterm sco; + Eterm scrpm; + Eterm scmgc; + + int is_initialized; + erts_mtx_t init_mutex; +}am; + +static void ERTS_INLINE atom_init(Eterm *atom, char *name) +{ + *atom = am_atom_put(name, strlen(name)); +} +#define AM_INIT(AM) atom_init(&am.AM, #AM) + +static void init_atoms(void) +{ + erts_mtx_lock(&am.init_mutex); + + if (!am.is_initialized) { + AM_INIT(total); + AM_INIT(total_sa); + AM_INIT(total_sua); + AM_INIT(used); + AM_INIT(used_sa); + AM_INIT(used_sua); + AM_INIT(max); + AM_INIT(allocated); + AM_INIT(reserved); + AM_INIT(sizes); + AM_INIT(free_segs); + AM_INIT(supercarrier); + AM_INIT(os); + AM_INIT(scs); + AM_INIT(sco); + AM_INIT(scrpm); + AM_INIT(scmgc); + am.is_initialized = 1; + } + erts_mtx_unlock(&am.init_mutex); +}; + + #ifdef HARD_DEBUG_MSEG static void hard_dbg_mseg_init(void); #endif @@ -2085,6 +2143,7 @@ erts_mmap_init(ErtsMMapInit *init) #endif erts_smp_mtx_init(&mmap_state.mtx, "erts_mmap"); + erts_mtx_init(&am.init_mutex, "mmap_init_atoms"); #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION if (init->virtual_range.start) { @@ -2246,56 +2305,6 @@ erts_mmap_init(ErtsMMapInit *init) #endif } -static struct { - Eterm total; - Eterm total_sa; - Eterm total_sua; - Eterm used; - Eterm used_sa; - Eterm used_sua; - Eterm max; - Eterm allocated; - Eterm reserved; - Eterm sizes; - Eterm free_segs; - Eterm supercarrier; - Eterm os; - Eterm scs; - Eterm sco; - Eterm scrpm; - Eterm scmgc; - - int is_initialized; -}am; - -static void ERTS_INLINE atom_init(Eterm *atom, char *name) -{ - *atom = am_atom_put(name, strlen(name)); -} -#define AM_INIT(AM) atom_init(&am.AM, #AM) - -static void init_atoms(void) -{ - AM_INIT(total); - AM_INIT(total_sa); - AM_INIT(total_sua); - AM_INIT(used); - AM_INIT(used_sa); - AM_INIT(used_sua); - AM_INIT(max); - AM_INIT(allocated); - AM_INIT(reserved); - AM_INIT(sizes); - AM_INIT(free_segs); - AM_INIT(supercarrier); - AM_INIT(os); - AM_INIT(scs); - AM_INIT(sco); - AM_INIT(scrpm); - AM_INIT(scmgc); - am.is_initialized = 1; -}; - static ERTS_INLINE void add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) -- cgit v1.2.3 From 5db2c05af6efaf636b2e41e3a1f305c592a71f34 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 3 Oct 2013 15:46:59 +0200 Subject: erts: Add test case for erts_mmap --- erts/emulator/test/alloc_SUITE.erl | 69 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) (limited to 'erts') diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 801ed0f85a..f6ff6bb813 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -29,6 +29,7 @@ bucket_mask/1, rbtree/1, mseg_clear_cache/1, + erts_mmap/1, cpool/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -41,7 +42,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [basic, coalesce, threads, realloc_copy, bucket_index, - bucket_mask, rbtree, mseg_clear_cache, cpool]. + bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool]. groups() -> []. @@ -110,6 +111,59 @@ cpool(suite) -> []; cpool(doc) -> []; cpool(Cfg) -> ?line drv_case(Cfg). +erts_mmap(Config) when is_list(Config) -> + case {?t:os_type(), is_halfword_vm()} of + {{unix, _}, false} -> + [erts_mmap_do(Config, SCO, SCRPM, SCMGC) + || SCO <-[true,false], SCMGC <-[1234,0], SCRPM <- [true,false]]; + + {_,true} -> + {skipped, "No supercarrier support on halfword vm"}; + {SkipOs,_} -> + ?line {skipped, + lists:flatten(["Not run on " + | io_lib:format("~p",[SkipOs])])} + end. + + +erts_mmap_do(Config, SCO, SCRPM, SCMGC) -> + SCS = 100, % Mb + O1 = "+MMscs" ++ integer_to_list(SCS) + ++ " +MMsco" ++ atom_to_list(SCO) + ++ " +MMscrpm" ++ atom_to_list(SCRPM), + Opts = case SCMGC of + 0 -> O1; + _ -> O1 ++ " +MMscmgc"++integer_to_list(SCMGC) + end, + {ok, Node} = start_node(Config, Opts), + Self = self(), + Ref = make_ref(), + F = fun () -> + SI = erlang:system_info({allocator,mseg_alloc}), + {erts_mmap,EM} = lists:keyfind(erts_mmap, 1, SI), + {supercarrier,SC} = lists:keyfind(supercarrier, 1, EM), + {sizes,Sizes} = lists:keyfind(sizes, 1, SC), + {free_segs,Segs} = lists:keyfind(free_segs,1,SC), + {total,Total} = lists:keyfind(total,1,Sizes), + Total = SCS*1024*1024, + + {reserved,Reserved} = lists:keyfind(reserved,1,Segs), + true = (Reserved >= SCMGC), + + case {SCO,lists:keyfind(os,1,EM)} of + {true, false} -> ok; + {false, {os,_}} -> ok + end, + + Self ! {Ref, ok} + end, + + spawn_link(Node, F), + Result = receive {Ref, Rslt} -> Rslt end, + stop_node(Node), + Result. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% Internal functions %% @@ -179,7 +233,9 @@ receive_drv_result(Port, CaseName) -> ?line {comment, Comment} end. -start_node(Config) when is_list(Config) -> +start_node(Config) -> + start_node(Config, []). +start_node(Config, Opts) when is_list(Config), is_list(Opts) -> ?line Pa = filename:dirname(code:which(?MODULE)), ?line {A, B, C} = now(), ?line Name = list_to_atom(atom_to_list(?MODULE) @@ -191,7 +247,14 @@ start_node(Config) when is_list(Config) -> ++ integer_to_list(B) ++ "-" ++ integer_to_list(C)), - ?line ?t:start_node(Name, slave, [{args, "-pa "++Pa}]). + ?line ?t:start_node(Name, slave, [{args, Opts++" -pa "++Pa}]). stop_node(Node) -> ?t:stop_node(Node). + +is_halfword_vm() -> + case {erlang:system_info({wordsize, internal}), + erlang:system_info({wordsize, external})} of + {4, 8} -> true; + {WS, WS} -> false + end. -- cgit v1.2.3