From 6cbd63b281a9690b2db5fead3f756d34140f8fb5 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 12 Apr 2016 20:51:49 +0200 Subject: erts: Tweak defaults for literal allocator Reduce main carrier size and number of free descriptors. --- erts/emulator/beam/erl_alloc.c | 4 ++-- erts/emulator/sys/common/erl_mmap.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 9cbe00d719..6b5f5bd63e 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -307,9 +307,9 @@ set_default_literal_alloc_opts(struct au_init *ip) ip->init.util.name_prefix = "literal_"; ip->init.util.alloc_no = ERTS_ALC_A_LITERAL; #ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 1024*1024; /* Main carrier size */ #else - ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ + ip->init.util.mmbcs = 256*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LITERAL; ip->init.util.asbcst = 0; diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 67e131b53e..7c5ddbec0b 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -53,7 +53,7 @@ typedef struct { #define ERTS_LITERAL_VIRTUAL_AREA_SIZE (UWORD_CONSTANT(1)*1024*1024*1024) #define ERTS_MMAP_INIT_LITERAL_INITER \ - {{NULL, NULL}, {NULL, NULL}, ERTS_LITERAL_VIRTUAL_AREA_SIZE, 1, (1 << 16), 0} + {{NULL, NULL}, {NULL, NULL}, ERTS_LITERAL_VIRTUAL_AREA_SIZE, 1, (1 << 10), 0} typedef struct ErtsMemMapper_ ErtsMemMapper; -- cgit v1.2.3 From 499a872d5f6ea09d23eb7b04ea5de2f6d3fabd98 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 11 Apr 2016 18:13:31 +0200 Subject: erts: Refactor callbacks for literal mseg alloc Make the callbacks more general to be usable for any allocator that that uses its own ErtsMemMapper. --- erts/emulator/beam/erl_alloc.c | 7 ++++--- erts/emulator/beam/erl_alloc_util.c | 14 ++++++++------ erts/emulator/beam/erl_alloc_util.h | 8 +++++--- erts/emulator/sys/common/erl_mmap.c | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 6b5f5bd63e..efe31193b9 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -329,9 +329,10 @@ set_default_literal_alloc_opts(struct au_init *ip) ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc; #elif defined(ARCH_64) # ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - ip->init.util.mseg_alloc = &erts_alcu_literal_64_mseg_alloc; - ip->init.util.mseg_realloc = &erts_alcu_literal_64_mseg_realloc; - ip->init.util.mseg_dealloc = &erts_alcu_literal_64_mseg_dealloc; + ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; + ip->init.util.mseg_mmapper = &erts_literal_mmapper; # endif #else # error Unknown architecture diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 6e682019ba..e362a03646 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -898,8 +898,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +/* Used by literal allocator that has its own mmapper (super carrier) */ void* -erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void* res; UWord size = (UWord) *size_p; @@ -907,33 +908,33 @@ erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) if (flags & ERTS_MSEG_FLG_2POW) mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; - res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size); + res = erts_mmap(allctr->mseg_mmapper, mmap_flags, &size); *size_p = (Uint)size; INC_CC(allctr->calls.mseg_alloc); return res; } void* -erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg, +erts_alcu_mmapper_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) { void *res; UWord new_size = (UWord) *new_size_p; - res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size); + res = erts_mremap(allctr->mseg_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size); *new_size_p = (Uint) new_size; INC_CC(allctr->calls.mseg_realloc); return res; } void -erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, +erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; if (flags & ERTS_MSEG_FLG_2POW) mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; - erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size); + erts_munmap(allctr->mseg_mmapper, mmap_flags, seg, (UWord)size); INC_CC(allctr->calls.mseg_dealloc); } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ @@ -6127,6 +6128,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mseg_alloc = init->mseg_alloc; allctr->mseg_realloc = init->mseg_realloc; allctr->mseg_dealloc = init->mseg_dealloc; + allctr->mseg_mmapper = init->mseg_mmapper; } else { ASSERT(!init->mseg_realloc && !init->mseg_dealloc); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index afdff1a71e..51af59e40c 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -72,6 +72,7 @@ typedef struct { void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); + ErtsMemMapper *mseg_mmapper; #endif void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); @@ -205,9 +206,9 @@ void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uin void erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -void* erts_alcu_literal_64_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); -void* erts_alcu_literal_64_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); -void erts_alcu_literal_64_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif #endif /* HAVE_ERTS_MSEG */ @@ -601,6 +602,7 @@ struct Allctr_t_ { void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); + ErtsMemMapper *mseg_mmapper; #endif void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 889d6f3868..756d038da4 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -299,7 +299,7 @@ struct ErtsMemMapper_ { int supercarrier; int no_os_mmap; /* - * Super unaligend area is located above super aligned + * Super unaligned area is located above super aligned * area. That is, `sa.bot` is beginning of the super * carrier, `sua.top` is the end of the super carrier, * and sa.top and sua.bot moves towards eachother. -- cgit v1.2.3 From 90641d82a4b07e6b0be142d07ac85107b8ebee9d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 12 Apr 2016 14:52:21 +0200 Subject: erts: Make ErtsMemMapper able to map memory with PROT_EXEC to prepare for hipe native code allocation. --- erts/emulator/sys/common/erl_mmap.c | 68 ++++++++++++++++++++++--------------- erts/emulator/sys/common/erl_mmap.h | 2 +- erts/emulator/sys/common/erl_mseg.c | 4 +-- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 756d038da4..f5f40b35a8 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -294,10 +294,11 @@ typedef struct { }ErtsFreeSegMap; struct ErtsMemMapper_ { - int (*reserve_physical)(char *, UWord); + int (*reserve_physical)(char *, UWord, int exec); void (*unreserve_physical)(char *, UWord); int supercarrier; int no_os_mmap; + int executable; /* is client a native code allocator? */ /* * Super unaligned area is located above super aligned * area. That is, `sa.bot` is beginning of the super @@ -1232,6 +1233,7 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map) #if HAVE_MMAP # define ERTS_MMAP_PROT (PROT_READ|PROT_WRITE) +# define ERTS_MMAP_PROT_EXEC (PROT_READ|PROT_WRITE|PROT_EXEC) # if defined(MAP_ANONYMOUS) # define ERTS_MMAP_FLAGS (MAP_ANON|MAP_PRIVATE) # define ERTS_MMAP_FD (-1) @@ -1245,24 +1247,26 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map) #endif static ERTS_INLINE void * -os_mmap(void *hint_ptr, UWord size, int try_superalign) +os_mmap(void *hint_ptr, UWord size, int try_superalign, int executable) { #if HAVE_MMAP + const int prot = executable ? ERTS_MMAP_PROT_EXEC : ERTS_MMAP_PROT; void *res; #ifdef MAP_ALIGN if (try_superalign) - res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, ERTS_MMAP_PROT, + res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, prot, ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0); else #endif - res = mmap((void *) hint_ptr, size, ERTS_MMAP_PROT, + res = mmap((void *) hint_ptr, size, prot, ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0); if (res == MAP_FAILED) return NULL; return res; #elif HAVE_VIRTUALALLOC + const DWORD prot = executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE; return (void *) VirtualAlloc(NULL, (SIZE_T) size, - MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE); + MEM_COMMIT|MEM_RESERVE, prot); #else # error "missing mmap() or similar" #endif @@ -1319,6 +1323,7 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign) #if HAVE_MMAP #define ERTS_MMAP_RESERVE_PROT (ERTS_MMAP_PROT) +#define ERTS_MMAP_RESERVE_PROT_EXEC (ERTS_MMAP_PROT_EXEC) #define ERTS_MMAP_RESERVE_FLAGS (ERTS_MMAP_FLAGS|MAP_FIXED) #define ERTS_MMAP_UNRESERVE_PROT (PROT_NONE) #define ERTS_MMAP_UNRESERVE_FLAGS (ERTS_MMAP_FLAGS|MAP_NORESERVE|MAP_FIXED) @@ -1326,9 +1331,10 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign) #define ERTS_MMAP_VIRTUAL_FLAGS (ERTS_MMAP_FLAGS|MAP_NORESERVE) static int -os_reserve_physical(char *ptr, UWord size) +os_reserve_physical(char *ptr, UWord size, int exec) { - void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_RESERVE_PROT, + const int prot = exec ? ERTS_MMAP_RESERVE_PROT_EXEC : ERTS_MMAP_RESERVE_PROT; + void *res = mmap((void *) ptr, (size_t) size, prot, ERTS_MMAP_RESERVE_FLAGS, ERTS_MMAP_FD, 0); if (res == (void *) MAP_FAILED) return 0; @@ -1361,7 +1367,7 @@ os_mmap_virtual(char *ptr, UWord size) #endif /* ERTS_HAVE_OS_MMAP */ -static int reserve_noop(char *ptr, UWord size) +static int reserve_noop(char *ptr, UWord size, int exec) { #ifdef ERTS_MMAP_DEBUG_FILL_AREAS Uint32 *uip, *end = (Uint32 *) (ptr + size); @@ -1404,7 +1410,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm, #if ERTS_HAVE_OS_MMAP if (!mm->no_os_mmap) { - ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0); + ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0, 0); if (ptr) { mm->desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE; ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE); @@ -1423,7 +1429,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm, da_map = &mm->sua.map; desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); if (desc) { - if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) + if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0)) ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE); else desc = NULL; @@ -1433,7 +1439,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm, da_map = &mm->sa.map; desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); if (desc) { - if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) + if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0)) ERTS_MMAP_SIZE_SC_SA_INC(ERTS_PAGEALIGNED_SIZE); else desc = NULL; @@ -1494,7 +1500,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) if (desc) { seg = desc->start; end = seg+asize; - if (!mm->reserve_physical(seg, asize)) + if (!mm->reserve_physical(seg, asize, mm->executable)) goto supercarrier_reserve_failure; if (desc->end == end) { delete_free_seg(&mm->sua.map, desc); @@ -1509,8 +1515,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) } if (asize <= mm->sua.bot - mm->sa.top) { - if (!mm->reserve_physical(mm->sua.bot - asize, - asize)) + if (!mm->reserve_physical(mm->sua.bot - asize, asize, + mm->executable)) goto supercarrier_reserve_failure; mm->sua.bot -= asize; seg = mm->sua.bot; @@ -1526,7 +1532,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) char *start = seg = desc->start; seg = (char *) ERTS_SUPERALIGNED_CEILING(seg); end = seg+asize; - if (!mm->reserve_physical(start, (UWord) (end - start))) + if (!mm->reserve_physical(start, (UWord) (end - start), + mm->executable)) goto supercarrier_reserve_failure; ERTS_MMAP_SIZE_SC_SA_INC(asize); if (desc->end == end) { @@ -1558,7 +1565,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) if (asize + (seg - start) <= mm->sua.bot - start) { end = seg + asize; - if (!mm->reserve_physical(start, (UWord) (end - start))) + if (!mm->reserve_physical(start, (UWord) (end - start), + mm->executable)) goto supercarrier_reserve_failure; mm->sa.top = end; ERTS_MMAP_SIZE_SC_SA_INC(asize); @@ -1580,7 +1588,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start); end = seg + asize; - if (!mm->reserve_physical(seg, (UWord) (org_end - seg))) + if (!mm->reserve_physical(seg, (UWord) (org_end - seg), + mm->executable)) goto supercarrier_reserve_failure; ERTS_MMAP_SIZE_SC_SUA_INC(asize); if (org_start != seg) { @@ -1613,13 +1622,13 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) /* Map using OS primitives */ if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mm->no_os_mmap) { if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) { - seg = os_mmap(NULL, asize, 0); + seg = os_mmap(NULL, asize, 0, mm->executable); if (!seg) goto failure; } else { asize = ERTS_SUPERALIGNED_CEILING(*sizep); - seg = os_mmap(NULL, asize, 1); + seg = os_mmap(NULL, asize, 1, mm->executable); if (!seg) goto failure; @@ -1629,7 +1638,8 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) os_munmap(seg, asize); - ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1); + ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1, + mm->executable); if (!ptr) goto failure; @@ -1964,7 +1974,8 @@ erts_mremap(ErtsMemMapper* mm, if (next && new_end <= next->end) { if (!mm->reserve_physical(((char *) ptr) + old_size, - asize - old_size)) + asize - old_size, + mm->executable)) goto supercarrier_reserve_failure; if (new_end < next->end) resize_free_seg(&mm->sua.map, next, new_end, next->end); @@ -1982,7 +1993,8 @@ erts_mremap(ErtsMemMapper* mm, if (end == mm->sa.top) { if (new_end <= mm->sua.bot) { if (!mm->reserve_physical(((char *) ptr) + old_size, - asize - old_size)) + asize - old_size, + mm->executable)) goto supercarrier_reserve_failure; mm->sa.top = new_end; new_ptr = ptr; @@ -1994,7 +2006,8 @@ erts_mremap(ErtsMemMapper* mm, adjacent_free_seg(&mm->sa.map, start, end, &prev, &next); if (next && new_end <= next->end) { if (!mm->reserve_physical(((char *) ptr) + old_size, - asize - old_size)) + asize - old_size, + mm->executable)) goto supercarrier_reserve_failure; if (new_end < next->end) resize_free_seg(&mm->sa.map, next, new_end, next->end); @@ -2112,7 +2125,7 @@ static void hard_dbg_mseg_init(void); #endif void -erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) +erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) { static int is_first_call = 1; int virtual_map = 0; @@ -2144,6 +2157,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) mm->supercarrier = 0; mm->reserve_physical = reserve_noop; mm->unreserve_physical = unreserve_noop; + mm->executable = executable; #if HAVE_MMAP && !defined(MAP_ANON) mm->mmap_fd = open("/dev/zero", O_RDWR); @@ -2200,7 +2214,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) * The whole supercarrier will by physically * reserved all the time. */ - start = os_mmap(NULL, sz, 1); + start = os_mmap(NULL, sz, 1, executable); } if (!start) erts_exit(1, @@ -2274,7 +2288,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) mm->sua.top -= ERTS_PAGEALIGNED_SIZE; mm->size.supercarrier.used.total += ERTS_PAGEALIGNED_SIZE; #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE)) + if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE, 0)) #endif add_free_desc_area(mm, mm->sua.top, end); mm->desc.reserved += (end - mm->sua.top) / sizeof(ErtsFreeSegDesc); @@ -2287,7 +2301,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) * will be used for free segment descritors. */ #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start)) + if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start, 0)) erts_exit(1, "erts_mmap: Failed to reserve physical memory for descriptors\n"); #endif mm->desc.unused_start = start; diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 7c5ddbec0b..30548bd0b0 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -61,7 +61,7 @@ void *erts_mmap(ErtsMemMapper*, Uint32 flags, UWord *sizep); void erts_munmap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord size); void *erts_mremap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord old_size, UWord *sizep); int erts_mmap_in_supercarrier(ErtsMemMapper*, void *ptr); -void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*); +void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*, int executable); struct erts_mmap_info_struct { UWord sizes[6]; diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 286130bece..f0fbb682b0 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1402,9 +1402,9 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); - erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap); + erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap, 0); #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) - erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap); + erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap, 0); #endif if (!IS_2POW(GET_PAGE_SIZE)) -- cgit v1.2.3 From 85a6623152988c267cea008d20616b61ea9c223c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 12 Apr 2016 20:41:58 +0200 Subject: erts: Add 'exec_alloc' for hipe code that uses its own super carrier (erts_exec_mmapper) to guarantee low addressed and executable memory (PROT_EXEC). Currently only used on x86_64 that needs low memory for HiPE/AMD64's small code model. By initializing erts_exec_mapper early we secure its low memory area before erts_literal_mmapper might steal it. --- erts/configure.in | 7 +++ erts/doc/src/erts_alloc.xml | 18 +++++- erts/emulator/beam/erl_alloc.c | 77 +++++++++++++++++++++++- erts/emulator/beam/erl_alloc.types | 12 ++++ erts/emulator/beam/erl_bif_info.c | 1 + erts/emulator/hipe/hipe_amd64.c | 116 ++++-------------------------------- erts/emulator/sys/common/erl_mmap.c | 45 ++++++++++++-- erts/emulator/sys/common/erl_mmap.h | 16 ++++- erts/emulator/sys/common/erl_mseg.c | 7 +++ erts/emulator/sys/common/erl_mseg.h | 4 +- erts/etc/common/erlexec.c | 2 + 11 files changed, 188 insertions(+), 117 deletions(-) diff --git a/erts/configure.in b/erts/configure.in index 0257079c3b..462a13269e 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -3351,6 +3351,13 @@ if test X${enable_hipe} = Xyes; then AC_DEFINE(HIPE,[1],[Define to enable HiPE]) HIPE_HELPERS="xmerl syntax_tools edoc" ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS hipe" + case "$ARCH" in + amd64) + # For now exec_alloc is only used for hipe on amd64 + AC_MSG_NOTICE([Enable exec_alloc for hipe code allocation]) + ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS exec_alloc" + ;; + esac fi fi AC_SUBST(HIPE_HELPERS) diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index 0965f9b49c..c54082f8c8 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -63,6 +63,9 @@ fix_alloc A fast allocator used for some frequently used fixed size data types. + exec_alloc + Allocator used by hipe for native executable code + on specific architectures (x86_64). std_alloc Allocator used for most memory blocks not allocated via any of the other allocators described above. @@ -80,7 +83,8 @@ the number of system calls made.

sys_alloc and literal_alloc are always enabled and - cannot be disabled. mseg_alloc is always enabled if it is + cannot be disabled. exec_alloc is only available if it is needed + and cannot be disabled. mseg_alloc is always enabled if it is available and an allocator that uses it is enabled. All other allocators can be enabled or disabled. By default all allocators are enabled. @@ -248,16 +252,17 @@ the currently present allocators:

B: binary_alloc - I: literal_alloc D: std_alloc E: ets_alloc F: fix_alloc H: eheap_alloc + I: literal_alloc L: ll_alloc M: mseg_alloc R: driver_alloc S: sl_alloc T: temp_alloc + X: exec_alloc Y: sys_alloc

The following flags are available for configuration of @@ -576,6 +581,15 @@ and is usually sufficient. The flag is ignored on 32-bit architectures. +

The following flag is special for exec_alloc:

+ + ]]> + + exec_alloc super carrier size (in MB). The amount of + virtual address space reserved for native executable code + used by hipe on specific architectures (x86_64). The default is 512 MB. + +

Instrumentation flags:

+Mim true|false diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index efe31193b9..00911ff027 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -30,6 +30,7 @@ #endif #define ERTS_ALLOC_C__ #define ERTS_ALC_INTERNAL__ +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" @@ -131,6 +132,9 @@ static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; static ErtsAllocatorState_t literal_alloc_state; +#ifdef ERTS_ALC_A_EXEC +static ErtsAllocatorState_t exec_alloc_state; +#endif static ErtsAllocatorState_t test_alloc_state; typedef struct { @@ -216,6 +220,7 @@ typedef struct { struct au_init driver_alloc; struct au_init fix_alloc; struct au_init literal_alloc; + struct au_init exec_alloc; struct au_init test_alloc; } erts_alc_hndl_args_init_t; @@ -339,6 +344,38 @@ set_default_literal_alloc_opts(struct au_init *ip) #endif } +#ifdef ERTS_ALC_A_EXEC +static void +set_default_exec_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 1; + ip->thr_spec = 0; + ip->disable_allowed = 0; + ip->thr_spec_allowed = 0; + ip->carrier_migration_allowed = 0; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.ramv = 0; + ip->init.util.mmsbc = 0; + ip->init.util.sbct = ~((UWord) 0); + ip->init.util.name_prefix = "exec_"; + ip->init.util.alloc_no = ERTS_ALC_A_EXEC; + ip->init.util.mmbcs = 0; /* No main carrier */ + ip->init.util.ts = ERTS_ALC_MTA_EXEC; + ip->init.util.asbcst = 0; + ip->init.util.rsbcst = 0; + ip->init.util.rsbcmt = 0; + ip->init.util.rmbcmt = 0; + ip->init.util.acul = 0; + + ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; + ip->init.util.mseg_mmapper = &erts_exec_mmapper; +} +#endif /* ERTS_ALC_A_EXEC */ + static void set_default_temp_alloc_opts(struct au_init *ip) { @@ -660,6 +697,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); set_default_literal_alloc_opts(&init.literal_alloc); +#ifdef ERTS_ALC_A_EXEC + set_default_exec_alloc_opts(&init.exec_alloc); +#endif set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) @@ -689,6 +729,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; init.literal_alloc.thr_spec = 0; +#ifdef ERTS_ALC_A_EXEC + init.exec_alloc.thr_spec = 0; +#endif #endif /* Make adjustments for carrier migration support */ @@ -702,6 +745,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_carrier_migration_support(&init.driver_alloc); adjust_carrier_migration_support(&init.fix_alloc); adjust_carrier_migration_support(&init.literal_alloc); +#ifdef ERTS_ALC_A_EXEC + adjust_carrier_migration_support(&init.exec_alloc); +#endif if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ @@ -717,6 +763,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; init.literal_alloc.thr_spec = 0; +#ifdef ERTS_ALC_A_EXEC + init.exec_alloc.thr_spec = 0; +#endif /* No carrier migration */ init.temp_alloc.init.util.acul = 0; @@ -729,6 +778,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.driver_alloc.init.util.acul = 0; init.fix_alloc.init.util.acul = 0; init.literal_alloc.init.util.acul = 0; +#ifdef ERTS_ALC_A_EXEC + init.exec_alloc.init.util.acul = 0; +#endif } #ifdef ERTS_SMP @@ -746,6 +798,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.driver_alloc, erts_no_schedulers); adjust_tpref(&init.fix_alloc, erts_no_schedulers); adjust_tpref(&init.literal_alloc, erts_no_schedulers); +#ifdef ERTS_ALC_A_EXEC + adjust_tpref(&init.exec_alloc, erts_no_schedulers); +#endif #else /* No thread specific if not smp */ @@ -765,6 +820,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.driver_alloc); refuse_af_strategy(&init.fix_alloc); refuse_af_strategy(&init.literal_alloc); +#ifdef ERTS_ALC_A_EXEC + refuse_af_strategy(&init.exec_alloc); +#endif #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) @@ -809,6 +867,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu); +#ifdef ERTS_ALC_A_EXEC + set_au_allocator(ERTS_ALC_A_EXEC, &init.exec_alloc, ncpu); +#endif set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { @@ -865,7 +926,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, &literal_alloc_state); - +#ifdef ERTS_ALC_A_EXEC + start_au_allocator(ERTS_ALC_A_EXEC, + &init.exec_alloc, + &exec_alloc_state); +#endif start_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, &test_alloc_state); @@ -1500,6 +1565,16 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) else handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0); break; + case 'X': + if (has_prefix("scs", argv[i]+3)) { +#ifdef ERTS_ALC_A_EXEC + init->mseg.exec_mmap.scs = +#endif + get_mb_value(argv[i]+6, argv, &i); + } + else + handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0); + break; case 'D': handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0); break; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 14067283bd..0649ed903a 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -87,6 +87,9 @@ allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc allocator LITERAL true literal_alloc ++if exec_alloc +allocator EXEC true exec_alloc ++endif +else # Non smp build @@ -98,6 +101,9 @@ allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc allocator LITERAL false literal_alloc ++if exec_alloc +allocator EXEC false exec_alloc ++endif +endif @@ -335,8 +341,14 @@ type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths # Currently most hipe code use this type. type HIPE SYSTEM SYSTEM hipe_data ++if exec_alloc +type HIPE_EXEC EXEC CODE hipe_code ++endif + +endif + + +if heap_frag_elim_test type SSB SHORT_LIVED PROCESSES ssb diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 0b2c26a548..163070e85a 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -22,6 +22,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_vm.h" #include "global.h" diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 1bb336637d..9d7ada18bc 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -83,21 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on AMD64 because some Linux kernels - * (including 2.6.10-rc1 and newer www.kernel.org ones) - * default to non-executable memory mappings, causing - * ordinary malloc() memory to be non-executable. - * - * Implementing this properly also allows us to ensure that - * executable code ends up in the low 2GB of the address space, - * as required by HiPE/AMD64's small code model. - */ -static unsigned int code_bytes; -static char *code_next; - #if 0 /* change to non-zero to get allocation statistics at exit() */ static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; static unsigned int atexit_done; @@ -121,101 +106,20 @@ static void atexit_alloc_code_stats(void) #define ALLOC_CODE_STATS(X) do{}while(0) #endif -/* FreeBSD 6.1 breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; -#if !defined(MAP_32BIT) - /* FreeBSD doesn't have MAP_32BIT, and it doesn't respect - a plain map_hint (returns high mappings even though the - hint refers to a free area), so we have to use both map_hint - and MAP_FIXED to get addresses below the 2GB boundary. - This is even worse than the Linux/ppc64 case. - Similarly, Solaris 10 doesn't have MAP_32BIT, - and it doesn't respect a plain map_hint. */ - if (!map_hint) /* first call */ - map_hint = (char*)(512*1024*1024); /* 0.5GB */ -#endif - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#if defined(MAP_32BIT) - |MAP_32BIT -#elif defined(__FreeBSD__) || defined(__sun__) - |MAP_FIXED -#endif - , - -1, 0); - ALLOC_CODE_STATS(fprintf(stderr, "%s: mmap(%p,%u,...) == %p\r\n", __FUNCTION__, map_hint, map_bytes, map_start)); -#if !defined(MAP_32BIT) - if (map_start != MAP_FAILED && - (((unsigned long)map_start + (map_bytes-1)) & ~0x7FFFFFFFUL)) { - fprintf(stderr, "mmap with hint %p returned code memory %p\r\n", map_hint, map_start); - abort(); - } -#endif - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; -#if !defined(MAP_32BIT) - if (!code_next) /* first call */ - code_next = map_start; -#endif - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - +/* + * Memory allocator for executable code. + * + * We use a dedicated allocator for executable code (from OTP 19.0) + * to make sure the memory we get is executable (PROT_EXEC) + * and to ensure that executable code ends up in the low 2GB + * of the address space, as required by HiPE/AMD64's small code model. + */ static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; ALLOC_CODE_STATS(++nr_allocs); ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index f5f40b35a8..38970bd24e 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -356,6 +356,12 @@ char* erts_literals_start; UWord erts_literals_size; #endif +#ifdef ERTS_ALC_A_EXEC +ErtsMemMapper erts_exec_mmapper; +#endif + + + #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ do { \ mm->size.supercarrier.used.total += (SZ); \ @@ -1351,10 +1357,39 @@ os_unreserve_physical(char *ptr, UWord size) } static void * -os_mmap_virtual(char *ptr, UWord size) +os_mmap_virtual(char *ptr, UWord size, int exec) { - void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_VIRTUAL_PROT, - ERTS_MMAP_VIRTUAL_FLAGS, ERTS_MMAP_FD, 0); + int flags = ERTS_MMAP_VIRTUAL_FLAGS; + void* res; + +#ifdef ERTS_ALC_A_EXEC + if (exec) { + ASSERT(!ptr); + /* OTP-19.0: Nice hack below cut-and-pasted from hipe_amd64.c */ + +# ifdef MAP_32BIT + /* If we got MAP_32BIT (Linux), then use that to ask for low memory */ + flags |= MAP_32BIT; +# else + /* FreeBSD doesn't have MAP_32BIT, and it doesn't respect + a plain map_hint (returns high mappings even though the + hint refers to a free area), so we have to use both map_hint + and MAP_FIXED to get addresses below the 2GB boundary. + This is even worse than the Linux/ppc64 case. + Similarly, Solaris 10 doesn't have MAP_32BIT, + and it doesn't respect a plain map_hint. */ + ptr = (char*)(512*1024*1024); /* 0.5GB */ + +# if defined(__FreeBSD__) || defined(__sun__) + flags |= MAP_FIXED; +# endif +# endif /* !MAP_32BIT */ + } +#else /* !ERTS_ALC_A_EXEC */ + ASSERT(!exec); +#endif + res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_VIRTUAL_PROT, + flags, ERTS_MMAP_FD, 0); if (res == (void *) MAP_FAILED) return NULL; return res; @@ -2177,7 +2212,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) ptr = (char *) ERTS_PAGEALIGNED_CEILING(init->virtual_range.start); end = (char *) ERTS_PAGEALIGNED_FLOOR(init->virtual_range.end); sz = end - ptr; - start = os_mmap_virtual(ptr, sz); + start = os_mmap_virtual(ptr, sz, executable); if (!start || start > ptr || start >= end) erts_exit(1, "erts_mmap: Failed to create virtual range for super carrier\n"); @@ -2202,7 +2237,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) sz = ERTS_PAGEALIGNED_CEILING(init->scs); #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION if (!init->scrpm) { - start = os_mmap_virtual(NULL, sz); + start = os_mmap_virtual(NULL, sz, executable); mm->reserve_physical = os_reserve_physical; mm->unreserve_physical = os_unreserve_physical; virtual_map = 1; diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 30548bd0b0..8a7530d943 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -55,6 +55,11 @@ typedef struct { #define ERTS_MMAP_INIT_LITERAL_INITER \ {{NULL, NULL}, {NULL, NULL}, ERTS_LITERAL_VIRTUAL_AREA_SIZE, 1, (1 << 10), 0} +#define ERTS_HIPE_EXEC_VIRTUAL_AREA_SIZE (UWORD_CONSTANT(512)*1024*1024) + +#define ERTS_MMAP_INIT_HIPE_EXEC_INITER \ + {{NULL, NULL}, {NULL, NULL}, ERTS_HIPE_EXEC_VIRTUAL_AREA_SIZE, 1, (1 << 10), 0} + typedef struct ErtsMemMapper_ ErtsMemMapper; void *erts_mmap(ErtsMemMapper*, Uint32 flags, UWord *sizep); @@ -126,10 +131,17 @@ Eterm erts_mmap_debug_info(ErtsMemMapper*, struct process*); # define ERTS_HAVE_OS_MMAP 1 #endif +#ifdef ERTS_WANT_MEM_MAPPERS +# include "erl_alloc_types.h" + extern ErtsMemMapper erts_dflt_mmapper; -#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) extern ErtsMemMapper erts_literal_mmapper; -#endif +# endif +# ifdef ERTS_ALC_A_EXEC +extern ErtsMemMapper erts_exec_mmapper; +# endif +#endif /* ERTS_WANT_MEM_MAPPERS */ /*#define HARD_DEBUG_MSEG*/ #ifdef HARD_DEBUG_MSEG diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index f0fbb682b0..f373302e65 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -31,6 +31,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_mseg.h" #include "global.h" @@ -1402,6 +1403,12 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); +#ifdef ERTS_ALC_A_EXEC + /* Initialize erts_exec_mapper *FIRST*, to increase probability + * of getting low memory for HiPE AMD64's small code model. + */ + erts_mmap_init(&erts_exec_mmapper, &init->exec_mmap, 1); +#endif erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap, 0); #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap, 0); diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index 2acd8f8505..556a135798 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -61,6 +61,7 @@ typedef struct { Uint nos; ErtsMMapInit dflt_mmap; ErtsMMapInit literal_mmap; + ErtsMMapInit exec_mmap; } ErtsMsegInit_t; #define ERTS_MSEG_INIT_DEFAULT_INITIALIZER \ @@ -70,7 +71,8 @@ typedef struct { 10, /* mcs: Max cache size */ \ 1000, /* cci: Cache check interval */ \ ERTS_MMAP_INIT_DEFAULT_INITER, \ - ERTS_MMAP_INIT_LITERAL_INITER \ + ERTS_MMAP_INIT_LITERAL_INITER, \ + ERTS_MMAP_INIT_HIPE_EXEC_INITER \ } typedef struct { diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 54da59e50d..96259f8fa6 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -74,6 +74,7 @@ static const char plusM_au_allocs[]= { 'R', /* driver_alloc */ 'S', /* sl_alloc */ 'T', /* temp_alloc */ + 'X', /* exec_alloc */ 'Z', /* test_alloc */ '\0' }; @@ -123,6 +124,7 @@ static char *plusM_other_switches[] = { "Ytp", "Ytt", "Iscs", + "Xscs", NULL }; -- cgit v1.2.3 From b3c7a581070180750a368ddf3bd55db1bafc11ea Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 15 Apr 2016 18:39:59 +0200 Subject: erts: Make sure literal MBCs have super aligned sizes on 32-bit, as the granularity of the literal bit vector is super-alignment. --- erts/emulator/beam/erl_alloc_util.c | 53 ++++++++++++++++++++++++------------- erts/emulator/beam/erl_alloc_util.h | 16 +++++------ 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index e362a03646..680899c919 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -859,28 +859,34 @@ void* erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void* res; + Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p); ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && !allctr->t && allctr->thread_safe); - res = erts_alcu_mseg_alloc(allctr, size_p, flags); - if (res) - set_literal_range(res, *size_p); + res = erts_alcu_mseg_alloc(allctr, &sz, flags); + if (res) { + set_literal_range(res, sz); + *size_p = sz; + } return res; } void* erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg, - Uint old_size, Uint *new_size_p) + Uint old_size, Uint *new_size_p) { void* res; + Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p); ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && !allctr->t && allctr->thread_safe); if (seg && old_size) clear_literal_range(seg, old_size); - res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); - if (res) - set_literal_range(res, *new_size_p); + res = erts_alcu_mseg_realloc(allctr, seg, old_size, &new_sz); + if (res) { + set_literal_range(res, new_sz); + *new_size_p = new_sz; + } return res; } @@ -942,9 +948,10 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #endif /* HAVE_ERTS_MSEG */ void* -erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) { void *res; + const Uint size = *size_p; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size); @@ -958,9 +965,10 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) } void* -erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign) { void *res; + const Uint size = *size_p; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) @@ -995,30 +1003,37 @@ erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) #ifdef ARCH_32 void* -erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) { void* res; + Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && !allctr->t && allctr->thread_safe); - res = erts_alcu_sys_alloc(allctr, size, 1); - if (res) + res = erts_alcu_sys_alloc(allctr, &size, 1); + if (res) { set_literal_range(res, size); + *size_p = size; + } return res; } void* -erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint old_size, int superalign) { void* res; + Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && !allctr->t && allctr->thread_safe); if (ptr && old_size) clear_literal_range(ptr, old_size); - res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1); - if (res) + res = erts_alcu_sys_realloc(allctr, ptr, &size, old_size, 1); + if (res) { set_literal_range(res, size); + *size_p = size; + } return res; } @@ -3865,12 +3880,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -4028,7 +4043,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, - new_crr_sz, + &new_crr_sz, old_crr_sz, 0); if (new_crr) { @@ -4049,7 +4064,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, - new_crr_sz, + &new_crr_sz, old_crr_sz, 0); if (new_crr) diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 51af59e40c..b3ae4c3281 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -74,8 +74,8 @@ typedef struct { void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); ErtsMemMapper *mseg_mmapper; #endif - void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); - void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); } AllctrInit_t; @@ -212,12 +212,12 @@ void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags # endif #endif /* HAVE_ERTS_MSEG */ -void* erts_alcu_sys_alloc(Allctr_t*, Uint size, int superalign); -void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); +void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #ifdef ARCH_32 -void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint size, int superalign); -void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint *size_p, int superalign); +void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint old_size, int superalign); void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #endif @@ -604,8 +604,8 @@ struct Allctr_t_ { void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); ErtsMemMapper *mseg_mmapper; #endif - void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); - void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void* (*sys_alloc)(Allctr_t *allctr, Uint *size_p, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); void (*init_atoms) (void); -- cgit v1.2.3