aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--erts/emulator/sys/common/erl_mmap.c102
-rw-r--r--erts/emulator/sys/common/erl_mmap.h2
-rw-r--r--erts/emulator/sys/common/erl_mseg.c70
3 files changed, 67 insertions, 107 deletions
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index eae1a1a410..73874759f5 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -1260,30 +1260,53 @@ erts_mmap(Uint32 flags, UWord *sizep)
desc = lookup_free_seg(&mmap_state.sa.map, asize);
if (desc) {
- seg = desc->start;
+ char *start = seg = desc->start;
+ seg = (char *) ERTS_SUPERALIGNED_CEILING(seg);
end = seg+asize;
- if (!mmap_state.reserve_physical(seg, asize))
+ if (!mmap_state.reserve_physical(start, (UWord) (end - start)))
goto supercarrier_reserve_failure;
+ ERTS_MMAP_SIZE_SC_SA_INC(asize);
if (desc->end == end) {
- delete_free_seg(&mmap_state.sa.map, desc);
- free_desc(desc);
+ if (start != seg)
+ resize_free_seg(&mmap_state.sa.map, desc, start, seg);
+ else {
+ delete_free_seg(&mmap_state.sa.map, desc);
+ free_desc(desc);
+ }
}
else {
ERTS_MMAP_ASSERT(end < desc->end);
resize_free_seg(&mmap_state.sa.map, desc, end, desc->end);
+ if (start != seg) {
+ UWord ad_sz;
+ ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map,
+ start, seg);
+ start += ad_sz;
+ if (start != seg)
+ mmap_state.unreserve_physical(start, (UWord) (seg - start));
+ }
}
- ERTS_MMAP_SIZE_SC_SA_INC(asize);
goto supercarrier_success;
}
if (superaligned) {
+ seg = (char *) ERTS_SUPERALIGNED_CEILING(mmap_state.sa.top);
- if (asize <= mmap_state.sua.bot - mmap_state.sa.top) {
- seg = (void *) mmap_state.sa.top;
- if (!mmap_state.reserve_physical(seg, asize))
+ if (asize <= mmap_state.sua.bot - seg) {
+ char *start = mmap_state.sa.top;
+ end = seg + asize;
+ if (!mmap_state.reserve_physical(start, (UWord) (end - start)))
goto supercarrier_reserve_failure;
- mmap_state.sa.top += asize;
+ mmap_state.sa.top = end;
ERTS_MMAP_SIZE_SC_SA_INC(asize);
+ if (start != seg) {
+ UWord ad_sz;
+ ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map,
+ start, seg);
+ start += ad_sz;
+ if (start != seg)
+ mmap_state.unreserve_physical(start, (UWord) (seg - start));
+ }
goto supercarrier_success;
}
@@ -1294,7 +1317,7 @@ erts_mmap(Uint32 flags, UWord *sizep)
seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start);
end = seg + asize;
- if (!mmap_state.reserve_physical(seg, asize))
+ if (!mmap_state.reserve_physical(seg, (UWord) (org_end - seg)))
goto supercarrier_reserve_failure;
ERTS_MMAP_SIZE_SC_SUA_INC(asize);
if (org_start != seg) {
@@ -1303,12 +1326,17 @@ erts_mmap(Uint32 flags, UWord *sizep)
desc = NULL;
}
if (end != org_end) {
+ UWord ad_sz = 0;
ERTS_MMAP_ASSERT(end < org_end);
if (desc)
resize_free_seg(&mmap_state.sua.map, desc, end, org_end);
else
- alloc_desc_insert_free_seg(&mmap_state.sua.map,
- end, org_end);
+ ad_sz = alloc_desc_insert_free_seg(&mmap_state.sua.map,
+ end, org_end);
+ end += ad_sz;
+ if (end != org_end)
+ mmap_state.unreserve_physical(end,
+ (UWord) (org_end - end));
}
goto supercarrier_success;
}
@@ -1363,8 +1391,7 @@ erts_mmap(Uint32 flags, UWord *sizep)
supercarrier_success:
#ifdef ERTS_MMAP_DEBUG
- if ((ERTS_MMAPFLG_SUPERALIGNED & flags)
- || ERTS_MMAP_IN_SUPERALIGNED_AREA(seg)) {
+ if (ERTS_MMAPFLG_SUPERALIGNED & flags) {
ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(seg));
ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize));
}
@@ -1388,10 +1415,8 @@ supercarrier_reserve_failure:
}
void
-erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
+erts_munmap(Uint32 flags, void *ptr, UWord size)
{
- void *ptr = *ptrp;
- UWord size = *sizep;
ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(size));
if (!ERTS_MMAP_IN_SUPERCARRIER(ptr)) {
@@ -1416,13 +1441,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) {
- start = (char *) ERTS_SUPERALIGNED_CEILING(start);
- end = (char *) ERTS_SUPERALIGNED_FLOOR(end);
-
- size = (UWord) (end - start);
- *ptrp = start;
- *sizep = size;
-
map = &mmap_state.sa.map;
adjacent_free_seg(map, start, end, &prev, &next);
@@ -1439,8 +1457,6 @@ erts_munmap(Uint32 flags, void **ptrp, UWord *sizep)
}
}
else {
- ERTS_MMAP_ASSERT(ERTS_MMAP_IN_SUPERUNALIGNED_AREA(ptr));
-
map = &mmap_state.sua.map;
adjacent_free_seg(map, start, end, &prev, &next);
@@ -1497,8 +1513,6 @@ static void *
remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
{
UWord size = *sizep;
- UWord um_size = old_size;
- void *um_ptr = ptr;
void *new_ptr = erts_mmap(flags, &size);
if (!new_ptr)
return NULL;
@@ -1506,9 +1520,7 @@ remap_move(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
if (old_size < size)
size = old_size;
sys_memcpy(new_ptr, ptr, (size_t) size);
- erts_munmap(flags, &um_ptr, &um_size);
- ERTS_MMAP_ASSERT(um_ptr == ptr);
- ERTS_MMAP_ASSERT(um_size == old_size);
+ erts_munmap(flags, ptr, old_size);
return new_ptr;
}
@@ -1627,28 +1639,18 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
end = start + old_size;
new_end = start+asize;
+ ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
+ ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size));
+ ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize));
+
if (asize < old_size) {
UWord unres_sz;
new_ptr = ptr;
if (!ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr)) {
- ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(ptr));
- ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(old_size));
- ERTS_MMAP_ASSERT(ERTS_IS_PAGEALIGNED(asize));
map = &mmap_state.sua.map;
ERTS_MMAP_SIZE_SC_SUA_DEC(old_size - asize);
}
else {
- ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr));
- ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size));
- if (!superaligned) {
- /* must be a superaligned size in this area */
- asize = ERTS_SUPERALIGNED_CEILING(asize);
- ERTS_MMAP_ASSERT(asize <= old_size);
- if (asize == old_size)
- goto supercarrier_resize_success;
- new_end = start+asize;
- }
- ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize));
if (end == mmap_state.sa.top) {
mmap_state.sa.top = new_end;
mmap_state.unreserve_physical(((char *) ptr) + asize,
@@ -1696,16 +1698,6 @@ erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep)
}
}
else { /* Superaligned area */
- ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(ptr));
- ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(old_size));
-
- if (!superaligned) {
- /* must be a superaligned size in this area */
- asize = ERTS_PAGEALIGNED_CEILING(asize);
- new_end = start+asize;
- }
-
- ERTS_MMAP_ASSERT(ERTS_IS_SUPERALIGNED(asize));
if (end == mmap_state.sa.top) {
if (new_end <= mmap_state.sua.bot) {
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index b75200f4e9..6cb51fb0b4 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -53,7 +53,7 @@ typedef struct {
{{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1}
void *erts_mmap(Uint32 flags, UWord *sizep);
-void erts_munmap(Uint32 flags, void **ptrp, UWord *sizep);
+void erts_munmap(Uint32 flags, void *ptr, UWord size);
void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep);
int erts_mmap_in_supercarrier(void *ptr);
void erts_mmap_init(ErtsMMapInit*);
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 09035ca73e..2474015bcb 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -330,7 +330,7 @@ mseg_create(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, UWord *sizep)
}
static ERTS_INLINE void
-mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord *size_p) {
+mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void *seg_p, UWord size) {
Uint32 mmap_flags = 0;
#if HALFWORD_HEAP
@@ -341,11 +341,11 @@ mseg_destroy(ErtsMsegAllctr_t *ma, Uint flags, MemKind* mk, void **seg_pp, UWord
if (MSEG_FLG_IS_2POW(flags))
mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
- erts_munmap(mmap_flags, seg_pp, size_p);
+ erts_munmap(mmap_flags, seg_p, size);
#ifdef ERTS_PRINT_ERTS_MMAP
erts_fprintf(stderr, "erts_munmap(%s, %p, %bpu);\n",
(mmap_flags & ERTS_MMAPFLG_SUPERALIGNED) ? "sa" : "sua",
- *seg_pp, *size_p);
+ seg_p, *size);
#endif
INC_CC(ma, destroy);
@@ -446,19 +446,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui
return 1;
} else if (!MSEG_FLG_IS_2POW(flags) && !erts_circleq_is_empty(&(mk->cache_unpowered_node))) {
- void *destr_seg;
- UWord destr_size;
/* No free slots.
* Evict oldest slot from unpowered cache so we can cache an unpowered (sbc) segment */
c = erts_circleq_tail(&(mk->cache_unpowered_node));
erts_circleq_remove(c);
- destr_seg = c->seg;
- destr_size = c->size;
- mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, &destr_seg, &destr_size);
- ASSERT(destr_seg == c->seg);
- ASSERT(destr_size == c->size);
+ mseg_destroy(mk->ma, ERTS_MSEG_FLG_NONE, mk, c->seg, c->size);
mseg_cache_clear_node(c);
c->seg = seg;
@@ -478,19 +472,13 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size, Ui
int i;
for( i = 0; i < CACHE_AREAS; i++) {
- void *destr_seg;
- UWord destr_size;
if (erts_circleq_is_empty(&(mk->cache_powered_node[i])))
continue;
c = erts_circleq_tail(&(mk->cache_powered_node[i]));
erts_circleq_remove(c);
- destr_seg = seg;
- destr_size = c->size;
- mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, &destr_seg, &destr_size);
- ASSERT(destr_seg == c->seg);
- ASSERT(destr_size == c->size);
+ mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, c->seg, c->size);
mseg_cache_clear_node(c);
@@ -544,13 +532,8 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
ASSERT(!(mk->cache_size < 0));
- if (csize != size) {
- char *destr_seg = seg + size;
- UWord destr_size = csize - size;
- mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, (void**)&destr_seg, &destr_size);
- *size_p = (UWord) (destr_seg - seg);
- ASSERT(seg + csize == destr_seg + destr_size);
- }
+ if (csize != size)
+ mseg_destroy(mk->ma, ERTS_MSEG_FLG_2POW, mk, seg + size, csize - size);
return seg;
}
@@ -625,8 +608,6 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, UWord *size_p, Uint flag
*/
static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags, cache_t *head) {
- void *destr_seg;
- UWord destr_size;
cache_t *c = NULL;
c = erts_circleq_tail(head);
@@ -635,11 +616,7 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, Uint flags
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
- destr_seg = c->seg;
- destr_size = c->size;
- mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size);
- ASSERT(destr_seg == c->seg);
- ASSERT(destr_size == c->size);
+ mseg_destroy(mk->ma, flags, mk, c->seg, c->size);
mseg_cache_clear_node(c);
erts_circleq_push_head(&(mk->cache_free), c);
@@ -655,8 +632,6 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca
cache_t *c = NULL;
while (!erts_circleq_is_empty(head)) {
- void *destr_seg;
- UWord destr_size;
c = erts_circleq_tail(head);
erts_circleq_remove(c);
@@ -664,11 +639,7 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, Uint flags, ca
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
- destr_seg = c->seg;
- destr_size = c->size;
- mseg_destroy(mk->ma, flags, mk, &destr_seg, &destr_size);
- ASSERT(destr_seg == c->seg);
- ASSERT(destr_size == c->size);
+ mseg_destroy(mk->ma, flags, mk, c->seg, c->size);
mseg_cache_clear_node(c);
erts_circleq_push_head(&(mk->cache_free), c);
@@ -813,12 +784,14 @@ mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, UWord *size_p,
INC_CC(ma, alloc);
- if (!MSEG_FLG_IS_2POW(flags) && !IS_2POW(size))
+ if (!MSEG_FLG_IS_2POW(flags))
size = ERTS_PAGEALIGNED_CEILING(*size_p);
else {
size = ALIGNED_CEILING(*size_p);
- /* Cache optim (if applicable) */
- size = ceil_2pow(size);
+ if (!IS_2POW(size)) {
+ /* Cache optim (if applicable) */
+ size = ceil_2pow(size);
+ }
}
if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size, flags)) != NULL)
@@ -845,8 +818,6 @@ static void
mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size,
Uint flags, const ErtsMsegOpt_t *opt)
{
- void *destr_seg;
- UWord destr_size;
MemKind* mk = memkind(ma, opt);
ERTS_MSEG_DEALLOC_STAT(mk,size);
@@ -859,11 +830,7 @@ mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, UWord size,
if (erts_mtrace_enabled)
erts_mtrace_crr_free(atype, SEGTYPE, seg);
- destr_seg = seg;
- destr_size = size;
- mseg_destroy(ma, flags, mk, &destr_seg, &destr_size);
- ASSERT(destr_seg == seg);
- ASSERT(destr_size == size);
+ mseg_destroy(ma, flags, mk, seg, size);
done:
@@ -896,13 +863,14 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg,
mk = memkind(ma, opt);
new_seg = seg;
-
if (!MSEG_FLG_IS_2POW(flags))
new_size = ERTS_PAGEALIGNED_CEILING(*new_size_p);
else {
new_size = ALIGNED_CEILING(*new_size_p);
- /* Cache optim (if applicable) */
- new_size = ceil_2pow(new_size);
+ if (!IS_2POW(new_size)) {
+ /* Cache optim (if applicable) */
+ new_size = ceil_2pow(new_size);
+ }
}
if (new_size > old_size) {
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-24 09:35:57 +0000