aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_alloc_util.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_alloc_util.c')
-rw-r--r--erts/emulator/beam/erl_alloc_util.c142
1 files changed, 99 insertions, 43 deletions
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 3230b6ef34..2995f2f822 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2016. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -52,6 +52,7 @@
#ifdef ERTS_ENABLE_LOCK_COUNT
#include "erl_lock_count.h"
#endif
+#include "lttng-wrapper.h"
#if defined(ERTS_ALLOC_UTIL_HARD_DEBUG) && defined(__GNUC__)
#warning "* * * * * * * * * *"
@@ -858,28 +859,36 @@ void*
erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
- res = erts_alcu_mseg_alloc(allctr, size_p, flags);
- if (res)
- set_literal_range(res, *size_p);
+ res = erts_alcu_mseg_alloc(allctr, &sz, flags);
+ if (res) {
+ set_literal_range(res, sz);
+ *size_p = sz;
+ }
return res;
}
void*
erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg,
- Uint old_size, Uint *new_size_p)
+ Uint old_size, Uint *new_size_p)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
if (seg && old_size)
clear_literal_range(seg, old_size);
- res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p);
- if (res)
- set_literal_range(res, *new_size_p);
+ res = erts_alcu_mseg_realloc(allctr, seg, old_size, &new_sz);
+ if (res) {
+ set_literal_range(res, new_sz);
+ *new_size_p = new_sz;
+ }
return res;
}
@@ -887,8 +896,9 @@ void
erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
Uint flags)
{
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
erts_alcu_mseg_dealloc(allctr, seg, size, flags);
@@ -897,8 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
#elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
+/* Used by literal allocator that has its own mmapper (super carrier) */
void*
-erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
{
void* res;
UWord size = (UWord) *size_p;
@@ -906,33 +917,33 @@ erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
if (flags & ERTS_MSEG_FLG_2POW)
mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
- res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size);
+ res = erts_mmap(allctr->mseg_mmapper, mmap_flags, &size);
*size_p = (Uint)size;
INC_CC(allctr->calls.mseg_alloc);
return res;
}
void*
-erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg,
+erts_alcu_mmapper_mseg_realloc(Allctr_t *allctr, void *seg,
Uint old_size, Uint *new_size_p)
{
void *res;
UWord new_size = (UWord) *new_size_p;
- res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
+ res = erts_mremap(allctr->mseg_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size);
*new_size_p = (Uint) new_size;
INC_CC(allctr->calls.mseg_realloc);
return res;
}
void
-erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
+erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
Uint flags)
{
Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY;
if (flags & ERTS_MSEG_FLG_2POW)
mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED;
- erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size);
+ erts_munmap(allctr->mseg_mmapper, mmap_flags, seg, (UWord)size);
INC_CC(allctr->calls.mseg_dealloc);
}
#endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
@@ -940,9 +951,10 @@ erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
#endif /* HAVE_ERTS_MSEG */
void*
-erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+erts_alcu_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
{
void *res;
+ const Uint size = *size_p;
#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
if (superalign)
res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size);
@@ -956,9 +968,10 @@ erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
}
void*
-erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign)
{
void *res;
+ const Uint size = *size_p;
#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
if (superalign)
@@ -993,38 +1006,48 @@ erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
#ifdef ARCH_32
void*
-erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign)
+erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
- res = erts_alcu_sys_alloc(allctr, size, 1);
- if (res)
+ res = erts_alcu_sys_alloc(allctr, &size, 1);
+ if (res) {
set_literal_range(res, size);
+ *size_p = size;
+ }
return res;
}
void*
-erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign)
+erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint old_size, int superalign)
{
void* res;
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ Uint size = ERTS_SUPERALIGNED_CEILING(*size_p);
+
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
if (ptr && old_size)
clear_literal_range(ptr, old_size);
- res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1);
- if (res)
+ res = erts_alcu_sys_realloc(allctr, ptr, &size, old_size, 1);
+ if (res) {
set_literal_range(res, size);
+ *size_p = size;
+ }
return res;
}
void
erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign)
{
- ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL
- && !allctr->t && allctr->thread_safe);
+ ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL &&
+ allctr->t == 0);
+ ERTS_SMP_LC_ASSERT(allctr->thread_safe);
erts_alcu_sys_dealloc(allctr, ptr, size, 1);
@@ -3125,6 +3148,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr)
erts_smp_atomic_set_wb(&crr->allctr,
((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL);
+ LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr));
}
static void
@@ -3240,6 +3264,7 @@ cpool_fetch(Allctr_t *allctr, UWord size)
first_old_traitor = allctr->cpool.traitor_list.next;
cpool_entrance = NULL;
+ LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size);
/*
* Search my own pooled_list,
* i.e my abandoned carriers that were in the pool last time I checked.
@@ -3861,12 +3886,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
? UNIT_CEILING(bcrr_sz)
: SYS_ALLOC_CARRIER_CEILING(bcrr_sz));
- crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+ crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC);
if (!crr) {
if (crr_sz > UNIT_CEILING(bcrr_sz)) {
crr_sz = UNIT_CEILING(bcrr_sz);
- crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC);
+ crr = (Carrier_t *) allctr->sys_alloc(allctr, &crr_sz, flags & CFLG_MBC);
}
if (!crr) {
#if HAVE_ERTS_MSEG
@@ -3925,6 +3950,21 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
}
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+ if (LTTNG_ENABLED(carrier_create)) {
+ lttng_decl_carrier_stats(mbc_stats);
+ lttng_decl_carrier_stats(sbc_stats);
+ LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats);
+ LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats);
+ LTTNG5(carrier_create,
+ ERTS_ALC_A2AD(allctr->alloc_no),
+ allctr->ix,
+ crr_sz,
+ mbc_stats,
+ sbc_stats);
+ }
+#endif
+
DEBUG_SAVE_ALIGNMENT(crr);
return blk;
}
@@ -4009,7 +4049,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
(void *) old_crr,
- new_crr_sz,
+ &new_crr_sz,
old_crr_sz,
0);
if (new_crr) {
@@ -4030,7 +4070,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
new_crr_sz = UNIT_CEILING(new_crr_sz);
new_crr = (Carrier_t *) allctr->sys_realloc(allctr,
(void *) old_crr,
- new_crr_sz,
+ &new_crr_sz,
old_crr_sz,
0);
if (new_crr)
@@ -4148,6 +4188,21 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp)
allctr->remove_mbc(allctr, crr);
}
+#ifdef USE_LTTNG_VM_TRACEPOINTS
+ if (LTTNG_ENABLED(carrier_destroy)) {
+ lttng_decl_carrier_stats(mbc_stats);
+ lttng_decl_carrier_stats(sbc_stats);
+ LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->mbcs), mbc_stats);
+ LTTNG_CARRIER_STATS_TO_LTTNG_STATS(&(allctr->sbcs), sbc_stats);
+ LTTNG5(carrier_destroy,
+ ERTS_ALC_A2AD(allctr->alloc_no),
+ allctr->ix,
+ crr_sz,
+ mbc_stats,
+ sbc_stats);
+ }
+#endif
+
#ifdef ERTS_SMP
schedule_dealloc_carrier(allctr, crr);
#else
@@ -4718,7 +4773,7 @@ make_name_atoms(Allctr_t *allctr)
size_t prefix_len = strlen(allctr->name_prefix);
if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1)
- erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix);
+ erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix);
memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len);
@@ -5912,7 +5967,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
/* erts_alcu_start assumes that allctr has been zeroed */
if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) {
- erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n",
+ erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n",
__FILE__, __LINE__);
}
@@ -6094,6 +6149,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->mseg_alloc = init->mseg_alloc;
allctr->mseg_realloc = init->mseg_realloc;
allctr->mseg_dealloc = init->mseg_dealloc;
+ allctr->mseg_mmapper = init->mseg_mmapper;
}
else {
ASSERT(!init->mseg_realloc && !init->mseg_dealloc);
@@ -6128,7 +6184,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (allctr->thread_safe)
erts_mtx_destroy(&allctr->mutex);
#endif
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"Failed to create main carrier for %salloc\n",
init->name_prefix);
}
@@ -6273,7 +6329,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2)
case 0x019: return (UWord) PREV_BLK((Block_t *) a1);
case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2);
case 0x01b: return (UWord) sizeof(Unit_t);
- case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1);
+ case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1);
case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break;
#ifdef ERTS_SMP
@@ -6344,7 +6400,7 @@ erts_alcu_verify_unused(Allctr_t *allctr)
if (no) {
UWord sz = allctr->sbcs.blocks.curr.size;
sz += allctr->mbcs.blocks.curr.size;
- erl_exit(ERTS_ABORT_EXIT,
+ erts_exit(ERTS_ABORT_EXIT,
"%salloc() used when expected to be unused!\n"
"Total amount of blocks allocated: %bpu\n"
"Total amount of bytes allocated: %bpu\n",
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-03 23:16:53 +0000