aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--erts/emulator/sys/common/erl_mseg.c115
1 files changed, 103 insertions, 12 deletions
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index c8dd50bf7a..7db9da6aba 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -183,6 +183,7 @@ typedef struct ErtsMsegAllctr_t_ ErtsMsegAllctr_t;
struct mem_kind_t {
cache_t cache[MAX_CACHE_SIZE];
+ cache_t *cache_unpowered;
cache_t *cache_area[CACHE_AREAS];
cache_t *cache_free;
@@ -529,13 +530,15 @@ static ERTS_INLINE void mseg_cache_free_descriptor(MemKind *mk, cache_t *c) {
static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size) {
+ cache_t *c;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- if (IS_2POW(size)) {
- int ix = SIZE_TO_CACHE_AREA_IDX(size);
- cache_t *c;
- if (ix < CACHE_AREAS && mk->cache_free) {
- ASSERT( (1 << (ix + MSEG_ALIGN_BITS)) == size);
+ if (mk->cache_free) {
+ if (IS_2POW(size)) {
+ int ix = SIZE_TO_CACHE_AREA_IDX(size);
+
+ ASSERT(ix < CACHE_AREAS);
+ ASSERT((1 << (ix + MSEG_ALIGN_BITS)) == size);
/* unlink from free cache list */
c = mseg_cache_alloc_descriptor(mk);
@@ -548,6 +551,23 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size) {
mk->cache_area[ix] = c;
mk->cache_size++;
+ ASSERT(mk->cache_size <= mk->ma->max_cache_size);
+
+ return 1;
+ } else {
+ /* unlink from free cache list */
+ c = mseg_cache_alloc_descriptor(mk);
+
+ /* link to cache area */
+ c->seg = seg;
+ c->size = size;
+ c->next = mk->cache_unpowered;
+
+ mk->cache_unpowered = c;
+ mk->cache_size++;
+
+ ASSERT(mk->cache_size <= mk->ma->max_cache_size);
+
return 1;
}
}
@@ -555,7 +575,9 @@ static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size) {
return 0;
}
-static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint size) {
+static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p) {
+
+ Uint size = *size_p;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
if (IS_2POW(size)) {
@@ -598,8 +620,46 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint size) {
ASSERT(csize == size);
return seg;
}
- }
+ }
+ else if (mk->cache_unpowered) {
+ void *seg;
+ cache_t *c, *pc;
+ Uint csize;
+ Uint bad_max_abs = mk->ma->abs_max_cache_bad_fit;
+ /* Uint bad_max_rel = mk->ma->rel_max_cache_bad_fit; */
+
+ c = mk->cache_unpowered;
+ pc = c;
+
+ while (c) {
+ csize = c->size;
+ if (csize >= size && (csize - size) < bad_max_abs ) {
+
+ /* unlink from cache area */
+ seg = c->seg;
+
+
+ if (pc == c) {
+ mk->cache_unpowered = c->next;
+ } else {
+ pc->next = c->next;
+ }
+ c->next = NULL;
+ mk->cache_size--;
+ mk->cache_hits++;
+
+ /* link to free cache list */
+ mseg_cache_free_descriptor(mk, c);
+ *size_p = csize;
+
+ return seg;
+ }
+
+ pc = c;
+ c = c->next;
+ }
+ }
return NULL;
}
@@ -607,6 +667,30 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint size) {
* Slowly remove segments cached in the allocator by
* using callbacks from aux-work in the scheduler.
*/
+
+static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t **head) {
+ cache_t *c = NULL;
+
+ c = *head;
+
+ ASSERT( c != NULL );
+
+ *head = c->next;
+
+ if (erts_mtrace_enabled)
+ erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
+
+ mseg_destroy(mk->ma, mk, c->seg, c->size);
+ mseg_cache_free_descriptor(mk, c);
+
+ mk->segments.current.watermark--;
+ mk->cache_size--;
+
+ ASSERT( mk->cache_size >= 0 );
+
+ return mk->cache_size;
+}
+
static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, int ix) {
cache_t *c = NULL, *next = NULL;
@@ -644,12 +728,15 @@ static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, int ix) {
static Uint mseg_check_memkind_cache(MemKind *mk) {
int i;
- /* remove biggest first (less likly to be reused) */
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- for (i = CACHE_AREAS - 1; i > 0; i--) {
+
+ if (mk->cache_unpowered)
+ return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_unpowered));
+
+ for (i = 0; i < CACHE_AREAS; i++) {
if (mk->cache_area[i] != NULL)
- return mseg_drop_memkind_cache_size(mk, i);
+ return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_area[i]));
}
return 0;
@@ -769,7 +856,7 @@ mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, Uint *size_p,
ma->min_seg_size = size;
#endif
- if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, size)) != NULL)
+ if (opt->cache && mk->cache_size > 0 && (seg = cache_get_segment(mk, &size)) != NULL)
goto done;
if ((seg = mseg_create(ma, mk, size)) == NULL)
@@ -1451,13 +1538,17 @@ static void mem_kind_init(ErtsMsegAllctr_t *ma, MemKind* mk, const char* name)
mk->cache_free = NULL;
- for (i = 0; i < MAX_CACHE_SIZE; i++) {
+ ASSERT(ma->max_cache_size <= MAX_CACHE_SIZE);
+
+ for (i = 0; i < ma->max_cache_size; i++) {
mk->cache[i].seg = NULL;
mk->cache[i].size = 0;
mk->cache[i].next = mk->cache_free;
mk->cache_free = &(mk->cache[i]);
}
+ mk->cache_unpowered = NULL;
+
mk->cache_size = 0;
mk->cache_hits = 0;