aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/sys/common/erl_mseg.c248
1 files changed, 130 insertions, 118 deletions
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 538eea88d1..772f1df0ce 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -175,6 +175,7 @@ struct cache_t_ {
Uint size;
void *seg;
cache_t *next;
+ cache_t *prev;
};
@@ -183,9 +184,9 @@ typedef struct ErtsMsegAllctr_t_ ErtsMsegAllctr_t;
struct mem_kind_t {
cache_t cache[MAX_CACHE_SIZE];
- cache_t *cache_unpowered;
- cache_t *cache_area[CACHE_AREAS];
- cache_t *cache_free;
+ cache_t cache_unpowered_node;
+ cache_t cache_powered_node[CACHE_AREAS];
+ cache_t cache_free;
Sint cache_size;
Uint cache_hits;
@@ -516,70 +517,102 @@ do { \
#define ERTS_DBG_MK_CHK_THR_ACCESS(MK)
#endif
-/* NEW CACHE interface */
+/* Cache interface */
-static ERTS_INLINE cache_t *mseg_cache_alloc_descriptor(MemKind *mk) {
- cache_t *c = mk->cache_free;
- ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- if (c)
- mk->cache_free = c->next;
+static ERTS_INLINE void mseg_cache_clear_node(cache_t *c) {
+ c->seg = NULL;
+ c->size = 0;
+ c->next = c;
+ c->prev = c;
+}
- return c;
+static ERTS_INLINE void mseg_cache_unlink_node(cache_t *c) {
+ c->next->prev = c->prev;
+ c->prev->next = c->next;
+
+ c->next = c;
+ c->prev = c;
}
-static ERTS_INLINE void mseg_cache_free_descriptor(MemKind *mk, cache_t *c) {
- ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- ASSERT(c);
+static ERTS_INLINE void mseg_cache_push_node_front(cache_t *head, cache_t *c) {
+ c->next = head->next;
+ c->prev = head;
- c->seg = NULL;
- c->size = 0;
- c->next = mk->cache_free;
- mk->cache_free = c;
+ head->next->prev = c;
+ head->next = c;
+
+ ASSERT(c->next != c);
+ ASSERT(c->prev != c);
+}
+
+/* not needed: mseg_cache_push_node_back */
+
+static ERTS_INLINE cache_t *mseg_cache_pop_node_front(cache_t *head) {
+ cache_t *c;
+
+ ASSERT(head->next != head && head->prev != head);
+
+ c = head->next;
+
+ mseg_cache_unlink_node(c);
+
+ return c;
}
+static ERTS_INLINE cache_t *mseg_cache_pop_node_back(cache_t *head) {
+ cache_t *c;
+
+ ASSERT(head->next != head && head->prev != head);
+
+ c = head->prev;
+
+ mseg_cache_unlink_node(c);
+
+ return c;
+}
+
+static ERTS_INLINE int mseg_cache_is_empty(cache_t *head) {
+
+ if (head->next == head) {
+ ASSERT(head->prev == head);
+ return 1;
+ }
+
+ ASSERT(head->next != head && head->prev != head);
+
+ return 0;
+}
+
+
static ERTS_INLINE int cache_bless_segment(MemKind *mk, void *seg, Uint size) {
cache_t *c;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
- if (mk->cache_free && MAP_IS_ALIGNED(seg)) {
+ if (!mseg_cache_is_empty(&(mk->cache_free)) && MAP_IS_ALIGNED(seg)) {
+ c = mseg_cache_pop_node_front(&(mk->cache_free));
+
+ c->seg = seg;
+ c->size = size;
+
if (IS_2POW(size)) {
int ix = SIZE_TO_CACHE_AREA_IDX(size);
ASSERT(ix < CACHE_AREAS);
ASSERT((1 << (ix + MSEG_ALIGN_BITS)) == size);
- /* unlink from free cache list */
- c = mseg_cache_alloc_descriptor(mk);
-
- /* link to cache area */
- c->seg = seg;
- c->size = size;
- c->next = mk->cache_area[ix];
-
- mk->cache_area[ix] = c;
- mk->cache_size++;
-
- ASSERT(mk->cache_size <= mk->ma->max_cache_size);
+ mseg_cache_push_node_front(&(mk->cache_powered_node[ix]), c);
- return 1;
} else {
- /* unlink from free cache list */
- c = mseg_cache_alloc_descriptor(mk);
-
- /* link to cache area */
- c->seg = seg;
- c->size = size;
- c->next = mk->cache_unpowered;
- mk->cache_unpowered = c;
- mk->cache_size++;
+ mseg_cache_push_node_front(&(mk->cache_unpowered_node), c);
+ }
- ASSERT(mk->cache_size <= mk->ma->max_cache_size);
+ mk->cache_size++;
+ ASSERT(mk->cache_size <= mk->ma->max_cache_size);
- return 1;
- }
+ return 1;
}
return 0;
@@ -590,6 +623,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p) {
Uint size = *size_p;
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
+
if (IS_2POW(size)) {
int i, ix = SIZE_TO_CACHE_AREA_IDX(size);
@@ -599,76 +633,63 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p) {
for( i = ix; i < CACHE_AREAS; i++) {
- if ((c = mk->cache_area[i]) == NULL)
+ if (mseg_cache_is_empty(&(mk->cache_powered_node[i])))
continue;
+ c = mseg_cache_pop_node_front(&(mk->cache_powered_node[i]));
+
ASSERT(IS_2POW(c->size));
- /* unlink from cache area */
- csize = c->size;
- seg = c->seg;
- mk->cache_area[i] = c->next;
- c->next = NULL;
+ csize = c->size;
+ seg = c->seg;
+
mk->cache_size--;
mk->cache_hits++;
/* link to free cache list */
- mseg_cache_free_descriptor(mk, c);
+ mseg_cache_clear_node(c);
+ mseg_cache_push_node_front(&(mk->cache_free), c);
ASSERT(!(mk->cache_size < 0));
- /* divvy up the cache - if needed */
- while( i > ix) {
- csize = csize >> 1;
- /* try to cache half of it */
- if (!cache_bless_segment(mk, (char *)seg + csize, csize)) {
- /* wouldn't cache .. destroy it instead */
- mseg_destroy(mk->ma, mk, (char *)seg + csize, csize);
- }
- i--;
+ if (csize != size) {
+ mseg_destroy(mk->ma, mk, (char *)seg + size, csize - size);
}
- ASSERT(csize == size);
+
return seg;
}
}
- else if (mk->cache_unpowered) {
+ else if (!mseg_cache_is_empty(&(mk->cache_unpowered_node))) {
void *seg;
- cache_t *c, *pc;
+ cache_t *c;
Uint csize;
Uint bad_max_abs = mk->ma->abs_max_cache_bad_fit;
Uint bad_max_rel = mk->ma->rel_max_cache_bad_fit;
- c = mk->cache_unpowered;
- pc = c;
+ c = mk->cache_unpowered_node.next;
- while (c) {
+ while (c != &(mk->cache_unpowered_node)) {
csize = c->size;
- if (csize >= size &&
- ((csize - size)*100 < bad_max_rel*size) &&
- (csize - size) < bad_max_abs ) {
+ if (csize >= size &&
+ ((csize - size)*100 < bad_max_rel*size) &&
+ (csize - size) < bad_max_abs ) {
- /* unlink from cache area */
seg = c->seg;
- if (pc == c) {
- mk->cache_unpowered = c->next;
- } else {
- pc->next = c->next;
- }
+ mseg_cache_unlink_node(c);
- c->next = NULL;
mk->cache_size--;
mk->cache_hits++;
/* link to free cache list */
- mseg_cache_free_descriptor(mk, c);
+ mseg_cache_clear_node(c);
+ mseg_cache_push_node_front(&(mk->cache_free), c);
+
*size_p = csize;
return seg;
}
-
- pc = c;
- c = c->next;
+ c = c->next;
}
}
return NULL;
@@ -679,20 +700,17 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p) {
* using callbacks from aux-work in the scheduler.
*/
-static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t **head) {
+static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t *head) {
cache_t *c = NULL;
- c = *head;
-
- ASSERT( c != NULL );
-
- *head = c->next;
+ c = mseg_cache_pop_node_back(head);
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
mseg_destroy(mk->ma, mk, c->seg, c->size);
- mseg_cache_free_descriptor(mk, c);
+ mseg_cache_clear_node(c);
+ mseg_cache_push_node_front(&(mk->cache_free), c);
mk->segments.current.watermark--;
mk->cache_size--;
@@ -702,30 +720,26 @@ static ERTS_INLINE Uint mseg_drop_one_memkind_cache_size(MemKind *mk, cache_t **
return mk->cache_size;
}
-static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, cache_t **head) {
- cache_t *c = NULL, *next = NULL;
-
- c = *head;
- ASSERT( c != NULL );
+static ERTS_INLINE Uint mseg_drop_memkind_cache_size(MemKind *mk, cache_t *head) {
+ cache_t *c = NULL;
- while (c) {
+ while (!mseg_cache_is_empty(head)) {
- next = c->next;
+ c = mseg_cache_pop_node_back(head);
if (erts_mtrace_enabled)
erts_mtrace_crr_free(SEGTYPE, SEGTYPE, c->seg);
mseg_destroy(mk->ma, mk, c->seg, c->size);
- mseg_cache_free_descriptor(mk, c);
+
+ mseg_cache_clear_node(c);
+ mseg_cache_push_node_front(&(mk->cache_free), c);
mk->segments.current.watermark--;
mk->cache_size--;
- c = next;
}
- *head = NULL;
-
ASSERT( mk->cache_size >= 0 );
return mk->cache_size;
@@ -743,12 +757,12 @@ static Uint mseg_check_memkind_cache(MemKind *mk) {
ERTS_DBG_MK_CHK_THR_ACCESS(mk);
for (i = 0; i < CACHE_AREAS; i++) {
- if (mk->cache_area[i] != NULL)
- return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_area[i]));
+ if (!mseg_cache_is_empty(&(mk->cache_powered_node[i])))
+ return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_powered_node[i]));
}
- if (mk->cache_unpowered)
- return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_unpowered));
+ if (!mseg_cache_is_empty(&(mk->cache_unpowered_node)))
+ return mseg_drop_one_memkind_cache_size(mk, &(mk->cache_unpowered_node));
return 0;
}
@@ -804,17 +818,17 @@ static void mseg_clear_memkind_cache(MemKind *mk) {
/* drop pow2 caches */
for (i = 0; i < CACHE_AREAS; i++) {
- if (mk->cache_area[i] == NULL)
+ if (mseg_cache_is_empty(&(mk->cache_powered_node[i])))
continue;
- mseg_drop_memkind_cache_size(mk, &(mk->cache_area[i]));
- ASSERT(mk->cache_area[i] == NULL);
+ mseg_drop_memkind_cache_size(mk, &(mk->cache_powered_node[i]));
+ ASSERT(mseg_cache_is_empty(&(mk->cache_powered_node[i])) == 1);
}
/* drop varied caches */
- if(mk->cache_unpowered)
- mseg_drop_memkind_cache_size(mk, &(mk->cache_unpowered));
+ if (!mseg_cache_is_empty(&(mk->cache_unpowered_node)))
+ mseg_drop_memkind_cache_size(mk, &(mk->cache_unpowered_node));
- ASSERT(mk->cache_unpowered == NULL);
+ ASSERT(mseg_cache_is_empty(&(mk->cache_unpowered_node)) == 1);
ASSERT(mk->cache_size == 0);
}
@@ -1556,27 +1570,28 @@ erts_mseg_unit_size(void)
return MSEG_ALIGNED_SIZE;
}
+
static void mem_kind_init(ErtsMsegAllctr_t *ma, MemKind* mk, const char* name)
{
int i;
+ /* Clear all cache headers */
+ mseg_cache_clear_node(&(mk->cache_free));
+ mseg_cache_clear_node(&(mk->cache_unpowered_node));
+
for (i = 0; i < CACHE_AREAS; i++) {
- mk->cache_area[i] = NULL;
+ mseg_cache_clear_node(&(mk->cache_powered_node[i]));
}
- mk->cache_free = NULL;
+ /* Populate cache free list */
ASSERT(ma->max_cache_size <= MAX_CACHE_SIZE);
for (i = 0; i < ma->max_cache_size; i++) {
- mk->cache[i].seg = NULL;
- mk->cache[i].size = 0;
- mk->cache[i].next = mk->cache_free;
- mk->cache_free = &(mk->cache[i]);
+ mseg_cache_clear_node(&(mk->cache[i]));
+ mseg_cache_push_node_front(&(mk->cache_free), &(mk->cache[i]));
}
- mk->cache_unpowered = NULL;
-
mk->cache_size = 0;
mk->cache_hits = 0;
@@ -1594,9 +1609,6 @@ static void mem_kind_init(ErtsMsegAllctr_t *ma, MemKind* mk, const char* name)
ma->mk_list = mk;
}
-
-
-
void
erts_mseg_init(ErtsMsegInit_t *init)
{
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-14 06:29:30 +0000