aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_process_dict.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_process_dict.c')
-rw-r--r--erts/emulator/beam/erl_process_dict.c251
1 files changed, 144 insertions, 107 deletions
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 8606371bdf..84cd81aecf 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -51,16 +51,19 @@
#define INITIAL_SIZE (erts_pd_initial_size)
/* Hash utility macros */
-#define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition)
+#define HASH_RANGE(PDict) ((PDict)->usedSlots)
-#define MAKE_HASH(Term) \
-((is_small(Term)) ? unsigned_val(Term) : \
- ((is_atom(Term)) ? \
- (atom_tab(atom_val(term))->slot.bucket.hvalue) : \
- make_hash2(Term)))
+#define MAKE_HASH(Term) \
+ ((is_small(Term)) ? unsigned_val(Term) : \
+ ((is_atom(Term)) ? \
+ (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \
+ make_internal_hash(Term)))
#define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm))
+#define pd_hash_value(Pdict, Key) \
+ pd_hash_value_to_ix(Pdict, MAKE_HASH((Key)))
+
/* Memory allocation macros */
#define PD_ALLOC(Sz) \
erts_alloc(ERTS_ALC_T_PROC_DICT, (Sz))
@@ -74,14 +77,19 @@
#define TCDR(Term) CDR(list_val(Term))
/* Array access macro */
-#define ARRAY_GET(PDict, Index) (((PDict)->size > (Index)) ? \
- (PDict)->data[Index] : NIL)
+#define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \
+ (PDict)->data[Index])
+#define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->arraySize), \
+ (PDict)->data[Index] = (Val))
+
+#define IS_POW2(X) ((X) && !((X) & ((X)-1)))
/*
* Forward decalarations
*/
static void pd_hash_erase(Process *p, Eterm id, Eterm *ret);
static void pd_hash_erase_all(Process *p);
+static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id);
static Eterm pd_hash_get_keys(Process *p, Eterm value);
static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd);
static Eterm pd_hash_get_all(Process *p, ProcDict *pd);
@@ -91,9 +99,9 @@ static void shrink(Process *p, Eterm* ret);
static void grow(Process *p);
static void array_shrink(ProcDict **ppd, unsigned int need);
-static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term);
+static void ensure_array_size(ProcDict**, unsigned int size);
-static unsigned int pd_hash_value(ProcDict *pdict, Eterm term);
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx);
static unsigned int next_array_size(unsigned int need);
/*
@@ -134,6 +142,16 @@ static void pd_check(ProcDict *pd);
** External interface
*/
+int
+erts_pd_set_initial_size(int size)
+{
+ if (size <= 0)
+ return 0;
+
+ erts_pd_initial_size = 1 << erts_fit_in_bits_uint(size-1);
+ return 1;
+}
+
/*
* Called from break handler
*/
@@ -146,9 +164,9 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd)
/*PD_CHECK(pd);*/
if (pd == NULL)
return;
- erts_print(to, to_arg, "(size = %d, used = %d, homeSize = %d, "
+ erts_print(to, to_arg, "(size = %d, usedSlots = %d, "
"splitPosition = %d, numElements = %d)\n",
- pd->size, pd->used, pd->homeSize,
+ pd->arraySize, pd->usedSlots,
pd->splitPosition, (unsigned int) pd->numElements);
for (i = 0; i < HASH_RANGE(pd); ++i) {
erts_print(to, to_arg, "%d: %T\n", i, ARRAY_GET(pd, i));
@@ -203,7 +221,7 @@ erts_dicts_mem_size(Process *p)
{
Uint size = 0;
if (p->dictionary)
- size += PD_SZ2BYTES(p->dictionary->size);
+ size += PD_SZ2BYTES(p->dictionary->arraySize);
return size;
}
@@ -345,7 +363,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret)
if (is_boxed(old)) { /* Tuple */
ASSERT(is_tuple(old));
if (EQ(tuple_val(old)[1], id)) {
- array_put(&(p->dictionary), hval, NIL);
+ ARRAY_PUT(p->dictionary, hval, NIL);
--(p->dictionary->numElements);
*ret = tuple_val(old)[2];
}
@@ -365,7 +383,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret)
old = ARRAY_GET(p->dictionary, hval);
ASSERT(is_list(old));
if (is_nil(TCDR(old))) {
- array_put(&p->dictionary, hval, TCAR(old));
+ ARRAY_PUT(p->dictionary, hval, TCAR(old));
}
} else if (is_not_nil(old)) {
#ifdef DEBUG
@@ -390,40 +408,56 @@ static void pd_hash_erase_all(Process *p)
}
}
+Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id)
+{
+ unsigned int hval;
+ ProcDict *pd = p->dictionary;
+
+ ASSERT(hx == MAKE_HASH(id));
+ if (pd == NULL)
+ return am_undefined;
+ hval = pd_hash_value_to_ix(pd, hx);
+ return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id);
+}
+
Eterm erts_pd_hash_get(Process *p, Eterm id)
{
unsigned int hval;
- Eterm tmp;
ProcDict *pd = p->dictionary;
if (pd == NULL)
return am_undefined;
hval = pd_hash_value(pd, id);
- tmp = ARRAY_GET(pd, hval);
- if (is_boxed(tmp)) { /* Tuple */
- ASSERT(is_tuple(tmp));
- if (EQ(tuple_val(tmp)[1], id)) {
- return tuple_val(tmp)[2];
+ return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id);
+}
+
+Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id)
+{
+ if (is_boxed(bucket)) { /* Tuple */
+ ASSERT(is_tuple(bucket));
+ if (EQ(tuple_val(bucket)[1], id)) {
+ return tuple_val(bucket)[2];
}
- } else if (is_list(tmp)) {
- for (; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) {
+ } else if (is_list(bucket)) {
+ for (; bucket != NIL && !EQ(tuple_val(TCAR(bucket))[1], id); bucket = TCDR(bucket)) {
;
}
- if (tmp != NIL) {
- return tuple_val(TCAR(tmp))[2];
+ if (bucket != NIL) {
+ return tuple_val(TCAR(bucket))[2];
}
- } else if (is_not_nil(tmp)) {
+ } else if (is_not_nil(bucket)) {
#ifdef DEBUG
erts_fprintf(stderr,
"Process dictionary for process %T is broken, trying to "
"display term found in line %d:\n"
- "%T\n", p->common.id, __LINE__, tmp);
+ "%T\n", p->common.id, __LINE__, bucket);
#endif
erl_exit(1, "Damaged process dictionary found during get/1.");
}
return am_undefined;
}
+
#define PD_GET_TKEY(Dst,Src) \
do { \
ASSERT(is_tuple((Src))); \
@@ -549,8 +583,11 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
if (p->dictionary == NULL) {
/* Create it */
- array_put(&(p->dictionary), INITIAL_SIZE - 1, NIL);
- p->dictionary->homeSize = INITIAL_SIZE;
+ ensure_array_size(&p->dictionary, INITIAL_SIZE);
+ p->dictionary->usedSlots = INITIAL_SIZE;
+ p->dictionary->sizeMask = INITIAL_SIZE*2 - 1;
+ p->dictionary->splitPosition = 0;
+ p->dictionary->numElements = 0;
}
hval = pd_hash_value(p->dictionary, id);
old = ARRAY_GET(p->dictionary, hval);
@@ -583,7 +620,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
root[0] = id;
root[1] = value;
root[2] = old;
- BUMP_REDS(p, erts_garbage_collect(p, needed, root, 3));
+ erts_garbage_collect(p, needed, root, 3);
id = root[0];
value = root[1];
old = root[2];
@@ -602,19 +639,19 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
* Update the dictionary.
*/
if (is_nil(old)) {
- array_put(&(p->dictionary), hval, tpl);
+ ARRAY_PUT(p->dictionary, hval, tpl);
++(p->dictionary->numElements);
} else if (is_boxed(old)) {
ASSERT(is_tuple(old));
if (EQ(tuple_val(old)[1],id)) {
- array_put(&(p->dictionary), hval, tpl);
+ ARRAY_PUT(p->dictionary, hval, tpl);
return tuple_val(old)[2];
} else {
hp = HeapOnlyAlloc(p, 4);
tmp = CONS(hp, old, NIL);
hp += 2;
++(p->dictionary->numElements);
- array_put(&(p->dictionary), hval, CONS(hp, tpl, tmp));
+ ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, tmp));
hp += 2;
ASSERT(hp <= hp_limit);
}
@@ -624,7 +661,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
* New key. Simply prepend the tuple to the beginning of the list.
*/
hp = HeapOnlyAlloc(p, 2);
- array_put(&(p->dictionary), hval, CONS(hp, tpl, old));
+ ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, old));
hp += 2;
ASSERT(hp <= hp_limit);
++(p->dictionary->numElements);
@@ -659,7 +696,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
nlist = CONS(hp, tpl, nlist);
hp += 2;
ASSERT(hp <= hp_limit);
- array_put(&(p->dictionary), hval, nlist);
+ ARRAY_PUT(p->dictionary, hval, nlist);
return tuple_val(TCAR(tmp))[2];
}
} else {
@@ -684,6 +721,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
static void shrink(Process *p, Eterm* ret)
{
+ ProcDict *pd = p->dictionary;
unsigned int range = HASH_RANGE(p->dictionary);
unsigned int steps = (range*3) / 10;
Eterm hi, lo, tmp;
@@ -698,25 +736,26 @@ static void shrink(Process *p, Eterm* ret)
}
for (i = 0; i < steps; ++i) {
- ProcDict *pd = p->dictionary;
if (pd->splitPosition == 0) {
- pd->homeSize /= 2;
- pd->splitPosition = pd->homeSize;
+ ASSERT(IS_POW2(pd->usedSlots));
+ pd->sizeMask = pd->usedSlots - 1;
+ pd->splitPosition = pd->usedSlots / 2;
}
--(pd->splitPosition);
- hi = ARRAY_GET(pd, (pd->splitPosition + pd->homeSize));
+ /* Must wait to decrement 'usedSlots' for GC rootset below */
+ hi = ARRAY_GET(pd, pd->usedSlots - 1);
lo = ARRAY_GET(pd, pd->splitPosition);
if (hi != NIL) {
if (lo == NIL) {
- array_put(&(p->dictionary), pd->splitPosition, hi);
+ ARRAY_PUT(pd, pd->splitPosition, hi);
} else {
int needed = 4;
if (is_list(hi) && is_list(lo)) {
needed = 2*erts_list_length(hi);
}
if (HeapWordsLeft(p) < needed) {
- BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1));
- hi = pd->data[(pd->splitPosition + pd->homeSize)];
+ erts_garbage_collect(p, needed, ret, 1);
+ hi = pd->data[pd->usedSlots - 1];
lo = pd->data[pd->splitPosition];
}
#ifdef DEBUG
@@ -727,13 +766,13 @@ static void shrink(Process *p, Eterm* ret)
hp = HeapOnlyAlloc(p, 4);
tmp = CONS(hp, hi, NIL);
hp += 2;
- array_put(&(p->dictionary), pd->splitPosition,
+ ARRAY_PUT(pd, pd->splitPosition,
CONS(hp,lo,tmp));
hp += 2;
ASSERT(hp <= hp_limit);
} else { /* hi is a list */
hp = HeapOnlyAlloc(p, 2);
- array_put(&(p->dictionary), pd->splitPosition,
+ ARRAY_PUT(pd, pd->splitPosition,
CONS(hp, lo, hi));
hp += 2;
ASSERT(hp <= hp_limit);
@@ -741,7 +780,7 @@ static void shrink(Process *p, Eterm* ret)
} else { /* lo is a list */
if (is_tuple(hi)) {
hp = HeapOnlyAlloc(p, 2);
- array_put(&(p->dictionary), pd->splitPosition,
+ ARRAY_PUT(pd, pd->splitPosition,
CONS(hp, hi, lo));
hp += 2;
ASSERT(hp <= hp_limit);
@@ -753,14 +792,15 @@ static void shrink(Process *p, Eterm* ret)
hp += 2;
}
ASSERT(hp <= hp_limit);
- array_put(&(p->dictionary), pd->splitPosition, lo);
+ ARRAY_PUT(pd, pd->splitPosition, lo);
}
}
}
}
- array_put(&(p->dictionary), (pd->splitPosition + pd->homeSize), NIL);
+ --pd->usedSlots;
+ ARRAY_PUT(pd, pd->usedSlots, NIL);
}
- if (HASH_RANGE(p->dictionary) <= (p->dictionary->size / 4)) {
+ if (HASH_RANGE(p->dictionary) <= (p->dictionary->arraySize / 4)) {
array_shrink(&(p->dictionary), (HASH_RANGE(p->dictionary) * 3) / 2);
}
}
@@ -768,14 +808,14 @@ static void shrink(Process *p, Eterm* ret)
static void grow(Process *p)
{
unsigned int i,j;
- unsigned int steps = p->dictionary->homeSize / 5;
+ unsigned int steps = (p->dictionary->usedSlots / 4) & 0xf;
Eterm l1,l2;
Eterm l;
Eterm *hp;
unsigned int pos;
unsigned int homeSize;
int needed = 0;
- ProcDict *pd;
+ ProcDict *pd = p->dictionary;
#ifdef DEBUG
Eterm *hp_limit;
#endif
@@ -784,18 +824,20 @@ static void grow(Process *p)
if (steps == 0)
steps = 1;
/* Dont grow over MAX_HASH */
- if ((MAX_HASH - steps) <= HASH_RANGE(p->dictionary)) {
+ if ((MAX_HASH - steps) <= HASH_RANGE(pd)) {
return;
}
+ ensure_array_size(&p->dictionary, HASH_RANGE(pd) + steps);
+ pd = p->dictionary;
+
/*
* Calculate total number of heap words needed, and garbage collect
* if necessary.
*/
- pd = p->dictionary;
pos = pd->splitPosition;
- homeSize = pd->homeSize;
+ homeSize = pd->usedSlots - pd->splitPosition;
for (i = 0; i < steps; ++i) {
if (pos == homeSize) {
homeSize *= 2;
@@ -811,7 +853,7 @@ static void grow(Process *p)
}
}
if (HeapWordsLeft(p) < needed) {
- BUMP_REDS(p, erts_garbage_collect(p, needed, 0, 0));
+ erts_garbage_collect(p, needed, 0, 0);
}
#ifdef DEBUG
hp_limit = p->htop + needed;
@@ -820,21 +862,22 @@ static void grow(Process *p)
/*
* Now grow.
*/
-
+ homeSize = pd->usedSlots - pd->splitPosition;
for (i = 0; i < steps; ++i) {
- ProcDict *pd = p->dictionary;
- if (pd->splitPosition == pd->homeSize) {
- pd->homeSize *= 2;
- pd->splitPosition = 0;
+ if (pd->splitPosition == homeSize) {
+ homeSize *= 2;
+ pd->sizeMask = homeSize*2 - 1;
+ pd->splitPosition = 0;
}
pos = pd->splitPosition;
++pd->splitPosition; /* For the hashes */
+ ++pd->usedSlots;
+ ASSERT(pos + homeSize == pd->usedSlots - 1);
l = ARRAY_GET(pd, pos);
if (is_tuple(l)) {
if (pd_hash_value(pd, tuple_val(l)[1]) != pos) {
- array_put(&(p->dictionary), pos +
- p->dictionary->homeSize, l);
- array_put(&(p->dictionary), pos, NIL);
+ ARRAY_PUT(pd, pos + homeSize, l);
+ ARRAY_PUT(pd, pos, NIL);
}
} else {
l2 = NIL;
@@ -856,10 +899,8 @@ static void grow(Process *p)
if (l2 != NIL && TCDR(l2) == NIL)
l2 = TCAR(l2);
ASSERT(hp <= hp_limit);
- /* After array_put pd is no longer valid */
- array_put(&(p->dictionary), pos, l1);
- array_put(&(p->dictionary), pos +
- p->dictionary->homeSize, l2);
+ ARRAY_PUT(pd, pos, l1);
+ ARRAY_PUT(pd, pos + homeSize, l2);
}
}
@@ -876,73 +917,65 @@ static void array_shrink(ProcDict **ppd, unsigned int need)
{
unsigned int siz = next_array_size(need);
- HDEBUGF(("array_shrink: size = %d, used = %d, need = %d",
- (*ppd)->size, (*ppd)->used, need));
+ HDEBUGF(("array_shrink: size = %d, need = %d",
+ (*ppd)->arraySize, need));
- if (siz > (*ppd)->size)
+ if (siz >= (*ppd)->arraySize)
return; /* Only shrink */
*ppd = PD_REALLOC(((void *) *ppd),
- PD_SZ2BYTES((*ppd)->size),
+ PD_SZ2BYTES((*ppd)->arraySize),
PD_SZ2BYTES(siz));
- (*ppd)->size = siz;
- if ((*ppd)->size < (*ppd)->used)
- (*ppd)->used = (*ppd)->size;
+ (*ppd)->arraySize = siz;
}
-static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term)
+static void ensure_array_size(ProcDict **ppdict, unsigned int size)
{
+ ProcDict *pd = *ppdict;
unsigned int i;
- Eterm ret;
- if (*ppdict == NULL) {
- Uint siz = next_array_size(ndx+1);
- ProcDict *p;
- p = PD_ALLOC(PD_SZ2BYTES(siz));
+ if (pd == NULL) {
+ Uint siz = next_array_size(size);
+
+ pd = PD_ALLOC(PD_SZ2BYTES(siz));
for (i = 0; i < siz; ++i)
- p->data[i] = NIL;
- p->size = siz;
- p->homeSize = p->splitPosition = p->numElements = p->used = 0;
- *ppdict = p;
- } else if (ndx >= (*ppdict)->size) {
- Uint osize = (*ppdict)->size;
- Uint nsize = next_array_size(ndx+1);
- *ppdict = PD_REALLOC(((void *) *ppdict),
+ pd->data[i] = NIL;
+ pd->arraySize = siz;
+ *ppdict = pd;
+ } else if (size > pd->arraySize) {
+ Uint osize = pd->arraySize;
+ Uint nsize = next_array_size(size);
+ pd = PD_REALLOC(((void *) pd),
PD_SZ2BYTES(osize),
PD_SZ2BYTES(nsize));
for (i = osize; i < nsize; ++i)
- (*ppdict)->data[i] = NIL;
- (*ppdict)->size = nsize;
+ pd->data[i] = NIL;
+ pd->arraySize = nsize;
+ *ppdict = pd;
}
- ret = (*ppdict)->data[ndx];
- (*ppdict)->data[ndx] = term;
- if ((ndx + 1) > (*ppdict)->used)
- (*ppdict)->used = ndx + 1;
-#ifdef HARDDEBUG
- HDEBUGF(("array_put: (*ppdict)->size = %d, (*ppdict)->used = %d, ndx = %d",
- (*ppdict)->size, (*ppdict)->used, ndx));
- erts_fprintf(stderr, "%T", term);
-#endif /* HARDDEBUG */
- return ret;
}
/*
** Basic utilities
*/
-static unsigned int pd_hash_value(ProcDict *pdict, Eterm term)
+static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx)
{
- Uint hash, high;
+ Uint high;
- hash = MAKE_HASH(term);
- high = hash % (pdict->homeSize*2);
+ ASSERT(IS_POW2(pdict->sizeMask+1));
+ ASSERT(HASH_RANGE(pdict) >= (pdict->sizeMask >> 1));
+ ASSERT(HASH_RANGE(pdict) <= (pdict->sizeMask + 1));
+
+ high = hx & pdict->sizeMask;
if (high >= HASH_RANGE(pdict))
- return hash % pdict->homeSize;
+ return hx & (pdict->sizeMask >> 1);
return high;
}
+
static unsigned int next_array_size(unsigned int need)
{
static unsigned int tab[] =
@@ -1002,24 +1035,28 @@ static unsigned int next_array_size(unsigned int need)
static void pd_check(ProcDict *pd)
{
unsigned int i;
+ unsigned int used;
Uint num;
if (pd == NULL)
return;
- ASSERT(pd->size >= pd->used);
+ used = HASH_RANGE(pd);
+ ASSERT(pd->arraySize >= used);
ASSERT(HASH_RANGE(pd) <= MAX_HASH);
- for (i = 0, num = 0; i < pd->used; ++i) {
+ for (i = 0, num = 0; i < used; ++i) {
Eterm t = pd->data[i];
if (is_nil(t)) {
continue;
} else if (is_tuple(t)) {
++num;
ASSERT(arityval(*tuple_val(t)) == 2);
+ ASSERT(pd_hash_value(pd, tuple_val(t)[1]) == i);
continue;
} else if (is_list(t)) {
while (t != NIL) {
++num;
ASSERT(is_tuple(TCAR(t)));
ASSERT(arityval(*(tuple_val(TCAR(t)))) == 2);
+ ASSERT(pd_hash_value(pd, tuple_val(TCAR(t))[1]) == i);
t = TCDR(t);
}
continue;
@@ -1030,7 +1067,7 @@ static void pd_check(ProcDict *pd)
}
}
ASSERT(num == pd->numElements);
- ASSERT(pd->splitPosition <= pd->homeSize);
+ ASSERT(pd->usedSlots >= pd->splitPosition*2);
}
#endif /* DEBUG */
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-14 23:31:18 +0000