aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/utils.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/utils.c')
-rw-r--r--erts/emulator/beam/utils.c723
1 files changed, 576 insertions, 147 deletions
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 0bbae65e28..88cdcc2675 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1069,54 +1069,237 @@ do { \
#define HCONST 0x9e3779b9UL /* the golden ratio; an arbitrary value */
-static Uint32
-block_hash(byte *k, Uint length, Uint32 initval)
+typedef struct {
+ Uint32 a,b,c;
+} ErtsBlockHashHelperCtx;
+
+#define BLOCK_HASH_BYTES_PER_ITER 12
+
+/* The three functions below are separated into different functions even
+ though they are always used together to make trapping and handling
+ of unaligned binaries easier. Examples of how they are used can be
+ found in block_hash and make_hash2_helper.*/
+static ERTS_INLINE
+void block_hash_setup(Uint32 initval,
+ ErtsBlockHashHelperCtx* ctx /* out parameter */)
+{
+ ctx->a = ctx->b = HCONST;
+ ctx->c = initval; /* the previous hash value */
+}
+
+static ERTS_INLINE
+void block_hash_buffer(byte *buf,
+ Uint buf_length,
+ ErtsBlockHashHelperCtx* ctx /* out parameter */)
{
- Uint32 a,b,c;
- Uint len;
-
- /* Set up the internal state */
- len = length;
- a = b = HCONST;
- c = initval; /* the previous hash value */
-
- while (len >= 12)
- {
- a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24));
- b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24));
- c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24));
- MIX(a,b,c);
- k += 12; len -= 12;
- }
-
- c += length;
- switch(len) /* all the case statements fall through */
- {
- case 11: c+=((Uint32)k[10]<<24);
- case 10: c+=((Uint32)k[9]<<16);
- case 9 : c+=((Uint32)k[8]<<8);
- /* the first byte of c is reserved for the length */
- case 8 : b+=((Uint32)k[7]<<24);
- case 7 : b+=((Uint32)k[6]<<16);
- case 6 : b+=((Uint32)k[5]<<8);
- case 5 : b+=k[4];
- case 4 : a+=((Uint32)k[3]<<24);
- case 3 : a+=((Uint32)k[2]<<16);
- case 2 : a+=((Uint32)k[1]<<8);
- case 1 : a+=k[0];
- /* case 0: nothing left to add */
- }
- MIX(a,b,c);
- return c;
+ Uint len = buf_length;
+ byte *k = buf;
+ ASSERT(buf_length % BLOCK_HASH_BYTES_PER_ITER == 0);
+ while (len >= BLOCK_HASH_BYTES_PER_ITER) {
+ ctx->a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24));
+ ctx->b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24));
+ ctx->c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24));
+ MIX(ctx->a,ctx->b,ctx->c);
+ k += BLOCK_HASH_BYTES_PER_ITER; len -= BLOCK_HASH_BYTES_PER_ITER;
+ }
}
+static ERTS_INLINE
+Uint32 block_hash_final_bytes(byte *buf,
+ Uint buf_length,
+ Uint full_length,
+ ErtsBlockHashHelperCtx* ctx)
+{
+ Uint len = buf_length;
+ byte *k = buf;
+ ctx->c += full_length;
+ switch(len)
+ { /* all the case statements fall through */
+ case 11: ctx->c+=((Uint32)k[10]<<24);
+ case 10: ctx->c+=((Uint32)k[9]<<16);
+ case 9 : ctx->c+=((Uint32)k[8]<<8);
+ /* the first byte of c is reserved for the length */
+ case 8 : ctx->b+=((Uint32)k[7]<<24);
+ case 7 : ctx->b+=((Uint32)k[6]<<16);
+ case 6 : ctx->b+=((Uint32)k[5]<<8);
+ case 5 : ctx->b+=k[4];
+ case 4 : ctx->a+=((Uint32)k[3]<<24);
+ case 3 : ctx->a+=((Uint32)k[2]<<16);
+ case 2 : ctx->a+=((Uint32)k[1]<<8);
+ case 1 : ctx->a+=k[0];
+ /* case 0: nothing left to add */
+ }
+ MIX(ctx->a,ctx->b,ctx->c);
+ return ctx->c;
+}
+
+static
Uint32
-make_hash2(Eterm term)
+block_hash(byte *block, Uint block_length, Uint32 initval)
{
+ ErtsBlockHashHelperCtx ctx;
+ Uint no_bytes_not_in_loop =
+ (block_length % BLOCK_HASH_BYTES_PER_ITER);
+ Uint no_bytes_to_process_in_loop =
+ block_length - no_bytes_not_in_loop;
+ byte *final_bytes = block + no_bytes_to_process_in_loop;
+ block_hash_setup(initval, &ctx);
+ block_hash_buffer(block,
+ no_bytes_to_process_in_loop,
+ &ctx);
+ return block_hash_final_bytes(final_bytes,
+ no_bytes_not_in_loop,
+ block_length,
+ &ctx);
+}
+
+typedef enum {
+ tag_primary_list,
+ arityval_subtag,
+ hamt_subtag_head_flatmap,
+ map_subtag,
+ fun_subtag,
+ neg_big_subtag,
+ sub_binary_subtag_1,
+ sub_binary_subtag_2,
+ hash2_common_1,
+ hash2_common_2,
+ hash2_common_3,
+} ErtsMakeHash2TrapLocation;
+
+typedef struct {
+ int c;
+ Uint32 sh;
+ Eterm* ptr;
+} ErtsMakeHash2Context_TAG_PRIMARY_LIST;
+
+typedef struct {
+ int i;
+ int arity;
+ Eterm* elem;
+} ErtsMakeHash2Context_ARITYVAL_SUBTAG;
+
+typedef struct {
+ Eterm *ks;
+ Eterm *vs;
+ int i;
+ Uint size;
+} ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP;
+
+typedef struct {
+ Eterm* ptr;
+ int i;
+} ErtsMakeHash2Context_MAP_SUBTAG;
+
+typedef struct {
+ Uint num_free;
+ Eterm* bptr;
+} ErtsMakeHash2Context_FUN_SUBTAG;
+
+typedef struct {
+ Eterm* ptr;
+ Uint i;
+ Uint n;
+ Uint32 con;
+} ErtsMakeHash2Context_NEG_BIG_SUBTAG;
+
+typedef struct {
+ byte* bptr;
+ Uint sz;
+ Uint bitsize;
+ Uint bitoffs;
+ Uint no_bytes_processed;
+ ErtsBlockHashHelperCtx block_hash_ctx;
+ /* The following fields are only used when bitoffs != 0 */
+ byte* buf;
+ int done;
+
+} ErtsMakeHash2Context_SUB_BINARY_SUBTAG;
+
+typedef struct {
+ int dummy__; /* Empty structs are not supported on all platforms */
+} ErtsMakeHash2Context_EMPTY;
+
+typedef struct {
+ ErtsMakeHash2TrapLocation trap_location;
+ /* specific to the trap location: */
+ union {
+ ErtsMakeHash2Context_TAG_PRIMARY_LIST tag_primary_list;
+ ErtsMakeHash2Context_ARITYVAL_SUBTAG arityval_subtag;
+ ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP hamt_subtag_head_flatmap;
+ ErtsMakeHash2Context_MAP_SUBTAG map_subtag;
+ ErtsMakeHash2Context_FUN_SUBTAG fun_subtag;
+ ErtsMakeHash2Context_NEG_BIG_SUBTAG neg_big_subtag;
+ ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_1;
+ ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_2;
+ ErtsMakeHash2Context_EMPTY hash2_common_1;
+ ErtsMakeHash2Context_EMPTY hash2_common_2;
+ ErtsMakeHash2Context_EMPTY hash2_common_3;
+ } trap_location_state;
+ /* same for all trap locations: */
+ Eterm term;
Uint32 hash;
Uint32 hash_xor_pairs;
- DeclareTmpHeapNoproc(tmp_big,2);
+ ErtsEStack stack;
+} ErtsMakeHash2Context;
+
+static int make_hash2_ctx_bin_dtor(Binary *context_bin) {
+ ErtsMakeHash2Context* context = ERTS_MAGIC_BIN_DATA(context_bin);
+ DESTROY_SAVED_ESTACK(&context->stack);
+ if (context->trap_location == sub_binary_subtag_2 &&
+ context->trap_location_state.sub_binary_subtag_2.buf != NULL) {
+ erts_free(ERTS_ALC_T_PHASH2_TRAP, context->trap_location_state.sub_binary_subtag_2.buf);
+ }
+ return 1;
+}
+/* hash2_save_trap_state is called seldom so we want to avoid inlining */
+static ERTS_NOINLINE
+Eterm hash2_save_trap_state(Eterm state_mref,
+ Uint32 hash_xor_pairs,
+ Uint32 hash,
+ Process* p,
+ Eterm term,
+ Eterm* ESTK_DEF_STACK(s),
+ ErtsEStack s,
+ ErtsMakeHash2TrapLocation trap_location,
+ void* trap_location_state_ptr,
+ size_t trap_location_state_size) {
+ Binary* state_bin;
+ ErtsMakeHash2Context* context;
+ if (state_mref == THE_NON_VALUE) {
+ Eterm* hp;
+ state_bin = erts_create_magic_binary(sizeof(ErtsMakeHash2Context),
+ make_hash2_ctx_bin_dtor);
+ hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+ state_mref = erts_mk_magic_ref(&hp, &MSO(p), state_bin);
+ } else {
+ state_bin = erts_magic_ref2bin(state_mref);
+ }
+ context = ERTS_MAGIC_BIN_DATA(state_bin);
+ context->term = term;
+ context->hash = hash;
+ context->hash_xor_pairs = hash_xor_pairs;
+ ESTACK_SAVE(s, &context->stack);
+ context->trap_location = trap_location;
+ sys_memcpy(&context->trap_location_state,
+ trap_location_state_ptr,
+ trap_location_state_size);
+ erts_set_gc_state(p, 0);
+ BUMP_ALL_REDS(p);
+ return state_mref;
+}
+#undef NOINLINE_HASH2_SAVE_TRAP_STATE
+
+/* Writes back a magic reference to *state_mref_write_back when the
+ function traps */
+static ERTS_INLINE Uint32
+make_hash2_helper(Eterm term_param, const int can_trap, Eterm* state_mref_write_back, Process* p)
+{
+ static const Uint ITERATIONS_PER_RED = 64;
+ Uint32 hash;
+ Uint32 hash_xor_pairs;
+ Eterm term = term_param;
ERTS_UNDEF(hash_xor_pairs, 0);
/* (HCONST * {2, ..., 22}) mod 2^32 */
@@ -1168,12 +1351,63 @@ make_hash2(Eterm term)
#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
+#define NOT_SSMALL28_HASH(SMALL) \
+ do { \
+ Uint64 t; \
+ Uint32 x, y; \
+ Uint32 con; \
+ if (SMALL < 0) { \
+ con = HCONST_10; \
+ t = (Uint64)(SMALL * (-1)); \
+ } else { \
+ con = HCONST_11; \
+ t = SMALL; \
+ } \
+ x = t & 0xffffffff; \
+ y = t >> 32; \
+ UINT32_HASH_2(x, y, con); \
+ } while(0)
+
#ifdef ARCH_64
# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst)
#else
# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst)
#endif
+#define TRAP_LOCATION_NO_RED(location_name) \
+ do { \
+ if(can_trap && iterations_until_trap <= 0) { \
+ *state_mref_write_back = \
+ hash2_save_trap_state(state_mref, \
+ hash_xor_pairs, \
+ hash, \
+ p, \
+ term, \
+ ESTK_DEF_STACK(s), \
+ s, \
+ location_name, \
+ &ctx, \
+ sizeof(ctx)); \
+ return 0; \
+ L_##location_name: \
+ ctx = context->trap_location_state. location_name; \
+ } \
+ } while(0)
+
+#define TRAP_LOCATION(location_name) \
+ do { \
+ if (can_trap) { \
+ iterations_until_trap--; \
+ TRAP_LOCATION_NO_RED(location_name); \
+ } \
+ } while(0)
+
+#define TRAP_LOCATION_NO_CTX(location_name) \
+ do { \
+ ErtsMakeHash2Context_EMPTY ctx; \
+ TRAP_LOCATION(location_name); \
+ } while(0)
+
/* Optimization. Simple cases before declaration of estack. */
if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
switch (term & _TAG_IMMED1_MASK) {
@@ -1186,51 +1420,94 @@ make_hash2(Eterm term)
break;
case _TAG_IMMED1_SMALL:
{
- Sint x = signed_val(term);
-
- if (SMALL_BITS > 28 && !IS_SSMALL28(x)) {
- term = small_to_big(x, tmp_big);
- break;
+ Sint small = signed_val(term);
+ if (SMALL_BITS > 28 && !IS_SSMALL28(small)) {
+ hash = 0;
+ NOT_SSMALL28_HASH(small);
+ return hash;
}
hash = 0;
- SINT32_HASH(x, HCONST);
+ SINT32_HASH(small, HCONST);
return hash;
}
}
};
{
Eterm tmp;
+ long max_iterations = 0;
+ long iterations_until_trap = 0;
+ Eterm state_mref = THE_NON_VALUE;
+ ErtsMakeHash2Context* context = NULL;
DECLARE_ESTACK(s);
-
- UseTmpHeapNoproc(2);
+ ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+ if(can_trap){
+#ifdef DEBUG
+ (void)ITERATIONS_PER_RED;
+ iterations_until_trap = max_iterations =
+ (1103515245 * (ERTS_BIF_REDS_LEFT(p)) + 12345) % 227;
+#else
+ iterations_until_trap = max_iterations =
+ ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(p);
+#endif
+ }
+ if (can_trap && is_internal_magic_ref(term)) {
+ Binary* state_bin;
+ state_mref = term;
+ state_bin = erts_magic_ref2bin(state_mref);
+ if (ERTS_MAGIC_BIN_DESTRUCTOR(state_bin) == make_hash2_ctx_bin_dtor) {
+ /* Restore state after a trap */
+ context = ERTS_MAGIC_BIN_DATA(state_bin);
+ term = context->term;
+ hash = context->hash;
+ hash_xor_pairs = context->hash_xor_pairs;
+ ESTACK_RESTORE(s, &context->stack);
+ ASSERT(p->flags & F_DISABLE_GC);
+ erts_set_gc_state(p, 1);
+ switch (context->trap_location) {
+ case hash2_common_3: goto L_hash2_common_3;
+ case tag_primary_list: goto L_tag_primary_list;
+ case arityval_subtag: goto L_arityval_subtag;
+ case hamt_subtag_head_flatmap: goto L_hamt_subtag_head_flatmap;
+ case map_subtag: goto L_map_subtag;
+ case fun_subtag: goto L_fun_subtag;
+ case neg_big_subtag: goto L_neg_big_subtag;
+ case sub_binary_subtag_1: goto L_sub_binary_subtag_1;
+ case sub_binary_subtag_2: goto L_sub_binary_subtag_2;
+ case hash2_common_1: goto L_hash2_common_1;
+ case hash2_common_2: goto L_hash2_common_2;
+ }
+ }
+ }
hash = 0;
for (;;) {
switch (primary_tag(term)) {
case TAG_PRIMARY_LIST:
{
- int c = 0;
- Uint32 sh = 0;
- Eterm* ptr = list_val(term);
- while (is_byte(*ptr)) {
+ ErtsMakeHash2Context_TAG_PRIMARY_LIST ctx = {
+ .c = 0,
+ .sh = 0,
+ .ptr = list_val(term)};
+ while (is_byte(*ctx.ptr)) {
/* Optimization for strings. */
- sh = (sh << 8) + unsigned_val(*ptr);
- if (c == 3) {
- UINT32_HASH(sh, HCONST_4);
- c = sh = 0;
+ ctx.sh = (ctx.sh << 8) + unsigned_val(*ctx.ptr);
+ if (ctx.c == 3) {
+ UINT32_HASH(ctx.sh, HCONST_4);
+ ctx.c = ctx.sh = 0;
} else {
- c++;
+ ctx.c++;
}
- term = CDR(ptr);
+ term = CDR(ctx.ptr);
if (is_not_list(term))
break;
- ptr = list_val(term);
+ ctx.ptr = list_val(term);
+ TRAP_LOCATION(tag_primary_list);
}
- if (c > 0)
- UINT32_HASH(sh, HCONST_4);
+ if (ctx.c > 0)
+ UINT32_HASH(ctx.sh, HCONST_4);
if (is_list(term)) {
- tmp = CDR(ptr);
+ tmp = CDR(ctx.ptr);
ESTACK_PUSH(s, tmp);
- term = CAR(ptr);
+ term = CAR(ctx.ptr);
}
}
break;
@@ -1241,34 +1518,39 @@ make_hash2(Eterm term)
switch (hdr & _TAG_HEADER_MASK) {
case ARITYVAL_SUBTAG:
{
- int i;
- int arity = header_arity(hdr);
- Eterm* elem = tuple_val(term);
- UINT32_HASH(arity, HCONST_9);
- if (arity == 0) /* Empty tuple */
+ ErtsMakeHash2Context_ARITYVAL_SUBTAG ctx = {
+ .i = 0,
+ .arity = header_arity(hdr),
+ .elem = tuple_val(term)};
+ UINT32_HASH(ctx.arity, HCONST_9);
+ if (ctx.arity == 0) /* Empty tuple */
goto hash2_common;
- for (i = arity; ; i--) {
- term = elem[i];
- if (i == 1)
+ for (ctx.i = ctx.arity; ; ctx.i--) {
+ term = ctx.elem[ctx.i];
+ if (ctx.i == 1)
break;
ESTACK_PUSH(s, term);
+ TRAP_LOCATION(arityval_subtag);
}
}
break;
case MAP_SUBTAG:
{
- Eterm* ptr = boxed_val(term) + 1;
Uint size;
- int i;
+ ErtsMakeHash2Context_MAP_SUBTAG ctx = {
+ .ptr = boxed_val(term) + 1,
+ .i = 0};
switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
case HAMT_SUBTAG_HEAD_FLATMAP:
{
flatmap_t *mp = (flatmap_t *)flatmap_val(term);
- Eterm *ks = flatmap_get_keys(mp);
- Eterm *vs = flatmap_get_values(mp);
- size = flatmap_get_size(mp);
- UINT32_HASH(size, HCONST_16);
- if (size == 0)
+ ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP ctx = {
+ .ks = flatmap_get_keys(mp),
+ .vs = flatmap_get_values(mp),
+ .i = 0,
+ .size = flatmap_get_size(mp)};
+ UINT32_HASH(ctx.size, HCONST_16);
+ if (ctx.size == 0)
goto hash2_common;
/* We want a portable hash function that is *independent* of
@@ -1281,17 +1563,18 @@ make_hash2(Eterm term)
ESTACK_PUSH(s, HASH_MAP_TAIL);
hash = 0;
hash_xor_pairs = 0;
- for (i = size - 1; i >= 0; i--) {
+ for (ctx.i = ctx.size - 1; ctx.i >= 0; ctx.i--) {
ESTACK_PUSH(s, HASH_MAP_PAIR);
- ESTACK_PUSH(s, vs[i]);
- ESTACK_PUSH(s, ks[i]);
+ ESTACK_PUSH(s, ctx.vs[ctx.i]);
+ ESTACK_PUSH(s, ctx.ks[ctx.i]);
+ TRAP_LOCATION(hamt_subtag_head_flatmap);
}
goto hash2_common;
}
case HAMT_SUBTAG_HEAD_ARRAY:
case HAMT_SUBTAG_HEAD_BITMAP:
- size = *ptr++;
+ size = *ctx.ptr++;
UINT32_HASH(size, HCONST_16);
if (size == 0)
goto hash2_common;
@@ -1303,27 +1586,28 @@ make_hash2(Eterm term)
}
switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
case HAMT_SUBTAG_HEAD_ARRAY:
- i = 16;
+ ctx.i = 16;
break;
case HAMT_SUBTAG_HEAD_BITMAP:
case HAMT_SUBTAG_NODE_BITMAP:
- i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ ctx.i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
break;
default:
erts_exit(ERTS_ERROR_EXIT, "bad header");
}
- while (i) {
- if (is_list(*ptr)) {
- Eterm* cons = list_val(*ptr);
+ while (ctx.i) {
+ if (is_list(*ctx.ptr)) {
+ Eterm* cons = list_val(*ctx.ptr);
ESTACK_PUSH(s, HASH_MAP_PAIR);
ESTACK_PUSH(s, CDR(cons));
ESTACK_PUSH(s, CAR(cons));
}
else {
- ASSERT(is_boxed(*ptr));
- ESTACK_PUSH(s, *ptr);
+ ASSERT(is_boxed(*ctx.ptr));
+ ESTACK_PUSH(s, *ctx.ptr);
}
- i--; ptr++;
+ ctx.i--; ctx.ptr++;
+ TRAP_LOCATION(map_subtag);
}
goto hash2_common;
}
@@ -1344,22 +1628,25 @@ make_hash2(Eterm term)
case FUN_SUBTAG:
{
ErlFunThing* funp = (ErlFunThing *) fun_val(term);
- Uint num_free = funp->num_free;
+ ErtsMakeHash2Context_FUN_SUBTAG ctx = {
+ .num_free = funp->num_free,
+ .bptr = NULL};
UINT32_HASH_2
- (num_free,
+ (ctx.num_free,
atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue,
HCONST);
UINT32_HASH_2
(funp->fe->old_index, funp->fe->old_uniq, HCONST);
- if (num_free == 0) {
+ if (ctx.num_free == 0) {
goto hash2_common;
} else {
- Eterm* bptr = funp->env + num_free - 1;
- while (num_free-- > 1) {
- term = *bptr--;
+ ctx.bptr = funp->env + ctx.num_free - 1;
+ while (ctx.num_free-- > 1) {
+ term = *ctx.bptr--;
ESTACK_PUSH(s, term);
+ TRAP_LOCATION(fun_subtag);
}
- term = *bptr;
+ term = *ctx.bptr;
}
}
break;
@@ -1367,70 +1654,190 @@ make_hash2(Eterm term)
case HEAP_BINARY_SUBTAG:
case SUB_BINARY_SUBTAG:
{
- byte* bptr;
- unsigned sz = binary_size(term);
+#define BYTE_BITS 8
+ ErtsMakeHash2Context_SUB_BINARY_SUBTAG ctx = {
+ .bptr = 0,
+ /* !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!!
+ *
+ * The size is truncated to 32 bits on the line
+ * below so that the code is compatible with old
+ * versions of the code. This means that hash
+ * values for binaries with a size greater than
+ * 4GB do not take all bytes in consideration.
+ *
+ * !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!!
+ */
+ .sz = (0xFFFFFFFF & binary_size(term)),
+ .bitsize = 0,
+ .bitoffs = 0,
+ .no_bytes_processed = 0
+ };
Uint32 con = HCONST_13 + hash;
- Uint bitoffs;
- Uint bitsize;
-
- ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize);
- if (sz == 0 && bitsize == 0) {
+ Uint iters_for_bin = MAX(1, ctx.sz / BLOCK_HASH_BYTES_PER_ITER);
+ ERTS_GET_BINARY_BYTES(term, ctx.bptr, ctx.bitoffs, ctx.bitsize);
+ if (ctx.sz == 0 && ctx.bitsize == 0) {
hash = con;
- } else {
- if (bitoffs == 0) {
- hash = block_hash(bptr, sz, con);
- if (bitsize > 0) {
- UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)),
- HCONST_15);
- }
- } else {
- byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP,
- sz + (bitsize != 0));
- erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize);
- hash = block_hash(buf, sz, con);
- if (bitsize > 0) {
- UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)),
- HCONST_15);
- }
- erts_free(ERTS_ALC_T_TMP, (void *) buf);
- }
+ } else if (ctx.bitoffs == 0 &&
+ (!can_trap ||
+ (iterations_until_trap - iters_for_bin) > 0)) {
+ /* No need to trap while hashing binary */
+ if (can_trap) iterations_until_trap -= iters_for_bin;
+ hash = block_hash(ctx.bptr, ctx.sz, con);
+ if (ctx.bitsize > 0) {
+ UINT32_HASH_2(ctx.bitsize,
+ (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ } else if (ctx.bitoffs == 0) {
+ /* Need to trap while hashing binary */
+ ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx;
+ block_hash_setup(con, block_hash_ctx);
+ do {
+ Uint max_bytes_to_process =
+ iterations_until_trap <= 0 ? BLOCK_HASH_BYTES_PER_ITER :
+ iterations_until_trap * BLOCK_HASH_BYTES_PER_ITER;
+ Uint bytes_left = ctx.sz - ctx.no_bytes_processed;
+ Uint even_bytes_left =
+ bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER);
+ Uint bytes_to_process =
+ MIN(max_bytes_to_process, even_bytes_left);
+ block_hash_buffer(&ctx.bptr[ctx.no_bytes_processed],
+ bytes_to_process,
+ block_hash_ctx);
+ ctx.no_bytes_processed += bytes_to_process;
+ iterations_until_trap -=
+ MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER);
+ TRAP_LOCATION_NO_RED(sub_binary_subtag_1);
+ block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */
+ } while ((ctx.sz - ctx.no_bytes_processed) >=
+ BLOCK_HASH_BYTES_PER_ITER);
+ hash = block_hash_final_bytes(ctx.bptr +
+ ctx.no_bytes_processed,
+ ctx.sz - ctx.no_bytes_processed,
+ ctx.sz,
+ block_hash_ctx);
+ if (ctx.bitsize > 0) {
+ UINT32_HASH_2(ctx.bitsize,
+ (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ } else if (/* ctx.bitoffs != 0 && */
+ (!can_trap ||
+ (iterations_until_trap - iters_for_bin) > 0)) {
+ /* No need to trap while hashing binary */
+ Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0);
+ byte *buf = erts_alloc(ERTS_ALC_T_TMP, nr_of_bytes);
+ Uint nr_of_bits_to_copy = ctx.sz*BYTE_BITS+ctx.bitsize;
+ if (can_trap) iterations_until_trap -= iters_for_bin;
+ erts_copy_bits(ctx.bptr,
+ ctx.bitoffs, 1, buf, 0, 1, nr_of_bits_to_copy);
+ hash = block_hash(buf, ctx.sz, con);
+ if (ctx.bitsize > 0) {
+ UINT32_HASH_2(ctx.bitsize,
+ (buf[ctx.sz] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ erts_free(ERTS_ALC_T_TMP, buf);
+ } else /* ctx.bitoffs != 0 && */ {
+#ifdef DEBUG
+#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 3)
+#else
+#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 256)
+#endif
+#define BINARY_BUF_SIZE_BITS (BINARY_BUF_SIZE*BYTE_BITS)
+ /* Need to trap while hashing binary */
+ ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx;
+ Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0);
+ ERTS_CT_ASSERT(BINARY_BUF_SIZE % BLOCK_HASH_BYTES_PER_ITER == 0);
+ ctx.buf = erts_alloc(ERTS_ALC_T_PHASH2_TRAP,
+ MIN(nr_of_bytes, BINARY_BUF_SIZE));
+ block_hash_setup(con, block_hash_ctx);
+ do {
+ Uint bytes_left =
+ ctx.sz - ctx.no_bytes_processed;
+ Uint even_bytes_left =
+ bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER);
+ Uint bytes_to_process =
+ MIN(BINARY_BUF_SIZE, even_bytes_left);
+ Uint nr_of_bits_left =
+ (ctx.sz*BYTE_BITS+ctx.bitsize) -
+ ctx.no_bytes_processed*BYTE_BITS;
+ Uint nr_of_bits_to_copy =
+ MIN(nr_of_bits_left, BINARY_BUF_SIZE_BITS);
+ ctx.done = nr_of_bits_left == nr_of_bits_to_copy;
+ erts_copy_bits(ctx.bptr + ctx.no_bytes_processed,
+ ctx.bitoffs, 1, ctx.buf, 0, 1,
+ nr_of_bits_to_copy);
+ block_hash_buffer(ctx.buf,
+ bytes_to_process,
+ block_hash_ctx);
+ ctx.no_bytes_processed += bytes_to_process;
+ iterations_until_trap -=
+ MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER);
+ TRAP_LOCATION_NO_RED(sub_binary_subtag_2);
+ block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */
+ } while (!ctx.done);
+ nr_of_bytes = ctx.sz + (ctx.bitsize != 0);
+ hash = block_hash_final_bytes(ctx.buf +
+ (ctx.no_bytes_processed -
+ ((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE),
+ ctx.sz - ctx.no_bytes_processed,
+ ctx.sz,
+ block_hash_ctx);
+ if (ctx.bitsize > 0) {
+ Uint last_byte_index =
+ nr_of_bytes - (((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE) -1;
+ UINT32_HASH_2(ctx.bitsize,
+ (ctx.buf[last_byte_index] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ erts_free(ERTS_ALC_T_PHASH2_TRAP, ctx.buf);
+ context->trap_location_state.sub_binary_subtag_2.buf = NULL;
}
goto hash2_common;
+#undef BYTE_BITS
+#undef BINARY_BUF_SIZE
+#undef BINARY_BUF_SIZE_BITS
}
break;
case POS_BIG_SUBTAG:
case NEG_BIG_SUBTAG:
{
- Eterm* ptr = big_val(term);
- Uint i = 0;
- Uint n = BIG_SIZE(ptr);
- Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11;
+ Eterm* big_val_ptr = big_val(term);
+ ErtsMakeHash2Context_NEG_BIG_SUBTAG ctx = {
+ .ptr = big_val_ptr,
+ .i = 0,
+ .n = BIG_SIZE(big_val_ptr),
+ .con = BIG_SIGN(big_val_ptr) ? HCONST_10 : HCONST_11};
#if D_EXP == 16
do {
Uint32 x, y;
- x = i < n ? BIG_DIGIT(ptr, i++) : 0;
- x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
- y = i < n ? BIG_DIGIT(ptr, i++) : 0;
- y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
+ x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ x += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16;
+ y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ y += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16;
+ UINT32_HASH_2(x, y, ctx.con);
+ TRAP_LOCATION(neg_big_subtag);
+ } while (ctx.i < ctx.n);
#elif D_EXP == 32
do {
Uint32 x, y;
- x = i < n ? BIG_DIGIT(ptr, i++) : 0;
- y = i < n ? BIG_DIGIT(ptr, i++) : 0;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
+ x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ UINT32_HASH_2(x, y, ctx.con);
+ TRAP_LOCATION(neg_big_subtag);
+ } while (ctx.i < ctx.n);
#elif D_EXP == 64
do {
Uint t;
Uint32 x, y;
- ASSERT(i < n);
- t = BIG_DIGIT(ptr, i++);
+ ASSERT(ctx.i < ctx.n);
+ t = BIG_DIGIT(ctx.ptr, ctx.i++);
x = t & 0xffffffff;
y = t >> 32;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
+ UINT32_HASH_2(x, y, ctx.con);
+ TRAP_LOCATION(neg_big_subtag);
+ } while (ctx.i < ctx.n);
#else
#error "unsupported D_EXP size"
#endif
@@ -1508,13 +1915,13 @@ make_hash2(Eterm term)
}
case _TAG_IMMED1_SMALL:
{
- Sint x = signed_val(term);
+ Sint small = signed_val(term);
+ if (SMALL_BITS > 28 && !IS_SSMALL28(small)) {
+ NOT_SSMALL28_HASH(small);
+ } else {
+ SINT32_HASH(small, HCONST);
+ }
- if (SMALL_BITS > 28 && !IS_SSMALL28(x)) {
- term = small_to_big(x, tmp_big);
- break;
- }
- SINT32_HASH(x, HCONST);
goto hash2_common;
}
}
@@ -1529,7 +1936,10 @@ make_hash2(Eterm term)
if (ESTACK_ISEMPTY(s)) {
DESTROY_ESTACK(s);
- UnUseTmpHeapNoproc(2);
+ if (can_trap) {
+ BUMP_REDS(p, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
+ ASSERT(!(p->flags & F_DISABLE_GC));
+ }
return hash;
}
@@ -1540,18 +1950,37 @@ make_hash2(Eterm term)
hash = (Uint32) ESTACK_POP(s);
UINT32_HASH(hash_xor_pairs, HCONST_19);
hash_xor_pairs = (Uint32) ESTACK_POP(s);
+ TRAP_LOCATION_NO_CTX(hash2_common_1);
goto hash2_common;
}
case HASH_MAP_PAIR:
hash_xor_pairs ^= hash;
hash = 0;
+ TRAP_LOCATION_NO_CTX(hash2_common_2);
goto hash2_common;
default:
break;
}
+
}
+ TRAP_LOCATION_NO_CTX(hash2_common_3);
}
}
+#undef TRAP_LOCATION_NO_RED
+#undef TRAP_LOCATION
+#undef TRAP_LOCATION_NO_CTX
+}
+
+Uint32
+make_hash2(Eterm term)
+{
+ return make_hash2_helper(term, 0, NULL, NULL);
+}
+
+Uint32
+trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p)
+{
+ return make_hash2_helper(term, 1, state_mref_write_back, p);
}
/* Term hash function for internal use.