aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_load.c73
-rw-r--r--erts/emulator/beam/bif.c22
-rw-r--r--erts/emulator/beam/big.c18
-rw-r--r--erts/emulator/beam/big.h2
-rw-r--r--erts/emulator/beam/dist.c14
-rw-r--r--erts/emulator/beam/dist.h6
-rw-r--r--erts/emulator/beam/erl_alloc.types1
-rw-r--r--erts/emulator/beam/erl_bif_info.c5
-rw-r--r--erts/emulator/beam/erl_bif_trace.c13
-rw-r--r--erts/emulator/beam/erl_db_hash.c250
-rw-r--r--erts/emulator/beam/erl_db_hash.h3
-rw-r--r--erts/emulator/beam/erl_fun.c54
-rw-r--r--erts/emulator/beam/erl_fun.h1
-rw-r--r--erts/emulator/beam/erl_message.c2
-rw-r--r--erts/emulator/beam/erl_nif.c2
-rw-r--r--erts/emulator/beam/erl_node_tables.c4
-rw-r--r--erts/emulator/beam/erl_node_tables.h2
-rw-r--r--erts/emulator/beam/erl_proc_sig_queue.c2
-rw-r--r--erts/emulator/beam/erl_process.c62
-rw-r--r--erts/emulator/beam/erl_process.h2
-rw-r--r--erts/emulator/beam/erl_trace.c3
-rw-r--r--erts/emulator/beam/erl_trace.h4
-rw-r--r--erts/emulator/beam/erl_utils.h1
-rw-r--r--erts/emulator/beam/external.c160
-rw-r--r--erts/emulator/beam/instrs.tab9
-rw-r--r--erts/emulator/beam/ops.tab69
-rw-r--r--erts/emulator/beam/sys.h9
-rw-r--r--erts/emulator/beam/utils.c735
28 files changed, 998 insertions, 530 deletions
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 35f2ea6688..3d5683f19f 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -315,6 +315,7 @@ typedef struct LoaderState {
* (or 0 if there is no on_load function)
*/
int otp_20_or_higher; /* Compiled with OTP 20 or higher */
+ unsigned max_opcode; /* Highest opcode used in module */
/*
* Atom table.
@@ -1588,6 +1589,17 @@ static int
read_lambda_table(LoaderState* stp)
{
unsigned int i;
+ unsigned int otp_22_or_lower;
+
+ /*
+ * Determine whether this module was compiled with OTP 22 or lower
+ * by looking at the max opcode number. The compiler in OTP 23 will
+ * always set the max opcode to the opcode for `swap` (whether
+ * actually used or not) so that a module compiled for OTP 23
+ * cannot be loaded in earlier versions.
+ */
+
+ otp_22_or_lower = stp->max_opcode < genop_swap_2;
GetInt(stp, 4, stp->num_lambdas);
if (stp->num_lambdas > stp->lambdas_allocated) {
@@ -1619,6 +1631,29 @@ read_lambda_table(LoaderState* stp)
GetInt(stp, 4, Index);
GetInt(stp, 4, stp->lambdas[i].num_free);
GetInt(stp, 4, OldUniq);
+
+ /*
+ * Fun entries are now keyed by the explicit ("new") index in
+ * the fun entry. That allows multiple make_fun2 instructions
+ * to share the same fun entry (when the `fun F/A` syntax is
+ * used). Before OTP 23, fun entries were keyed by the old
+ * index, which is the order of the entries in the fun
+ * chunk. Each make_fun2 needed to refer to its own fun entry.
+ *
+ * Modules compiled before OTP 23 can safely be loaded if the
+ * old index and the new index are equal. That is true for all
+ * modules compiled with OTP R15 and later.
+ */
+ if (otp_22_or_lower && i != Index) {
+ /*
+ * Compiled with a compiler before OTP R15B. The new indices
+ * are not reliable, so it is not safe to load this module.
+ */
+ LoadError2(stp, "please re-compile this module with an "
+ ERLANG_OTP_RELEASE " compiler "
+ "(old-style fun with indices: %d/%d)",
+ i, Index);
+ }
fe = erts_put_fun_entry2(stp->module, OldUniq, i, stp->mod_md5,
Index, arity-stp->lambdas[i].num_free);
stp->lambdas[i].fe = fe;
@@ -1839,7 +1874,6 @@ read_code_header(LoaderState* stp)
{
unsigned head_size;
unsigned version;
- unsigned opcode_max;
int i;
/*
@@ -1871,8 +1905,8 @@ read_code_header(LoaderState* stp)
/*
* Verify the number of the highest opcode used.
*/
- GetInt(stp, 4, opcode_max);
- if (opcode_max > MAX_GENERIC_OPCODE) {
+ GetInt(stp, 4, stp->max_opcode);
+ if (stp->max_opcode > MAX_GENERIC_OPCODE) {
LoadError2(stp,
"This BEAM file was compiled for a later version"
" of the run-time system than " ERLANG_OTP_RELEASE ".\n"
@@ -1880,7 +1914,7 @@ read_code_header(LoaderState* stp)
ERLANG_OTP_RELEASE " compiler.\n"
" (Use of opcode %d; this emulator supports "
"only up to %d.)",
- opcode_max, MAX_GENERIC_OPCODE);
+ stp->max_opcode, MAX_GENERIC_OPCODE);
}
GetInt(stp, 4, stp->num_labels);
@@ -3131,27 +3165,6 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
return 0;
}
-static int
-is_killed_apply(LoaderState* stp, GenOpArg Reg, GenOpArg Live)
-{
- return Reg.type == TAG_x && Live.type == TAG_u &&
- Live.val+2 <= Reg.val;
-}
-
-static int
-is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live)
-{
- return Reg.type == TAG_x && Live.type == TAG_u &&
- Live.val <= Reg.val;
-}
-
-static int
-is_killed_by_call_fun(LoaderState* stp, GenOpArg Reg, GenOpArg Live)
-{
- return Reg.type == TAG_x && Live.type == TAG_u &&
- Live.val+1 <= Reg.val;
-}
-
/*
* Test whether register Reg is killed by make_fun instruction that
* creates the fun given by index idx.
@@ -3172,16 +3185,6 @@ is_killed_by_make_fun(LoaderState* stp, GenOpArg Reg, GenOpArg idx)
}
/*
- * Test whether register Reg is killed by the send instruction that follows.
- */
-
-static int
-is_killed_by_send(LoaderState* stp, GenOpArg Reg)
-{
- return Reg.type == TAG_x && 2 <= Reg.val;
-}
-
-/*
* Generate an instruction for element/2.
*/
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index b81056c774..9f67f46b31 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1915,7 +1915,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp,
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
erts_dsprintf(dsbufp,
"Discarding message %T from %T to %T in an old "
- "incarnation (%d) of this node (%d)\n",
+ "incarnation (%u) of this node (%u)\n",
msg,
p->common.id,
to,
@@ -1959,7 +1959,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp,
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
erts_dsprintf(dsbufp,
"Discarding message %T from %T to %T in an old "
- "incarnation (%d) of this node (%d)\n",
+ "incarnation (%u) of this node (%u)\n",
msg,
p->common.id,
to,
@@ -1987,7 +1987,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp,
trace_send(p, portid, msg);
if (have_seqtrace(SEQ_TRACE_TOKEN(p))) {
- seq_trace_update_send(p);
+ seq_trace_update_serial(p);
seq_trace_output(SEQ_TRACE_TOKEN(p), msg,
SEQ_TRACE_SEND, portid, p);
}
@@ -4866,9 +4866,13 @@ BIF_RETTYPE phash_2(BIF_ALIST_2)
BIF_RETTYPE phash2_1(BIF_ALIST_1)
{
Uint32 hash;
-
- hash = make_hash2(BIF_ARG_1);
- BIF_RET(make_small(hash & ((1L << 27) - 1)));
+ Eterm trap_state = THE_NON_VALUE;
+ hash = trapping_make_hash2(BIF_ARG_1, &trap_state, BIF_P);
+ if (trap_state == THE_NON_VALUE) {
+ BIF_RET(make_small(hash & ((1L << 27) - 1)));
+ } else {
+ BIF_TRAP1(bif_export[BIF_phash2_1], BIF_P, trap_state);
+ }
}
BIF_RETTYPE phash2_2(BIF_ALIST_2)
@@ -4876,6 +4880,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
Uint32 hash;
Uint32 final_hash;
Uint32 range;
+ Eterm trap_state = THE_NON_VALUE;
/* Check for special case 2^32 */
if (term_equals_2pow32(BIF_ARG_2)) {
@@ -4887,7 +4892,10 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
}
range = (Uint32) u;
}
- hash = make_hash2(BIF_ARG_1);
+ hash = trapping_make_hash2(BIF_ARG_1, &trap_state, BIF_P);
+ if (trap_state != THE_NON_VALUE) {
+ BIF_TRAP2(bif_export[BIF_phash2_2], BIF_P, trap_state, BIF_ARG_2);
+ }
if (range) {
final_hash = hash % range; /* [0..range-1] */
} else {
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 522f50287a..7666f23a4f 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -2176,6 +2176,24 @@ term_to_Uint64(Eterm term, Uint64 *up)
#endif
}
+int
+term_to_Uint32(Eterm term, Uint32 *up)
+{
+#if ERTS_SIZEOF_ETERM == 4
+ return term_to_Uint(term,up);
+#else
+ if (is_small(term)) {
+ Sint i = signed_val(term);
+ if (i >= 0) {
+ *up = (Uint32) i;
+ return 1;
+ }
+ }
+ *up = BADARG;
+ return 0;
+#endif
+}
+
int term_to_Sint(Eterm term, Sint *sp)
{
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index ad19cce395..3fed076419 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -168,6 +168,8 @@ Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *);
int term_to_Uint64(Eterm, Uint64*);
int term_to_Sint64(Eterm, Sint64*);
#endif
+int term_to_Uint32(Eterm, Uint32*);
+
Uint32 big_to_uint32(Eterm b);
int term_equals_2pow32(Eterm);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index eb9e749a08..dafe805a6f 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1051,7 +1051,7 @@ erts_dsig_send_msg(ErtsDSigSendContext* ctx, Eterm remote, Eterm message)
#endif
if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) {
- seq_trace_update_send(sender);
+ seq_trace_update_serial(sender);
token = SEQ_TRACE_TOKEN(sender);
seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender);
}
@@ -1125,7 +1125,7 @@ erts_dsig_send_reg_msg(ErtsDSigSendContext* ctx, Eterm remote_name, Eterm messag
#endif
if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) {
- seq_trace_update_send(sender);
+ seq_trace_update_serial(sender);
token = SEQ_TRACE_TOKEN(sender);
seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender);
}
@@ -1184,7 +1184,7 @@ erts_dsig_send_exit_tt(ErtsDSigSendContext *ctx, Eterm local, Eterm remote,
msg = reason;
if (have_seqtrace(token)) {
- seq_trace_update_send(ctx->c_p);
+ seq_trace_update_serial(ctx->c_p);
seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local);
if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) {
ctl = TUPLE4(&ctx->ctl_heap[0],
@@ -3762,12 +3762,10 @@ int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */
BIF_RETTYPE setnode_2(BIF_ALIST_2)
{
Process *net_kernel;
- Uint creation;
+ Uint32 creation;
/* valid creation ? */
- if(!term_to_Uint(BIF_ARG_2, &creation))
- goto error;
- if(creation > 3)
+ if(!term_to_Uint32(BIF_ARG_2, &creation))
goto error;
/* valid node name ? */
@@ -3811,7 +3809,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_thr_progress_block();
inc_no_nodes();
- erts_set_this_node(BIF_ARG_1, (Uint32) creation);
+ erts_set_this_node(BIF_ARG_1, creation);
erts_is_alive = 1;
send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL);
erts_thr_progress_unblock();
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index a33fb7efcf..37ec88cc55 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -54,11 +54,12 @@
#define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \
| DFLAG_EXTENDED_PIDS_PORTS \
| DFLAG_UTF8_ATOMS \
- | DFLAG_NEW_FUN_TAGS)
+ | DFLAG_NEW_FUN_TAGS \
+ | DFLAG_BIG_CREATION)
/*
* Additional optimistic flags when encoding toward pending connection.
- * If remote node (erl_interface) does not supporting these then we may need
+ * If remote node (erl_interface) does not support these then we may need
* to transcode messages enqueued before connection setup was finished.
*/
#define DFLAG_DIST_HOPEFULLY (DFLAG_EXPORT_PTR_TAG \
@@ -75,7 +76,6 @@
| DFLAG_SMALL_ATOM_TAGS \
| DFLAG_UTF8_ATOMS \
| DFLAG_MAP_TAG \
- | DFLAG_BIG_CREATION \
| DFLAG_SEND_SENDER \
| DFLAG_BIG_SEQTRACE_LABELS \
| DFLAG_EXIT_PAYLOAD \
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 21941ba96e..349977ebe7 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -278,6 +278,7 @@ type SETUP_CONN_ARG SHORT_LIVED PROCESSES setup_connection_argument
type LIST_TRAP SHORT_LIVED PROCESSES list_bif_trap_state
type CONT_EXIT_TRAP SHORT_LIVED PROCESSES continue_exit_trap_state
type SEQ_YIELD_STATE SHORT_LIVED SYSTEM dist_seq_yield_state
+type PHASH2_TRAP SHORT_LIVED PROCESSES phash2_trap_state
type ENVIRONMENT SYSTEM SYSTEM environment
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4d8c3eb9dd..d1ceffd23c 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2813,7 +2813,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
} else if (BIF_ARG_1 == am_threads) {
return am_true;
} else if (BIF_ARG_1 == am_creation) {
- return make_small(erts_this_node->creation);
+ Uint hsz = 0;
+ erts_bld_uint(NULL, &hsz, erts_this_node->creation);
+ hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
+ BIF_RET(erts_bld_uint(&hp, NULL, erts_this_node->creation));
} else if (BIF_ARG_1 == am_break_ignored) {
extern int ignore_break;
if (ignore_break)
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index b31d5b86cb..80ba7d1b3c 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1858,6 +1858,8 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2,
if (arg1 == am_send) {
current_flag = SEQ_TRACE_SEND;
+ } else if (arg1 == am_spawn) {
+ current_flag = SEQ_TRACE_SPAWN;
} else if (arg1 == am_receive) {
current_flag = SEQ_TRACE_RECEIVE;
} else if (arg1 == am_print) {
@@ -1976,8 +1978,9 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item)
}
if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) {
- if ((item == am_send) || (item == am_receive) ||
- (item == am_print) || (item == am_timestamp)
+ if ((item == am_send) || (item == am_spawn) ||
+ (item == am_receive) || (item == am_print)
+ || (item == am_timestamp)
|| (item == am_monotonic_timestamp)
|| (item == am_strict_monotonic_timestamp)) {
hp = HAlloc(p,3);
@@ -1992,6 +1995,8 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item)
if (item == am_send) {
current_flag = SEQ_TRACE_SEND;
+ } else if (item == am_spawn) {
+ current_flag = SEQ_TRACE_SPAWN;
} else if (item == am_receive) {
current_flag = SEQ_TRACE_RECEIVE;
} else if (item == am_print) {
@@ -2041,7 +2046,7 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1)
if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) {
BIF_RET(am_false);
}
- seq_trace_update_send(BIF_P);
+ seq_trace_update_serial(BIF_P);
seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_1,
SEQ_TRACE_PRINT, NIL, BIF_P);
BIF_RET(am_true);
@@ -2062,7 +2067,7 @@ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2)
}
if (!EQ(BIF_ARG_1, SEQ_TRACE_TOKEN_LABEL(BIF_P)))
BIF_RET(am_false);
- seq_trace_update_send(BIF_P);
+ seq_trace_update_serial(BIF_P);
seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_2,
SEQ_TRACE_PRINT, NIL, BIF_P);
BIF_RET(am_true);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 4904d3fa42..44ecf7cce5 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -93,11 +93,9 @@
erts_flxctr_dec_read_centralized(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID)
#define RESET_NITEMS(DB) \
erts_flxctr_reset(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID)
-/*
- * The following symbols can be manipulated to "tune" the linear hash array
- */
+
#define GROW_LIMIT(NACTIVE) ((NACTIVE)*1)
-#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2)
+#define SHRINK_LIMIT(TB) erts_atomic_read_nob(&(TB)->shrink_limit)
/*
** We want the first mandatory segment to be small (to reduce minimal footprint)
@@ -137,6 +135,11 @@
#define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK]
+#ifdef DEBUG
+# define DBG_BUCKET_INACTIVE ((HashDbTerm*)0xdead5107)
+#endif
+
+
/*
* When deleting a table, the number of records to delete.
* Approximate number, because we must delete entire buckets.
@@ -377,7 +380,7 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete
*/
static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix);
static void alloc_seg(DbTableHash *tb);
-static int free_seg(DbTableHash *tb, int free_records);
+static int free_seg(DbTableHash *tb);
static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
HashDbTerm *list);
static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
@@ -471,10 +474,8 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle);
static ERTS_INLINE void try_shrink(DbTableHash* tb)
{
- int nactive = NACTIVE(tb);
int nitems = NITEMS(tb);
- if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)
- && !IS_FIXED(tb)) {
+ if (nitems < SHRINK_LIMIT(tb) && !IS_FIXED(tb)) {
shrink(tb, nitems);
}
}
@@ -685,6 +686,7 @@ int db_create_hash(Process *p, DbTable *tbl)
erts_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK);
erts_atomic_init_nob(&tb->nactive, FIRST_SEGSZ);
+ erts_atomic_init_nob(&tb->shrink_limit, 0);
erts_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL);
erts_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL);
SET_SEGTAB(tb, tb->first_segtab);
@@ -771,7 +773,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
b = next_live(tb, &ix, &lck, b->next);
if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
while (b != 0) {
- if (!has_live_key(tb, b, key, hval)) {
+ if (!has_key(tb, b, key, hval)) {
break;
}
b = next_live(tb, &ix, &lck, b->next);
@@ -781,6 +783,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
*ret = am_EOT;
}
else {
+ ASSERT(!is_pseudo_deleted(b));
*ret = db_copy_key(p, tbl, &b->dbterm);
RUNLOCK_HASH(lck);
}
@@ -2466,7 +2469,7 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL);
while(tb->nslots != 0) {
- reds -= EXT_SEGSZ/64 + free_seg(tb, 1);
+ reds -= EXT_SEGSZ/64 + free_seg(tb);
/*
* If we have done enough work, get out here.
@@ -2664,6 +2667,34 @@ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix)
return est;
}
+static void calc_shrink_limit(DbTableHash* tb)
+{
+ erts_aint_t shrink_limit;
+
+ if (tb->nslots >= (FIRST_SEGSZ + 2*EXT_SEGSZ)) {
+ /*
+ * Start shrink when we can remove one extra segment
+ * and still remain below 50% load.
+ */
+ shrink_limit = (tb->nslots - EXT_SEGSZ) / 2;
+ }
+ else {
+ /*
+ * But don't shrink below two segments.
+ * Why? In order to have chance of getting rid of the last extra segment,
+ * and rehash it into the first small segment, we either have to start
+ * early and do speculative joining of buckets or we have to join a lot
+ * of buckets during each delete-op.
+ *
+ * Instead keep segment #2 once allocated. I also think it's a good bet
+ * a shrinking large table will grow large again.
+ */
+ shrink_limit = 0;
+ }
+ erts_atomic_set_nob(&tb->shrink_limit, shrink_limit);
+}
+
+
/* Extend table with one new segment
*/
static void alloc_seg(DbTableHash *tb)
@@ -2682,8 +2713,17 @@ static void alloc_seg(DbTableHash *tb)
segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG,
(DbTable *) tb,
SIZEOF_SEGMENT(EXT_SEGSZ));
- sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ));
+#ifdef DEBUG
+ {
+ int i;
+ for (i = 0; i < EXT_SEGSZ; i++) {
+ segtab[seg_ix]->buckets[i] = DBG_BUCKET_INACTIVE;
+ }
+ }
+#endif
tb->nslots += EXT_SEGSZ;
+
+ calc_shrink_limit(tb);
}
static void dealloc_ext_segtab(void* lop_data)
@@ -2693,10 +2733,19 @@ static void dealloc_ext_segtab(void* lop_data)
erts_free(ERTS_ALC_T_DB_SEG, est);
}
-/* Shrink table by freeing the top segment
+struct dealloc_seg_ops {
+ struct segment* segp;
+ Uint seg_sz;
+
+ struct ext_segtab* est;
+};
+
+/* Shrink table by removing the top segment
** free_records: 1=free any records in segment, 0=assume segment is empty
+** ds_ops: (out) Instructions for dealloc_seg().
*/
-static int free_seg(DbTableHash *tb, int free_records)
+static int remove_seg(DbTableHash *tb, int free_records,
+ struct dealloc_seg_ops *ds_ops)
{
const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1;
struct segment** const segtab = SEGTAB(tb);
@@ -2704,24 +2753,47 @@ static int free_seg(DbTableHash *tb, int free_records)
Uint seg_sz;
int nrecords = 0;
+ ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb) || tb->common.status & DB_DELETE
+ || erts_atomic_read_nob(&tb->is_resizing));
+
ASSERT(segp != NULL);
-#ifndef DEBUG
- if (free_records)
-#endif
- {
- int i = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ;
- while (i--) {
- HashDbTerm* p = segp->buckets[i];
+ if (free_records) {
+ int ix, n;
+ if (seg_ix == 0) {
+ /* First segment (always fully active) */
+ n = FIRST_SEGSZ;
+ ix = FIRST_SEGSZ-1;
+ }
+ else if (NACTIVE(tb) < tb->nslots) {
+ /* Last extended segment partially active */
+ n = (NACTIVE(tb) - FIRST_SEGSZ) & EXT_SEGSZ_MASK;
+ ix = (NACTIVE(tb)-1) & EXT_SEGSZ_MASK;
+ }
+ else {
+ /* Full extended segment */
+ n = EXT_SEGSZ;
+ ix = EXT_SEGSZ - 1;
+ }
+ for ( ; n > 0; n--, ix--) {
+ HashDbTerm* p = segp->buckets[ix & EXT_SEGSZ_MASK];
while(p != 0) {
HashDbTerm* nxt = p->next;
- ASSERT(free_records); /* segment not empty as assumed? */
free_term(tb, p);
p = nxt;
++nrecords;
}
}
}
-
+#ifdef DEBUG
+ else {
+ int ix = (seg_ix == 0) ? FIRST_SEGSZ-1 : EXT_SEGSZ-1;
+ for ( ; ix >= 0; ix--) {
+ ASSERT(segp->buckets[ix] == DBG_BUCKET_INACTIVE);
+ }
+ }
+#endif
+
+ ds_ops->est = NULL;
if (seg_ix >= NSEG_1) {
struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab);
@@ -2730,35 +2802,64 @@ static int free_seg(DbTableHash *tb, int free_records)
SET_SEGTAB(tb, est->prev_segtab);
tb->nsegs = est->prev_nsegs;
- if (!tb->common.is_thread_safe) {
- /*
- * Table is doing a graceful shrink operation and we must avoid
- * deallocating this segtab while it may still be read by other
- * threads. Schedule deallocation with thread progress to make
- * sure no lingering threads are still hanging in BUCKET macro
- * with an old segtab pointer.
- */
- erts_schedule_db_free(&tb->common, dealloc_ext_segtab,
- est, &est->lop,
- SIZEOF_EXT_SEGTAB(est->nsegs));
- }
- else
- erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est,
- SIZEOF_EXT_SEGTAB(est->nsegs));
+ ds_ops->est = est;
}
}
+
seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ;
- erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz));
+ tb->nslots -= seg_sz;
+ ASSERT(tb->nslots >= 0);
+
+ ds_ops->segp = segp;
+ ds_ops->seg_sz = seg_sz;
#ifdef DEBUG
if (seg_ix < tb->nsegs)
SEGTAB(tb)[seg_ix] = NULL;
#endif
- tb->nslots -= seg_sz;
- ASSERT(tb->nslots >= 0);
+ calc_shrink_limit(tb);
return nrecords;
}
+/*
+ * Deallocate segment removed by remove_seg()
+ */
+static void dealloc_seg(DbTableHash *tb, struct dealloc_seg_ops* ds_ops)
+{
+ struct ext_segtab* est = ds_ops->est;
+
+ if (est) {
+ if (!tb->common.is_thread_safe) {
+ /*
+ * Table is doing a graceful shrink operation and we must avoid
+ * deallocating this segtab while it may still be read by other
+ * threads. Schedule deallocation with thread progress to make
+ * sure no lingering threads are still hanging in BUCKET macro
+ * with an old segtab pointer.
+ */
+ erts_schedule_db_free(&tb->common, dealloc_ext_segtab,
+ est, &est->lop,
+ SIZEOF_EXT_SEGTAB(est->nsegs));
+ }
+ else
+ erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est,
+ SIZEOF_EXT_SEGTAB(est->nsegs));
+ }
+
+ erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb,
+ ds_ops->segp, SIZEOF_SEGMENT(ds_ops->seg_sz));
+}
+
+/* Remove and deallocate top segment and all its contained objects */
+static int free_seg(DbTableHash *tb)
+{
+ struct dealloc_seg_ops ds_ops;
+ int reds;
+
+ reds = remove_seg(tb, 1, &ds_ops);
+ dealloc_seg(tb, &ds_ops);
+ return reds;
+}
/*
** Copy terms from ptr1 until ptr2
@@ -2880,6 +2981,7 @@ static void grow(DbTableHash* tb, int nitems)
pnext = &BUCKET(tb, from_ix);
p = *pnext;
to_pnext = &BUCKET(tb, to_ix);
+ ASSERT(*to_pnext == DBG_BUCKET_INACTIVE);
while (p != NULL) {
if (is_pseudo_deleted(p)) { /* rare but possible with fine locking */
*pnext = p->next;
@@ -2916,19 +3018,21 @@ abort:
*/
static void shrink(DbTableHash* tb, int nitems)
{
- HashDbTerm** src_bp;
- HashDbTerm** dst_bp;
+ struct dealloc_seg_ops ds_ops;
+ HashDbTerm* src;
+ HashDbTerm* tail;
HashDbTerm** bp;
erts_rwmtx_t* lck;
int src_ix, dst_ix, low_szm;
int nactive;
int loop_limit = 5;
+ ds_ops.segp = NULL;
do {
if (!begin_resizing(tb))
return; /* already in progress */
nactive = NACTIVE(tb);
- if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) {
+ if (!(nitems < SHRINK_LIMIT(tb))) {
goto abort; /* already done (race) */
}
src_ix = nactive - 1;
@@ -2945,41 +3049,49 @@ static void shrink(DbTableHash* tb, int nitems)
goto abort;
}
- src_bp = &BUCKET(tb, src_ix);
- dst_bp = &BUCKET(tb, dst_ix);
- bp = src_bp;
-
- /*
- * We join lists by appending "dst" at the end of "src"
- * as we must step through "src" anyway to purge pseudo deleted.
- */
- while(*bp != NULL) {
- if (is_pseudo_deleted(*bp)) {
- HashDbTerm* deleted = *bp;
- *bp = deleted->next;
- free_term(tb, deleted);
- } else {
- bp = &(*bp)->next;
- }
- }
- *bp = *dst_bp;
- *dst_bp = *src_bp;
- *src_bp = NULL;
-
+ src = BUCKET(tb, src_ix);
+#ifdef DEBUG
+ BUCKET(tb, src_ix) = DBG_BUCKET_INACTIVE;
+#endif
nactive = src_ix;
erts_atomic_set_nob(&tb->nactive, nactive);
if (dst_ix == 0) {
erts_atomic_set_relb(&tb->szm, low_szm);
}
- WUNLOCK_HASH(lck);
-
if (tb->nslots - src_ix >= EXT_SEGSZ) {
- free_seg(tb, 0);
+ remove_seg(tb, 0, &ds_ops);
}
done_resizing(tb);
- } while (--loop_limit
- && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive));
+ if (src) {
+ /*
+ * We join buckets by appending "dst" list at the end of "src" list
+ * as we must step through "src" anyway to purge pseudo deleted.
+ */
+ bp = &BUCKET(tb, dst_ix);
+ tail = *bp;
+ *bp = src;
+
+ while(*bp != NULL) {
+ if (is_pseudo_deleted(*bp)) {
+ HashDbTerm* deleted = *bp;
+ *bp = deleted->next;
+ free_term(tb, deleted);
+ } else {
+ bp = &(*bp)->next;
+ }
+ }
+ *bp = tail;
+ }
+
+ WUNLOCK_HASH(lck);
+
+ if (ds_ops.segp) {
+ dealloc_seg(tb, &ds_ops);
+ ds_ops.segp = NULL;
+ }
+
+ } while (--loop_limit && nitems < SHRINK_LIMIT(tb));
return;
abort:
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index 9759d8b466..b26b82056f 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -63,9 +63,10 @@ typedef struct db_table_hash_fine_locks {
typedef struct db_table_hash {
DbTableCommon common;
- /* SMP: szm and nactive are write-protected by is_resizing or table write lock */
+ /* szm, nactive, shrink_limit are write-protected by is_resizing or table write lock */
erts_atomic_t szm; /* current size mask. */
erts_atomic_t nactive; /* Number of "active" slots */
+ erts_atomic_t shrink_limit; /* Shrink table when fewer objects than this */
erts_atomic_t segtab; /* The segment table (struct segment**) */
struct segment* first_segtab[1];
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 9c866250bb..257f9bf5b3 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -100,27 +100,6 @@ int erts_fun_table_sz(void)
}
ErlFunEntry*
-erts_put_fun_entry(Eterm mod, int uniq, int index)
-{
- ErlFunEntry template;
- ErlFunEntry* fe;
- erts_aint_t refc;
- ASSERT(is_atom(mod));
- template.old_uniq = uniq;
- template.old_index = index;
- template.module = mod;
- erts_fun_write_lock();
- fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template);
- sys_memset(fe->uniq, 0, sizeof(fe->uniq));
- fe->index = 0;
- refc = erts_refc_inctest(&fe->refc, 0);
- if (refc < 2) /* New or pending delete */
- erts_refc_inc(&fe->refc, 1);
- erts_fun_write_unlock();
- return fe;
-}
-
-ErlFunEntry*
erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
byte* uniq, int index, int arity)
{
@@ -130,12 +109,12 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
ASSERT(is_atom(mod));
template.old_uniq = old_uniq;
- template.old_index = old_index;
+ template.index = index;
template.module = mod;
erts_fun_write_lock();
fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template);
sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq));
- fe->index = index;
+ fe->old_index = old_index;
fe->arity = arity;
refc = erts_refc_inctest(&fe->refc, 0);
if (refc < 2) /* New or pending delete */
@@ -144,13 +123,6 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
return fe;
}
-struct my_key {
- Eterm mod;
- byte* uniq;
- int index;
- ErlFunEntry* fe;
-};
-
ErlFunEntry*
erts_get_fun_entry(Eterm mod, int uniq, int index)
{
@@ -159,7 +131,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index)
ASSERT(is_atom(mod));
template.old_uniq = uniq;
- template.old_index = index;
+ template.index = index;
template.module = mod;
erts_fun_read_lock();
ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template);
@@ -315,15 +287,27 @@ erts_dump_fun_entries(fmtfn_t to, void *to_arg)
static HashValue
fun_hash(ErlFunEntry* obj)
{
- return (HashValue) (obj->old_uniq ^ obj->old_index ^ atom_val(obj->module));
+ return (HashValue) (obj->old_uniq ^ obj->index ^ atom_val(obj->module));
}
static int
fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2)
{
- return !(obj1->module == obj2->module &&
+ /*
+ * OTP 23: Use 'index' (instead of 'old_index') when comparing fun
+ * entries. In OTP 23, multiple make_fun2 instructions may refer to the
+ * the same 'index' (for the wrapper function generated for the
+ * 'fun F/A' syntax).
+ *
+ * This is safe when loading code compiled with OTP R15 and later,
+ * because since R15 (2011), the 'index' has been reliably equal
+ * to 'old_index'. The loader refuses to load modules compiled before
+ * OTP R15.
+ */
+
+ return !(obj1->module == obj2->module &&
obj1->old_uniq == obj2->old_uniq &&
- obj1->old_index == obj2->old_index);
+ obj1->index == obj2->index);
}
static ErlFunEntry*
@@ -333,7 +317,7 @@ fun_alloc(ErlFunEntry* template)
sizeof(ErlFunEntry));
obj->old_uniq = template->old_uniq;
- obj->old_index = template->old_index;
+ obj->index = template->index;
obj->module = template->module;
erts_refc_init(&obj->refc, -1);
obj->address = unloaded_fun;
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index fb2901d866..eefc7a95bb 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -74,7 +74,6 @@ void erts_init_fun_table(void);
void erts_fun_info(fmtfn_t, void *);
int erts_fun_table_sz(void);
-ErlFunEntry* erts_put_fun_entry(Eterm mod, int uniq, int index);
ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index);
ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 1bebf6efe2..42a07a59d6 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -674,7 +674,7 @@ erts_send_message(Process* sender,
* Make sure we don't use the heap between those instances.
*/
if (have_seqtrace(stoken)) {
- seq_trace_update_send(sender);
+ seq_trace_update_serial(sender);
seq_trace_output(stoken, message, SEQ_TRACE_SEND,
receiver->common.id, sender);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 1fbe362330..ce43cb9e71 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -815,7 +815,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
}
#endif
if (have_seqtrace(stoken)) {
- seq_trace_update_send(c_p);
+ seq_trace_update_serial(c_p);
seq_trace_output(stoken, msg, SEQ_TRACE_SEND,
rp->common.id, c_p);
}
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 50e9812534..285252a53e 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -977,7 +977,7 @@ static void print_node(void *venp, void *vpndp)
if(pndp->sysname == NIL) {
erts_print(pndp->to, pndp->to_arg, "Name: %T ", enp->sysname);
}
- erts_print(pndp->to, pndp->to_arg, " %d", enp->creation);
+ erts_print(pndp->to, pndp->to_arg, " %u", enp->creation);
#ifdef DEBUG
erts_print(pndp->to, pndp->to_arg, " (refc=%ld)",
erts_refc_read(&enp->refc, 0));
@@ -1020,7 +1020,7 @@ void erts_print_node_info(fmtfn_t to,
/* ----------------------------------------------------------------------- */
void
-erts_set_this_node(Eterm sysname, Uint creation)
+erts_set_this_node(Eterm sysname, Uint32 creation)
{
ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
ASSERT(2 <= de_refc_read(erts_this_dist_entry, 2));
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index ffaafbbbea..beae2df75f 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -264,7 +264,7 @@ void erts_set_dist_entry_pending(DistEntry *);
void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint);
ErlNode *erts_find_or_insert_node(Eterm, Uint32, Eterm);
void erts_schedule_delete_node(ErlNode *);
-void erts_set_this_node(Eterm, Uint);
+void erts_set_this_node(Eterm, Uint32);
Uint erts_node_table_size(void);
void erts_init_node_tables(int);
void erts_node_table_info(fmtfn_t, void *);
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index d5e0e3b218..b60fb64342 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -995,7 +995,7 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag,
seq_trace = c_p && have_seqtrace(token);
if (seq_trace)
- seq_trace_update_send(c_p);
+ seq_trace_update_serial(c_p);
#ifdef USE_VM_PROBES
utag_sz = 0;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1c1ef1db84..e8c83276f5 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -10992,8 +10992,13 @@ erts_set_gc_state(Process *c_p, int enable)
ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p));
if (!enable) {
- c_p->flags |= F_DISABLE_GC;
- return 0;
+ /* Strictly speaking it's not illegal to disable the GC when it's
+ * already disabled, but we risk enabling the GC prematurely if (for
+ * example) a BIF were to blindly disable it when trapping and then
+ * re-enable it before returning its result. */
+ ASSERT(!(c_p->flags & F_DISABLE_GC));
+ c_p->flags |= F_DISABLE_GC;
+ return 0;
}
c_p->flags &= ~F_DISABLE_GC;
@@ -11583,9 +11588,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->mbuf_sz = 0;
erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL);
p->dictionary = NULL;
- p->seq_trace_lastcnt = 0;
- p->seq_trace_clock = 0;
- SEQ_TRACE_TOKEN(p) = NIL;
#ifdef USE_VM_PROBES
DT_UTAG(p) = NIL;
DT_UTAG_FLAGS(p) = 0;
@@ -11606,6 +11608,45 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->fp_exception = 0;
#endif
+ /* seq_trace is handled before regular tracing as the latter may touch the
+ * trace token. */
+ if (have_seqtrace(SEQ_TRACE_TOKEN(parent))) {
+ Eterm token;
+ Uint token_sz;
+ Eterm *hp;
+
+ ASSERT(SEQ_TRACE_TOKEN_ARITY(parent) == 5);
+ ASSERT(is_immed(SEQ_TRACE_TOKEN_FLAGS(parent)));
+ ASSERT(is_immed(SEQ_TRACE_TOKEN_SERIAL(parent)));
+ ASSERT(is_immed(SEQ_TRACE_TOKEN_LASTCNT(parent)));
+
+ seq_trace_update_serial(parent);
+
+ token = SEQ_TRACE_TOKEN(parent);
+ token_sz = size_object(token);
+
+ hp = HAlloc(p, token_sz);
+ SEQ_TRACE_TOKEN(p) = copy_struct(token, token_sz, &hp, &MSO(p));
+
+ /* The counters behave the same way on spawning as they do on messages;
+ * we don't inherit our parent's lastcnt. */
+ p->seq_trace_lastcnt = parent->seq_trace_clock;
+ p->seq_trace_clock = parent->seq_trace_clock;
+
+ ASSERT((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) ==
+ (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE));
+
+ locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+ erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+ erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+
+ seq_trace_output(token, NIL, SEQ_TRACE_SPAWN, p->common.id, parent);
+ } else {
+ SEQ_TRACE_TOKEN(p) = NIL;
+ p->seq_trace_lastcnt = 0;
+ p->seq_trace_clock = 0;
+ }
+
if (IS_TRACED(parent)) {
if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) {
ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS);
@@ -11627,9 +11668,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
}
}
if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) {
- locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
- erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
- erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+ /* The locks may already be released if seq_trace is enabled as
+ * well. */
+ if ((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE))
+ == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) {
+ locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+ erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+ erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE);
+ }
trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args);
if (so->flags & SPO_LINK)
trace_proc(parent, locks, parent, am_link, p->common.id);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c0d7cfd13d..405611c584 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1489,6 +1489,8 @@ extern int erts_system_profile_ts_type;
#define SEQ_TRACE_SEND (1 << 0)
#define SEQ_TRACE_RECEIVE (1 << 1)
#define SEQ_TRACE_PRINT (1 << 2)
+/* (This three-bit gap contains the timestamp.) */
+#define SEQ_TRACE_SPAWN (1 << 6)
#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index f6f177887c..5c46a10d64 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -830,7 +830,7 @@ trace_receive(Process* receiver,
}
int
-seq_trace_update_send(Process *p)
+seq_trace_update_serial(Process *p)
{
ErtsTracer seq_tracer = erts_get_system_seq_tracer();
ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p))));
@@ -898,6 +898,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
switch (type) {
case SEQ_TRACE_SEND: type_atom = am_send; break;
+ case SEQ_TRACE_SPAWN: type_atom = am_spawn; break;
case SEQ_TRACE_PRINT: type_atom = am_print; break;
case SEQ_TRACE_RECEIVE: type_atom = am_receive; break;
default:
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index af38ef52db..d25f97c656 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -163,7 +163,9 @@ seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom))
void seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
Eterm receiver, Process *process, Eterm exitfrom);
-int seq_trace_update_send(Process *process);
+/* Bump the sequence number if tracing is enabled; must be used before sending
+ * send/spawn trace messages. */
+int seq_trace_update_serial(Process *process);
Eterm erts_seq_trace(Process *process,
Eterm atom_type, Eterm atom_true_or_false,
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 430ac305c5..449243a9b7 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -70,6 +70,7 @@ int erts_fit_in_bits_uint(Uint);
Sint erts_list_length(Eterm);
int erts_is_builtin(Eterm, Eterm, int);
Uint32 make_hash2(Eterm);
+Uint32 trapping_make_hash2(Eterm, Eterm*, struct process*);
Uint32 make_hash(Eterm);
Uint32 make_internal_hash(Eterm, Uint32 salt);
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index ce61cdf040..5cea253ebe 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -51,18 +51,17 @@
#define MAX_STRING_LEN 0xffff
-/* MAX value for the creation field in pid, port and reference
- for the local node and for the current external format.
-
- Larger creation values than this are allowed in external pid, port and refs
- encoded with NEW_PID_EXT, NEW_PORT_EXT and NEWER_REFERENCE_EXT.
- The point here is to prepare for future upgrade to 32-bit creation.
- OTP-19 (erts-8.0) can handle big creation values from other (newer) nodes,
- but do not use big creation values for the local node yet,
- as we still may have to communicate with older nodes.
+/*
+ * MAX value for the creation field in pid, port and reference
+ * for the old PID_EXT, PORT_EXT, REFERENCE_EXT and NEW_REFERENCE_EXT.
+ * Older nodes (OTP 19-22) will send us these so we must be able to decode them.
+ *
+ * From OTP 23 DFLAG_BIG_CREATION is mandatory so this node will always
+ * encode with new big 32-bit creations using NEW_PID_EXT, NEW_PORT_EXT
+ * and NEWER_REFERENCE_EXT.
*/
-#define ERTS_MAX_LOCAL_CREATION (3)
-#define is_valid_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_LOCAL_CREATION)
+#define ERTS_MAX_TINY_CREATION (3)
+#define is_tiny_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_TINY_CREATION)
#undef ERTS_DEBUG_USE_DIST_SEP
#ifdef DEBUG
@@ -2469,7 +2468,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS))
? INTERNAL_LOCAL_SYSNAME : pid_node_name(pid));
Uint32 creation = pid_creation(pid);
- byte* tagp = ep++;
+
+ *ep++ = NEW_PID_EXT;
/* insert atom here containing host and sysname */
ep = enc_atom(acmp, sysname, ep, dflags);
@@ -2481,15 +2481,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
ep += 4;
put_int32(os, ep);
ep += 4;
- if (creation <= ERTS_MAX_LOCAL_CREATION) {
- *tagp = PID_EXT;
- *ep++ = creation;
- } else {
- ASSERT(is_external_pid(pid));
- *tagp = NEW_PID_EXT;
- put_int32(creation, ep);
- ep += 4;
- }
+ put_int32(creation, ep);
+ ep += 4;
return ep;
}
@@ -2609,7 +2602,7 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep,
if (tag == PID_EXT) {
cre = get_int8(ep);
ep += 1;
- if (!is_valid_creation(cre)) {
+ if (!is_tiny_creation(cre)) {
return NULL;
}
} else {
@@ -2870,25 +2863,18 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj))
? INTERNAL_LOCAL_SYSNAME : ref_node_name(obj));
Uint32 creation = ref_creation(obj);
- byte* tagp = ep++;
ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
erts_magic_ref_save_bin(obj);
+ *ep++ = NEWER_REFERENCE_EXT;
i = ref_no_numbers(obj);
put_int16(i, ep);
ep += 2;
ep = enc_atom(acmp, sysname, ep, dflags);
- if (creation <= ERTS_MAX_LOCAL_CREATION) {
- *tagp = NEW_REFERENCE_EXT;
- *ep++ = creation;
- } else {
- ASSERT(is_external_ref(obj));
- *tagp = NEWER_REFERENCE_EXT;
- put_int32(creation, ep);
- ep += 4;
- }
+ put_int32(creation, ep);
+ ep += 4;
ref_num = ref_numbers(obj);
for (j = 0; j < i; j++) {
put_int32(ref_num[j], ep);
@@ -2901,21 +2887,14 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep,
Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj))
? INTERNAL_LOCAL_SYSNAME : port_node_name(obj));
Uint32 creation = port_creation(obj);
- byte* tagp = ep++;
+ *ep++ = NEW_PORT_EXT;
ep = enc_atom(acmp, sysname, ep, dflags);
j = port_number(obj);
put_int32(j, ep);
ep += 4;
- if (creation <= ERTS_MAX_LOCAL_CREATION) {
- *tagp = PORT_EXT;
- *ep++ = creation;
- } else {
- ASSERT(is_external_port(obj));
- *tagp = NEW_PORT_EXT;
- put_int32(creation, ep);
- ep += 4;
- }
+ put_int32(creation, ep);
+ ep += 4;
break;
}
case LIST_DEF:
@@ -3610,7 +3589,7 @@ dec_term_atom_common:
if (tag == PORT_EXT) {
cre = get_int8(ep);
ep++;
- if (!is_valid_creation(cre)) {
+ if (!is_tiny_creation(cre)) {
goto error;
}
}
@@ -3657,7 +3636,7 @@ dec_term_atom_common:
cre = get_int8(ep);
ep += 1;
- if (!is_valid_creation(cre)) {
+ if (!is_tiny_creation(cre)) {
goto error;
}
goto ref_ext_common;
@@ -3671,7 +3650,7 @@ dec_term_atom_common:
cre = get_int8(ep);
ep += 1;
- if (!is_valid_creation(cre)) {
+ if (!is_tiny_creation(cre)) {
goto error;
}
r0 = get_int32(ep);
@@ -4066,73 +4045,6 @@ dec_term_atom_common:
next = &(funp->creator);
break;
}
- case FUN_EXT:
- {
- ErlFunThing* funp = (ErlFunThing *) hp;
- Eterm module;
- Sint old_uniq;
- Sint old_index;
- unsigned num_free;
- int i;
- Eterm temp;
-
- num_free = get_int32(ep);
- ep += 4;
- hp += ERL_FUN_SIZE;
- hp += num_free;
- factory->hp = hp;
- funp->thing_word = HEADER_FUN;
- funp->num_free = num_free;
- *objp = make_fun(funp);
-
- /* Creator pid */
- if ((*ep != PID_EXT && *ep != NEW_PID_EXT)
- || (ep = dec_pid(edep, factory, ep+1,
- &funp->creator, *ep))==NULL) {
- goto error;
- }
-
- /* Module */
- if ((ep = dec_atom(edep, ep, &module)) == NULL) {
- goto error;
- }
-
- /* Index */
- if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
- goto error;
- }
- if (!is_small(temp)) {
- goto error;
- }
- old_index = unsigned_val(temp);
-
- /* Uniq */
- if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) {
- goto error;
- }
- if (!is_small(temp)) {
- goto error;
- }
-
- /*
- * It is safe to link the fun into the fun list only when
- * no more validity tests can fail.
- */
- funp->next = factory->off_heap->first;
- factory->off_heap->first = (struct erl_off_heap_header*)funp;
- old_uniq = unsigned_val(temp);
-
- funp->fe = erts_put_fun_entry(module, old_uniq, old_index);
- funp->arity = funp->fe->address[-1] - num_free;
- hp = factory->hp;
-
- /* Environment */
- for (i = num_free-1; i >= 0; i--) {
- funp->env[i] = (Eterm) next;
- next = funp->env + i;
- }
- break;
- }
case ATOM_INTERNAL_REF2:
n = get_int16(ep);
ep += 2;
@@ -4401,30 +4313,21 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj,
result += 1 + 4 + 1 + i; /* tag,size,sign,digits */
break;
case EXTERNAL_PID_DEF:
- if (external_pid_creation(obj) > ERTS_MAX_LOCAL_CREATION)
- result += 3;
- /*fall through*/
case PID_DEF:
result += (1 + encode_size_struct2(acmp, pid_node_name(obj), dflags) +
- 4 + 4 + 1);
+ 4 + 4 + 4);
break;
case EXTERNAL_REF_DEF:
- if (external_ref_creation(obj) > ERTS_MAX_LOCAL_CREATION)
- result += 3;
- /*fall through*/
case REF_DEF:
ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
i = ref_no_numbers(obj);
result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) +
- 1 + 4*i);
+ 4 + 4*i);
break;
case EXTERNAL_PORT_DEF:
- if (external_port_creation(obj) > ERTS_MAX_LOCAL_CREATION)
- result += 3;
- /*fall through*/
case PORT_DEF:
result += (1 + encode_size_struct2(acmp, port_node_name(obj), dflags) +
- 4 + 1);
+ 4 + 4);
break;
case LIST_DEF: {
int is_str = is_external_string(obj, &m);
@@ -4891,9 +4794,6 @@ init_done:
total_size = get_int32(ep);
CHKSIZE(total_size);
ep += 1+16+4+4;
- /*FALLTHROUGH*/
-
- case FUN_EXT:
CHKSIZE(4);
num_free = get_int32(ep);
ep += 4;
@@ -4904,6 +4804,12 @@ init_done:
heap_size += ERL_FUN_SIZE + num_free;
break;
}
+ case FUN_EXT:
+ /*
+ * OTP 23: No longer support decoding the old fun
+ * representation.
+ */
+ goto error;
case ATOM_INTERNAL_REF2:
SKIP(2+atom_extra_skip);
atom_extra_skip = 0;
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 7cffe7fb5c..bc8c1189a8 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -683,10 +683,11 @@ swap(R1, R2) {
$R2 = V;
}
-swap_temp(R1, R2, Tmp) {
- Eterm V = $R1;
- $R1 = $R2;
- $R2 = $Tmp = V;
+swap2(R1, R2, R3) {
+ Eterm V = $R2;
+ $R2 = $R1;
+ $R1 = $R3;
+ $R3 = V;
}
test_heap(Nh, Live) {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b9d4f6afcc..f525d126e7 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -324,76 +324,15 @@ move_src_window2 y x x
move_src_window3 y x x x
move_src_window4 y x x x x
-# Swap registers.
-move R1=xy Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp
-
-# The compiler uses x(1022) when swapping registers. It will definitely
-# not be used again.
-swap_temp R1 R2 Tmp=x==1022 => swap R1 R2
-
-swap_temp R1 R2 Tmp | move Src Tmp => swap R1 R2 | move Src Tmp
-
-swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed_apply(Tmp, Live) => \
- swap R1 R2 | line Loc | apply Live
-swap_temp R1 R2 Tmp | line Loc | apply_last Live D | is_killed_apply(Tmp, Live) => \
- swap R1 R2 | line Loc | apply_last Live D
-
-swap_temp R1 R2 Tmp | line Loc | call_fun Live | is_killed_by_call_fun(Tmp, Live) => \
- swap R1 R2 | line Loc | call_fun Live
-swap_temp R1 R2 Tmp | make_fun2 OldIndex=u | is_killed_by_make_fun(Tmp, OldIndex) => \
- swap R1 R2 | make_fun2 OldIndex
-
-swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \
- swap R1 R2 | line Loc | call Live Addr
-swap_temp R1 R2 Tmp | call_only Live Addr | \
- is_killed(Tmp, Live) => swap R1 R2 | call_only Live Addr
-swap_temp R1 R2 Tmp | call_last Live Addr D | \
- is_killed(Tmp, Live) => swap R1 R2 | call_last Live Addr D
-
-swap_temp R1 R2 Tmp | line Loc | call_ext Live Addr | is_killed(Tmp, Live) => \
- swap R1 R2 | line Loc | call_ext Live Addr
-swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \
- is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_only Live Addr
-swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \
- is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D
-
-swap_temp R1 R2 Tmp | call_ext Live Addr | is_killed(Tmp, Live) => \
- swap R1 R2 | call_ext Live Addr
-swap_temp R1 R2 Tmp | call_ext_only Live Addr | is_killed(Tmp, Live) => \
- swap R1 R2 | call_ext_only Live Addr
-swap_temp R1 R2 Tmp | call_ext_last Live Addr D | is_killed(Tmp, Live) => \
- swap R1 R2 | call_ext_last Live Addr D
-
-swap_temp R1 R2 Tmp | move Src Any | line Loc | call Live Addr | \
- is_killed(Tmp, Live) | distinct(Tmp, Src) => \
- swap R1 R2 | move Src Any | line Loc | call Live Addr
-swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext Live Addr | \
- is_killed(Tmp, Live) | distinct(Tmp, Src) => \
- swap R1 R2 | move Src Any | line Loc | call_ext Live Addr
-swap_temp R1 R2 Tmp | move Src Any | call_only Live Addr | \
- is_killed(Tmp, Live) | distinct(Tmp, Src) => \
- swap R1 R2 | move Src Any | call_only Live Addr
-swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext_only Live Addr | \
- is_killed(Tmp, Live) | distinct(Tmp, Src) => \
- swap R1 R2 | move Src Any | line Loc | call_ext_only Live Addr
-swap_temp R1 R2 Tmp | move Src Any | line Loc | call_fun Live | \
- is_killed(Tmp, Live) | distinct(Tmp, Src) => \
- swap R1 R2 | move Src Any | line Loc | call_fun Live
-
-swap_temp R1 R2 Tmp | line Loc | send | is_killed_by_send(Tmp) => \
- swap R1 R2 | line Loc | send
-
-# swap_temp/3 with Y register operands are rare.
-swap_temp R1 R2=y Tmp => swap R1 R2 | move R2 Tmp
-swap_temp R1=y R2 Tmp => swap R1 R2 | move R2 Tmp
-
swap R1=x R2=y => swap R2 R1
-swap_temp x x x
-
swap xy x
swap y y
+swap R1=x R2=x | swap R3=x R1 => swap2 R1 R2 R3
+
+swap2 x x x
+
# move_shift
move SD=x D=x | move Src=cxy SD=x | distinct(D, Src) => move_shift Src SD D
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index c261c8e117..db07512cf7 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -92,6 +92,12 @@
# define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
#endif
+#ifdef __GNUC__
+# define ERTS_NOINLINE __attribute__((__noinline__))
+#else
+# define ERTS_NOINLINE
+#endif
+
#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
# define NO_FPE_SIGNALS
#endif
@@ -172,7 +178,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
# define ERTS_UNLIKELY(BOOL) (BOOL)
#endif
-#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
+/* AIX doesn't like this and claims section conflicts */
+#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) && !defined(_AIX)
#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__)
# define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") ))
#else
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 0bbae65e28..fb06d60768 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -66,7 +66,7 @@
#undef M_MMAP_THRESHOLD
#undef M_MMAP_MAX
-#if defined(__GLIBC__) && defined(HAVE_MALLOC_H)
+#if (defined(__GLIBC__) || defined(_AIX)) && defined(HAVE_MALLOC_H)
#include <malloc.h>
#endif
@@ -907,7 +907,7 @@ tail_recur:
hash = hash * FUNNY_NUMBER10 + num_free;
hash = hash*FUNNY_NUMBER1 +
(atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue);
- hash = hash*FUNNY_NUMBER2 + funp->fe->old_index;
+ hash = hash*FUNNY_NUMBER2 + funp->fe->index;
hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
if (num_free > 0) {
if (num_free > 1) {
@@ -1069,54 +1069,237 @@ do { \
#define HCONST 0x9e3779b9UL /* the golden ratio; an arbitrary value */
-static Uint32
-block_hash(byte *k, Uint length, Uint32 initval)
+typedef struct {
+ Uint32 a,b,c;
+} ErtsBlockHashHelperCtx;
+
+#define BLOCK_HASH_BYTES_PER_ITER 12
+
+/* The three functions below are separated into different functions even
+ though they are always used together to make trapping and handling
+ of unaligned binaries easier. Examples of how they are used can be
+ found in block_hash and make_hash2_helper.*/
+static ERTS_INLINE
+void block_hash_setup(Uint32 initval,
+ ErtsBlockHashHelperCtx* ctx /* out parameter */)
+{
+ ctx->a = ctx->b = HCONST;
+ ctx->c = initval; /* the previous hash value */
+}
+
+static ERTS_INLINE
+void block_hash_buffer(byte *buf,
+ Uint buf_length,
+ ErtsBlockHashHelperCtx* ctx /* out parameter */)
{
- Uint32 a,b,c;
- Uint len;
-
- /* Set up the internal state */
- len = length;
- a = b = HCONST;
- c = initval; /* the previous hash value */
-
- while (len >= 12)
- {
- a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24));
- b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24));
- c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24));
- MIX(a,b,c);
- k += 12; len -= 12;
- }
-
- c += length;
- switch(len) /* all the case statements fall through */
- {
- case 11: c+=((Uint32)k[10]<<24);
- case 10: c+=((Uint32)k[9]<<16);
- case 9 : c+=((Uint32)k[8]<<8);
- /* the first byte of c is reserved for the length */
- case 8 : b+=((Uint32)k[7]<<24);
- case 7 : b+=((Uint32)k[6]<<16);
- case 6 : b+=((Uint32)k[5]<<8);
- case 5 : b+=k[4];
- case 4 : a+=((Uint32)k[3]<<24);
- case 3 : a+=((Uint32)k[2]<<16);
- case 2 : a+=((Uint32)k[1]<<8);
- case 1 : a+=k[0];
- /* case 0: nothing left to add */
- }
- MIX(a,b,c);
- return c;
+ Uint len = buf_length;
+ byte *k = buf;
+ ASSERT(buf_length % BLOCK_HASH_BYTES_PER_ITER == 0);
+ while (len >= BLOCK_HASH_BYTES_PER_ITER) {
+ ctx->a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24));
+ ctx->b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24));
+ ctx->c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24));
+ MIX(ctx->a,ctx->b,ctx->c);
+ k += BLOCK_HASH_BYTES_PER_ITER; len -= BLOCK_HASH_BYTES_PER_ITER;
+ }
}
+static ERTS_INLINE
+Uint32 block_hash_final_bytes(byte *buf,
+ Uint buf_length,
+ Uint full_length,
+ ErtsBlockHashHelperCtx* ctx)
+{
+ Uint len = buf_length;
+ byte *k = buf;
+ ctx->c += full_length;
+ switch(len)
+ { /* all the case statements fall through */
+ case 11: ctx->c+=((Uint32)k[10]<<24);
+ case 10: ctx->c+=((Uint32)k[9]<<16);
+ case 9 : ctx->c+=((Uint32)k[8]<<8);
+ /* the first byte of c is reserved for the length */
+ case 8 : ctx->b+=((Uint32)k[7]<<24);
+ case 7 : ctx->b+=((Uint32)k[6]<<16);
+ case 6 : ctx->b+=((Uint32)k[5]<<8);
+ case 5 : ctx->b+=k[4];
+ case 4 : ctx->a+=((Uint32)k[3]<<24);
+ case 3 : ctx->a+=((Uint32)k[2]<<16);
+ case 2 : ctx->a+=((Uint32)k[1]<<8);
+ case 1 : ctx->a+=k[0];
+ /* case 0: nothing left to add */
+ }
+ MIX(ctx->a,ctx->b,ctx->c);
+ return ctx->c;
+}
+
+static
Uint32
-make_hash2(Eterm term)
+block_hash(byte *block, Uint block_length, Uint32 initval)
{
+ ErtsBlockHashHelperCtx ctx;
+ Uint no_bytes_not_in_loop =
+ (block_length % BLOCK_HASH_BYTES_PER_ITER);
+ Uint no_bytes_to_process_in_loop =
+ block_length - no_bytes_not_in_loop;
+ byte *final_bytes = block + no_bytes_to_process_in_loop;
+ block_hash_setup(initval, &ctx);
+ block_hash_buffer(block,
+ no_bytes_to_process_in_loop,
+ &ctx);
+ return block_hash_final_bytes(final_bytes,
+ no_bytes_not_in_loop,
+ block_length,
+ &ctx);
+}
+
+typedef enum {
+ tag_primary_list,
+ arityval_subtag,
+ hamt_subtag_head_flatmap,
+ map_subtag,
+ fun_subtag,
+ neg_big_subtag,
+ sub_binary_subtag_1,
+ sub_binary_subtag_2,
+ hash2_common_1,
+ hash2_common_2,
+ hash2_common_3,
+} ErtsMakeHash2TrapLocation;
+
+typedef struct {
+ int c;
+ Uint32 sh;
+ Eterm* ptr;
+} ErtsMakeHash2Context_TAG_PRIMARY_LIST;
+
+typedef struct {
+ int i;
+ int arity;
+ Eterm* elem;
+} ErtsMakeHash2Context_ARITYVAL_SUBTAG;
+
+typedef struct {
+ Eterm *ks;
+ Eterm *vs;
+ int i;
+ Uint size;
+} ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP;
+
+typedef struct {
+ Eterm* ptr;
+ int i;
+} ErtsMakeHash2Context_MAP_SUBTAG;
+
+typedef struct {
+ Uint num_free;
+ Eterm* bptr;
+} ErtsMakeHash2Context_FUN_SUBTAG;
+
+typedef struct {
+ Eterm* ptr;
+ Uint i;
+ Uint n;
+ Uint32 con;
+} ErtsMakeHash2Context_NEG_BIG_SUBTAG;
+
+typedef struct {
+ byte* bptr;
+ Uint sz;
+ Uint bitsize;
+ Uint bitoffs;
+ Uint no_bytes_processed;
+ ErtsBlockHashHelperCtx block_hash_ctx;
+ /* The following fields are only used when bitoffs != 0 */
+ byte* buf;
+ int done;
+
+} ErtsMakeHash2Context_SUB_BINARY_SUBTAG;
+
+typedef struct {
+ int dummy__; /* Empty structs are not supported on all platforms */
+} ErtsMakeHash2Context_EMPTY;
+
+typedef struct {
+ ErtsMakeHash2TrapLocation trap_location;
+ /* specific to the trap location: */
+ union {
+ ErtsMakeHash2Context_TAG_PRIMARY_LIST tag_primary_list;
+ ErtsMakeHash2Context_ARITYVAL_SUBTAG arityval_subtag;
+ ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP hamt_subtag_head_flatmap;
+ ErtsMakeHash2Context_MAP_SUBTAG map_subtag;
+ ErtsMakeHash2Context_FUN_SUBTAG fun_subtag;
+ ErtsMakeHash2Context_NEG_BIG_SUBTAG neg_big_subtag;
+ ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_1;
+ ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_2;
+ ErtsMakeHash2Context_EMPTY hash2_common_1;
+ ErtsMakeHash2Context_EMPTY hash2_common_2;
+ ErtsMakeHash2Context_EMPTY hash2_common_3;
+ } trap_location_state;
+ /* same for all trap locations: */
+ Eterm term;
Uint32 hash;
Uint32 hash_xor_pairs;
- DeclareTmpHeapNoproc(tmp_big,2);
+ ErtsEStack stack;
+} ErtsMakeHash2Context;
+
+static int make_hash2_ctx_bin_dtor(Binary *context_bin) {
+ ErtsMakeHash2Context* context = ERTS_MAGIC_BIN_DATA(context_bin);
+ DESTROY_SAVED_ESTACK(&context->stack);
+ if (context->trap_location == sub_binary_subtag_2 &&
+ context->trap_location_state.sub_binary_subtag_2.buf != NULL) {
+ erts_free(ERTS_ALC_T_PHASH2_TRAP, context->trap_location_state.sub_binary_subtag_2.buf);
+ }
+ return 1;
+}
+/* hash2_save_trap_state is called seldom so we want to avoid inlining */
+static ERTS_NOINLINE
+Eterm hash2_save_trap_state(Eterm state_mref,
+ Uint32 hash_xor_pairs,
+ Uint32 hash,
+ Process* p,
+ Eterm term,
+ Eterm* ESTK_DEF_STACK(s),
+ ErtsEStack s,
+ ErtsMakeHash2TrapLocation trap_location,
+ void* trap_location_state_ptr,
+ size_t trap_location_state_size) {
+ Binary* state_bin;
+ ErtsMakeHash2Context* context;
+ if (state_mref == THE_NON_VALUE) {
+ Eterm* hp;
+ state_bin = erts_create_magic_binary(sizeof(ErtsMakeHash2Context),
+ make_hash2_ctx_bin_dtor);
+ hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE);
+ state_mref = erts_mk_magic_ref(&hp, &MSO(p), state_bin);
+ } else {
+ state_bin = erts_magic_ref2bin(state_mref);
+ }
+ context = ERTS_MAGIC_BIN_DATA(state_bin);
+ context->term = term;
+ context->hash = hash;
+ context->hash_xor_pairs = hash_xor_pairs;
+ ESTACK_SAVE(s, &context->stack);
+ context->trap_location = trap_location;
+ sys_memcpy(&context->trap_location_state,
+ trap_location_state_ptr,
+ trap_location_state_size);
+ erts_set_gc_state(p, 0);
+ BUMP_ALL_REDS(p);
+ return state_mref;
+}
+#undef NOINLINE_HASH2_SAVE_TRAP_STATE
+
+/* Writes back a magic reference to *state_mref_write_back when the
+ function traps */
+static ERTS_INLINE Uint32
+make_hash2_helper(Eterm term_param, const int can_trap, Eterm* state_mref_write_back, Process* p)
+{
+ static const Uint ITERATIONS_PER_RED = 64;
+ Uint32 hash;
+ Uint32 hash_xor_pairs;
+ Eterm term = term_param;
ERTS_UNDEF(hash_xor_pairs, 0);
/* (HCONST * {2, ..., 22}) mod 2^32 */
@@ -1168,12 +1351,63 @@ make_hash2(Eterm term)
#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
+#define NOT_SSMALL28_HASH(SMALL) \
+ do { \
+ Uint64 t; \
+ Uint32 x, y; \
+ Uint32 con; \
+ if (SMALL < 0) { \
+ con = HCONST_10; \
+ t = (Uint64)(SMALL * (-1)); \
+ } else { \
+ con = HCONST_11; \
+ t = SMALL; \
+ } \
+ x = t & 0xffffffff; \
+ y = t >> 32; \
+ UINT32_HASH_2(x, y, con); \
+ } while(0)
+
#ifdef ARCH_64
# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst)
#else
# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst)
#endif
+#define TRAP_LOCATION_NO_RED(location_name) \
+ do { \
+ if(can_trap && iterations_until_trap <= 0) { \
+ *state_mref_write_back = \
+ hash2_save_trap_state(state_mref, \
+ hash_xor_pairs, \
+ hash, \
+ p, \
+ term, \
+ ESTK_DEF_STACK(s), \
+ s, \
+ location_name, \
+ &ctx, \
+ sizeof(ctx)); \
+ return 0; \
+ L_##location_name: \
+ ctx = context->trap_location_state. location_name; \
+ } \
+ } while(0)
+
+#define TRAP_LOCATION(location_name) \
+ do { \
+ if (can_trap) { \
+ iterations_until_trap--; \
+ TRAP_LOCATION_NO_RED(location_name); \
+ } \
+ } while(0)
+
+#define TRAP_LOCATION_NO_CTX(location_name) \
+ do { \
+ ErtsMakeHash2Context_EMPTY ctx; \
+ TRAP_LOCATION(location_name); \
+ } while(0)
+
/* Optimization. Simple cases before declaration of estack. */
if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
switch (term & _TAG_IMMED1_MASK) {
@@ -1186,51 +1420,94 @@ make_hash2(Eterm term)
break;
case _TAG_IMMED1_SMALL:
{
- Sint x = signed_val(term);
-
- if (SMALL_BITS > 28 && !IS_SSMALL28(x)) {
- term = small_to_big(x, tmp_big);
- break;
+ Sint small = signed_val(term);
+ if (SMALL_BITS > 28 && !IS_SSMALL28(small)) {
+ hash = 0;
+ NOT_SSMALL28_HASH(small);
+ return hash;
}
hash = 0;
- SINT32_HASH(x, HCONST);
+ SINT32_HASH(small, HCONST);
return hash;
}
}
};
{
Eterm tmp;
+ long max_iterations = 0;
+ long iterations_until_trap = 0;
+ Eterm state_mref = THE_NON_VALUE;
+ ErtsMakeHash2Context* context = NULL;
DECLARE_ESTACK(s);
-
- UseTmpHeapNoproc(2);
+ ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+ if(can_trap){
+#ifdef DEBUG
+ (void)ITERATIONS_PER_RED;
+ iterations_until_trap = max_iterations =
+ (1103515245 * (ERTS_BIF_REDS_LEFT(p)) + 12345) % 227;
+#else
+ iterations_until_trap = max_iterations =
+ ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(p);
+#endif
+ }
+ if (can_trap && is_internal_magic_ref(term)) {
+ Binary* state_bin;
+ state_mref = term;
+ state_bin = erts_magic_ref2bin(state_mref);
+ if (ERTS_MAGIC_BIN_DESTRUCTOR(state_bin) == make_hash2_ctx_bin_dtor) {
+ /* Restore state after a trap */
+ context = ERTS_MAGIC_BIN_DATA(state_bin);
+ term = context->term;
+ hash = context->hash;
+ hash_xor_pairs = context->hash_xor_pairs;
+ ESTACK_RESTORE(s, &context->stack);
+ ASSERT(p->flags & F_DISABLE_GC);
+ erts_set_gc_state(p, 1);
+ switch (context->trap_location) {
+ case hash2_common_3: goto L_hash2_common_3;
+ case tag_primary_list: goto L_tag_primary_list;
+ case arityval_subtag: goto L_arityval_subtag;
+ case hamt_subtag_head_flatmap: goto L_hamt_subtag_head_flatmap;
+ case map_subtag: goto L_map_subtag;
+ case fun_subtag: goto L_fun_subtag;
+ case neg_big_subtag: goto L_neg_big_subtag;
+ case sub_binary_subtag_1: goto L_sub_binary_subtag_1;
+ case sub_binary_subtag_2: goto L_sub_binary_subtag_2;
+ case hash2_common_1: goto L_hash2_common_1;
+ case hash2_common_2: goto L_hash2_common_2;
+ }
+ }
+ }
hash = 0;
for (;;) {
switch (primary_tag(term)) {
case TAG_PRIMARY_LIST:
{
- int c = 0;
- Uint32 sh = 0;
- Eterm* ptr = list_val(term);
- while (is_byte(*ptr)) {
+ ErtsMakeHash2Context_TAG_PRIMARY_LIST ctx = {
+ .c = 0,
+ .sh = 0,
+ .ptr = list_val(term)};
+ while (is_byte(*ctx.ptr)) {
/* Optimization for strings. */
- sh = (sh << 8) + unsigned_val(*ptr);
- if (c == 3) {
- UINT32_HASH(sh, HCONST_4);
- c = sh = 0;
+ ctx.sh = (ctx.sh << 8) + unsigned_val(*ctx.ptr);
+ if (ctx.c == 3) {
+ UINT32_HASH(ctx.sh, HCONST_4);
+ ctx.c = ctx.sh = 0;
} else {
- c++;
+ ctx.c++;
}
- term = CDR(ptr);
+ term = CDR(ctx.ptr);
if (is_not_list(term))
break;
- ptr = list_val(term);
+ ctx.ptr = list_val(term);
+ TRAP_LOCATION(tag_primary_list);
}
- if (c > 0)
- UINT32_HASH(sh, HCONST_4);
+ if (ctx.c > 0)
+ UINT32_HASH(ctx.sh, HCONST_4);
if (is_list(term)) {
- tmp = CDR(ptr);
+ tmp = CDR(ctx.ptr);
ESTACK_PUSH(s, tmp);
- term = CAR(ptr);
+ term = CAR(ctx.ptr);
}
}
break;
@@ -1241,34 +1518,39 @@ make_hash2(Eterm term)
switch (hdr & _TAG_HEADER_MASK) {
case ARITYVAL_SUBTAG:
{
- int i;
- int arity = header_arity(hdr);
- Eterm* elem = tuple_val(term);
- UINT32_HASH(arity, HCONST_9);
- if (arity == 0) /* Empty tuple */
+ ErtsMakeHash2Context_ARITYVAL_SUBTAG ctx = {
+ .i = 0,
+ .arity = header_arity(hdr),
+ .elem = tuple_val(term)};
+ UINT32_HASH(ctx.arity, HCONST_9);
+ if (ctx.arity == 0) /* Empty tuple */
goto hash2_common;
- for (i = arity; ; i--) {
- term = elem[i];
- if (i == 1)
+ for (ctx.i = ctx.arity; ; ctx.i--) {
+ term = ctx.elem[ctx.i];
+ if (ctx.i == 1)
break;
ESTACK_PUSH(s, term);
+ TRAP_LOCATION(arityval_subtag);
}
}
break;
case MAP_SUBTAG:
{
- Eterm* ptr = boxed_val(term) + 1;
Uint size;
- int i;
+ ErtsMakeHash2Context_MAP_SUBTAG ctx = {
+ .ptr = boxed_val(term) + 1,
+ .i = 0};
switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
case HAMT_SUBTAG_HEAD_FLATMAP:
{
flatmap_t *mp = (flatmap_t *)flatmap_val(term);
- Eterm *ks = flatmap_get_keys(mp);
- Eterm *vs = flatmap_get_values(mp);
- size = flatmap_get_size(mp);
- UINT32_HASH(size, HCONST_16);
- if (size == 0)
+ ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP ctx = {
+ .ks = flatmap_get_keys(mp),
+ .vs = flatmap_get_values(mp),
+ .i = 0,
+ .size = flatmap_get_size(mp)};
+ UINT32_HASH(ctx.size, HCONST_16);
+ if (ctx.size == 0)
goto hash2_common;
/* We want a portable hash function that is *independent* of
@@ -1281,17 +1563,18 @@ make_hash2(Eterm term)
ESTACK_PUSH(s, HASH_MAP_TAIL);
hash = 0;
hash_xor_pairs = 0;
- for (i = size - 1; i >= 0; i--) {
+ for (ctx.i = ctx.size - 1; ctx.i >= 0; ctx.i--) {
ESTACK_PUSH(s, HASH_MAP_PAIR);
- ESTACK_PUSH(s, vs[i]);
- ESTACK_PUSH(s, ks[i]);
+ ESTACK_PUSH(s, ctx.vs[ctx.i]);
+ ESTACK_PUSH(s, ctx.ks[ctx.i]);
+ TRAP_LOCATION(hamt_subtag_head_flatmap);
}
goto hash2_common;
}
case HAMT_SUBTAG_HEAD_ARRAY:
case HAMT_SUBTAG_HEAD_BITMAP:
- size = *ptr++;
+ size = *ctx.ptr++;
UINT32_HASH(size, HCONST_16);
if (size == 0)
goto hash2_common;
@@ -1303,27 +1586,28 @@ make_hash2(Eterm term)
}
switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
case HAMT_SUBTAG_HEAD_ARRAY:
- i = 16;
+ ctx.i = 16;
break;
case HAMT_SUBTAG_HEAD_BITMAP:
case HAMT_SUBTAG_NODE_BITMAP:
- i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ ctx.i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
break;
default:
erts_exit(ERTS_ERROR_EXIT, "bad header");
}
- while (i) {
- if (is_list(*ptr)) {
- Eterm* cons = list_val(*ptr);
+ while (ctx.i) {
+ if (is_list(*ctx.ptr)) {
+ Eterm* cons = list_val(*ctx.ptr);
ESTACK_PUSH(s, HASH_MAP_PAIR);
ESTACK_PUSH(s, CDR(cons));
ESTACK_PUSH(s, CAR(cons));
}
else {
- ASSERT(is_boxed(*ptr));
- ESTACK_PUSH(s, *ptr);
+ ASSERT(is_boxed(*ctx.ptr));
+ ESTACK_PUSH(s, *ctx.ptr);
}
- i--; ptr++;
+ ctx.i--; ctx.ptr++;
+ TRAP_LOCATION(map_subtag);
}
goto hash2_common;
}
@@ -1344,22 +1628,25 @@ make_hash2(Eterm term)
case FUN_SUBTAG:
{
ErlFunThing* funp = (ErlFunThing *) fun_val(term);
- Uint num_free = funp->num_free;
+ ErtsMakeHash2Context_FUN_SUBTAG ctx = {
+ .num_free = funp->num_free,
+ .bptr = NULL};
UINT32_HASH_2
- (num_free,
+ (ctx.num_free,
atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue,
HCONST);
UINT32_HASH_2
- (funp->fe->old_index, funp->fe->old_uniq, HCONST);
- if (num_free == 0) {
+ (funp->fe->index, funp->fe->old_uniq, HCONST);
+ if (ctx.num_free == 0) {
goto hash2_common;
} else {
- Eterm* bptr = funp->env + num_free - 1;
- while (num_free-- > 1) {
- term = *bptr--;
+ ctx.bptr = funp->env + ctx.num_free - 1;
+ while (ctx.num_free-- > 1) {
+ term = *ctx.bptr--;
ESTACK_PUSH(s, term);
+ TRAP_LOCATION(fun_subtag);
}
- term = *bptr;
+ term = *ctx.bptr;
}
}
break;
@@ -1367,70 +1654,190 @@ make_hash2(Eterm term)
case HEAP_BINARY_SUBTAG:
case SUB_BINARY_SUBTAG:
{
- byte* bptr;
- unsigned sz = binary_size(term);
+#define BYTE_BITS 8
+ ErtsMakeHash2Context_SUB_BINARY_SUBTAG ctx = {
+ .bptr = 0,
+ /* !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!!
+ *
+ * The size is truncated to 32 bits on the line
+ * below so that the code is compatible with old
+ * versions of the code. This means that hash
+ * values for binaries with a size greater than
+ * 4GB do not take all bytes in consideration.
+ *
+ * !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!!
+ */
+ .sz = (0xFFFFFFFF & binary_size(term)),
+ .bitsize = 0,
+ .bitoffs = 0,
+ .no_bytes_processed = 0
+ };
Uint32 con = HCONST_13 + hash;
- Uint bitoffs;
- Uint bitsize;
-
- ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize);
- if (sz == 0 && bitsize == 0) {
+ Uint iters_for_bin = MAX(1, ctx.sz / BLOCK_HASH_BYTES_PER_ITER);
+ ERTS_GET_BINARY_BYTES(term, ctx.bptr, ctx.bitoffs, ctx.bitsize);
+ if (ctx.sz == 0 && ctx.bitsize == 0) {
hash = con;
- } else {
- if (bitoffs == 0) {
- hash = block_hash(bptr, sz, con);
- if (bitsize > 0) {
- UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)),
- HCONST_15);
- }
- } else {
- byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP,
- sz + (bitsize != 0));
- erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize);
- hash = block_hash(buf, sz, con);
- if (bitsize > 0) {
- UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)),
- HCONST_15);
- }
- erts_free(ERTS_ALC_T_TMP, (void *) buf);
- }
+ } else if (ctx.bitoffs == 0 &&
+ (!can_trap ||
+ (iterations_until_trap - iters_for_bin) > 0)) {
+ /* No need to trap while hashing binary */
+ if (can_trap) iterations_until_trap -= iters_for_bin;
+ hash = block_hash(ctx.bptr, ctx.sz, con);
+ if (ctx.bitsize > 0) {
+ UINT32_HASH_2(ctx.bitsize,
+ (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ } else if (ctx.bitoffs == 0) {
+ /* Need to trap while hashing binary */
+ ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx;
+ block_hash_setup(con, block_hash_ctx);
+ do {
+ Uint max_bytes_to_process =
+ iterations_until_trap <= 0 ? BLOCK_HASH_BYTES_PER_ITER :
+ iterations_until_trap * BLOCK_HASH_BYTES_PER_ITER;
+ Uint bytes_left = ctx.sz - ctx.no_bytes_processed;
+ Uint even_bytes_left =
+ bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER);
+ Uint bytes_to_process =
+ MIN(max_bytes_to_process, even_bytes_left);
+ block_hash_buffer(&ctx.bptr[ctx.no_bytes_processed],
+ bytes_to_process,
+ block_hash_ctx);
+ ctx.no_bytes_processed += bytes_to_process;
+ iterations_until_trap -=
+ MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER);
+ TRAP_LOCATION_NO_RED(sub_binary_subtag_1);
+ block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */
+ } while ((ctx.sz - ctx.no_bytes_processed) >=
+ BLOCK_HASH_BYTES_PER_ITER);
+ hash = block_hash_final_bytes(ctx.bptr +
+ ctx.no_bytes_processed,
+ ctx.sz - ctx.no_bytes_processed,
+ ctx.sz,
+ block_hash_ctx);
+ if (ctx.bitsize > 0) {
+ UINT32_HASH_2(ctx.bitsize,
+ (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ } else if (/* ctx.bitoffs != 0 && */
+ (!can_trap ||
+ (iterations_until_trap - iters_for_bin) > 0)) {
+ /* No need to trap while hashing binary */
+ Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0);
+ byte *buf = erts_alloc(ERTS_ALC_T_TMP, nr_of_bytes);
+ Uint nr_of_bits_to_copy = ctx.sz*BYTE_BITS+ctx.bitsize;
+ if (can_trap) iterations_until_trap -= iters_for_bin;
+ erts_copy_bits(ctx.bptr,
+ ctx.bitoffs, 1, buf, 0, 1, nr_of_bits_to_copy);
+ hash = block_hash(buf, ctx.sz, con);
+ if (ctx.bitsize > 0) {
+ UINT32_HASH_2(ctx.bitsize,
+ (buf[ctx.sz] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ erts_free(ERTS_ALC_T_TMP, buf);
+ } else /* ctx.bitoffs != 0 && */ {
+#ifdef DEBUG
+#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 3)
+#else
+#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 256)
+#endif
+#define BINARY_BUF_SIZE_BITS (BINARY_BUF_SIZE*BYTE_BITS)
+ /* Need to trap while hashing binary */
+ ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx;
+ Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0);
+ ERTS_CT_ASSERT(BINARY_BUF_SIZE % BLOCK_HASH_BYTES_PER_ITER == 0);
+ ctx.buf = erts_alloc(ERTS_ALC_T_PHASH2_TRAP,
+ MIN(nr_of_bytes, BINARY_BUF_SIZE));
+ block_hash_setup(con, block_hash_ctx);
+ do {
+ Uint bytes_left =
+ ctx.sz - ctx.no_bytes_processed;
+ Uint even_bytes_left =
+ bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER);
+ Uint bytes_to_process =
+ MIN(BINARY_BUF_SIZE, even_bytes_left);
+ Uint nr_of_bits_left =
+ (ctx.sz*BYTE_BITS+ctx.bitsize) -
+ ctx.no_bytes_processed*BYTE_BITS;
+ Uint nr_of_bits_to_copy =
+ MIN(nr_of_bits_left, BINARY_BUF_SIZE_BITS);
+ ctx.done = nr_of_bits_left == nr_of_bits_to_copy;
+ erts_copy_bits(ctx.bptr + ctx.no_bytes_processed,
+ ctx.bitoffs, 1, ctx.buf, 0, 1,
+ nr_of_bits_to_copy);
+ block_hash_buffer(ctx.buf,
+ bytes_to_process,
+ block_hash_ctx);
+ ctx.no_bytes_processed += bytes_to_process;
+ iterations_until_trap -=
+ MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER);
+ TRAP_LOCATION_NO_RED(sub_binary_subtag_2);
+ block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */
+ } while (!ctx.done);
+ nr_of_bytes = ctx.sz + (ctx.bitsize != 0);
+ hash = block_hash_final_bytes(ctx.buf +
+ (ctx.no_bytes_processed -
+ ((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE),
+ ctx.sz - ctx.no_bytes_processed,
+ ctx.sz,
+ block_hash_ctx);
+ if (ctx.bitsize > 0) {
+ Uint last_byte_index =
+ nr_of_bytes - (((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE) -1;
+ UINT32_HASH_2(ctx.bitsize,
+ (ctx.buf[last_byte_index] >> (BYTE_BITS - ctx.bitsize)),
+ HCONST_15);
+ }
+ erts_free(ERTS_ALC_T_PHASH2_TRAP, ctx.buf);
+ context->trap_location_state.sub_binary_subtag_2.buf = NULL;
}
goto hash2_common;
+#undef BYTE_BITS
+#undef BINARY_BUF_SIZE
+#undef BINARY_BUF_SIZE_BITS
}
break;
case POS_BIG_SUBTAG:
case NEG_BIG_SUBTAG:
{
- Eterm* ptr = big_val(term);
- Uint i = 0;
- Uint n = BIG_SIZE(ptr);
- Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11;
+ Eterm* big_val_ptr = big_val(term);
+ ErtsMakeHash2Context_NEG_BIG_SUBTAG ctx = {
+ .ptr = big_val_ptr,
+ .i = 0,
+ .n = BIG_SIZE(big_val_ptr),
+ .con = BIG_SIGN(big_val_ptr) ? HCONST_10 : HCONST_11};
#if D_EXP == 16
do {
Uint32 x, y;
- x = i < n ? BIG_DIGIT(ptr, i++) : 0;
- x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
- y = i < n ? BIG_DIGIT(ptr, i++) : 0;
- y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
+ x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ x += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16;
+ y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ y += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16;
+ UINT32_HASH_2(x, y, ctx.con);
+ TRAP_LOCATION(neg_big_subtag);
+ } while (ctx.i < ctx.n);
#elif D_EXP == 32
do {
Uint32 x, y;
- x = i < n ? BIG_DIGIT(ptr, i++) : 0;
- y = i < n ? BIG_DIGIT(ptr, i++) : 0;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
+ x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0;
+ UINT32_HASH_2(x, y, ctx.con);
+ TRAP_LOCATION(neg_big_subtag);
+ } while (ctx.i < ctx.n);
#elif D_EXP == 64
do {
Uint t;
Uint32 x, y;
- ASSERT(i < n);
- t = BIG_DIGIT(ptr, i++);
+ ASSERT(ctx.i < ctx.n);
+ t = BIG_DIGIT(ctx.ptr, ctx.i++);
x = t & 0xffffffff;
y = t >> 32;
- UINT32_HASH_2(x, y, con);
- } while (i < n);
+ UINT32_HASH_2(x, y, ctx.con);
+ TRAP_LOCATION(neg_big_subtag);
+ } while (ctx.i < ctx.n);
#else
#error "unsupported D_EXP size"
#endif
@@ -1508,13 +1915,13 @@ make_hash2(Eterm term)
}
case _TAG_IMMED1_SMALL:
{
- Sint x = signed_val(term);
+ Sint small = signed_val(term);
+ if (SMALL_BITS > 28 && !IS_SSMALL28(small)) {
+ NOT_SSMALL28_HASH(small);
+ } else {
+ SINT32_HASH(small, HCONST);
+ }
- if (SMALL_BITS > 28 && !IS_SSMALL28(x)) {
- term = small_to_big(x, tmp_big);
- break;
- }
- SINT32_HASH(x, HCONST);
goto hash2_common;
}
}
@@ -1529,7 +1936,10 @@ make_hash2(Eterm term)
if (ESTACK_ISEMPTY(s)) {
DESTROY_ESTACK(s);
- UnUseTmpHeapNoproc(2);
+ if (can_trap) {
+ BUMP_REDS(p, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED);
+ ASSERT(!(p->flags & F_DISABLE_GC));
+ }
return hash;
}
@@ -1540,18 +1950,37 @@ make_hash2(Eterm term)
hash = (Uint32) ESTACK_POP(s);
UINT32_HASH(hash_xor_pairs, HCONST_19);
hash_xor_pairs = (Uint32) ESTACK_POP(s);
+ TRAP_LOCATION_NO_CTX(hash2_common_1);
goto hash2_common;
}
case HASH_MAP_PAIR:
hash_xor_pairs ^= hash;
hash = 0;
+ TRAP_LOCATION_NO_CTX(hash2_common_2);
goto hash2_common;
default:
break;
}
+
}
+ TRAP_LOCATION_NO_CTX(hash2_common_3);
}
}
+#undef TRAP_LOCATION_NO_RED
+#undef TRAP_LOCATION
+#undef TRAP_LOCATION_NO_CTX
+}
+
+Uint32
+make_hash2(Eterm term)
+{
+ return make_hash2_helper(term, 0, NULL, NULL);
+}
+
+Uint32
+trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p)
+{
+ return make_hash2_helper(term, 1, state_mref_write_back, p);
}
/* Term hash function for internal use.
@@ -1731,7 +2160,7 @@ make_internal_hash(Eterm term, Uint32 salt)
ErlFunThing* funp = (ErlFunThing *) fun_val(term);
Uint num_free = funp->num_free;
UINT32_HASH_2(num_free, funp->fe->module, HCONST_20);
- UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21);
+ UINT32_HASH_2(funp->fe->index, funp->fe->old_uniq, HCONST_21);
if (num_free == 0) {
goto pop_next;
} else {
@@ -2381,7 +2810,7 @@ tailrecur_ne:
f1 = (ErlFunThing *) fun_val(a);
f2 = (ErlFunThing *) fun_val(b);
if (f1->fe->module != f2->fe->module ||
- f1->fe->old_index != f2->fe->old_index ||
+ f1->fe->index != f2->fe->index ||
f1->fe->old_uniq != f2->fe->old_uniq ||
f1->num_free != f2->num_free) {
goto not_equal;
@@ -2976,7 +3405,7 @@ tailrecur_ne:
if (diff != 0) {
RETURN_NEQ(diff);
}
- diff = f1->fe->old_index - f2->fe->old_index;
+ diff = f1->fe->index - f2->fe->index;
if (diff != 0) {
RETURN_NEQ(diff);
}