aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/arith_instrs.tab83
-rw-r--r--erts/emulator/beam/beam_emu.c39
-rw-r--r--erts/emulator/beam/dist.c30
-rw-r--r--erts/emulator/beam/erl_bif_info.c2
-rw-r--r--erts/emulator/beam/erl_node_tables.h1
-rw-r--r--erts/emulator/beam/erl_proc_sig_queue.c2
-rw-r--r--erts/emulator/beam/erl_process.c6
-rw-r--r--erts/emulator/beam/external.c6
-rw-r--r--erts/emulator/beam/external.h8
-rw-r--r--erts/emulator/beam/global.h9
-rw-r--r--erts/emulator/beam/sys.h8
11 files changed, 136 insertions, 58 deletions
diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 5f23b2c168..f14b376419 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -51,11 +51,50 @@ plus.fetch(Op1, Op2) {
plus.execute(Fail, Dst) {
if (ERTS_LIKELY(is_both_small(PlusOp1, PlusOp2))) {
+#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
+ Sint lhs_tagged, rhs_untagged, res;
+
+ /* The value part of immediate integers start right after the tag and
+ * occupy the rest of the word, so if you squint a bit they look like
+ * fixed-point integers; as long as you mask the tag away you will get
+ * correct results from addition/subtraction since they share the same
+ * notion of zero. It's fairly easy to see that the following holds
+ * when (a + b) is in range:
+ *
+ * (a >> s) + (b >> s) == ((a & ~m) + (b & ~m)) >> s
+ *
+ * Where 's' is the tag size and 'm' is the tag mask.
+ *
+ * The left-hand side is our fallback in the #else clause and is the
+ * fastest way to do this safely in plain C. The actual addition will
+ * never overflow since `Sint` has a much greater range than our
+ * smalls, so we can use the IS_SSMALL macro to see if the result is
+ * within range.
+ *
+ * What we're doing below is an extension of the right-hand side. By
+ * treating `a` and `b` as fixed-point integers, all additions whose
+ * result is out of range will also overflow `Sint` and we can use the
+ * compiler's overflow intrinsics to check for this condition.
+ *
+ * In addition, since the tag lives in the lowest bits we can further
+ * optimize this by only stripping the tag from either side. The higher
+ * bits can't influence the tag bits since we bail on overflow, so the
+ * tag bits from the tagged side will simply appear in the result. */
+ lhs_tagged = PlusOp1;
+ rhs_untagged = PlusOp2 & ~_TAG_IMMED1_MASK;
+
+ if (ERTS_LIKELY(!__builtin_add_overflow(lhs_tagged, rhs_untagged, &res))) {
+ ASSERT(is_small(res));
+ $Dst = res;
+ $NEXT0();
+ }
+#else
Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
if (ERTS_LIKELY(IS_SSMALL(i))) {
$Dst = make_small(i);
$NEXT0();
}
+#endif
}
$OUTLINED_ARITH_2($Fail, mixed_plus, BIF_splus_2, PlusOp1, PlusOp2, $Dst);
}
@@ -73,11 +112,26 @@ minus.fetch(Op1, Op2) {
minus.execute(Fail, Dst) {
if (ERTS_LIKELY(is_both_small(MinusOp1, MinusOp2))) {
+#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
+ Sint lhs_tagged, rhs_untagged, res;
+
+ /* See plus.execute */
+ lhs_tagged = MinusOp1;
+ rhs_untagged = MinusOp2 & ~_TAG_IMMED1_MASK;
+
+ if (ERTS_LIKELY(!__builtin_sub_overflow(lhs_tagged, rhs_untagged, &res))) {
+ ASSERT(is_small(res));
+ $Dst = res;
+ $NEXT0();
+ }
+#else
Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
+
if (ERTS_LIKELY(IS_SSMALL(i))) {
$Dst = make_small(i);
$NEXT0();
}
+#endif
}
$OUTLINED_ARITH_2($Fail, mixed_minus, BIF_sminus_2, MinusOp1, MinusOp2, $Dst);
}
@@ -97,12 +151,27 @@ increment.execute(IncrementVal, Dst) {
Eterm result;
if (ERTS_LIKELY(is_small(increment_reg_val))) {
+#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
+ Sint lhs_tagged, rhs_untagged, res;
+
+ /* See plus.execute */
+ lhs_tagged = increment_reg_val;
+ rhs_untagged = (Sint)increment_val << _TAG_IMMED1_SIZE;
+
+ if (ERTS_LIKELY(!__builtin_add_overflow(lhs_tagged, rhs_untagged, &res))) {
+ ASSERT(is_small(res));
+ $Dst = res;
+ $NEXT0();
+ }
+#else
Sint i = signed_val(increment_reg_val) + increment_val;
if (ERTS_LIKELY(IS_SSMALL(i))) {
$Dst = make_small(i);
$NEXT0();
}
+#endif
}
+
result = erts_mixed_plus(c_p, increment_reg_val, make_small(increment_val));
ERTS_HOLE_CHECK(c_p);
if (ERTS_LIKELY(is_value(result))) {
@@ -118,11 +187,15 @@ i_times(Fail, Op1, Op2, Dst) {
Eterm op2 = $Op2;
#ifdef HAVE_OVERFLOW_CHECK_BUILTINS
if (ERTS_LIKELY(is_both_small(op1, op2))) {
- Sint a = signed_val(op1);
- Sint b = signed_val(op2);
- Sint res;
- if (ERTS_LIKELY(!__builtin_mul_overflow(a, b, &res) && IS_SSMALL(res))) {
- $Dst = make_small(res);
+ /* See plus.execute */
+ Sint lhs_untagged, rhs_actual, res;
+
+ lhs_untagged = op1 & ~_TAG_IMMED1_MASK;
+ rhs_actual = signed_val(op2);
+
+ if (ERTS_LIKELY(!__builtin_mul_overflow(lhs_untagged, rhs_actual, &res))) {
+ ASSERT(!(res & _TAG_IMMED1_MASK));
+ $Dst = res | _TAG_IMMED1_SMALL;
$NEXT0();
}
}
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index ea01ce597d..8e93e53003 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -375,44 +375,33 @@ do { \
/*
* process_main() is already huge, so we want to avoid inlining
- * into it. Especially functions that are seldom used.
+ * seldom used functions into it.
*/
-#ifdef __GNUC__
-# define NOINLINE __attribute__((__noinline__))
-#else
-# define NOINLINE
-#endif
-
-
-/*
- * The following functions are called directly by process_main().
- * Don't inline them.
- */
-static void init_emulator_finish(void) NOINLINE;
-static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE;
+static void init_emulator_finish(void) ERTS_NOINLINE;
+static ErtsCodeMFA *ubif2mfa(void* uf) ERTS_NOINLINE;
static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
- Eterm* reg, ErtsCodeMFA* bif_mfa) NOINLINE;
+ Eterm* reg, ErtsCodeMFA* bif_mfa) ERTS_NOINLINE;
static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
- Eterm* reg, Eterm func) NOINLINE;
+ Eterm* reg, Eterm func) ERTS_NOINLINE;
static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
- BeamInstr *I, Uint offs) NOINLINE;
+ BeamInstr *I, Uint offs) ERTS_NOINLINE;
static BeamInstr* apply(Process* p, Eterm* reg,
- BeamInstr *I, Uint offs) NOINLINE;
+ BeamInstr *I, Uint offs) ERTS_NOINLINE;
static BeamInstr* call_fun(Process* p, int arity,
- Eterm* reg, Eterm args) NOINLINE;
+ Eterm* reg, Eterm args) ERTS_NOINLINE;
static BeamInstr* apply_fun(Process* p, Eterm fun,
- Eterm args, Eterm* reg) NOINLINE;
+ Eterm args, Eterm* reg) ERTS_NOINLINE;
static Eterm new_fun(Process* p, Eterm* reg,
- ErlFunEntry* fe, int num_free) NOINLINE;
+ ErlFunEntry* fe, int num_free) ERTS_NOINLINE;
static int is_function2(Eterm Term, Uint arity);
static Eterm erts_gc_new_map(Process* p, Eterm* reg, Uint live,
- Uint n, BeamInstr* ptr) NOINLINE;
+ Uint n, BeamInstr* ptr) ERTS_NOINLINE;
static Eterm erts_gc_new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal,
- Uint live, BeamInstr* ptr) NOINLINE;
+ Uint live, BeamInstr* ptr) ERTS_NOINLINE;
static Eterm erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live,
- Uint n, BeamInstr* new_p) NOINLINE;
+ Uint n, BeamInstr* new_p) ERTS_NOINLINE;
static Eterm erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live,
- Uint n, Eterm* new_p) NOINLINE;
+ Uint n, Eterm* new_p) ERTS_NOINLINE;
static Eterm get_map_element(Eterm map, Eterm key);
static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx);
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index 8bbe6450eb..456b6fdac0 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -775,19 +775,25 @@ void init_dist(void)
static ERTS_INLINE ErtsDistOutputBuf *
alloc_dist_obuf(Uint size, Uint headers)
{
- int i;
+ Uint obuf_size = sizeof(ErtsDistOutputBuf)*(headers);
ErtsDistOutputBuf *obuf;
- Uint obuf_size = sizeof(ErtsDistOutputBuf)*(headers) +
- sizeof(byte)*size;
- Binary *bin = erts_bin_drv_alloc(obuf_size);
- obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[size];
+ Binary *bin;
+ byte *extp;
+ int i;
+
+ bin = erts_bin_drv_alloc(obuf_size + size);
erts_refc_add(&bin->intern.refc, headers - 1, 1);
+
+ obuf = (ErtsDistOutputBuf *)&bin->orig_bytes[0];
+ extp = (byte *)&bin->orig_bytes[obuf_size];
+
for (i = 0; i < headers; i++) {
obuf[i].bin = bin;
- obuf[i].extp = (byte *)&bin->orig_bytes[0];
+ obuf[i].extp = extp;
#ifdef DEBUG
obuf[i].dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN;
- obuf[i].alloc_endp = obuf->extp + size;
+ obuf[i].ext_startp = extp;
+ obuf[i].alloc_endp = &extp[size];
ASSERT(bin == ErtsDistOutputBuf2Binary(obuf));
#endif
}
@@ -1360,7 +1366,7 @@ erts_dist_seq_tree_foreach_delete_yielding(DistSeqNode **root,
limit);
if (res > 0) {
if (ysp != &ys)
- erts_free(ERTS_ALC_T_ML_YIELD_STATE, ysp);
+ erts_free(ERTS_ALC_T_SEQ_YIELD_STATE, ysp);
*vyspp = NULL;
}
else {
@@ -2341,7 +2347,8 @@ erts_dsig_send(ErtsDSigSendContext *ctx)
(ctx->fragments-1) * ERTS_DIST_FRAGMENT_HEADER_SIZE,
ctx->fragments);
ctx->obuf->ext_start = &ctx->obuf->extp[0];
- ctx->obuf->ext_endp = &ctx->obuf->extp[0] + ctx->max_finalize_prepend + ctx->dhdr_ext_size;
+ ctx->obuf->ext_endp = &ctx->obuf->extp[0] + ctx->max_finalize_prepend
+ + ctx->dhdr_ext_size;
/* Encode internal version of dist header */
ctx->obuf->extp = erts_encode_ext_dist_header_setup(
@@ -2380,8 +2387,8 @@ erts_dsig_send(ErtsDSigSendContext *ctx)
case ERTS_DSIG_SEND_PHASE_FIN: {
ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp);
- ASSERT(((byte*)&ctx->obuf->bin->orig_bytes[0]) <= ctx->obuf->extp - ctx->max_finalize_prepend);
- ASSERT(ctx->obuf->ext_endp <= ((byte*)ctx->obuf->bin->orig_bytes) + ctx->data_size + ctx->dhdr_ext_size);
+ ASSERT(ctx->obuf->ext_startp <= ctx->obuf->extp - ctx->max_finalize_prepend);
+ ASSERT(ctx->obuf->ext_endp <= (byte*)ctx->obuf->ext_startp + ctx->data_size + ctx->dhdr_ext_size);
ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp;
@@ -3457,6 +3464,7 @@ dist_ctrl_get_data_1(BIF_ALIST_1)
pb->bytes = (byte*) obuf->extp;
pb->flags = 0;
res = make_binary(pb);
+ hp += PROC_BIN_SIZE;
} else {
hp = HAlloc(BIF_P, PROC_BIN_SIZE * 2 + 4 + hsz);
pb = (ProcBin *) (char *) hp;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a7424bbcb8..39d42d9757 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2579,6 +2579,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
/* Need to be the only thread running... */
erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ BIF_P->scheduler_data->current_process = NULL;
erts_thr_progress_block();
if (BIF_ARG_1 == am_info)
@@ -2592,6 +2593,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
erts_thr_progress_unblock();
erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ BIF_P->scheduler_data->current_process = BIF_P;
ASSERT(dsbufp && dsbufp->str);
res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index c434926142..fc3e117463 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -95,6 +95,7 @@ enum dist_entry_state {
struct ErtsDistOutputBuf_ {
#ifdef DEBUG
Uint dbg_pattern;
+ byte *ext_startp;
byte *alloc_endp;
#endif
ErtsDistOutputBuf *next;
diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c
index 4e9f177e51..f58a606d57 100644
--- a/erts/emulator/beam/erl_proc_sig_queue.c
+++ b/erts/emulator/beam/erl_proc_sig_queue.c
@@ -1019,6 +1019,8 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag,
ref_sz = size_object(ref);
hsz += ref_sz;
+ reason_sz = 0; /* Set to silence gcc warning */
+
/* The reason was part of the control message,
just use copy it into the xsigd */
if (is_value(reason)) {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 9e662632b4..2b45d2d353 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -12141,10 +12141,9 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds)
reason);
switch (code) {
case ERTS_DSIG_SEND_CONTINUE:
+ case ERTS_DSIG_SEND_YIELD:
erts_set_gc_state(c_p, 0);
ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx);
- /* fall-through */
- case ERTS_DSIG_SEND_YIELD:
break;
case ERTS_DSIG_SEND_OK:
break;
@@ -12388,11 +12387,10 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds)
reason,
SEQ_TRACE_TOKEN(c_p));
switch (code) {
+ case ERTS_DSIG_SEND_YIELD:
case ERTS_DSIG_SEND_CONTINUE:
erts_set_gc_state(c_p, 0);
ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx);
- /* fall-through */
- case ERTS_DSIG_SEND_YIELD:
break;
case ERTS_DSIG_SEND_OK:
break;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 471c1c3938..fa8314c115 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -699,6 +699,7 @@ dist_ext_size(ErtsDistExternal *edep)
} else {
sz -= sizeof(ErtsAtomTranslationTable);
}
+ ASSERT(sz % 4 == 0);
return sz;
}
@@ -706,8 +707,9 @@ Uint
erts_dist_ext_size(ErtsDistExternal *edep)
{
Uint sz = dist_ext_size(edep);
+ sz += 4; /* may need to pad to 8-byte-align ErtsDistExternalData */
sz += edep->data[0].frag_id * sizeof(ErtsDistExternalData);
- return sz + ERTS_EXTRA_DATA_ALIGN_SZ(sz);
+ return sz;
}
Uint
@@ -749,6 +751,8 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, ErtsDistExternal *new_edep)
erts_ref_dist_entry(new_edep->dep);
ep += dist_ext_sz;
+ ep += (UWord)ep & 4; /* 8-byte alignment for ErtsDistExternalData */
+ ASSERT((UWord)ep % 8 == 0);
new_edep->data = (ErtsDistExternalData*)ep;
sys_memzero(new_edep->data, sizeof(ErtsDistExternalData) * edep->data->frag_id);
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index 396cd9f802..f2cc9bf98f 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -144,14 +144,6 @@ typedef struct erl_dist_external {
ErtsAtomTranslationTable attab;
} ErtsDistExternal;
-#define ERTS_DIST_EXT_SIZE(EDEP) \
- (sizeof(ErtsDistExternal) \
- - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \
- ? (ASSERT(0 <= (EDEP)->attab.size \
- && (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \
- sizeof(Eterm)*(ERTS_ATOM_CACHE_SIZE - (EDEP)->attab.size)) \
- : sizeof(ErtsAtomTranslationTable)))
-
typedef struct {
byte *extp;
int exttmp;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index f9bbe4167f..4c8d3d3dbe 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1216,10 +1216,11 @@ Uint64 erts_timestamp_millis(void);
Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex);
-void *erts_calc_stacklimit(char *prev_c, UWord stacksize);
-int erts_check_below_limit(char *ptr, char *limit);
-int erts_check_above_limit(char *ptr, char *limit);
-void *erts_ptr_id(void *ptr);
+/* ERTS_NOINLINE prevents link-time optimization across modules */
+void *erts_calc_stacklimit(char *prev_c, UWord stacksize) ERTS_NOINLINE;
+int erts_check_below_limit(char *ptr, char *limit) ERTS_NOINLINE;
+int erts_check_above_limit(char *ptr, char *limit) ERTS_NOINLINE;
+void *erts_ptr_id(void *ptr) ERTS_NOINLINE;
Eterm store_external_or_ref_in_proc_(Process *, Eterm);
Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index a6312293cc..c261c8e117 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -63,6 +63,14 @@
# endif
#endif
+#ifndef ERTS_NOINLINE
+# if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
+# define ERTS_NOINLINE __attribute__((__noinline__))
+# else
+# define ERTS_NOINLINE
+# endif
+#endif
+
#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
# undef ERTS_CAN_INLINE
# define ERTS_CAN_INLINE 0
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-08 23:57:25 +0000