aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/atom.names1
-rw-r--r--erts/emulator/beam/beam_bif_load.c4
-rw-r--r--erts/emulator/beam/beam_emu.c11
-rw-r--r--erts/emulator/beam/bif.c166
-rw-r--r--erts/emulator/beam/binary.c103
-rw-r--r--erts/emulator/beam/erl_bif_ddll.c13
-rw-r--r--erts/emulator/beam/erl_bif_info.c6
-rw-r--r--erts/emulator/beam/erl_bif_port.c9
-rw-r--r--erts/emulator/beam/erl_bif_re.c18
-rw-r--r--erts/emulator/beam/erl_db_hash.c23
-rw-r--r--erts/emulator/beam/erl_db_tree.c2
-rw-r--r--erts/emulator/beam/erl_drv_thread.c7
-rw-r--r--erts/emulator/beam/erl_nif.c4
-rw-r--r--erts/emulator/beam/erl_port_task.c3
-rw-r--r--erts/emulator/beam/erl_process.c138
-rw-r--r--erts/emulator/beam/erl_trace.c22
-rw-r--r--erts/emulator/beam/global.h21
-rw-r--r--erts/emulator/beam/io.c106
-rw-r--r--erts/emulator/beam/safe_hash.c2
-rw-r--r--erts/emulator/beam/utils.c37
20 files changed, 486 insertions, 210 deletions
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 327620772f..de76cb9680 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -76,6 +76,7 @@ atom allocator_sizes
atom alloc_util_allocators
atom allow_passive_connect
atom already_loaded
+atom amd64
atom anchored
atom and
atom andalso
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 1dbf6f9b92..d76a7d8e9f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -142,7 +142,7 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
if ((modp = erts_get_module(BIF_ARG_1)) == NULL) {
return am_undefined;
}
- return (is_native(modp->code) ||
+ return ((modp->code && is_native(modp->code)) ||
(modp->old_code != 0 && is_native(modp->old_code))) ?
am_true : am_false;
}
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 32ea8588d2..fb90a7d4f7 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1911,13 +1911,15 @@ void process_main(void)
* Note that for the halfword emulator, the two first elements
* of the array are used.
*/
- *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3;
+ BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+ *pi = I+3;
set_timer(c_p, unsigned_val(timeout_value));
} else if (timeout_value == am_infinity) {
c_p->flags |= F_TIMO;
#if !defined(ARCH_64) || HALFWORD_HEAP
} else if (term_to_Uint(timeout_value, &time_val)) {
- *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3;
+ BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+ *pi = I+3;
set_timer(c_p, time_val);
#endif
} else { /* Wrong time */
@@ -1974,7 +1976,8 @@ void process_main(void)
* we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
*/
if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
- *((BeamInstr **) (UWord) c_p->def_arg_reg) = I+3;
+ BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
+ *p = I+3;
set_timer(c_p, Arg(1));
}
goto wait2;
@@ -5322,7 +5325,7 @@ void process_main(void)
ep->code[3] = (BeamInstr) OpCode(apply_bif);
ep->code[4] = (BeamInstr) bif_table[i].f;
/* XXX: set func info for bifs */
- ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+ ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
}
return;
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 8c35644125..68b3350d7f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -2168,20 +2168,146 @@ BIF_RETTYPE tl_1(BIF_ALIST_1)
/**********************************************************************/
/* return the size of an I/O list */
-BIF_RETTYPE iolist_size_1(BIF_ALIST_1)
+static Eterm
+accumulate(Eterm acc, Uint size)
{
- Sint size = io_list_len(BIF_ARG_1);
+ if (is_non_value(acc)) {
+ /*
+ * There is no pre-existing accumulator. Allocate a
+ * bignum buffer with one extra word to be used if
+ * the bignum grows in the future.
+ */
+ Eterm* hp = (Eterm *) erts_alloc(ERTS_ALC_T_TEMP_TERM,
+ (BIG_UINT_HEAP_SIZE+1) *
+ sizeof(Eterm));
+ return uint_to_big(size, hp);
+ } else {
+ Eterm* big;
+ int need_heap;
- if (size == -1) {
- BIF_ERROR(BIF_P, BADARG);
- } else if (IS_USMALL(0, (Uint) size)) {
- BIF_RET(make_small(size));
+ /*
+ * Add 'size' to 'acc' in place. There is always one
+ * extra word allocated in case the bignum grows by one word.
+ */
+ big = big_val(acc);
+ need_heap = BIG_NEED_SIZE(BIG_SIZE(big));
+ acc = big_plus_small(acc, size, big);
+ if (BIG_NEED_SIZE(big_size(acc)) > need_heap) {
+ /*
+ * The extra word has been consumed. Grow the
+ * allocation by one word.
+ */
+ big = (Eterm *) erts_realloc(ERTS_ALC_T_TEMP_TERM,
+ big_val(acc),
+ (need_heap+1) * sizeof(Eterm));
+ acc = make_big(big);
+ }
+ return acc;
+ }
+}
+
+static Eterm
+consolidate(Process* p, Eterm acc, Uint size)
+{
+ Eterm* hp;
+
+ if (is_non_value(acc)) {
+ return erts_make_integer(size, p);
} else {
- Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
- BIF_RET(uint_to_big(size, hp));
+ Eterm* big;
+ Uint sz;
+ Eterm res;
+
+ acc = accumulate(acc, size);
+ big = big_val(acc);
+ sz = BIG_NEED_SIZE(BIG_SIZE(big));
+ hp = HAlloc(p, sz);
+ res = make_big(hp);
+ while (sz--) {
+ *hp++ = *big++;
+ }
+ erts_free(ERTS_ALC_T_TEMP_TERM, (void *) big_val(acc));
+ return res;
}
}
+BIF_RETTYPE iolist_size_1(BIF_ALIST_1)
+{
+ Eterm obj, hd;
+ Eterm* objp;
+ Uint size = 0;
+ Uint cur_size;
+ Uint new_size;
+ Eterm acc = THE_NON_VALUE;
+ DECLARE_ESTACK(s);
+
+ obj = BIF_ARG_1;
+ goto L_again;
+
+ while (!ESTACK_ISEMPTY(s)) {
+ obj = ESTACK_POP(s);
+ L_again:
+ if (is_list(obj)) {
+ L_iter_list:
+ objp = list_val(obj);
+ hd = CAR(objp);
+ obj = CDR(objp);
+ /* Head */
+ if (is_byte(hd)) {
+ size++;
+ if (size == 0) {
+ acc = accumulate(acc, (Uint) -1);
+ size = 1;
+ }
+ } else if (is_binary(hd) && binary_bitsize(hd) == 0) {
+ cur_size = binary_size(hd);
+ if ((new_size = size + cur_size) >= size) {
+ size = new_size;
+ } else {
+ acc = accumulate(acc, size);
+ size = cur_size;
+ }
+ } else if (is_list(hd)) {
+ ESTACK_PUSH(s, obj);
+ obj = hd;
+ goto L_iter_list;
+ } else if (is_not_nil(hd)) {
+ goto L_type_error;
+ }
+ /* Tail */
+ if (is_list(obj)) {
+ goto L_iter_list;
+ } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+ cur_size = binary_size(obj);
+ if ((new_size = size + cur_size) >= size) {
+ size = new_size;
+ } else {
+ acc = accumulate(acc, size);
+ size = cur_size;
+ }
+ } else if (is_not_nil(obj)) {
+ goto L_type_error;
+ }
+ } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+ cur_size = binary_size(obj);
+ if ((new_size = size + cur_size) >= size) {
+ size = new_size;
+ } else {
+ acc = accumulate(acc, size);
+ size = cur_size;
+ }
+ } else if (is_not_nil(obj)) {
+ goto L_type_error;
+ }
+ }
+
+ DESTROY_ESTACK(s);
+ BIF_RET(consolidate(BIF_P, acc, size));
+
+ L_type_error:
+ DESTROY_ESTACK(s);
+ BIF_ERROR(BIF_P, BADARG);
+}
/**********************************************************************/
@@ -3282,6 +3408,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
Eterm* dead_ports;
int alive, dead;
Uint32 next_ss;
+ int i;
/* To get a consistent snapshot...
* We add alive ports from start of the buffer
@@ -3293,21 +3420,18 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
erts_smp_atomic_set(&erts_dead_ports_ptr,
(erts_aint_t) (port_buf + erts_max_ports));
- next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot);
+ next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot);
- if (erts_smp_atomic_read(&erts_ports_alive) > 0) {
- erts_aint_t i;
- for (i = erts_max_ports-1; i >= 0; i--) {
- Port* prt = &erts_port[i];
- erts_smp_port_state_lock(prt);
- if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
- && prt->snapshot != next_ss) {
- ASSERT(prt->snapshot == next_ss - 1);
- *pp++ = prt->id;
- prt->snapshot = next_ss; /* Consumed by this snapshot */
- }
- erts_smp_port_state_unlock(prt);
+ for (i = erts_max_ports-1; i >= 0; i--) {
+ Port* prt = &erts_port[i];
+ erts_smp_port_state_lock(prt);
+ if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
+ && prt->snapshot != next_ss) {
+ ASSERT(prt->snapshot == next_ss - 1);
+ *pp++ = prt->id;
+ prt->snapshot = next_ss; /* Consumed by this snapshot */
}
+ erts_smp_port_state_unlock(prt);
}
dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr,
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 9486602633..1fb39c6c67 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -32,11 +32,11 @@
#include "erl_bits.h"
#ifdef DEBUG
-static int list_to_bitstr_buf(Eterm obj, char* buf, int len);
+static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len);
#else
static int list_to_bitstr_buf(Eterm obj, char* buf);
#endif
-static Sint bitstr_list_len(Eterm obj);
+static int bitstr_list_len(Eterm obj, Uint* num_bytes);
void
erts_init_binary(void)
@@ -355,21 +355,24 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
{
Eterm bin;
- int i;
+ Uint size;
int offset;
byte* bytes;
+
if (is_nil(arg)) {
BIF_RET(new_binary(p,(byte*)"",0));
}
if (is_not_list(arg)) {
goto error;
}
- if ((i = io_list_len(arg)) < 0) {
- goto error;
+ switch (erts_iolist_size(arg, &size)) {
+ case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT);
+ case ERTS_IOLIST_TYPE: goto error;
+ default: ;
}
- bin = new_binary(p, (byte *)NULL, i);
+ bin = new_binary(p, (byte *)NULL, size);
bytes = binary_bytes(bin);
- offset = io_list_to_buf(arg, (char*) bytes, i);
+ offset = io_list_to_buf(arg, (char*) bytes, size);
ASSERT(offset == 0);
BIF_RET(bin);
@@ -396,7 +399,8 @@ BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1)
BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
{
Eterm bin;
- int i,offset;
+ Uint sz;
+ int offset;
byte* bytes;
ErlSubBin* sb1;
Eterm* hp;
@@ -405,15 +409,19 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
BIF_RET(new_binary(BIF_P,(byte*)"",0));
}
if (is_not_list(BIF_ARG_1)) {
- goto error;
+ error:
+ BIF_ERROR(BIF_P, BADARG);
}
- if ((i = bitstr_list_len(BIF_ARG_1)) < 0) {
+ switch (bitstr_list_len(BIF_ARG_1, &sz)) {
+ case ERTS_IOLIST_TYPE:
goto error;
+ case ERTS_IOLIST_OVERFLOW:
+ BIF_ERROR(BIF_P, SYSTEM_LIMIT);
}
- bin = new_binary(BIF_P, (byte *)NULL, i);
+ bin = new_binary(BIF_P, (byte *)NULL, sz);
bytes = binary_bytes(bin);
#ifdef DEBUG
- offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, i);
+ offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz);
#else
offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes);
#endif
@@ -422,20 +430,16 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE);
sb1 = (ErlSubBin *) hp;
sb1->thing_word = HEADER_SUB_BIN;
- sb1->size = i-1;
+ sb1->size = sz-1;
sb1->offs = 0;
sb1->orig = bin;
sb1->bitoffs = 0;
sb1->bitsize = offset;
sb1->is_writable = 0;
- hp += ERL_SUB_BIN_SIZE;
bin = make_binary(sb1);
}
BIF_RET(bin);
-
- error:
- BIF_ERROR(BIF_P, BADARG);
}
BIF_RETTYPE split_binary_2(BIF_ALIST_2)
@@ -499,7 +503,7 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2)
*/
static int
#ifdef DEBUG
-list_to_bitstr_buf(Eterm obj, char* buf, int len)
+list_to_bitstr_buf(Eterm obj, char* buf, Uint len)
#else
list_to_bitstr_buf(Eterm obj, char* buf)
#endif
@@ -602,8 +606,8 @@ list_to_bitstr_buf(Eterm obj, char* buf)
return offset;
}
-static Sint
-bitstr_list_len(Eterm obj)
+static int
+bitstr_list_len(Eterm obj, Uint* num_bytes)
{
Eterm* objp;
Uint len = 0;
@@ -611,6 +615,26 @@ bitstr_list_len(Eterm obj)
DECLARE_ESTACK(s);
goto L_again;
+#define SAFE_ADD(Var, Val) \
+ do { \
+ Uint valvar = (Val); \
+ Var += valvar; \
+ if (Var < valvar) { \
+ goto L_overflow_error; \
+ } \
+ } while (0)
+
+#define SAFE_ADD_BITSIZE(Var, Bin) \
+ do { \
+ if (*binary_val(Bin) == HEADER_SUB_BIN) { \
+ Uint valvar = ((ErlSubBin *) binary_val(Bin))->bitsize; \
+ Var += valvar; \
+ if (Var < valvar) { \
+ goto L_overflow_error; \
+ } \
+ } \
+ } while (0)
+
while (!ESTACK_ISEMPTY(s)) {
obj = ESTACK_POP(s);
L_again:
@@ -621,9 +645,12 @@ bitstr_list_len(Eterm obj)
obj = CAR(objp);
if (is_byte(obj)) {
len++;
+ if (len == 0) {
+ goto L_overflow_error;
+ }
} else if (is_binary(obj)) {
- len += binary_size(obj);
- offs += binary_bitsize(obj);
+ SAFE_ADD(len, binary_size(obj));
+ SAFE_ADD_BITSIZE(offs, obj);
} else if (is_list(obj)) {
ESTACK_PUSH(s, CDR(objp));
goto L_iter_list; /* on head */
@@ -635,24 +662,44 @@ bitstr_list_len(Eterm obj)
if (is_list(obj))
goto L_iter_list; /* on tail */
else if (is_binary(obj)) {
- len += binary_size(obj);
- offs += binary_bitsize(obj);
+ SAFE_ADD(len, binary_size(obj));
+ SAFE_ADD_BITSIZE(offs, obj);
} else if (is_not_nil(obj)) {
goto L_type_error;
}
} else if (is_binary(obj)) {
- len += binary_size(obj);
- offs += binary_bitsize(obj);
+ SAFE_ADD(len, binary_size(obj));
+ SAFE_ADD_BITSIZE(offs, obj);
} else if (is_not_nil(obj)) {
goto L_type_error;
}
}
+#undef SAFE_ADD
+#undef SAFE_ADD_BITSIZE
DESTROY_ESTACK(s);
- return (Sint) (len + (offs/8) + ((offs % 8) != 0));
+
+ /*
+ * Make sure that the number of bits in the bitstring will fit
+ * in an Uint to ensure that the binary can be matched using
+ * the binary syntax.
+ */
+ if (len << 3 < len) {
+ goto L_overflow_error;
+ }
+ len += (offs >> 3) + ((offs & 7) != 0);
+ if (len << 3 < len) {
+ goto L_overflow_error;
+ }
+ *num_bytes = len;
+ return ERTS_IOLIST_OK;
L_type_error:
DESTROY_ESTACK(s);
- return (Sint) -1;
+ return ERTS_IOLIST_TYPE;
+
+ L_overflow_error:
+ DESTROY_ESTACK(s);
+ return ERTS_IOLIST_OVERFLOW;
}
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index c9cdcb87a6..9631fb50db 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -146,7 +146,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
Eterm name_term, Eterm options)
{
char *path = NULL;
- int path_len;
+ Uint path_len;
char *name = NULL;
DE_Handle *dh;
erts_driver_t *drv;
@@ -221,9 +221,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
goto error;
}
- path_len = io_list_len(path_term);
-
- if (path_len <= 0) {
+ if (erts_iolist_size(path_term, &path_len)) {
goto error;
}
path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1);
@@ -1878,7 +1876,7 @@ static Eterm mkatom(char *str)
static char *pick_list_or_atom(Eterm name_term)
{
char *name = NULL;
- int name_len;
+ Uint name_len;
if (is_atom(name_term)) {
Atom *ap = atom_tab(atom_val(name_term));
if (ap->len == 0) {
@@ -1890,8 +1888,7 @@ static char *pick_list_or_atom(Eterm name_term)
memcpy(name,ap->name,ap->len);
name[ap->len] = '\0';
} else {
- name_len = io_list_len(name_term);
- if (name_len <= 0) {
+ if (erts_iolist_size(name_term, &name_len)) {
goto error;
}
name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index e50fc18e64..f264bf44df 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1732,14 +1732,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */
# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML
# endif
#endif
- int buf_size = 8*1024; /* Try with 8KB first */
+ Uint buf_size = 8*1024; /* Try with 8KB first */
char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
if (r < 0) {
erts_free(ERTS_ALC_T_TMP, (void *) buf);
- buf_size = io_list_len(*tp);
- if (buf_size < 0)
+ if (erts_iolist_size(*tp, &buf_size)) {
goto badarg;
+ }
buf_size++;
buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index fbc92b9730..3fd35dd963 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -996,6 +996,7 @@ static byte* convert_environment(Process* p, Eterm env)
Eterm* hp;
Uint heap_size;
int n;
+ Uint size;
byte* bytes;
if ((n = list_length(env)) < 0) {
@@ -1039,15 +1040,15 @@ static byte* convert_environment(Process* p, Eterm env)
if (is_not_nil(env)) {
goto done;
}
- if ((n = io_list_len(all)) < 0) {
+ if (erts_iolist_size(all, &size)) {
goto done;
}
/*
* Put the result in a binary (no risk for a memory leak that way).
*/
- (void) erts_new_heap_binary(p, NULL, n, &bytes);
- io_list_to_buf(all, (char*)bytes, n);
+ (void) erts_new_heap_binary(p, NULL, size, &bytes);
+ io_list_to_buf(all, (char*)bytes, size);
done:
erts_free(ERTS_ALC_T_TMP, temp_heap);
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index d4a8a3aaa7..26891c4348 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -417,7 +417,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
BIF_RETTYPE
re_compile_2(BIF_ALIST_2)
{
- int slen;
+ Uint slen;
char *expr;
pcre *result;
int errcode = 0;
@@ -444,7 +444,7 @@ re_compile_2(BIF_ALIST_2)
BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2);
}
- if ((slen = io_list_len(BIF_ARG_1)) < 0) {
+ if (erts_iolist_size(BIF_ARG_1, &slen)) {
BIF_ERROR(BIF_P,BADARG);
}
expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
@@ -795,8 +795,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
memcpy(tmpb,ap->name,ap->len);
tmpb[ap->len] = '\0';
} else {
- int slen = io_list_len(val);
- if (slen < 0) {
+ Uint slen;
+ if (erts_iolist_size(val, &slen)) {
goto error;
}
if ((slen + 1) > tmpbsiz) {
@@ -851,7 +851,7 @@ re_run_3(BIF_ALIST_3)
const pcre *code_tmp;
RestartContext restart;
byte *temp_alloc = NULL;
- int slength;
+ Uint slength;
int startoffset = 0;
int options = 0, comp_options = 0;
int ovsize;
@@ -875,7 +875,7 @@ re_run_3(BIF_ALIST_3)
if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) {
if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) {
/* Compile from textual RE */
- int slen;
+ Uint slen;
char *expr;
pcre *result;
int errcode = 0;
@@ -889,7 +889,7 @@ re_run_3(BIF_ALIST_3)
BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
}
- if ((slen = io_list_len(BIF_ARG_2)) < 0) {
+ if (erts_iolist_size(BIF_ARG_2, &slen)) {
BIF_ERROR(BIF_P,BADARG);
}
@@ -1027,7 +1027,7 @@ re_run_3(BIF_ALIST_3)
restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY;
} else {
handle_iolist:
- if ((slength = io_list_len(BIF_ARG_1)) < 0) {
+ if (erts_iolist_size(BIF_ARG_1, &slength)) {
erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
if (restart.ret_info != NULL) {
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 694348e31d..fdc82c8b88 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -105,7 +105,7 @@
#define NSEG_2 256 /* Size of second segment table */
#define NSEG_INC 128 /* Number of segments to grow after that */
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab))
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab))
#define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive))
#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
@@ -122,8 +122,8 @@
*/
static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
{
- Uint mask = erts_smp_atomic_read(&tb->szm);
- Uint ix = hval & mask;
+ Uint mask = erts_smp_atomic_read_acqb(&tb->szm);
+ Uint ix = hval & mask;
if (ix >= erts_smp_atomic_read(&tb->nactive)) {
ix &= mask>>1;
ASSERT(ix < erts_smp_atomic_read(&tb->nactive));
@@ -668,6 +668,7 @@ int db_create_hash(Process *p, DbTable *tbl)
else { /* coarse locking */
tb->locks = NULL;
}
+ ERTS_THR_MEMORY_BARRIER;
#endif /* ERST_SMP */
return DB_ERROR_NONE;
}
@@ -2349,7 +2350,7 @@ static int alloc_seg(DbTableHash *tb)
struct ext_segment* eseg;
eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1];
MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment);
- erts_smp_atomic_set(&tb->segtab, (erts_aint_t) eseg->segtab);
+ erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab);
tb->nsegs = eseg->nsegs;
}
ASSERT(seg_ix < tb->nsegs);
@@ -2421,7 +2422,7 @@ static int free_seg(DbTableHash *tb, int free_records)
MY_ASSERT(newtop->s.is_ext_segment);
if (newtop->prev_segtab != NULL) {
/* Time to use a smaller segtab */
- erts_smp_atomic_set(&tb->segtab, (erts_aint_t)newtop->prev_segtab);
+ erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab);
tb->nsegs = seg_ix;
ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs);
}
@@ -2438,7 +2439,7 @@ static int free_seg(DbTableHash *tb, int free_records)
if (seg_ix > 0) {
if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL;
} else {
- erts_smp_atomic_set(&tb->segtab, (erts_aint_t)NULL);
+ erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL);
}
#endif
tb->nslots -= SEGSZ;
@@ -2533,9 +2534,9 @@ static void grow(DbTableHash* tb, int nactive)
}
erts_smp_atomic_inc(&tb->nactive);
if (from_ix == 0) {
- erts_smp_atomic_set(&tb->szm, szm);
+ erts_smp_atomic_set_relb(&tb->szm, szm);
}
- erts_smp_atomic_set(&tb->is_resizing, 0);
+ erts_smp_atomic_set_relb(&tb->is_resizing, 0);
/* Finally, let's split the bucket. We try to do it in a smart way
to keep link order and avoid unnecessary updates of next-pointers */
@@ -2567,7 +2568,7 @@ static void grow(DbTableHash* tb, int nactive)
return;
abort:
- erts_smp_atomic_set(&tb->is_resizing, 0);
+ erts_smp_atomic_set_relb(&tb->is_resizing, 0);
}
@@ -2611,7 +2612,7 @@ static void shrink(DbTableHash* tb, int nactive)
erts_smp_atomic_set(&tb->nactive, src_ix);
if (dst_ix == 0) {
- erts_smp_atomic_set(&tb->szm, low_szm);
+ erts_smp_atomic_set_relb(&tb->szm, low_szm);
}
WUNLOCK_HASH(lck);
@@ -2625,7 +2626,7 @@ static void shrink(DbTableHash* tb, int nactive)
}
/*else already done */
- erts_smp_atomic_set(&tb->is_resizing, 0);
+ erts_smp_atomic_set_relb(&tb->is_resizing, 0);
}
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index eb77d1281a..9a0ba3a418 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -111,7 +111,7 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack)
{
if (stack == &tb->static_stack) {
ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1);
- erts_smp_atomic_set(&tb->is_stack_busy, 0);
+ erts_smp_atomic_set_relb(&tb->is_stack_busy, 0);
}
else {
erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb,
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 39bbe9633b..dc578f6d2a 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -700,6 +700,13 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp)
extern int erts_darwin_main_thread_pipe[2];
extern int erts_darwin_main_thread_result_pipe[2];
+int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp);
+int erl_drv_steal_main_thread(char *name,
+ ErlDrvTid *dtid,
+ void* (*func)(void*),
+ void* arg,
+ ErlDrvThreadOpts *opts);
+
int
erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp)
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8b48444904..68421b4387 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -472,7 +472,7 @@ static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj)
int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
{
struct enif_tmp_obj_t* tobj;
- int sz;
+ Uint sz;
if (is_binary(term)) {
return enif_inspect_binary(env,term,bin);
}
@@ -483,7 +483,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
bin->ref_bin = NULL;
return 1;
}
- if ((sz = io_list_len(term)) < 0) {
+ if (erts_iolist_size(term, &sz)) {
return 0;
}
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 1b07024ca1..326021643f 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -658,8 +658,6 @@ erts_port_task_free_port(Port *pp)
when scheduled out... */
ErtsPortTask *ptp = port_task_alloc();
erts_smp_port_state_lock(pp);
- ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0);
- erts_smp_atomic_dec(&erts_ports_alive);
pp->status &= ~ERTS_PORT_SFLG_CLOSING;
pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
erts_may_save_closed_port(pp);
@@ -681,7 +679,6 @@ erts_port_task_free_port(Port *pp)
port_is_dequeued = 1;
}
erts_smp_port_state_lock(pp);
- erts_smp_atomic_dec(&erts_ports_alive);
pp->status &= ~ERTS_PORT_SFLG_CLOSING;
pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
erts_may_save_closed_port(pp);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 8a56976905..2704359a8f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -166,9 +166,8 @@ static struct {
static struct {
erts_smp_mtx_t update_mtx;
- erts_smp_atomic32_t active_runqs;
+ erts_smp_atomic32_t no_runqs;
int last_active_runqs;
- erts_smp_atomic32_t used_runqs;
int forced_check_balance;
erts_smp_atomic32_t checking_balance;
int halftime;
@@ -965,7 +964,7 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount)
erts_aint32_t flgs;
do {
- flgs = erts_smp_atomic32_read(&ssi->flags);
+ flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
!= (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
break;
@@ -1114,7 +1113,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
else {
erts_aint_t dt;
- erts_smp_atomic32_set(&function_calls, 0);
+ erts_smp_atomic32_set_relb(&function_calls, 0);
*fcalls = 0;
sched_waiting_sys(esdp->no, rq);
@@ -1147,7 +1146,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
nonblockable_aux_work(esdp, ssi, aux_work);
#endif
- flgs = erts_smp_atomic32_read(&ssi->flags);
+ flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
if (!(flgs & ERTS_SSI_FLG_WAITING)) {
ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
goto sys_woken;
@@ -1333,6 +1332,76 @@ wake_all_schedulers(void)
}
}
+#define ERTS_NO_USED_RUNQS_SHIFT 16
+#define ERTS_NO_RUNQS_MASK 0xffff
+
+#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK
+# error "Too large amount of schedulers allowed"
+#endif
+
+static ERTS_INLINE void
+init_no_runqs(int active, int used)
+{
+ erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK);
+ no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT);
+ erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs);
+}
+
+static ERTS_INLINE void
+get_no_runqs(int *active, int *used)
+{
+ erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs);
+ if (active)
+ *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK);
+ if (used)
+ *used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK);
+}
+
+static ERTS_INLINE void
+set_no_used_runqs(int used)
+{
+ erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+ while (1) {
+ erts_aint32_t act, new;
+ new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK);
+ act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+ if (act == exp)
+ break;
+ exp = act;
+ }
+}
+
+static ERTS_INLINE void
+set_no_active_runqs(int active)
+{
+ erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+ while (1) {
+ erts_aint32_t act, new;
+ new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active;
+ act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+ if (act == exp)
+ break;
+ exp = act;
+ }
+}
+
+static ERTS_INLINE int
+try_inc_no_active_runqs(int active)
+{
+ erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+ if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active)
+ return 0;
+ if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) {
+ erts_aint32_t new, act;
+ new = (exp & ~ERTS_NO_RUNQS_MASK) | active;
+ act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+ if (act == exp)
+ return 1;
+ }
+ return 0;
+}
+
+
static ERTS_INLINE int
chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
{
@@ -1344,9 +1413,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
iflgs = erts_smp_atomic32_read(&wrq->info_flags);
if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) {
if (activate) {
- if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs,
- ix+1,
- ix)) {
+ if (try_inc_no_active_runqs(ix+1)) {
erts_smp_xrunq_lock(crq, wrq);
wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE;
erts_smp_xrunq_unlock(crq, wrq);
@@ -1363,8 +1430,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq)
{
int ix = crq->ix;
int stop_ix = ix;
- int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs);
- int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs);
+ int active_ix, balance_ix;
+
+ get_no_runqs(&active_ix, &balance_ix);
if (active_ix > balance_ix)
active_ix = balance_ix;
@@ -1416,7 +1484,7 @@ erts_sched_notify_check_cpu_bind(void)
int ix;
if (erts_common_run_queue) {
for (ix = 0; ix < erts_no_schedulers; ix++)
- erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1);
+ erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1);
wake_all_schedulers();
}
else {
@@ -1871,8 +1939,7 @@ try_steal_task(ErtsRunQueue *rq)
ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked);
- active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs);
- blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs);
+ get_no_runqs(&active_rqs, &blnc_rqs);
if (active_rqs > blnc_rqs)
active_rqs = blnc_rqs;
@@ -1883,7 +1950,7 @@ try_steal_task(ErtsRunQueue *rq)
if (active_rqs < blnc_rqs) {
int no = blnc_rqs - active_rqs;
int stop_ix = vix = active_rqs + rq->ix % no;
- while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) {
+ while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
res = check_possible_steal_victim(rq, &rq_locked, vix);
if (res)
goto done;
@@ -1898,7 +1965,7 @@ try_steal_task(ErtsRunQueue *rq)
vix = rq->ix;
/* ... then try to steal a job from another active queue... */
- while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) {
+ while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
vix++;
if (vix >= active_rqs)
vix = 0;
@@ -1999,7 +2066,7 @@ check_balance(ErtsRunQueue *c_rq)
return;
}
- blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs);
+ get_no_runqs(NULL, &blnc_no_rqs);
if (blnc_no_rqs == 1) {
c_rq->check_balance_reds = INT_MAX;
erts_smp_atomic32_set(&balance_info.checking_balance, 0);
@@ -2038,7 +2105,8 @@ check_balance(ErtsRunQueue *c_rq)
forced = balance_info.forced_check_balance;
balance_info.forced_check_balance = 0;
- blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs);
+ get_no_runqs(&current_active, &blnc_no_rqs);
+
if (blnc_no_rqs == 1) {
erts_smp_mtx_unlock(&balance_info.update_mtx);
erts_smp_runq_lock(c_rq);
@@ -2052,8 +2120,6 @@ check_balance(ErtsRunQueue *c_rq)
if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE)
balance_info.full_reds_history_index = 0;
- current_active = erts_smp_atomic32_read(&balance_info.active_runqs);
-
/* Read balance information for all run queues */
for (qix = 0; qix < blnc_no_rqs; qix++) {
ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
@@ -2387,7 +2453,7 @@ erts_fprintf(stderr, "--------------------------------\n");
}
balance_info.last_active_runqs = active;
- erts_smp_atomic32_set(&balance_info.active_runqs, active);
+ set_no_active_runqs(active);
balance_info.halftime = 1;
erts_smp_atomic32_set(&balance_info.checking_balance, 0);
@@ -2695,9 +2761,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0);
erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers);
schdlr_sspnd.msb.procs = NULL;
- erts_smp_atomic32_set(&balance_info.used_runqs,
- erts_common_run_queue ? 1 : no_schedulers_online);
- erts_smp_atomic32_init(&balance_info.active_runqs, no_schedulers);
+ init_no_runqs(no_schedulers,
+ erts_common_run_queue ? 1 : no_schedulers_online);
balance_info.last_active_runqs = no_schedulers;
erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update");
balance_info.forced_check_balance = 0;
@@ -2939,7 +3004,7 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount)
erts_aint32_t flgs;
do {
- flgs = erts_smp_atomic32_read(&ssi->flags);
+ flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
if ((flgs & (ERTS_SSI_FLG_SLEEPING
| ERTS_SSI_FLG_WAITING
| ERTS_SSI_FLG_SUSPENDED))
@@ -3068,7 +3133,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
wake = 0;
}
- flgs = erts_smp_atomic32_read(&ssi->flags);
+ flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
break;
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
@@ -3292,7 +3357,7 @@ erts_set_schedulers_online(Process *p,
ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no);
evacuate_run_queue(from_rq, to_rq);
}
- erts_smp_atomic32_set(&balance_info.used_runqs, no);
+ set_no_used_runqs(no);
erts_smp_mtx_unlock(&balance_info.update_mtx);
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
}
@@ -3346,7 +3411,7 @@ erts_set_schedulers_online(Process *p,
for (ix = erts_no_run_queues-1; ix >= no; ix--)
evacuate_run_queue(ERTS_RUNQ_IX(ix),
ERTS_RUNQ_IX(ix % no));
- erts_smp_atomic32_set(&balance_info.used_runqs, no);
+ set_no_used_runqs(no);
erts_smp_mtx_unlock(&balance_info.update_mtx);
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
for (ix = no; ix < online; ix++) {
@@ -3443,7 +3508,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
else {
erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
erts_smp_mtx_lock(&balance_info.update_mtx);
- erts_smp_atomic32_set(&balance_info.used_runqs, 1);
+ set_no_used_runqs(1);
for (ix = 0; ix < online; ix++) {
ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
erts_smp_runq_lock(rq);
@@ -3580,7 +3645,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
evacuate_run_queue(ERTS_RUNQ_IX(ix),
ERTS_RUNQ_IX(ix % online));
- erts_smp_atomic32_set(&balance_info.used_runqs, online);
+ set_no_used_runqs(online);
/* Make sure that we balance soon... */
balance_info.forced_check_balance = 1;
erts_smp_runq_lock(ERTS_RUNQ_IX(0));
@@ -5131,7 +5196,7 @@ Process *schedule(Process *p, int calls)
esdp = erts_get_scheduler_data();
rq = erts_get_runq_current(esdp);
ASSERT(esdp);
- fcalls = (int) erts_smp_atomic32_read(&function_calls);
+ fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
actual_reds = reds = 0;
erts_smp_runq_lock(rq);
} else {
@@ -5282,14 +5347,14 @@ Process *schedule(Process *p, int calls)
| ERTS_RUNQ_FLG_CHK_CPU_BIND
| ERTS_RUNQ_FLG_SUSPENDED)) {
if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
- || (erts_smp_atomic32_read(&esdp->ssi->flags)
+ || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED)) {
ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED);
suspend_scheduler(esdp);
}
if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND)
- || erts_smp_atomic32_read(&esdp->chk_cpu_bind)) {
+ || erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) {
erts_sched_check_cpu_bind(esdp);
}
}
@@ -5340,7 +5405,7 @@ Process *schedule(Process *p, int calls)
if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ
| ERTS_RUNQ_FLG_SUSPENDED)) {
if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
- || (erts_smp_atomic32_read(&esdp->ssi->flags)
+ || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED)) {
ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED);
@@ -5384,7 +5449,7 @@ Process *schedule(Process *p, int calls)
* Schedule system-level activities.
*/
- erts_smp_atomic32_set(&function_calls, 0);
+ erts_smp_atomic32_set_relb(&function_calls, 0);
fcalls = 0;
ASSERT(!erts_port_task_have_outstanding_io_tasks());
@@ -5426,7 +5491,7 @@ Process *schedule(Process *p, int calls)
if (erts_common_run_queue->waiting)
wake_scheduler(erts_common_run_queue, 0, 1);
}
- else if (erts_smp_atomic32_read(&no_empty_run_queues) != 0) {
+ else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
wake_scheduler_on_empty_runq(rq);
rq->wakeup_other = 0;
}
@@ -7637,7 +7702,8 @@ continue_exit_process(Process *p
static void
timeout_proc(Process* p)
{
- p->i = *((BeamInstr **) (UWord) p->def_arg_reg);
+ BeamInstr** pi = (BeamInstr **) p->def_arg_reg;
+ p->i = *pi;
p->flags |= F_TIMO;
p->flags &= ~F_INSLPQUEUE;
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index c0397ca6c3..8833137112 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1538,8 +1538,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
Eterm tracee;
#endif
Eterm transformed_args[MAX_ARG];
- DeclareTmpHeap(sub_bin_heap_et,ERL_SUB_BIN_SIZE,p);
- ErlSubBin *sub_bin_heap = (ErlSubBin *) sub_bin_heap_et;
+ DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p);
ASSERT(tracer_pid);
if (*tracer_pid == am_true) {
@@ -1600,21 +1599,20 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) {
ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg);
ErlBinMatchBuffer* mb = &ms->mb;
- ErlSubBin* sb = sub_bin_heap;
Uint bit_size;
ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */
bit_size = mb->size - mb->offset;
- sb->thing_word = HEADER_SUB_BIN;
- sb->size = BYTE_OFFSET(bit_size);
- sb->bitsize = BIT_OFFSET(bit_size);
- sb->offs = BYTE_OFFSET(mb->offset);
- sb->bitoffs = BIT_OFFSET(mb->offset);
- sb->is_writable = 0;
- sb->orig = mb->orig;
-
- arg = make_binary(sb);
+ sub_bin_heap->thing_word = HEADER_SUB_BIN;
+ sub_bin_heap->size = BYTE_OFFSET(bit_size);
+ sub_bin_heap->bitsize = BIT_OFFSET(bit_size);
+ sub_bin_heap->offs = BYTE_OFFSET(mb->offset);
+ sub_bin_heap->bitoffs = BIT_OFFSET(mb->offset);
+ sub_bin_heap->is_writable = 0;
+ sub_bin_heap->orig = mb->orig;
+
+ arg = make_binary(sub_bin_heap);
}
transformed_args[i] = arg;
}
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 96da894d90..499bdd77ba 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -183,7 +183,7 @@ struct port {
process to get (line oriented I/O)*/
Uint32 status; /* Status and type flags */
int control_flags; /* Flags for port_control() */
- Uint32 snapshot; /* Next snapshot that port should be part of */
+ erts_aint32_t snapshot; /* Next snapshot that port should be part of */
struct reg_proc *reg;
ErlDrvPDL port_data_lock;
@@ -527,11 +527,10 @@ union erl_off_heap_ptr {
/* arrays that get malloced at startup */
extern Port* erts_port;
-extern erts_smp_atomic_t erts_ports_alive;
extern Uint erts_max_ports;
extern Uint erts_port_tab_index_mask;
-extern erts_smp_atomic_t erts_ports_snapshot;
+extern erts_smp_atomic32_t erts_ports_snapshot;
extern erts_smp_atomic_t erts_dead_ports_ptr;
ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
@@ -541,12 +540,12 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt)
{
ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck));
- if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) {
+ if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) {
/* Dead ports are added from the end of the snapshot buffer */
Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr,
-(erts_aint_t)sizeof(Eterm));
ASSERT(tombstone+1 != NULL);
- ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1);
+ ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1);
*tombstone = prt->id;
}
/*else no ongoing snapshot or port was already included or created after snapshot */
@@ -1653,10 +1652,14 @@ struct Sint_buf {
};
char* Sint_to_buf(Sint, struct Sint_buf*);
+#define ERTS_IOLIST_OK 0
+#define ERTS_IOLIST_OVERFLOW 1
+#define ERTS_IOLIST_TYPE 2
+
Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */
int io_list_to_buf(Eterm, char*, int);
int io_list_to_buf2(Eterm, char*, int);
-int io_list_len(Eterm);
+int erts_iolist_size(Eterm, Uint *);
int is_string(Eterm);
void erl_at_exit(void (*) (void*), void*);
Eterm collect_memory(Process *);
@@ -1890,6 +1893,8 @@ erts_alloc_message_heap(Uint size,
# if defined(DEBUG)
# define DeclareTmpHeap(VariableName,Size,Process) \
Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process)
+# define DeclareTypedTmpHeap(Type,VariableName,Process) \
+ Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process)
# define DeclareTmpHeapNoproc(VariableName,Size) \
Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL)
# define UseTmpHeap(Size,Proc) \
@@ -1911,6 +1916,8 @@ erts_alloc_message_heap(Uint size,
# else
# define DeclareTmpHeap(VariableName,Size,Process) \
Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
+# define DeclareTypedTmpHeap(Type,VariableName,Process) \
+ Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
# define DeclareTmpHeapNoproc(VariableName,Size) \
Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used)
# define UseTmpHeap(Size,Proc) \
@@ -1936,6 +1943,8 @@ erts_alloc_message_heap(Uint size,
#else
# define DeclareTmpHeap(VariableName,Size,Process) \
Eterm VariableName[Size]
+# define DeclareTypedTmpHeap(Type,VariableName,Process) \
+ Type VariableName[1]
# define DeclareTmpHeapNoproc(VariableName,Size) \
Eterm VariableName[Size]
# define UseTmpHeap(Size,Proc) /* Nothing */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index f619c6f88b..df5f8b22a3 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -56,7 +56,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o
per thread basis (for BC interfaces) */
Port* erts_port; /* The port table */
-erts_smp_atomic_t erts_ports_alive;
erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */
erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */
@@ -82,6 +81,9 @@ static void driver_monitor_unlock_pdl(Port *p);
#define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */
#endif
+#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
+#define SMALL_WRITE_VEC 16
+
static ERTS_INLINE ErlIOQueue*
drvport2ioq(ErlDrvPort drvport)
{
@@ -190,7 +192,7 @@ typedef struct line_buf_context {
static erts_smp_spinlock_t get_free_port_lck;
static Uint last_port_num;
static Uint port_num_mask;
-erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
+erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
static ERTS_INLINE void
@@ -421,10 +423,9 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1);
sys_strcpy(new_name, name);
erts_smp_runq_lock(runq);
- erts_smp_atomic_inc(&erts_ports_alive);
erts_smp_port_state_lock(prt);
prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus;
- prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot);
+ prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot);
old_name = prt->name;
prt->name = new_name;
#ifdef ERTS_SMP
@@ -954,13 +955,14 @@ do { \
int _bitoffs; \
int _bitsize; \
ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \
- ASSERT(_bitsize == 0); \
+ if (_bitsize != 0) goto L_type_error; \
if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \
_bitoffs == 0) { \
b_size += _size; \
+ if (b_size < _size) goto L_overflow_error; \
in_clist = 0; \
v_size++; \
- if (_size >= bin_limit) { \
+ if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \
p_in_clist = 0; \
p_v_size++; \
} else { \
@@ -972,6 +974,7 @@ do { \
} \
} else { \
c_size += _size; \
+ if (c_size < _size) goto L_overflow_error; \
if (!in_clist) { \
in_clist = 1; \
v_size++; \
@@ -986,29 +989,30 @@ do { \
/*
-** Size of a io list in bytes
-** return -1 if error
-** returns: - Total size of io list
-** vsize - SysIOVec size needed for a writev
-** csize - Number of bytes not in binary (in the common binary)
-** pvsize - SysIOVec size needed if packing small binaries
-** pcsize - Number of bytes in the common binary if packing
-*/
+ * Returns 0 if successful and a non-zero value otherwise.
+ *
+ * Return values through pointers:
+ * *vsize - SysIOVec size needed for a writev
+ * *csize - Number of bytes not in binary (in the common binary)
+ * *pvsize - SysIOVec size needed if packing small binaries
+ * *pcsize - Number of bytes in the common binary if packing
+ * *total_size - Total size of iolist in bytes
+ */
static int
-io_list_vec_len(Eterm obj, int* vsize, int* csize,
- int bin_limit, /* small binaries limit */
- int * pvsize, int * pcsize)
+io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
+ Uint* pvsize, Uint* pcsize, Uint* total_size)
{
DECLARE_ESTACK(s);
Eterm* objp;
- int v_size = 0;
- int c_size = 0;
- int b_size = 0;
- int in_clist = 0;
- int p_v_size = 0;
- int p_c_size = 0;
- int p_in_clist = 0;
+ Uint v_size = 0;
+ Uint c_size = 0;
+ Uint b_size = 0;
+ Uint in_clist = 0;
+ Uint p_v_size = 0;
+ Uint p_c_size = 0;
+ Uint p_in_clist = 0;
+ Uint total;
goto L_jump_start; /* avoid a push */
@@ -1022,6 +1026,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
if (is_byte(obj)) {
c_size++;
+ if (c_size == 0) {
+ goto L_overflow_error;
+ }
if (!in_clist) {
in_clist = 1;
v_size++;
@@ -1061,32 +1068,31 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
}
}
+ total = c_size + b_size;
+ if (total < c_size) {
+ goto L_overflow_error;
+ }
+ *total_size = total;
+
DESTROY_ESTACK(s);
- if (vsize != NULL)
- *vsize = v_size;
- if (csize != NULL)
- *csize = c_size;
- if (pvsize != NULL)
- *pvsize = p_v_size;
- if (pcsize != NULL)
- *pcsize = p_c_size;
- return c_size + b_size;
+ *vsize = v_size;
+ *csize = c_size;
+ *pvsize = p_v_size;
+ *pcsize = p_c_size;
+ return 0;
L_type_error:
+ L_overflow_error:
DESTROY_ESTACK(s);
- return -1;
+ return 1;
}
-#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
-#define SMALL_WRITE_VEC 16
-
-
/* write data to a port */
int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
{
char *buf;
erts_driver_t *drv = p->drv_ptr;
- int size;
+ Uint size;
int fpe_was_unmasked;
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
@@ -1094,10 +1100,10 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
p->caller = caller_id;
if (drv->outputv != NULL) {
- int vsize;
- int csize;
- int pvsize;
- int pcsize;
+ Uint vsize;
+ Uint csize;
+ Uint pvsize;
+ Uint pcsize;
int blimit;
SysIOVec iv[SMALL_WRITE_VEC];
ErlDrvBinary* bv[SMALL_WRITE_VEC];
@@ -1106,9 +1112,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
ErlDrvBinary* cbin;
ErlIOVec ev;
- if ((size = io_list_vec_len(list, &vsize, &csize,
- ERL_SMALL_IO_BIN_LIMIT,
- &pvsize, &pcsize)) < 0) {
+ if (io_list_vec_len(list, &vsize, &csize,
+ &pvsize, &pcsize, &size)) {
goto bad_value;
}
/* To pack or not to pack (small binaries) ...? */
@@ -1183,7 +1188,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
else {
ASSERT(r == -1); /* Overflow */
erts_free(ERTS_ALC_T_TMP, buf);
- if ((size = io_list_len(list)) < 0) {
+ if (erts_iolist_size(list, &size)) {
goto bad_value;
}
@@ -1274,7 +1279,6 @@ void init_io(void)
erts_smp_atomic_init(&erts_bytes_out, 0);
erts_smp_atomic_init(&erts_bytes_in, 0);
- erts_smp_atomic_init(&erts_ports_alive, 0);
for (i = 0; i < erts_max_ports; i++) {
erts_port_task_init_sched(&erts_port[i].sched);
@@ -1296,7 +1300,7 @@ void init_io(void)
erts_port[i].port_data_lock = NULL;
}
- erts_smp_atomic_init(&erts_ports_snapshot, (erts_aint_t) 0);
+ erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0);
last_port_num = 0;
erts_smp_spinlock_init(&get_free_port_lck, "get_free_port");
@@ -2147,7 +2151,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
byte* to_port = NULL; /* Buffer to write to port. */
/* Initialization is for shutting up
warning about use before set. */
- int to_len = 0; /* Length of buffer. */
+ Uint to_len = 0; /* Length of buffer. */
int must_free = 0; /* True if the buffer should be freed. */
char port_result[ERL_ONHEAP_BIN_LIMIT]; /* Default buffer for result from port. */
char* port_resp; /* Pointer to result buffer. */
@@ -2192,7 +2196,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
} else {
ASSERT(r == -1); /* Overflow */
erts_free(ERTS_ALC_T_TMP, (void *) to_port);
- if ((to_len = io_list_len(iolist)) < 0) { /* Type error */
+ if (erts_iolist_size(iolist, &to_len)) { /* Type error */
return THE_NON_VALUE;
}
must_free = 1;
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 21d6ce9304..3e9243c77d 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit)
erts_free(h->type, (void *) old_tab);
}
/*else already done */
- erts_smp_atomic_set(&h->is_rehashing, 0);
+ erts_smp_atomic_set_relb(&h->is_rehashing, 0);
}
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 6b4f3b3b36..a17de717bc 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3021,13 +3021,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len)
return -1;
}
-int io_list_len(Eterm obj)
+/*
+ * Return 0 if successful, and non-zero if unsuccessful.
+ */
+int erts_iolist_size(Eterm obj, Uint* sizep)
{
Eterm* objp;
- Sint len = 0;
+ Uint size = 0;
DECLARE_ESTACK(s);
goto L_again;
+#define SAFE_ADD(Var, Val) \
+ do { \
+ Uint valvar = (Val); \
+ Var += valvar; \
+ if (Var < valvar) { \
+ goto L_overflow_error; \
+ } \
+ } while (0)
+
while (!ESTACK_ISEMPTY(s)) {
obj = ESTACK_POP(s);
L_again:
@@ -3037,9 +3049,12 @@ int io_list_len(Eterm obj)
/* Head */
obj = CAR(objp);
if (is_byte(obj)) {
- len++;
+ size++;
+ if (size == 0) {
+ goto L_overflow_error;
+ }
} else if (is_binary(obj) && binary_bitsize(obj) == 0) {
- len += binary_size(obj);
+ SAFE_ADD(size, binary_size(obj));
} else if (is_list(obj)) {
ESTACK_PUSH(s, CDR(objp));
goto L_iter_list; /* on head */
@@ -3051,23 +3066,29 @@ int io_list_len(Eterm obj)
if (is_list(obj))
goto L_iter_list; /* on tail */
else if (is_binary(obj) && binary_bitsize(obj) == 0) {
- len += binary_size(obj);
+ SAFE_ADD(size, binary_size(obj));
} else if (is_not_nil(obj)) {
goto L_type_error;
}
} else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
- len += binary_size(obj);
+ SAFE_ADD(size, binary_size(obj));
} else if (is_not_nil(obj)) {
goto L_type_error;
}
}
+#undef SAFE_ADD
DESTROY_ESTACK(s);
- return len;
+ *sizep = size;
+ return ERTS_IOLIST_OK;
+
+ L_overflow_error:
+ DESTROY_ESTACK(s);
+ return ERTS_IOLIST_OVERFLOW;
L_type_error:
DESTROY_ESTACK(s);
- return -1;
+ return ERTS_IOLIST_TYPE;
}
/* return 0 if item is not a non-empty flat list of bytes */
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-01 21:01:13 +0000