aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/utils.c
diff options
context:
space:
mode:
authorSverker Eriksson <[email protected]>2017-08-30 20:55:08 +0200
committerSverker Eriksson <[email protected]>2017-08-30 20:55:08 +0200
commit7c67bbddb53c364086f66260701bc54a61c9659c (patch)
tree92ab0d4b91d5e2f6e7a3f9d61ea25089e8a71fe0 /erts/emulator/beam/utils.c
parent97dc5e7f396129222419811c173edc7fa767b0f8 (diff)
parent3b7a6ffddc819bf305353a593904cea9e932e7dc (diff)
downloadotp-7c67bbddb53c364086f66260701bc54a61c9659c.tar.gz
otp-7c67bbddb53c364086f66260701bc54a61c9659c.tar.bz2
otp-7c67bbddb53c364086f66260701bc54a61c9659c.zip
Merge tag 'OTP-19.0' into sverker/19/binary_to_atom-utf8-crash/ERL-474/OTP-14590
Diffstat (limited to 'erts/emulator/beam/utils.c')
-rw-r--r--erts/emulator/beam/utils.c2714
1 files changed, 1999 insertions, 715 deletions
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 62caa67ce1..675fafa726 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,18 +1,19 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2016. All Rights Reserved.
*
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*
* %CopyrightEnd%
*/
@@ -31,6 +32,7 @@
#include "bif.h"
#include "erl_binary.h"
#include "erl_bits.h"
+#include "erl_map.h"
#include "packet_parser.h"
#include "erl_gc.h"
#define ERTS_WANT_DB_INTERNAL__
@@ -47,6 +49,13 @@
#include "erl_sched_spec_pre_alloc.h"
#include "beam_bp.h"
#include "erl_ptab.h"
+#include "erl_check_io.h"
+#include "erl_bif_unique.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
+#ifdef HIPE
+# include "hipe_mode_switch.h"
+#endif
#undef M_TRIM_THRESHOLD
#undef M_TOP_PAD
@@ -62,46 +71,10 @@
#define HAVE_MALLOPT 0
#endif
-/* profile_scheduler mini message queue */
-
-typedef struct {
- Uint scheduler_id;
- Uint no_schedulers;
- Uint Ms;
- Uint s;
- Uint us;
- Eterm state;
-} profile_sched_msg;
-
-typedef struct {
- profile_sched_msg msg[2];
- Uint n;
-} profile_sched_msg_q;
-
-#ifdef ERTS_SMP
-
-#if 0 /* Unused */
-static void
-dispatch_profile_msg_q(profile_sched_msg_q *psmq)
-{
- int i = 0;
- profile_sched_msg *msg = NULL;
- ASSERT(psmq != NULL);
- for (i = 0; i < psmq->n; i++) {
- msg = &(psmq->msg[i]);
- profile_scheduler_q(make_small(msg->scheduler_id), msg->state, am_undefined, msg->Ms, msg->s, msg->us);
- }
-}
-#endif
-
-#endif
-
-
Eterm*
erts_heap_alloc(Process* p, Uint need, Uint xtra)
{
ErlHeapFragment* bp;
- Eterm* htop;
Uint n;
#if defined(DEBUG) || defined(CHECK_FOR_HOLES)
Uint i;
@@ -147,16 +120,6 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra)
n--;
#endif
- /*
- * When we have created a heap fragment, we are no longer allowed
- * to store anything more on the heap.
- */
- htop = HEAP_TOP(p);
- if (htop < HEAP_LIMIT(p)) {
- *htop = make_pos_bignum_header(HEAP_LIMIT(p)-htop-1);
- HEAP_TOP(p) = HEAP_LIMIT(p);
- }
-
bp->next = MBUF(p);
MBUF(p) = bp;
bp->alloc_size = n;
@@ -185,39 +148,104 @@ erts_set_hole_marker(Eterm* ptr, Uint sz)
* Helper function for the ESTACK macros defined in global.h.
*/
void
-erl_grow_stack(ErtsAlcType_t a_type, Eterm** start, Eterm** sp, Eterm** end)
+erl_grow_estack(ErtsEStack* s, Uint need)
{
- Uint old_size = (*end - *start);
- Uint new_size = old_size * 2;
- Uint sp_offs = *sp - *start;
- if (new_size > 2 * DEF_ESTACK_SIZE) {
- *start = erts_realloc(a_type, (void *) *start, new_size*sizeof(Eterm));
+ Uint old_size = (s->end - s->start);
+ Uint new_size;
+ Uint sp_offs = s->sp - s->start;
+
+ if (need < old_size)
+ new_size = 2*old_size;
+ else
+ new_size = ((need / old_size) + 2) * old_size;
+
+ if (s->start != s->edefault) {
+ s->start = erts_realloc(s->alloc_type, s->start,
+ new_size*sizeof(Eterm));
} else {
- Eterm* new_ptr = erts_alloc(a_type, new_size*sizeof(Eterm));
- sys_memcpy(new_ptr, *start, old_size*sizeof(Eterm));
- *start = new_ptr;
+ Eterm* new_ptr = erts_alloc(s->alloc_type, new_size*sizeof(Eterm));
+ sys_memcpy(new_ptr, s->start, old_size*sizeof(Eterm));
+ s->start = new_ptr;
}
- *end = *start + new_size;
- *sp = *start + sp_offs;
+ s->end = s->start + new_size;
+ s->sp = s->start + sp_offs;
}
/*
- * Helper function for the ESTACK macros defined in global.h.
+ * Helper function for the WSTACK macros defined in global.h.
*/
void
-erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end)
+erl_grow_wstack(ErtsWStack* s, Uint need)
{
- Uint old_size = (*end - *start);
- Uint new_size = old_size * 2;
- Uint sp_offs = *sp - *start;
- if (new_size > 2 * DEF_ESTACK_SIZE) {
- *start = erts_realloc(a_type, (void *) *start, new_size*sizeof(UWord));
+ Uint old_size = (s->wend - s->wstart);
+ Uint new_size;
+ Uint sp_offs = s->wsp - s->wstart;
+
+ if (need < old_size)
+ new_size = 2 * old_size;
+ else
+ new_size = ((need / old_size) + 2) * old_size;
+
+ if (s->wstart != s->wdefault) {
+ s->wstart = erts_realloc(s->alloc_type, s->wstart,
+ new_size*sizeof(UWord));
} else {
- UWord* new_ptr = erts_alloc(a_type, new_size*sizeof(UWord));
- sys_memcpy(new_ptr, *start, old_size*sizeof(UWord));
- *start = new_ptr;
+ UWord* new_ptr = erts_alloc(s->alloc_type, new_size*sizeof(UWord));
+ sys_memcpy(new_ptr, s->wstart, old_size*sizeof(UWord));
+ s->wstart = new_ptr;
}
- *end = *start + new_size;
- *sp = *start + sp_offs;
+ s->wend = s->wstart + new_size;
+ s->wsp = s->wstart + sp_offs;
+}
+
+/*
+ * Helper function for the PSTACK macros defined in global.h.
+ */
+void
+erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes)
+{
+ Uint old_size = s->pend - s->pstart;
+ Uint new_size;
+ Uint sp_offs = s->psp - s->pstart;
+
+ if (need_bytes < old_size)
+ new_size = 2 * old_size;
+ else
+ new_size = ((need_bytes / old_size) + 2) * old_size;
+
+ if (s->pstart != default_pstack) {
+ s->pstart = erts_realloc(s->alloc_type, s->pstart, new_size);
+ } else {
+ byte* new_ptr = erts_alloc(s->alloc_type, new_size);
+ sys_memcpy(new_ptr, s->pstart, old_size);
+ s->pstart = new_ptr;
+ }
+ s->pend = s->pstart + new_size;
+ s->psp = s->pstart + sp_offs;
+}
+
+/*
+ * Helper function for the EQUEUE macros defined in global.h.
+ */
+
+void
+erl_grow_equeue(ErtsEQueue* q, Eterm* default_equeue)
+{
+ Uint old_size = (q->end - q->start);
+ Uint new_size = old_size * 2;
+ Uint first_part = (q->end - q->front);
+ Uint second_part = (q->back - q->start);
+ Eterm* new_ptr = erts_alloc(q->alloc_type, new_size*sizeof(Eterm));
+ ASSERT(q->back == q->front); // of course the queue is full now!
+ if (first_part > 0)
+ sys_memcpy(new_ptr, q->front, first_part*sizeof(Eterm));
+ if (second_part > 0)
+ sys_memcpy(new_ptr+first_part, q->start, second_part*sizeof(Eterm));
+ if (q->start != default_equeue)
+ erts_free(q->alloc_type, q->start);
+ q->start = new_ptr;
+ q->end = q->start + new_size;
+ q->front = q->start;
+ q->back = q->start + old_size;
}
/* CTYPE macros */
@@ -254,10 +282,10 @@ erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end)
* Calculate length of a list.
* Returns -1 if not a proper list (i.e. not terminated with NIL)
*/
-int
-list_length(Eterm list)
+Sint
+erts_list_length(Eterm list)
{
- int i = 0;
+ Sint i = 0;
while(is_list(list)) {
i++;
@@ -307,6 +335,17 @@ int erts_fit_in_bits_int32(Sint32 value)
return fit_in_bits((Sint64) (Uint32) value, 4);
}
+int erts_fit_in_bits_uint(Uint value)
+{
+#if ERTS_SIZEOF_ETERM == 4
+ return fit_in_bits((Sint64) (Uint32) value, 4);
+#elif ERTS_SIZEOF_ETERM == 8
+ return fit_in_bits(value, 5);
+#else
+# error "No way, Jose"
+#endif
+}
+
int
erts_print(int to, void *arg, char *format, ...)
{
@@ -339,9 +378,6 @@ erts_print(int to, void *arg, char *format, ...)
case ERTS_PRINT_DSBUF:
res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list);
break;
- case ERTS_PRINT_INVALID:
- res = -EINVAL;
- break;
default:
res = erts_vfdprintf((int) to, format, arg_list);
break;
@@ -576,8 +612,8 @@ erts_bld_2tup_list(Uint **hpp, Uint *szp,
}
Eterm
-erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp,
- Sint length, Eterm atoms[], Uint uints[])
+erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp,
+ Sint length, Eterm atoms[], UWord uints[])
{
Sint i;
Eterm res = THE_NON_VALUE;
@@ -703,7 +739,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
** If N < 0, Y = FUNNY_NUMBER4 else Y = FUNNY_NUMBER3.
** The hash value is Y*h(J) mod 2^32 where h(J) is calculated like
** h(0) = <initial hash>
-** h(i) = h(i-i)*X + B(i-1)
+** h(i) = h(i-1)*X + B(i-1)
** The above should hold regardless of internal representation.
** Pids are hashed like small numbers but with differrent constants, as are
** ports.
@@ -732,6 +768,8 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
#define FUNNY_NUMBER10 268440479
#define FUNNY_NUMBER11 268440577
#define FUNNY_NUMBER12 268440581
+#define FUNNY_NUMBER13 268440593
+#define FUNNY_NUMBER14 268440611
static Uint32
hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash)
@@ -782,11 +820,10 @@ Uint32 make_hash(Eterm term_arg)
Eterm hash = 0;
unsigned op;
- /* Must not collide with the real tag_val_def's: */
-#define MAKE_HASH_TUPLE_OP 0x10
-#define MAKE_HASH_FUN_OP 0x11
-#define MAKE_HASH_CDR_PRE_OP 0x12
-#define MAKE_HASH_CDR_POST_OP 0x13
+#define MAKE_HASH_TUPLE_OP (FIRST_VACANT_TAG_DEF)
+#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1)
+#define MAKE_HASH_CDR_PRE_OP (FIRST_VACANT_TAG_DEF+2)
+#define MAKE_HASH_CDR_POST_OP (FIRST_VACANT_TAG_DEF+3)
/*
** Convenience macro for calculating a bytewise hash on an unsigned 32 bit
@@ -836,7 +873,7 @@ tail_recur:
Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
UINT32_HASH_STEP(y2, FUNNY_NUMBER2);
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
if (y2 >> 32)
UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2);
#endif
@@ -875,7 +912,7 @@ tail_recur:
hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
if (num_free > 0) {
if (num_free > 1) {
- WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+ WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP);
}
term = funp->env[0];
goto tail_recur;
@@ -896,12 +933,15 @@ tail_recur:
UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10);
case FLOAT_DEF:
{
- FloatDef ff;
- GET_DOUBLE(term, ff);
- hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
- break;
+ FloatDef ff;
+ GET_DOUBLE(term, ff);
+ if (ff.fd == 0.0f) {
+ /* ensure positive 0.0 */
+ ff.fd = erts_get_positive_zero_float();
+ }
+ hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
+ break;
}
-
case MAKE_HASH_CDR_PRE_OP:
term = (Eterm) WSTACK_POP(stack);
if (is_not_list(term)) {
@@ -954,7 +994,7 @@ tail_recur:
}
d = BIG_DIGIT(ptr, k);
k = sizeof(ErtsDigit);
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
if (!(d >> 32))
k /= 2;
#endif
@@ -965,6 +1005,9 @@ tail_recur:
hash *= is_neg ? FUNNY_NUMBER4 : FUNNY_NUMBER3;
break;
}
+ case MAP_DEF:
+ hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
+ break;
case TUPLE_DEF:
{
Eterm* ptr = tuple_val(term);
@@ -974,7 +1017,7 @@ tail_recur:
op = MAKE_HASH_TUPLE_OP;
}/*fall through*/
case MAKE_HASH_TUPLE_OP:
- case MAKE_HASH_FUN_OP:
+ case MAKE_HASH_TERM_ARRAY_OP:
{
Uint i = (Uint) WSTACK_POP(stack);
Eterm* ptr = (Eterm*) WSTACK_POP(stack);
@@ -991,7 +1034,7 @@ tail_recur:
}
default:
- erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
return 0;
}
if (WSTACK_ISEMPTY(stack)) break;
@@ -1068,9 +1111,12 @@ Uint32
make_hash2(Eterm term)
{
Uint32 hash;
+ Uint32 hash_xor_pairs;
DeclareTmpHeapNoproc(tmp_big,2);
-/* (HCONST * {2, ..., 14}) mod 2^32 */
+ ERTS_UNDEF(hash_xor_pairs, 0);
+
+/* (HCONST * {2, ..., 22}) mod 2^32 */
#define HCONST_2 0x3c6ef372UL
#define HCONST_3 0xdaa66d2bUL
#define HCONST_4 0x78dde6e4UL
@@ -1085,6 +1131,17 @@ make_hash2(Eterm term)
#define HCONST_13 0x08d12e65UL
#define HCONST_14 0xa708a81eUL
#define HCONST_15 0x454021d7UL
+#define HCONST_16 0xe3779b90UL
+#define HCONST_17 0x81af1549UL
+#define HCONST_18 0x1fe68f02UL
+#define HCONST_19 0xbe1e08bbUL
+#define HCONST_20 0x5c558274UL
+#define HCONST_21 0xfa8cfc2dUL
+#define HCONST_22 0x98c475e6UL
+
+#define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF))
+#define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF))
+#define HASH_CDR (_make_header(3,_TAG_HEADER_REF))
#define UINT32_HASH_2(Expr1, Expr2, AConst) \
do { \
@@ -1107,6 +1164,13 @@ make_hash2(Eterm term)
} while(0)
#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
+
+#ifdef ARCH_64
+# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst)
+#else
+# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst)
+#endif
+
/* Optimization. Simple cases before declaration of estack. */
if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
switch (term & _TAG_IMMED1_MASK) {
@@ -1161,9 +1225,9 @@ make_hash2(Eterm term)
if (c > 0)
UINT32_HASH(sh, HCONST_4);
if (is_list(term)) {
- term = *ptr;
- tmp = *++ptr;
- ESTACK_PUSH(s, tmp);
+ tmp = CDR(ptr);
+ ESTACK_PUSH(s, tmp);
+ term = CAR(ptr);
}
}
break;
@@ -1180,17 +1244,90 @@ make_hash2(Eterm term)
UINT32_HASH(arity, HCONST_9);
if (arity == 0) /* Empty tuple */
goto hash2_common;
- for (i = arity; i >= 2; i--) {
- tmp = elem[i];
- ESTACK_PUSH(s, tmp);
+ for (i = arity; ; i--) {
+ term = elem[i];
+ if (i == 1)
+ break;
+ ESTACK_PUSH(s, term);
}
- term = elem[1];
}
break;
+ case MAP_SUBTAG:
+ {
+ Eterm* ptr = boxed_val(term) + 1;
+ Uint size;
+ int i;
+ switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+ case HAMT_SUBTAG_HEAD_FLATMAP:
+ {
+ flatmap_t *mp = (flatmap_t *)flatmap_val(term);
+ Eterm *ks = flatmap_get_keys(mp);
+ Eterm *vs = flatmap_get_values(mp);
+ size = flatmap_get_size(mp);
+ UINT32_HASH(size, HCONST_16);
+ if (size == 0)
+ goto hash2_common;
+
+ /* We want a portable hash function that is *independent* of
+ * the order in which keys and values are encountered.
+ * We therefore calculate context independent hashes for all .
+ * key-value pairs and then xor them together.
+ */
+ ESTACK_PUSH(s, hash_xor_pairs);
+ ESTACK_PUSH(s, hash);
+ ESTACK_PUSH(s, HASH_MAP_TAIL);
+ hash = 0;
+ hash_xor_pairs = 0;
+ for (i = size - 1; i >= 0; i--) {
+ ESTACK_PUSH(s, HASH_MAP_PAIR);
+ ESTACK_PUSH(s, vs[i]);
+ ESTACK_PUSH(s, ks[i]);
+ }
+ goto hash2_common;
+ }
+
+ case HAMT_SUBTAG_HEAD_ARRAY:
+ case HAMT_SUBTAG_HEAD_BITMAP:
+ size = *ptr++;
+ UINT32_HASH(size, HCONST_16);
+ if (size == 0)
+ goto hash2_common;
+ ESTACK_PUSH(s, hash_xor_pairs);
+ ESTACK_PUSH(s, hash);
+ ESTACK_PUSH(s, HASH_MAP_TAIL);
+ hash = 0;
+ hash_xor_pairs = 0;
+ }
+ switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+ case HAMT_SUBTAG_HEAD_ARRAY:
+ i = 16;
+ break;
+ case HAMT_SUBTAG_HEAD_BITMAP:
+ case HAMT_SUBTAG_NODE_BITMAP:
+ i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ break;
+ default:
+ erts_exit(ERTS_ERROR_EXIT, "bad header");
+ }
+ while (i) {
+ if (is_list(*ptr)) {
+ Eterm* cons = list_val(*ptr);
+ ESTACK_PUSH(s, HASH_MAP_PAIR);
+ ESTACK_PUSH(s, CDR(cons));
+ ESTACK_PUSH(s, CAR(cons));
+ }
+ else {
+ ASSERT(is_boxed(*ptr));
+ ESTACK_PUSH(s, *ptr);
+ }
+ i--; ptr++;
+ }
+ goto hash2_common;
+ }
+ break;
case EXPORT_SUBTAG:
{
Export* ep = *((Export **) (export_val(term) + 1));
-
UINT32_HASH_2
(ep->code[2],
atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue,
@@ -1205,7 +1342,6 @@ make_hash2(Eterm term)
{
ErlFunThing* funp = (ErlFunThing *) fun_val(term);
Uint num_free = funp->num_free;
-
UINT32_HASH_2
(num_free,
atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue,
@@ -1286,7 +1422,8 @@ make_hash2(Eterm term)
do {
Uint t;
Uint32 x, y;
- t = i < n ? BIG_DIGIT(ptr, i++) : 0;
+ ASSERT(i < n);
+ t = BIG_DIGIT(ptr, i++);
x = t & 0xffffffff;
y = t >> 32;
UINT32_HASH_2(x, y, con);
@@ -1319,7 +1456,11 @@ make_hash2(Eterm term)
{
FloatDef ff;
GET_DOUBLE(term, ff);
-#if defined(WORDS_BIGENDIAN)
+ if (ff.fd == 0.0f) {
+ /* ensure positive 0.0 */
+ ff.fd = erts_get_positive_zero_float();
+ }
+#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN)
UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
#else
UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12);
@@ -1329,7 +1470,7 @@ make_hash2(Eterm term)
break;
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
}
}
break;
@@ -1360,7 +1501,7 @@ make_hash2(Eterm term)
UINT32_HASH(NIL_DEF, HCONST_2);
goto hash2_common;
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
}
case _TAG_IMMED1_SMALL:
{
@@ -1376,17 +1517,427 @@ make_hash2(Eterm term)
}
break;
default:
- erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term);
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
hash2_common:
+
+ /* Uint32 hash always has the hash value of the previous term,
+ * compounded or otherwise.
+ */
+
if (ESTACK_ISEMPTY(s)) {
DESTROY_ESTACK(s);
UnUseTmpHeapNoproc(2);
return hash;
}
+
+ term = ESTACK_POP(s);
+
+ switch (term) {
+ case HASH_MAP_TAIL: {
+ hash = (Uint32) ESTACK_POP(s);
+ UINT32_HASH(hash_xor_pairs, HCONST_19);
+ hash_xor_pairs = (Uint32) ESTACK_POP(s);
+ goto hash2_common;
+ }
+ case HASH_MAP_PAIR:
+ hash_xor_pairs ^= hash;
+ hash = 0;
+ goto hash2_common;
+ default:
+ break;
+ }
+ }
+ }
+ }
+}
+
+/* Term hash function for internal use.
+ *
+ * Limitation #1: Is not "portable" in any way between different VM instances.
+ *
+ * Limitation #2: The hash value is only valid as long as the term exists
+ * somewhere in the VM. Why? Because external pids, ports and refs are hashed
+ * by mixing the node *pointer* value. If a node disappears and later reappears
+ * with a new ErlNode struct, externals from that node will hash different than
+ * before.
+ *
+ * One IMPORTANT property must hold (for hamt).
+ * EVERY BIT of the term that is significant for equality (see EQ)
+ * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a
+ * chance of hashing different when salted: hash([Salt|A]) vs hash([Salt|B]).
+ *
+ * This is why we can not use cached hash values for atoms for example.
+ *
+ */
+
+#define CONST_HASH(AConst) \
+do { /* Lightweight mixing of constant (type info) */ \
+ hash ^= AConst; \
+ hash = (hash << 17) ^ (hash >> (32-17)); \
+} while (0)
+
+Uint32
+make_internal_hash(Eterm term)
+{
+ Uint32 hash;
+ Uint32 hash_xor_pairs;
+
+ ERTS_UNDEF(hash_xor_pairs, 0);
+
+ /* Optimization. Simple cases before declaration of estack. */
+ if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
+ hash = 0;
+ #if ERTS_SIZEOF_ETERM == 8
+ UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST);
+ #elif ERTS_SIZEOF_ETERM == 4
+ UINT32_HASH(term, HCONST);
+ #else
+ # error "No you don't"
+ #endif
+ return hash;
+ }
+ {
+ Eterm tmp;
+ DECLARE_ESTACK(s);
+
+ hash = 0;
+ for (;;) {
+ switch (primary_tag(term)) {
+ case TAG_PRIMARY_LIST:
+ {
+ int c = 0;
+ Uint32 sh = 0;
+ Eterm* ptr = list_val(term);
+ while (is_byte(*ptr)) {
+ /* Optimization for strings. */
+ sh = (sh << 8) + unsigned_val(*ptr);
+ if (c == 3) {
+ UINT32_HASH(sh, HCONST_4);
+ c = sh = 0;
+ } else {
+ c++;
+ }
+ term = CDR(ptr);
+ if (is_not_list(term))
+ break;
+ ptr = list_val(term);
+ }
+ if (c > 0)
+ UINT32_HASH_2(sh, (Uint32)c, HCONST_22);
+
+ if (is_list(term)) {
+ tmp = CDR(ptr);
+ CONST_HASH(HCONST_17); /* Hash CAR in cons cell */
+ ESTACK_PUSH(s, tmp);
+ if (is_not_list(tmp)) {
+ ESTACK_PUSH(s, HASH_CDR);
+ }
+ term = CAR(ptr);
+ }
+ }
+ break;
+ case TAG_PRIMARY_BOXED:
+ {
+ Eterm hdr = *boxed_val(term);
+ ASSERT(is_header(hdr));
+ switch (hdr & _TAG_HEADER_MASK) {
+ case ARITYVAL_SUBTAG:
+ {
+ int i;
+ int arity = header_arity(hdr);
+ Eterm* elem = tuple_val(term);
+ UINT32_HASH(arity, HCONST_9);
+ if (arity == 0) /* Empty tuple */
+ goto pop_next;
+ for (i = arity; ; i--) {
+ term = elem[i];
+ if (i == 1)
+ break;
+ ESTACK_PUSH(s, term);
+ }
+ }
+ break;
+
+ case MAP_SUBTAG:
+ {
+ Eterm* ptr = boxed_val(term) + 1;
+ Uint size;
+ int i;
+ switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+ case HAMT_SUBTAG_HEAD_FLATMAP:
+ {
+ flatmap_t *mp = (flatmap_t *)flatmap_val(term);
+ Eterm *ks = flatmap_get_keys(mp);
+ Eterm *vs = flatmap_get_values(mp);
+ size = flatmap_get_size(mp);
+ UINT32_HASH(size, HCONST_16);
+ if (size == 0)
+ goto pop_next;
+
+ /* We want a hash function that is *independent* of
+ * the order in which keys and values are encountered.
+ * We therefore calculate context independent hashes for all .
+ * key-value pairs and then xor them together.
+ */
+ ESTACK_PUSH(s, hash_xor_pairs);
+ ESTACK_PUSH(s, hash);
+ ESTACK_PUSH(s, HASH_MAP_TAIL);
+ hash = 0;
+ hash_xor_pairs = 0;
+ for (i = size - 1; i >= 0; i--) {
+ ESTACK_PUSH(s, HASH_MAP_PAIR);
+ ESTACK_PUSH(s, vs[i]);
+ ESTACK_PUSH(s, ks[i]);
+ }
+ goto pop_next;
+ }
+ case HAMT_SUBTAG_HEAD_ARRAY:
+ case HAMT_SUBTAG_HEAD_BITMAP:
+ size = *ptr++;
+ UINT32_HASH(size, HCONST_16);
+ if (size == 0)
+ goto pop_next;
+ ESTACK_PUSH(s, hash_xor_pairs);
+ ESTACK_PUSH(s, hash);
+ ESTACK_PUSH(s, HASH_MAP_TAIL);
+ hash = 0;
+ hash_xor_pairs = 0;
+ }
+ switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+ case HAMT_SUBTAG_HEAD_ARRAY:
+ i = 16;
+ break;
+ case HAMT_SUBTAG_HEAD_BITMAP:
+ case HAMT_SUBTAG_NODE_BITMAP:
+ i = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ break;
+ default:
+ erts_exit(ERTS_ERROR_EXIT, "bad header");
+ }
+ while (i) {
+ if (is_list(*ptr)) {
+ Eterm* cons = list_val(*ptr);
+ ESTACK_PUSH(s, HASH_MAP_PAIR);
+ ESTACK_PUSH(s, CDR(cons));
+ ESTACK_PUSH(s, CAR(cons));
+ }
+ else {
+ ASSERT(is_boxed(*ptr));
+ ESTACK_PUSH(s, *ptr);
+ }
+ i--; ptr++;
+ }
+ goto pop_next;
+ }
+ break;
+ case EXPORT_SUBTAG:
+ {
+ Export* ep = *((Export **) (export_val(term) + 1));
+ /* Assumes Export entries never moves */
+ POINTER_HASH(ep, HCONST_14);
+ goto pop_next;
+ }
+
+ case FUN_SUBTAG:
+ {
+ ErlFunThing* funp = (ErlFunThing *) fun_val(term);
+ Uint num_free = funp->num_free;
+ UINT32_HASH_2(num_free, funp->fe->module, HCONST_20);
+ UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21);
+ if (num_free == 0) {
+ goto pop_next;
+ } else {
+ Eterm* bptr = funp->env + num_free - 1;
+ while (num_free-- > 1) {
+ term = *bptr--;
+ ESTACK_PUSH(s, term);
+ }
+ term = *bptr;
+ }
+ }
+ break;
+ case REFC_BINARY_SUBTAG:
+ case HEAP_BINARY_SUBTAG:
+ case SUB_BINARY_SUBTAG:
+ {
+ byte* bptr;
+ unsigned sz = binary_size(term);
+ Uint32 con = HCONST_13 + hash;
+ Uint bitoffs;
+ Uint bitsize;
+
+ ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize);
+ if (sz == 0 && bitsize == 0) {
+ hash = con;
+ } else {
+ if (bitoffs == 0) {
+ hash = block_hash(bptr, sz, con);
+ if (bitsize > 0) {
+ UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)),
+ HCONST_15);
+ }
+ } else {
+ byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP,
+ sz + (bitsize != 0));
+ erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize);
+ hash = block_hash(buf, sz, con);
+ if (bitsize > 0) {
+ UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)),
+ HCONST_15);
+ }
+ erts_free(ERTS_ALC_T_TMP, (void *) buf);
+ }
+ }
+ goto pop_next;
+ }
+ break;
+ case POS_BIG_SUBTAG:
+ case NEG_BIG_SUBTAG:
+ {
+ Eterm* ptr = big_val(term);
+ Uint i = 0;
+ Uint n = BIG_SIZE(ptr);
+ Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11;
+#if D_EXP == 16
+ do {
+ Uint32 x, y;
+ x = i < n ? BIG_DIGIT(ptr, i++) : 0;
+ x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
+ y = i < n ? BIG_DIGIT(ptr, i++) : 0;
+ y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16;
+ UINT32_HASH_2(x, y, con);
+ } while (i < n);
+#elif D_EXP == 32
+ do {
+ Uint32 x, y;
+ x = i < n ? BIG_DIGIT(ptr, i++) : 0;
+ y = i < n ? BIG_DIGIT(ptr, i++) : 0;
+ UINT32_HASH_2(x, y, con);
+ } while (i < n);
+#elif D_EXP == 64
+ do {
+ Uint t;
+ Uint32 x, y;
+ ASSERT(i < n);
+ t = BIG_DIGIT(ptr, i++);
+ x = t & 0xffffffff;
+ y = t >> 32;
+ UINT32_HASH_2(x, y, con);
+ } while (i < n);
+#else
+#error "unsupported D_EXP size"
+#endif
+ goto pop_next;
+ }
+ break;
+ case REF_SUBTAG:
+ UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7);
+ ASSERT(internal_ref_no_of_numbers(term) == 3);
+ UINT32_HASH_2(internal_ref_numbers(term)[1],
+ internal_ref_numbers(term)[2], HCONST_8);
+ goto pop_next;
+
+ case EXTERNAL_REF_SUBTAG:
+ {
+ ExternalThing* thing = external_thing_ptr(term);
+
+ ASSERT(external_thing_ref_no_of_numbers(thing) == 3);
+ /* See limitation #2 */
+ #ifdef ARCH_64
+ POINTER_HASH(thing->node, HCONST_7);
+ UINT32_HASH(external_thing_ref_numbers(thing)[0], HCONST_7);
+ #else
+ UINT32_HASH_2(thing->node,
+ external_thing_ref_numbers(thing)[0], HCONST_7);
+ #endif
+ UINT32_HASH_2(external_thing_ref_numbers(thing)[1],
+ external_thing_ref_numbers(thing)[2], HCONST_8);
+ goto pop_next;
+ }
+ case EXTERNAL_PID_SUBTAG: {
+ ExternalThing* thing = external_thing_ptr(term);
+ /* See limitation #2 */
+ #ifdef ARCH_64
+ POINTER_HASH(thing->node, HCONST_5);
+ UINT32_HASH(thing->data.ui[0], HCONST_5);
+ #else
+ UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_5);
+ #endif
+ goto pop_next;
+ }
+ case EXTERNAL_PORT_SUBTAG: {
+ ExternalThing* thing = external_thing_ptr(term);
+ /* See limitation #2 */
+ #ifdef ARCH_64
+ POINTER_HASH(thing->node, HCONST_6);
+ UINT32_HASH(thing->data.ui[0], HCONST_6);
+ #else
+ UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_6);
+ #endif
+ goto pop_next;
+ }
+ case FLOAT_SUBTAG:
+ {
+ FloatDef ff;
+ GET_DOUBLE(term, ff);
+ if (ff.fd == 0.0f) {
+ /* ensure positive 0.0 */
+ ff.fd = erts_get_positive_zero_float();
+ }
+ UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12);
+ goto pop_next;
+ }
+ default:
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
+ }
+ }
+ break;
+ case TAG_PRIMARY_IMMED1:
+ #if ERTS_SIZEOF_ETERM == 8
+ UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST);
+ #else
+ UINT32_HASH(term, HCONST);
+ #endif
+ goto pop_next;
+
+ default:
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term);
+
+ pop_next:
+ if (ESTACK_ISEMPTY(s)) {
+ DESTROY_ESTACK(s);
+ return hash;
+ }
+
term = ESTACK_POP(s);
+
+ switch (term) {
+ case HASH_MAP_TAIL: {
+ hash = (Uint32) ESTACK_POP(s);
+ UINT32_HASH(hash_xor_pairs, HCONST_19);
+ hash_xor_pairs = (Uint32) ESTACK_POP(s);
+ goto pop_next;
+ }
+ case HASH_MAP_PAIR:
+ hash_xor_pairs ^= hash;
+ hash = 0;
+ goto pop_next;
+
+ case HASH_CDR:
+ CONST_HASH(HCONST_18); /* Hash CDR i cons cell */
+ goto pop_next;
+ default:
+ break;
+ }
}
}
}
+
+#undef CONST_HASH
+#undef HASH_MAP_TAIL
+#undef HASH_MAP_PAIR
+#undef HASH_CDR
+
#undef UINT32_HASH_2
#undef UINT32_HASH
#undef SINT32_HASH
@@ -1413,7 +1964,7 @@ tail_recur:
(atom_tab(atom_val(term))->slot.bucket.hvalue);
break;
case SMALL_DEF:
-#if defined(ARCH_64) && !HALFWORD_HEAP
+#if defined(ARCH_64)
{
Sint y1 = signed_val(term);
Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
@@ -1488,7 +2039,7 @@ tail_recur:
hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
if (num_free > 0) {
if (num_free > 1) {
- WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+ WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP);
}
term = funp->env[0];
goto tail_recur;
@@ -1516,12 +2067,15 @@ tail_recur:
break;
case FLOAT_DEF:
{
- FloatDef ff;
- GET_DOUBLE(term, ff);
- hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
+ FloatDef ff;
+ GET_DOUBLE(term, ff);
+ if (ff.fd == 0.0f) {
+ /* ensure positive 0.0 */
+ ff.fd = erts_get_positive_zero_float();
+ }
+ hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]);
}
break;
-
case MAKE_HASH_CDR_PRE_OP:
term = (Eterm) WSTACK_POP(stack);
if (is_not_list(term)) {
@@ -1601,6 +2155,9 @@ tail_recur:
}
break;
+ case MAP_DEF:
+ hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term);
+ break;
case TUPLE_DEF:
{
Eterm* ptr = tuple_val(term);
@@ -1610,7 +2167,7 @@ tail_recur:
op = MAKE_HASH_TUPLE_OP;
}/*fall through*/
case MAKE_HASH_TUPLE_OP:
- case MAKE_HASH_FUN_OP:
+ case MAKE_HASH_TERM_ARRAY_OP:
{
Uint i = (Uint) WSTACK_POP(stack);
Eterm* ptr = (Eterm*) WSTACK_POP(stack);
@@ -1627,7 +2184,7 @@ tail_recur:
}
default:
- erl_exit(1, "Invalid tag in make_broken_hash\n");
+ erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n");
return 0;
}
if (WSTACK_ISEMPTY(stack)) break;
@@ -1638,106 +2195,160 @@ tail_recur:
return hash;
#undef MAKE_HASH_TUPLE_OP
-#undef MAKE_HASH_FUN_OP
+#undef MAKE_HASH_TERM_ARRAY_OP
#undef MAKE_HASH_CDR_PRE_OP
#undef MAKE_HASH_CDR_POST_OP
}
-static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
+static Eterm
+do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp,
+ ErlHeapFragment **bp, Process **p, Uint sz)
{
- /* error_logger !
- {notify,{info_msg,gleader,{emulator,"~s~n",[<message as list>]}}} |
- {notify,{error,gleader,{emulator,"~s~n",[<message as list>]}}} |
- {notify,{warning_msg,gleader,{emulator,"~s~n",[<message as list>}]}} */
- Eterm* hp;
- Uint sz;
Uint gl_sz;
- Eterm gl;
- Eterm list,plist,format,tuple1,tuple2,tuple3;
- ErlOffHeap *ohp;
- ErlHeapFragment *bp = NULL;
-#if !defined(ERTS_SMP)
- Process *p;
-#endif
-
- ASSERT(is_atom(tag));
-
- if (len <= 0) {
- return -1;
- }
+ gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader);
+ sz = sz + gl_sz;
#ifndef ERTS_SMP
#ifdef USE_THREADS
- p = NULL;
- if (erts_get_scheduler_data()) /* Must be scheduler thread */
+ if (!erts_get_scheduler_data()) /* Must be scheduler thread */
+ *p = NULL;
+ else
#endif
{
- p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
- if (p) {
- erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state);
- if (state & ERTS_PSFLG_RUNNING)
- p = NULL;
+ *p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0);
+ if (*p) {
+ erts_aint32_t state = erts_smp_atomic32_read_acqb(&(*p)->state);
+ if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))
+ *p = NULL;
}
}
- if (!p) {
- /* buf *always* points to a null terminated string */
- erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n",
- tag, buf);
- return 0;
+ if (!*p) {
+ return NIL;
}
+
/* So we have an error logger, lets build the message */
#endif
- gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader);
- sz = len * 2 /* message list */+ 2 /* cons surrounding message list */
- + gl_sz +
- 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */ +
- 8 /* "~s~n" */;
+ *bp = new_message_buffer(sz);
+ *ohp = &(*bp)->off_heap;
+ *hp = (*bp)->mem;
-#ifndef ERTS_SMP
- if (sz <= HeapWordsLeft(p)) {
- ohp = &MSO(p);
- hp = HEAP_TOP(p);
- HEAP_TOP(p) += sz;
- } else {
-#endif
- bp = new_message_buffer(sz);
- ohp = &bp->off_heap;
- hp = bp->mem;
-#ifndef ERTS_SMP
- }
-#endif
- gl = (is_nil(gleader)
+ return (is_nil(gleader)
? am_noproc
: (IS_CONST(gleader)
? gleader
- : copy_struct(gleader,gl_sz,&hp,ohp)));
- list = buf_to_intlist(&hp, buf, len, NIL);
- plist = CONS(hp,list,NIL);
- hp += 2;
- format = buf_to_intlist(&hp, "~s~n", 4, NIL);
- tuple1 = TUPLE3(hp, am_emulator, format, plist);
- hp += 4;
- tuple2 = TUPLE3(hp, tag, gl, tuple1);
- hp += 4;
- tuple3 = TUPLE2(hp, am_notify, tuple2);
+ : copy_struct(gleader,gl_sz,hp,*ohp)));
+}
+
+static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *bp,
+ Process *p, Eterm message)
+{
#ifdef HARDDEBUG
- erts_fprintf(stderr, "%T\n", tuple3);
+ erts_fprintf(stderr, "%T\n", message);
#endif
#ifdef ERTS_SMP
{
Eterm from = erts_get_current_pid();
if (is_not_internal_pid(from))
from = NIL;
- erts_queue_error_logger_message(from, tuple3, bp);
+ erts_queue_error_logger_message(from, message, bp);
}
#else
- erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL
-#ifdef USE_VM_PROBES
- , NIL
-#endif
- );
+ {
+ ErtsMessage *mp = erts_alloc_message(0, NULL);
+ mp->data.heap_frag = bp;
+ erts_queue_message(p, 0, mp, message, am_system);
+ }
#endif
+}
+
+/* error_logger !
+ {notify,{info_msg,gleader,{emulator,format,[args]}}} |
+ {notify,{error,gleader,{emulator,format,[args]}}} |
+ {notify,{warning_msg,gleader,{emulator,format,[args}]}} */
+static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
+{
+ Uint sz;
+ Eterm gl;
+ Eterm list,args,format,tuple1,tuple2,tuple3;
+
+ Eterm *hp = NULL;
+ ErlOffHeap *ohp = NULL;
+ ErlHeapFragment *bp = NULL;
+ Process *p = NULL;
+
+ ASSERT(is_atom(tag));
+
+ if (len <= 0) {
+ return -1;
+ }
+
+ sz = len * 2 /* message list */ + 2 /* cons surrounding message list */
+ + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */
+ + 8 /* "~s~n" */;
+
+ /* gleader size is accounted and allocated next */
+ gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz);
+
+ if(is_nil(gl)) {
+ /* buf *always* points to a null terminated string */
+ erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n",
+ tag, buf);
+ return 0;
+ }
+
+ list = buf_to_intlist(&hp, buf, len, NIL);
+ args = CONS(hp,list,NIL);
+ hp += 2;
+ format = buf_to_intlist(&hp, "~s~n", 4, NIL);
+ tuple1 = TUPLE3(hp, am_emulator, format, args);
+ hp += 4;
+ tuple2 = TUPLE3(hp, tag, gl, tuple1);
+ hp += 4;
+ tuple3 = TUPLE2(hp, am_notify, tuple2);
+
+ do_send_logger_message(hp, ohp, bp, p, tuple3);
+ return 0;
+}
+
+static int do_send_term_to_logger(Eterm tag, Eterm gleader,
+ char *buf, int len, Eterm args)
+{
+ Uint sz;
+ Eterm gl;
+ Uint args_sz;
+ Eterm format,tuple1,tuple2,tuple3;
+
+ Eterm *hp = NULL;
+ ErlOffHeap *ohp = NULL;
+ ErlHeapFragment *bp = NULL;
+ Process *p = NULL;
+
+ ASSERT(is_atom(tag));
+
+ args_sz = size_object(args);
+ sz = len * 2 /* format */ + args_sz
+ + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */;
+
+ /* gleader size is accounted and allocated next */
+ gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz);
+
+ if(is_nil(gl)) {
+ /* buf *always* points to a null terminated string */
+ erts_fprintf(stderr, "(no error logger present) %T: \"%s\" %T\n",
+ tag, buf, args);
+ return 0;
+ }
+
+ format = buf_to_intlist(&hp, buf, len, NIL);
+ args = copy_struct(args, args_sz, &hp, ohp);
+ tuple1 = TUPLE3(hp, am_emulator, format, args);
+ hp += 4;
+ tuple2 = TUPLE3(hp, tag, gl, tuple1);
+ hp += 4;
+ tuple3 = TUPLE2(hp, am_notify, tuple2);
+
+ do_send_logger_message(hp, ohp, bp, p, tuple3);
return 0;
}
@@ -1765,6 +2376,12 @@ send_error_to_logger(Eterm gleader, char *buf, int len)
return do_send_to_logger(am_error, gleader, buf, len);
}
+static ERTS_INLINE int
+send_error_term_to_logger(Eterm gleader, char *buf, int len, Eterm args)
+{
+ return do_send_term_to_logger(am_error, gleader, buf, len, args);
+}
+
#define LOGGER_DSBUF_INC_SZ 256
static erts_dsprintf_buf_t *
@@ -1840,6 +2457,15 @@ erts_send_error_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp)
}
int
+erts_send_error_term_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp, Eterm args)
+{
+ int res;
+ res = send_error_term_to_logger(gleader, dsbufp->str, dsbufp->str_len, args);
+ destroy_logger_dsbuf(dsbufp);
+ return res;
+}
+
+int
erts_send_info_to_logger_str(Eterm gleader, char *str)
{
return send_info_to_logger(gleader, str, sys_strlen(str));
@@ -1945,11 +2571,7 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp)
* Test for equality of two terms.
* Returns 0 if not equal, or a non-zero value otherwise.
*/
-#if HALFWORD_HEAP
-int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
-#else
int eq(Eterm a, Eterm b)
-#endif
{
DECLARE_WSTACK(stack);
Sint sz;
@@ -1957,18 +2579,18 @@ int eq(Eterm a, Eterm b)
Eterm* bb;
tailrecur:
- if (is_same(a, a_base, b, b_base)) goto pop_next;
+ if (is_same(a, b)) goto pop_next;
tailrecur_ne:
switch (primary_tag(a)) {
case TAG_PRIMARY_LIST:
if (is_list(b)) {
- Eterm* aval = list_val_rel(a, a_base);
- Eterm* bval = list_val_rel(b, b_base);
+ Eterm* aval = list_val(a);
+ Eterm* bval = list_val(b);
while (1) {
Eterm atmp = CAR(aval);
Eterm btmp = CAR(bval);
- if (!is_same(atmp,a_base,btmp,b_base)) {
+ if (!is_same(atmp,btmp)) {
WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval));
a = atmp;
b = btmp;
@@ -1976,7 +2598,7 @@ tailrecur_ne:
}
atmp = CDR(aval);
btmp = CDR(bval);
- if (is_same(atmp,a_base,btmp,b_base)) {
+ if (is_same(atmp,btmp)) {
goto pop_next;
}
if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -1984,22 +2606,22 @@ tailrecur_ne:
b = btmp;
goto tailrecur_ne;
}
- aval = list_val_rel(atmp, a_base);
- bval = list_val_rel(btmp, b_base);
+ aval = list_val(atmp);
+ bval = list_val(btmp);
}
}
break; /* not equal */
case TAG_PRIMARY_BOXED:
{
- Eterm hdr = *boxed_val_rel(a,a_base);
+ Eterm hdr = *boxed_val(a);
switch (hdr & _TAG_HEADER_MASK) {
case ARITYVAL_SUBTAG:
{
- aa = tuple_val_rel(a, a_base);
- if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
+ aa = tuple_val(a);
+ if (!is_boxed(b) || *boxed_val(b) != *aa)
goto not_equal;
- bb = tuple_val_rel(b,b_base);
+ bb = tuple_val(b);
if ((sz = arityval(*aa)) == 0) goto pop_next;
++aa;
++bb;
@@ -2018,16 +2640,16 @@ tailrecur_ne:
Uint a_bitoffs;
Uint b_bitoffs;
- if (!is_binary_rel(b,b_base)) {
+ if (!is_binary(b)) {
goto not_equal;
}
- a_size = binary_size_rel(a,a_base);
- b_size = binary_size_rel(b,b_base);
+ a_size = binary_size(a);
+ b_size = binary_size(b);
if (a_size != b_size) {
goto not_equal;
}
- ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
- ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
+ ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
+ ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next;
} else if (a_bitsize == b_bitsize) {
@@ -2038,9 +2660,9 @@ tailrecur_ne:
}
case EXPORT_SUBTAG:
{
- if (is_export_rel(b,b_base)) {
- Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
- Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
+ if (is_export(b)) {
+ Export* a_exp = *((Export **) (export_val(a) + 1));
+ Export* b_exp = *((Export **) (export_val(b) + 1));
if (a_exp == b_exp) goto pop_next;
}
break; /* not equal */
@@ -2050,10 +2672,10 @@ tailrecur_ne:
ErlFunThing* f1;
ErlFunThing* f2;
- if (!is_fun_rel(b,b_base))
+ if (!is_fun(b))
goto not_equal;
- f1 = (ErlFunThing *) fun_val_rel(a,a_base);
- f2 = (ErlFunThing *) fun_val_rel(b,b_base);
+ f1 = (ErlFunThing *) fun_val(a);
+ f2 = (ErlFunThing *) fun_val(b);
if (f1->fe->module != f2->fe->module ||
f1->fe->old_index != f2->fe->old_index ||
f1->fe->old_uniq != f2->fe->old_uniq ||
@@ -2071,15 +2693,15 @@ tailrecur_ne:
ExternalThing *ap;
ExternalThing *bp;
- if(!is_external_rel(b,b_base))
+ if(!is_external(b))
goto not_equal;
- ap = external_thing_ptr_rel(a,a_base);
- bp = external_thing_ptr_rel(b,b_base);
+ ap = external_thing_ptr(a);
+ bp = external_thing_ptr(b);
if(ap->header == bp->header && ap->node == bp->node) {
- ASSERT(1 == external_data_words_rel(a,a_base));
- ASSERT(1 == external_data_words_rel(b,b_base));
+ ASSERT(1 == external_data_words(a));
+ ASSERT(1 == external_data_words(b));
if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next;
}
@@ -2100,11 +2722,11 @@ tailrecur_ne:
ExternalThing* athing;
ExternalThing* bthing;
- if(!is_external_ref_rel(b,b_base))
+ if(!is_external_ref(b))
goto not_equal;
- athing = external_thing_ptr_rel(a,a_base);
- bthing = external_thing_ptr_rel(b,b_base);
+ athing = external_thing_ptr(a);
+ bthing = external_thing_ptr(b);
if(athing->node != bthing->node)
goto not_equal;
@@ -2116,12 +2738,12 @@ tailrecur_ne:
goto ref_common;
case REF_SUBTAG:
- if (!is_internal_ref_rel(b,b_base))
+ if (!is_internal_ref(b))
goto not_equal;
{
- RefThing* athing = ref_thing_ptr_rel(a,a_base);
- RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+ RefThing* athing = ref_thing_ptr(a);
+ RefThing* bthing = ref_thing_ptr(b);
alen = internal_thing_ref_no_of_numbers(athing);
blen = internal_thing_ref_no_of_numbers(bthing);
anum = internal_thing_ref_numbers(athing);
@@ -2171,10 +2793,10 @@ tailrecur_ne:
{
int i;
- if (!is_big_rel(b,b_base))
+ if (!is_big(b))
goto not_equal;
- aa = big_val_rel(a,a_base);
- bb = big_val_rel(b,b_base);
+ aa = big_val(a);
+ bb = big_val(b);
if (*aa != *bb)
goto not_equal;
i = BIG_ARITY(aa);
@@ -2189,13 +2811,53 @@ tailrecur_ne:
FloatDef af;
FloatDef bf;
- if (is_float_rel(b,b_base)) {
- GET_DOUBLE_REL(a, af, a_base);
- GET_DOUBLE_REL(b, bf, b_base);
+ if (is_float(b)) {
+ GET_DOUBLE(a, af);
+ GET_DOUBLE(b, bf);
if (af.fd == bf.fd) goto pop_next;
}
break; /* not equal */
}
+ case MAP_SUBTAG:
+ if (is_flatmap(a)) {
+ aa = flatmap_val(a);
+ if (!is_boxed(b) || *boxed_val(b) != *aa)
+ goto not_equal;
+ bb = flatmap_val(b);
+ sz = flatmap_get_size((flatmap_t*)aa);
+
+ if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal;
+ if (sz == 0) goto pop_next;
+
+ aa += 2;
+ bb += 2;
+ sz += 1; /* increment for tuple-keys */
+ goto term_array;
+
+ } else {
+ if (!is_boxed(b) || *boxed_val(b) != hdr)
+ goto not_equal;
+
+ aa = hashmap_val(a) + 1;
+ bb = hashmap_val(b) + 1;
+ switch (hdr & _HEADER_MAP_SUBTAG_MASK) {
+ case HAMT_SUBTAG_HEAD_ARRAY:
+ aa++; bb++;
+ sz = 16;
+ break;
+ case HAMT_SUBTAG_HEAD_BITMAP:
+ aa++; bb++;
+ case HAMT_SUBTAG_NODE_BITMAP:
+ sz = hashmap_bitcount(MAP_HEADER_VAL(hdr));
+ ASSERT(sz > 0 && sz < 17);
+ break;
+ default:
+ erts_exit(ERTS_ERROR_EXIT, "Unknown hashmap subsubtag\n");
+ }
+ goto term_array;
+ }
+ default:
+ ASSERT(!"Unknown boxed subtab in EQ");
}
break;
}
@@ -2210,7 +2872,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */
Eterm* bp = bb;
Sint i = sz;
for (;;) {
- if (!is_same(*ap,a_base,*bp,b_base)) break;
+ if (!is_same(*ap,*bp)) break;
if (--i == 0) goto pop_next;
++ap;
++bp;
@@ -2279,7 +2941,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2)
*
* According to the Erlang Standard, types are orderered as follows:
* numbers < (characters) < atoms < refs < funs < ports < pids <
- * tuples < [] < conses < binaries.
+ * tuples < maps < [] < conses < binaries.
*
* Note that characters are currently not implemented.
*
@@ -2288,7 +2950,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2)
#define float_comp(x,y) (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1))
-static int cmp_atoms(Eterm a, Eterm b)
+int erts_cmp_atoms(Eterm a, Eterm b)
{
Atom *aa = atom_tab(atom_val(a));
Atom *bb = atom_tab(atom_val(b));
@@ -2299,13 +2961,50 @@ static int cmp_atoms(Eterm a, Eterm b)
bb->name+3, bb->len-3);
}
-#if HALFWORD_HEAP
-Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
-#else
+/* cmp(Eterm a, Eterm b)
+ * For compatibility with HiPE - arith-based compare.
+ */
Sint cmp(Eterm a, Eterm b)
-#endif
{
- DECLARE_WSTACK(stack);
+ return erts_cmp(a, b, 0, 0);
+}
+
+Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
+
+Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
+{
+ if (is_atom(a) && is_atom(b)) {
+ return erts_cmp_atoms(a, b);
+ } else if (is_both_small(a, b)) {
+ return (signed_val(a) - signed_val(b));
+ } else if (is_float(a) && is_float(b)) {
+ FloatDef af, bf;
+ GET_DOUBLE(a, af);
+ GET_DOUBLE(b, bf);
+ return float_comp(af.fd, bf.fd);
+ }
+ return erts_cmp_compound(a,b,exact,eq_only);
+}
+
+
+/* erts_cmp(Eterm a, Eterm b, int exact)
+ * exact = 1 -> term-based compare
+ * exact = 0 -> arith-based compare
+ */
+Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
+{
+#define PSTACK_TYPE struct erts_cmp_hashmap_state
+ struct erts_cmp_hashmap_state {
+ Sint wstack_rollback;
+ int was_exact;
+ Eterm *ap;
+ Eterm *bp;
+ Eterm min_key;
+ Sint cmp_res; /* result so far -1,0,+1 */
+ };
+ PSTACK_DECLARE(hmap_stack, 1);
+ WSTACK_DECLARE(stack);
+ WSTACK_DECLARE(b_stack); /* only used by hashmaps */
Eterm* aa;
Eterm* bb;
int i;
@@ -2321,6 +3020,26 @@ Sint cmp(Eterm a, Eterm b)
Uint32 *anum;
Uint32 *bnum;
+/* The WSTACK contains naked Eterms and Operations marked with header-tags */
+#define OP_BITS 4
+#define OP_MASK 0xF
+#define TERM_ARRAY_OP 0
+#define SWITCH_EXACT_OFF_OP 1
+#define HASHMAP_PHASE1_ARE_KEYS_EQUAL 2
+#define HASHMAP_PHASE1_IS_MIN_KEY 3
+#define HASHMAP_PHASE1_CMP_VALUES 4
+#define HASHMAP_PHASE2_ARE_KEYS_EQUAL 5
+#define HASHMAP_PHASE2_IS_MIN_KEY_A 6
+#define HASHMAP_PHASE2_IS_MIN_KEY_B 7
+
+
+#define OP_WORD(OP) (((OP) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER)
+#define TERM_ARRAY_OP_WORD(SZ) OP_WORD(((SZ) << OP_BITS) | TERM_ARRAY_OP)
+
+#define GET_OP(WORD) (ASSERT(is_header(WORD)), ((WORD) >> _TAG_PRIMARY_SIZE) & OP_MASK)
+#define GET_OP_ARG(WORD) (ASSERT(is_header(WORD)), ((WORD) >> (OP_BITS + _TAG_PRIMARY_SIZE)))
+
+
#define RETURN_NEQ(cmp) { j=(cmp); ASSERT(j != 0); goto not_equal; }
#define ON_CMP_GOTO(cmp) if ((j=(cmp)) == 0) goto pop_next; else goto not_equal
@@ -2329,15 +3048,17 @@ Sint cmp(Eterm a, Eterm b)
do { \
if((AN) != (BN)) { \
if((AN)->sysname != (BN)->sysname) \
- RETURN_NEQ(cmp_atoms((AN)->sysname, (BN)->sysname)); \
+ RETURN_NEQ(erts_cmp_atoms((AN)->sysname, (BN)->sysname)); \
ASSERT((AN)->creation != (BN)->creation); \
RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1); \
} \
} while (0)
+bodyrecur:
+ j = 0;
tailrecur:
- if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */
+ if (is_same(a,b)) { /* Equal values or pointers. */
goto pop_next;
}
tailrecur_ne:
@@ -2345,7 +3066,7 @@ tailrecur_ne:
/* deal with majority (?) cases by brute-force */
if (is_atom(a)) {
if (is_atom(b)) {
- ON_CMP_GOTO(cmp_atoms(a, b));
+ ON_CMP_GOTO(erts_cmp_atoms(a, b));
}
} else if (is_both_small(a, b)) {
ON_CMP_GOTO(signed_val(a) - signed_val(b));
@@ -2363,9 +3084,9 @@ tailrecur_ne:
if (is_internal_port(b)) {
bnode = erts_this_node;
bdata = internal_port_data(b);
- } else if (is_external_port_rel(b,b_base)) {
- bnode = external_port_node_rel(b,b_base);
- bdata = external_port_data_rel(b,b_base);
+ } else if (is_external_port(b)) {
+ bnode = external_port_node(b);
+ bdata = external_port_data(b);
} else {
a_tag = PORT_DEF;
goto mixed_types;
@@ -2381,9 +3102,9 @@ tailrecur_ne:
if (is_internal_pid(b)) {
bnode = erts_this_node;
bdata = internal_pid_data(b);
- } else if (is_external_pid_rel(b,b_base)) {
- bnode = external_pid_node_rel(b,b_base);
- bdata = external_pid_data_rel(b,b_base);
+ } else if (is_external_pid(b)) {
+ bnode = external_pid_node(b);
+ bdata = external_pid_data(b);
} else {
a_tag = PID_DEF;
goto mixed_types;
@@ -2416,12 +3137,12 @@ tailrecur_ne:
a_tag = LIST_DEF;
goto mixed_types;
}
- aa = list_val_rel(a,a_base);
- bb = list_val_rel(b,b_base);
+ aa = list_val(a);
+ bb = list_val(b);
while (1) {
Eterm atmp = CAR(aa);
Eterm btmp = CAR(bb);
- if (!is_same(atmp,a_base,btmp,b_base)) {
+ if (!is_same(atmp,btmp)) {
WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa));
a = atmp;
b = btmp;
@@ -2429,7 +3150,7 @@ tailrecur_ne:
}
atmp = CDR(aa);
btmp = CDR(bb);
- if (is_same(atmp,a_base,btmp,b_base)) {
+ if (is_same(atmp,btmp)) {
goto pop_next;
}
if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -2437,20 +3158,20 @@ tailrecur_ne:
b = btmp;
goto tailrecur_ne;
}
- aa = list_val_rel(atmp,a_base);
- bb = list_val_rel(btmp,b_base);
+ aa = list_val(atmp);
+ bb = list_val(btmp);
}
case TAG_PRIMARY_BOXED:
{
- Eterm ahdr = *boxed_val_rel(a,a_base);
+ Eterm ahdr = *boxed_val(a);
switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):
- if (!is_tuple_rel(b,b_base)) {
+ if (!is_tuple(b)) {
a_tag = TUPLE_DEF;
goto mixed_types;
}
- aa = tuple_val_rel(a,a_base);
- bb = tuple_val_rel(b,b_base);
+ aa = tuple_val(a);
+ bb = tuple_val(b);
/* compare the arities */
i = arityval(ahdr); /* get the arity*/
if (i != arityval(*bb)) {
@@ -2462,50 +3183,139 @@ tailrecur_ne:
++aa;
++bb;
goto term_array;
-
+ case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) :
+ {
+ struct erts_cmp_hashmap_state* sp;
+ if (is_flatmap_header(ahdr)) {
+ if (!is_flatmap(b)) {
+ if (is_hashmap(b)) {
+ aa = (Eterm *)flatmap_val(a);
+ i = flatmap_get_size((flatmap_t*)aa) - hashmap_size(b);
+ ASSERT(i != 0);
+ RETURN_NEQ(i);
+ }
+ a_tag = MAP_DEF;
+ goto mixed_types;
+ }
+ aa = (Eterm *)flatmap_val(a);
+ bb = (Eterm *)flatmap_val(b);
+
+ i = flatmap_get_size((flatmap_t*)aa);
+ if (i != flatmap_get_size((flatmap_t*)bb)) {
+ RETURN_NEQ((int)(i - flatmap_get_size((flatmap_t*)bb)));
+ }
+ if (i == 0) {
+ goto pop_next;
+ }
+ aa += 2;
+ bb += 2;
+ if (exact) {
+ i += 1; /* increment for tuple-keys */
+ goto term_array;
+ }
+ else {
+ /* Value array */
+ WSTACK_PUSH3(stack,(UWord)(bb+1),(UWord)(aa+1),TERM_ARRAY_OP_WORD(i));
+ /* Switch back from 'exact' key compare */
+ WSTACK_PUSH(stack,OP_WORD(SWITCH_EXACT_OFF_OP));
+ /* Now do 'exact' compare of key tuples */
+ a = *aa;
+ b = *bb;
+ exact = 1;
+ goto bodyrecur;
+ }
+ }
+ if (!is_hashmap(b)) {
+ if (is_flatmap(b)) {
+ bb = (Eterm *)flatmap_val(b);
+ i = hashmap_size(a) - flatmap_get_size((flatmap_t*)bb);
+ ASSERT(i != 0);
+ RETURN_NEQ(i);
+ }
+ a_tag = MAP_DEF;
+ goto mixed_types;
+ }
+ i = hashmap_size(a) - hashmap_size(b);
+ if (i) {
+ RETURN_NEQ(i);
+ }
+ if (hashmap_size(a) == 0) {
+ goto pop_next;
+ }
+
+ /* Hashmap compare strategy:
+ Phase 1. While keys are identical
+ Do synchronous stepping through leafs of both trees in hash
+ order. Maintain value compare result of minimal key.
+
+ Phase 2. If key diff was found in phase 1
+ Ignore values from now on.
+ Continue iterate trees by always advancing the one
+ lagging behind hash-wise. Identical keys are skipped.
+ A minimal key can only be candidate as tie-breaker if we
+ have passed that hash value in the other tree (which means
+ the key did not exist in the other tree).
+ */
+
+ sp = PSTACK_PUSH(hmap_stack);
+ hashmap_iterator_init(&stack, a, 0);
+ hashmap_iterator_init(&b_stack, b, 0);
+ sp->ap = hashmap_iterator_next(&stack);
+ sp->bp = hashmap_iterator_next(&b_stack);
+ sp->cmp_res = 0;
+ ASSERT(sp->ap && sp->bp);
+
+ a = CAR(sp->ap);
+ b = CAR(sp->bp);
+ sp->was_exact = exact;
+ exact = 1;
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL));
+ sp->wstack_rollback = WSTACK_COUNT(stack);
+ goto bodyrecur;
+ }
case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
- if (!is_float_rel(b,b_base)) {
+ if (!is_float(b)) {
a_tag = FLOAT_DEF;
goto mixed_types;
} else {
FloatDef af;
FloatDef bf;
- GET_DOUBLE_REL(a, af, a_base);
- GET_DOUBLE_REL(b, bf, b_base);
+ GET_DOUBLE(a, af);
+ GET_DOUBLE(b, bf);
ON_CMP_GOTO(float_comp(af.fd, bf.fd));
}
case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
- if (!is_big_rel(b,b_base)) {
+ if (!is_big(b)) {
a_tag = BIG_DEF;
goto mixed_types;
}
- ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base)));
+ ON_CMP_GOTO(big_comp(a, b));
case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE):
- if (!is_export_rel(b,b_base)) {
+ if (!is_export(b)) {
a_tag = EXPORT_DEF;
goto mixed_types;
} else {
- Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
- Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
+ Export* a_exp = *((Export **) (export_val(a) + 1));
+ Export* b_exp = *((Export **) (export_val(b) + 1));
- if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
+ if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
RETURN_NEQ(j);
}
- if ((j = cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) {
+ if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) {
RETURN_NEQ(j);
}
ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]);
}
break;
case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):
- if (!is_fun_rel(b,b_base)) {
+ if (!is_fun(b)) {
a_tag = FUN_DEF;
goto mixed_types;
} else {
- ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base);
- ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base);
+ ErlFunThing* f1 = (ErlFunThing *) fun_val(a);
+ ErlFunThing* f2 = (ErlFunThing *) fun_val(b);
Sint diff;
diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name,
@@ -2537,29 +3347,29 @@ tailrecur_ne:
if (is_internal_pid(b)) {
bnode = erts_this_node;
bdata = internal_pid_data(b);
- } else if (is_external_pid_rel(b,b_base)) {
- bnode = external_pid_node_rel(b,b_base);
- bdata = external_pid_data_rel(b,b_base);
+ } else if (is_external_pid(b)) {
+ bnode = external_pid_node(b);
+ bdata = external_pid_data(b);
} else {
a_tag = EXTERNAL_PID_DEF;
goto mixed_types;
}
- anode = external_pid_node_rel(a,a_base);
- adata = external_pid_data_rel(a,a_base);
+ anode = external_pid_node(a);
+ adata = external_pid_data(a);
goto pid_common;
case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):
if (is_internal_port(b)) {
bnode = erts_this_node;
bdata = internal_port_data(b);
- } else if (is_external_port_rel(b,b_base)) {
- bnode = external_port_node_rel(b,b_base);
- bdata = external_port_data_rel(b,b_base);
+ } else if (is_external_port(b)) {
+ bnode = external_port_node(b);
+ bdata = external_port_data(b);
} else {
a_tag = EXTERNAL_PORT_DEF;
goto mixed_types;
}
- anode = external_port_node_rel(a,a_base);
- adata = external_port_data_rel(a,a_base);
+ anode = external_port_node(a);
+ adata = external_port_data(a);
goto port_common;
case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
/*
@@ -2567,14 +3377,13 @@ tailrecur_ne:
* (32-bit words), *not* ref data words.
*/
-
- if (is_internal_ref_rel(b,b_base)) {
- RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+ if (is_internal_ref(b)) {
+ RefThing* bthing = ref_thing_ptr(b);
bnode = erts_this_node;
bnum = internal_thing_ref_numbers(bthing);
blen = internal_thing_ref_no_of_numbers(bthing);
- } else if(is_external_ref_rel(b,b_base)) {
- ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+ } else if(is_external_ref(b)) {
+ ExternalThing* bthing = external_thing_ptr(b);
bnode = bthing->node;
bnum = external_thing_ref_numbers(bthing);
blen = external_thing_ref_no_of_numbers(bthing);
@@ -2583,7 +3392,7 @@ tailrecur_ne:
goto mixed_types;
}
{
- RefThing* athing = ref_thing_ptr_rel(a,a_base);
+ RefThing* athing = ref_thing_ptr(a);
anode = erts_this_node;
anum = internal_thing_ref_numbers(athing);
alen = internal_thing_ref_no_of_numbers(athing);
@@ -2616,13 +3425,13 @@ tailrecur_ne:
RETURN_NEQ((Sint32) (anum[i] - bnum[i]));
goto pop_next;
case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
- if (is_internal_ref_rel(b,b_base)) {
- RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+ if (is_internal_ref(b)) {
+ RefThing* bthing = ref_thing_ptr(b);
bnode = erts_this_node;
bnum = internal_thing_ref_numbers(bthing);
blen = internal_thing_ref_no_of_numbers(bthing);
- } else if (is_external_ref_rel(b,b_base)) {
- ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+ } else if (is_external_ref(b)) {
+ ExternalThing* bthing = external_thing_ptr(b);
bnode = bthing->node;
bnum = external_thing_ref_numbers(bthing);
blen = external_thing_ref_no_of_numbers(bthing);
@@ -2631,7 +3440,7 @@ tailrecur_ne:
goto mixed_types;
}
{
- ExternalThing* athing = external_thing_ptr_rel(a,a_base);
+ ExternalThing* athing = external_thing_ptr(a);
anode = athing->node;
anum = external_thing_ref_numbers(athing);
alen = external_thing_ref_no_of_numbers(athing);
@@ -2639,13 +3448,13 @@ tailrecur_ne:
goto ref_common;
default:
/* Must be a binary */
- ASSERT(is_binary_rel(a,a_base));
- if (!is_binary_rel(b,b_base)) {
+ ASSERT(is_binary(a));
+ if (!is_binary(b)) {
a_tag = BINARY_DEF;
goto mixed_types;
} else {
- Uint a_size = binary_size_rel(a,a_base);
- Uint b_size = binary_size_rel(b,b_base);
+ Uint a_size = binary_size(a);
+ Uint b_size = binary_size(b);
Uint a_bitsize;
Uint b_bitsize;
Uint a_bitoffs;
@@ -2654,8 +3463,8 @@ tailrecur_ne:
int cmp;
byte* a_ptr;
byte* b_ptr;
- ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
- ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
+ ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
+ ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
min_size = (a_size < b_size) ? a_size : b_size;
if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) {
@@ -2686,21 +3495,13 @@ tailrecur_ne:
{
FloatDef f1, f2;
Eterm big;
-#if HEAP_ON_C_STACK
- Eterm big_buf[CMP_TMP_HEAP_SIZE]; /* If HEAP_ON_C_STACK */
-#else
- Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap;
-#endif
-#if HALFWORD_HEAP
- Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base);
- Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base);
-#else
Eterm aw = a;
Eterm bw = b;
-#endif
#define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2))
#define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1))
#define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */
+ Eterm big_buf[BIG_NEED_SIZE(BIG_ARITY_FLOAT_MAX)];
+
b_tag = tag_val_def(bw);
switch(_NUMBER_CODE(a_tag, b_tag)) {
@@ -2711,13 +3512,15 @@ tailrecur_ne:
j = big_sign(aw) ? -1 : 1;
break;
case SMALL_FLOAT:
+ if (exact) goto exact_fall_through;
GET_DOUBLE(bw, f2);
if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) {
/* Float is within the no loss limit */
f1.fd = signed_val(aw);
j = float_comp(f1.fd, f2.fd);
+ }
#if ERTS_SIZEOF_ETERM == 8
- } else if (f2.fd > (double) (MAX_SMALL + 1)) {
+ else if (f2.fd > (double) (MAX_SMALL + 1)) {
/* Float is a positive bignum, i.e. bigger */
j = -1;
} else if (f2.fd < (double) (MIN_SMALL - 1)) {
@@ -2728,19 +3531,21 @@ tailrecur_ne:
j = signed_val(aw) - (Sint) f2.fd;
}
#else
- } else {
+ else {
/* If float is positive it is bigger than small */
j = (f2.fd > 0.0) ? -1 : 1;
}
#endif /* ERTS_SIZEOF_ETERM == 8 */
break;
case FLOAT_BIG:
+ if (exact) goto exact_fall_through;
{
Wterm tmp = aw;
aw = bw;
bw = tmp;
}/* fall through */
case BIG_FLOAT:
+ if (exact) goto exact_fall_through;
GET_DOUBLE(bw, f2);
if ((f2.fd < (double) (MAX_SMALL + 1))
&& (f2.fd > (double) (MIN_SMALL - 1))) {
@@ -2762,7 +3567,7 @@ tailrecur_ne:
j = float_comp(f1.fd, f2.fd);
}
} else {
- big = double_to_big(f2.fd, big_buf);
+ big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm));
j = big_comp(aw, big);
}
if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) {
@@ -2770,13 +3575,15 @@ tailrecur_ne:
}
break;
case FLOAT_SMALL:
+ if (exact) goto exact_fall_through;
GET_DOUBLE(aw, f1);
if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) {
/* Float is within the no loss limit */
f2.fd = signed_val(bw);
j = float_comp(f1.fd, f2.fd);
+ }
#if ERTS_SIZEOF_ETERM == 8
- } else if (f1.fd > (double) (MAX_SMALL + 1)) {
+ else if (f1.fd > (double) (MAX_SMALL + 1)) {
/* Float is a positive bignum, i.e. bigger */
j = 1;
} else if (f1.fd < (double) (MIN_SMALL - 1)) {
@@ -2787,12 +3594,13 @@ tailrecur_ne:
j = (Sint) f1.fd - signed_val(bw);
}
#else
- } else {
+ else {
/* If float is positive it is bigger than small */
j = (f1.fd > 0.0) ? 1 : -1;
}
#endif /* ERTS_SIZEOF_ETERM == 8 */
break;
+exact_fall_through:
default:
j = b_tag - a_tag;
}
@@ -2808,9 +3616,9 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
while (--i) {
a = *aa++;
b = *bb++;
- if (!is_same(a,a_base, b,b_base)) {
+ if (!is_same(a, b)) {
if (is_atom(a) && is_atom(b)) {
- if ((j = cmp_atoms(a, b)) != 0) {
+ if ((j = erts_cmp_atoms(a, b)) != 0) {
goto not_equal;
}
} else if (is_both_small(a, b)) {
@@ -2818,8 +3626,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
goto not_equal;
}
} else {
- /* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */
- WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER);
+ WSTACK_PUSH3(stack, (UWord)bb, (UWord)aa, TERM_ARRAY_OP_WORD(i));
goto tailrecur_ne;
}
}
@@ -2831,22 +3638,179 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
pop_next:
if (!WSTACK_ISEMPTY(stack)) {
UWord something = WSTACK_POP(stack);
- if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */
- aa = (Eterm*) something;
- bb = (Eterm*) WSTACK_POP(stack);
- i = WSTACK_POP(stack);
- goto term_array;
+ struct erts_cmp_hashmap_state* sp;
+ if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* an operation */
+ switch (GET_OP(something)) {
+ case TERM_ARRAY_OP:
+ i = GET_OP_ARG(something);
+ aa = (Eterm*)WSTACK_POP(stack);
+ bb = (Eterm*) WSTACK_POP(stack);
+ goto term_array;
+
+ case SWITCH_EXACT_OFF_OP:
+ /* Done with exact compare of map keys, switch back */
+ ASSERT(exact);
+ exact = 0;
+ goto pop_next;
+
+ case HASHMAP_PHASE1_ARE_KEYS_EQUAL: {
+ sp = PSTACK_TOP(hmap_stack);
+ if (j) {
+ /* Key diff found, enter phase 2 */
+ if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) {
+ sp->min_key = CAR(sp->ap);
+ sp->cmp_res = -1;
+ sp->ap = hashmap_iterator_next(&stack);
+ }
+ else {
+ sp->min_key = CAR(sp->bp);
+ sp->cmp_res = 1;
+ sp->bp = hashmap_iterator_next(&b_stack);
+ }
+ exact = 1; /* only exact key compares in phase 2 */
+ goto case_HASHMAP_PHASE2_LOOP;
+ }
+
+ /* No key diff found so far, compare values if min key */
+
+ if (sp->cmp_res) {
+ a = CAR(sp->ap);
+ b = sp->min_key;
+ exact = 1;
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_IS_MIN_KEY));
+ sp->wstack_rollback = WSTACK_COUNT(stack);
+ goto bodyrecur;
+ }
+ /* no min key-value found yet */
+ a = CDR(sp->ap);
+ b = CDR(sp->bp);
+ exact = sp->was_exact;
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES));
+ sp->wstack_rollback = WSTACK_COUNT(stack);
+ goto bodyrecur;
+ }
+ case HASHMAP_PHASE1_IS_MIN_KEY:
+ sp = PSTACK_TOP(hmap_stack);
+ if (j < 0) {
+ a = CDR(sp->ap);
+ b = CDR(sp->bp);
+ exact = sp->was_exact;
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES));
+ sp->wstack_rollback = WSTACK_COUNT(stack);
+ goto bodyrecur;
+ }
+ goto case_HASHMAP_PHASE1_LOOP;
+
+ case HASHMAP_PHASE1_CMP_VALUES:
+ sp = PSTACK_TOP(hmap_stack);
+ if (j) {
+ sp->cmp_res = j;
+ sp->min_key = CAR(sp->ap);
+ }
+ case_HASHMAP_PHASE1_LOOP:
+ sp->ap = hashmap_iterator_next(&stack);
+ sp->bp = hashmap_iterator_next(&b_stack);
+ if (!sp->ap) {
+ /* end of maps with identical keys */
+ ASSERT(!sp->bp);
+ j = sp->cmp_res;
+ exact = sp->was_exact;
+ (void) PSTACK_POP(hmap_stack);
+ ON_CMP_GOTO(j);
+ }
+ a = CAR(sp->ap);
+ b = CAR(sp->bp);
+ exact = 1;
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL));
+ sp->wstack_rollback = WSTACK_COUNT(stack);
+ goto bodyrecur;
+
+ case_HASHMAP_PHASE2_LOOP:
+ if (sp->ap && sp->bp) {
+ a = CAR(sp->ap);
+ b = CAR(sp->bp);
+ ASSERT(exact);
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_ARE_KEYS_EQUAL));
+ sp->wstack_rollback = WSTACK_COUNT(stack);
+ goto bodyrecur;
+ }
+ goto case_HASHMAP_PHASE2_NEXT_STEP;
+
+ case HASHMAP_PHASE2_ARE_KEYS_EQUAL:
+ sp = PSTACK_TOP(hmap_stack);
+ if (j == 0) {
+ /* keys are equal, skip them */
+ sp->ap = hashmap_iterator_next(&stack);
+ sp->bp = hashmap_iterator_next(&b_stack);
+ goto case_HASHMAP_PHASE2_LOOP;
+ }
+ /* fall through */
+ case_HASHMAP_PHASE2_NEXT_STEP:
+ if (sp->ap || sp->bp) {
+ if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) {
+ ASSERT(sp->ap);
+ a = CAR(sp->ap);
+ b = sp->min_key;
+ ASSERT(exact);
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_A));
+ }
+ else { /* hash_cmp > 0 */
+ ASSERT(sp->bp);
+ a = CAR(sp->bp);
+ b = sp->min_key;
+ ASSERT(exact);
+ WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_B));
+ }
+ sp->wstack_rollback = WSTACK_COUNT(stack);
+ goto bodyrecur;
+ }
+ /* End of both maps */
+ j = sp->cmp_res;
+ exact = sp->was_exact;
+ (void) PSTACK_POP(hmap_stack);
+ ON_CMP_GOTO(j);
+
+ case HASHMAP_PHASE2_IS_MIN_KEY_A:
+ sp = PSTACK_TOP(hmap_stack);
+ if (j < 0) {
+ sp->min_key = CAR(sp->ap);
+ sp->cmp_res = -1;
+ }
+ sp->ap = hashmap_iterator_next(&stack);
+ goto case_HASHMAP_PHASE2_LOOP;
+
+ case HASHMAP_PHASE2_IS_MIN_KEY_B:
+ sp = PSTACK_TOP(hmap_stack);
+ if (j < 0) {
+ sp->min_key = CAR(sp->bp);
+ sp->cmp_res = 1;
+ }
+ sp->bp = hashmap_iterator_next(&b_stack);
+ goto case_HASHMAP_PHASE2_LOOP;
+
+ default:
+ ASSERT(!"Invalid cmp op");
+ } /* switch */
}
a = (Eterm) something;
b = (Eterm) WSTACK_POP(stack);
goto tailrecur;
}
- DESTROY_WSTACK(stack);
+ ASSERT(PSTACK_IS_EMPTY(hmap_stack));
+ PSTACK_DESTROY(hmap_stack);
+ WSTACK_DESTROY(stack);
+ WSTACK_DESTROY(b_stack);
return 0;
not_equal:
- DESTROY_ESTACK(stack);
+ if (!PSTACK_IS_EMPTY(hmap_stack) && !eq_only) {
+ WSTACK_ROLLBACK(stack, PSTACK_TOP(hmap_stack)->wstack_rollback);
+ goto pop_next;
+ }
+ PSTACK_DESTROY(hmap_stack);
+ WSTACK_DESTROY(stack);
+ WSTACK_DESTROY(b_stack);
return j;
#undef CMP_NODES
@@ -2923,13 +3887,15 @@ void bin_write(int to, void *to_arg, byte* buf, size_t sz)
}
/* Fill buf with the contents of bytelist list
- return number of chars in list or -1 for error */
-
-int
-intlist_to_buf(Eterm list, char *buf, int len)
+ * return number of chars in list
+ * or -1 for type error
+ * or -2 for not enough buffer space (buffer contains truncated result)
+ */
+Sint
+intlist_to_buf(Eterm list, char *buf, Sint len)
{
Eterm* listptr;
- int sz = 0;
+ Sint sz = 0;
if (is_nil(list))
return 0;
@@ -2947,7 +3913,7 @@ intlist_to_buf(Eterm list, char *buf, int len)
return -1;
listptr = list_val(*(listptr + 1));
}
- return -1; /* not enough space */
+ return -2; /* not enough space */
}
/*
@@ -2983,7 +3949,7 @@ char* Sint_to_buf(Sint n, struct Sint_buf *buf)
*/
Eterm
-buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail)
+buf_to_intlist(Eterm** hpp, const char *buf, size_t len, Eterm tail)
{
Eterm* hp = *hpp;
size_t i = len;
@@ -3021,118 +3987,349 @@ buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail)
** Return remaining bytes in buffer on success
** ERTS_IOLIST_TO_BUF_OVERFLOW on overflow
** ERTS_IOLIST_TO_BUF_TYPE_ERROR on type error (including that result would not be a whole number of bytes)
+**
+** Note!
+** Do not detect indata errors in this fiunction that are not detected by erts_iolist_size!
+**
+** A caller should be able to rely on a successful return from erts_iolist_to_buf
+** if erts_iolist_size is previously successfully called and erts_iolist_to_buf
+** is called with a buffer at least as large as the value given by erts_iolist_size.
+**
*/
-ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
+typedef enum {
+ ERTS_IL2B_BCOPY_OK,
+ ERTS_IL2B_BCOPY_YIELD,
+ ERTS_IL2B_BCOPY_OVERFLOW,
+ ERTS_IL2B_BCOPY_TYPE_ERROR
+} ErtsIL2BBCopyRes;
+
+static ErtsIL2BBCopyRes
+iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp);
+
+static ERTS_INLINE ErlDrvSizeT
+iolist_to_buf(const int yield_support,
+ ErtsIOList2BufState *state,
+ Eterm obj,
+ char* buf,
+ ErlDrvSizeT alloced_len)
{
- ErlDrvSizeT len = (ErlDrvSizeT) alloced_len;
- Eterm* objp;
+#undef IOLIST_TO_BUF_BCOPY
+#define IOLIST_TO_BUF_BCOPY(CONSP) \
+do { \
+ size_t size = binary_size(obj); \
+ if (size > 0) { \
+ Uint bitsize; \
+ byte* bptr; \
+ Uint bitoffs; \
+ Uint num_bits; \
+ if (yield_support) { \
+ size_t max_size = ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \
+ if (yield_count > 0) \
+ max_size *= yield_count+1; \
+ if (size > max_size) { \
+ state->objp = CONSP; \
+ goto L_bcopy_yield; \
+ } \
+ if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { \
+ int cost = (int) size; \
+ cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \
+ yield_count -= cost; \
+ } \
+ } \
+ if (len < size) \
+ goto L_overflow; \
+ ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); \
+ if (bitsize != 0) \
+ goto L_type_error; \
+ num_bits = 8*size; \
+ copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); \
+ buf += size; \
+ len -= size; \
+ } \
+} while (0)
+
+ ErlDrvSizeT res, len;
+ Eterm* objp = NULL;
+ int init_yield_count;
+ int yield_count;
DECLARE_ESTACK(s);
- goto L_again;
-
- while (!ESTACK_ISEMPTY(s)) {
- obj = ESTACK_POP(s);
- L_again:
- if (is_list(obj)) {
- L_iter_list:
- objp = list_val(obj);
- obj = CAR(objp);
- if (is_byte(obj)) {
- if (len == 0) {
- goto L_overflow;
- }
- *buf++ = unsigned_val(obj);
- len--;
- } else if (is_binary(obj)) {
- byte* bptr;
- size_t size = binary_size(obj);
- Uint bitsize;
- Uint bitoffs;
- Uint num_bits;
-
- if (len < size) {
+
+ len = (ErlDrvSizeT) alloced_len;
+
+ if (!yield_support) {
+ yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */
+ goto L_again;
+ }
+ else {
+
+ if (state->iolist.reds_left <= 0)
+ return ERTS_IOLIST_TO_BUF_YIELD;
+
+ ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+ init_yield_count = (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED
+ * state->iolist.reds_left);
+ yield_count = init_yield_count;
+
+ if (!state->iolist.estack.start)
+ goto L_again;
+ else {
+ int chk_stack;
+ /* Restart; restore state... */
+ ESTACK_RESTORE(s, &state->iolist.estack);
+
+ if (!state->bcopy.bptr)
+ chk_stack = 0;
+ else {
+ chk_stack = 1;
+ switch (iolist_to_buf_bcopy(state, THE_NON_VALUE, &yield_count)) {
+ case ERTS_IL2B_BCOPY_OK:
+ break;
+ case ERTS_IL2B_BCOPY_YIELD:
+ BUMP_ALL_REDS(state->iolist.c_p);
+ state->iolist.reds_left = 0;
+ ESTACK_SAVE(s, &state->iolist.estack);
+ return ERTS_IOLIST_TO_BUF_YIELD;
+ case ERTS_IL2B_BCOPY_OVERFLOW:
goto L_overflow;
- }
- ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
- if (bitsize != 0) {
+ case ERTS_IL2B_BCOPY_TYPE_ERROR:
goto L_type_error;
}
- num_bits = 8*size;
- copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
- buf += size;
- len -= size;
- } else if (is_list(obj)) {
- ESTACK_PUSH(s, CDR(objp));
- goto L_iter_list; /* on head */
- } else if (is_not_nil(obj)) {
- goto L_type_error;
}
- obj = CDR(objp);
- if (is_list(obj)) {
- goto L_iter_list; /* on tail */
- } else if (is_binary(obj)) {
- byte* bptr;
- size_t size = binary_size(obj);
- Uint bitsize;
- Uint bitoffs;
- Uint num_bits;
- if (len < size) {
- goto L_overflow;
+ obj = state->iolist.obj;
+ buf = state->buf;
+ len = state->len;
+ objp = state->objp;
+ state->objp = NULL;
+ if (objp)
+ goto L_tail;
+ if (!chk_stack)
+ goto L_again;
+ /* check stack */
+ }
+ }
+
+ while (!ESTACK_ISEMPTY(s)) {
+ obj = ESTACK_POP(s);
+ L_again:
+ if (is_list(obj)) {
+ while (1) { /* Tail loop */
+ while (1) { /* Head loop */
+ if (yield_support && --yield_count <= 0)
+ goto L_yield;
+ objp = list_val(obj);
+ obj = CAR(objp);
+ if (is_byte(obj)) {
+ if (len == 0) {
+ goto L_overflow;
+ }
+ *buf++ = unsigned_val(obj);
+ len--;
+ } else if (is_binary(obj)) {
+ IOLIST_TO_BUF_BCOPY(objp);
+ } else if (is_list(obj)) {
+ ESTACK_PUSH(s, CDR(objp));
+ continue; /* Head loop */
+ } else if (is_not_nil(obj)) {
+ goto L_type_error;
+ }
+ break;
}
- ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
- if (bitsize != 0) {
+
+ L_tail:
+
+ obj = CDR(objp);
+
+ if (is_list(obj)) {
+ continue; /* Tail loop */
+ } else if (is_binary(obj)) {
+ IOLIST_TO_BUF_BCOPY(NULL);
+ } else if (is_not_nil(obj)) {
goto L_type_error;
}
- num_bits = 8*size;
- copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
- buf += size;
- len -= size;
- } else if (is_not_nil(obj)) {
- goto L_type_error;
+ break;
}
} else if (is_binary(obj)) {
- byte* bptr;
- size_t size = binary_size(obj);
- Uint bitsize;
- Uint bitoffs;
- Uint num_bits;
- if (len < size) {
- goto L_overflow;
- }
- ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
- if (bitsize != 0) {
- goto L_type_error;
- }
- num_bits = 8*size;
- copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
- buf += size;
- len -= size;
+ IOLIST_TO_BUF_BCOPY(NULL);
} else if (is_not_nil(obj)) {
goto L_type_error;
- }
+ } else if (yield_support && --yield_count <= 0)
+ goto L_yield;
}
+ res = len;
+
+ L_return:
+
DESTROY_ESTACK(s);
- return len;
+
+ if (yield_support) {
+ int reds;
+ CLEAR_SAVED_ESTACK(&state->iolist.estack);
+ reds = ((init_yield_count - yield_count - 1)
+ / ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED) + 1;
+ BUMP_REDS(state->iolist.c_p, reds);
+ state->iolist.reds_left -= reds;
+ if (state->iolist.reds_left < 0)
+ state->iolist.reds_left = 0;
+ }
+
+
+ return res;
L_type_error:
- DESTROY_ESTACK(s);
- return ERTS_IOLIST_TO_BUF_TYPE_ERROR;
+ res = ERTS_IOLIST_TO_BUF_TYPE_ERROR;
+ goto L_return;
L_overflow:
- DESTROY_ESTACK(s);
- return ERTS_IOLIST_TO_BUF_OVERFLOW;
+ res = ERTS_IOLIST_TO_BUF_OVERFLOW;
+ goto L_return;
+
+ L_bcopy_yield:
+
+ state->buf = buf;
+ state->len = len;
+
+ switch (iolist_to_buf_bcopy(state, obj, &yield_count)) {
+ case ERTS_IL2B_BCOPY_OK:
+ ERTS_INTERNAL_ERROR("Missing yield");
+ case ERTS_IL2B_BCOPY_YIELD:
+ BUMP_ALL_REDS(state->iolist.c_p);
+ state->iolist.reds_left = 0;
+ ESTACK_SAVE(s, &state->iolist.estack);
+ return ERTS_IOLIST_TO_BUF_YIELD;
+ case ERTS_IL2B_BCOPY_OVERFLOW:
+ goto L_overflow;
+ case ERTS_IL2B_BCOPY_TYPE_ERROR:
+ goto L_type_error;
+ }
+
+ L_yield:
+
+ BUMP_ALL_REDS(state->iolist.c_p);
+ state->iolist.reds_left = 0;
+ state->iolist.obj = obj;
+ state->buf = buf;
+ state->len = len;
+ ESTACK_SAVE(s, &state->iolist.estack);
+ return ERTS_IOLIST_TO_BUF_YIELD;
+
+#undef IOLIST_TO_BUF_BCOPY
+}
+
+static ErtsIL2BBCopyRes
+iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp)
+{
+ ErtsIL2BBCopyRes res;
+ char *buf = state->buf;
+ ErlDrvSizeT len = state->len;
+ byte* bptr;
+ size_t size;
+ size_t max_size;
+ Uint bitoffs;
+ Uint num_bits;
+ int yield_count = *yield_countp;
+
+ if (state->bcopy.bptr) {
+ bptr = state->bcopy.bptr;
+ size = state->bcopy.size;
+ bitoffs = state->bcopy.bitoffs;
+ state->bcopy.bptr = NULL;
+ }
+ else {
+ Uint bitsize;
+
+ ASSERT(is_binary(obj));
+
+ size = binary_size(obj);
+ if (size <= 0)
+ return ERTS_IL2B_BCOPY_OK;
+
+ if (len < size)
+ return ERTS_IL2B_BCOPY_OVERFLOW;
+
+ ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize);
+ if (bitsize != 0)
+ return ERTS_IL2B_BCOPY_TYPE_ERROR;
+ }
+
+ ASSERT(size > 0);
+ max_size = (size_t) ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;
+ if (yield_count > 0)
+ max_size *= (size_t) (yield_count+1);
+
+ if (size <= max_size) {
+ if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) {
+ int cost = (int) size;
+ cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT;
+ yield_count -= cost;
+ }
+ res = ERTS_IL2B_BCOPY_OK;
+ }
+ else {
+ ASSERT(0 < max_size && max_size < size);
+ yield_count = 0;
+ state->bcopy.bptr = bptr + max_size;
+ state->bcopy.bitoffs = bitoffs;
+ state->bcopy.size = size - max_size;
+ size = max_size;
+ res = ERTS_IL2B_BCOPY_YIELD;
+ }
+
+ num_bits = 8*size;
+ copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits);
+ state->buf += size;
+ state->len -= size;
+ *yield_countp = yield_count;
+
+ return res;
+}
+
+ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *state)
+{
+ return iolist_to_buf(1, state, state->iolist.obj, state->buf, state->len);
+}
+
+ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len)
+{
+ return iolist_to_buf(0, NULL, obj, buf, alloced_len);
}
/*
* Return 0 if successful, and non-zero if unsuccessful.
+ *
+ * It is vital that if erts_iolist_to_buf would return an error for
+ * any type of term data, this function should do so as well.
+ * Any input term error detected in erts_iolist_to_buf should also
+ * be detected in this function!
*/
-int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
+
+static ERTS_INLINE int
+iolist_size(const int yield_support, ErtsIOListState *state, Eterm obj, ErlDrvSizeT* sizep)
{
+ int res, init_yield_count, yield_count;
Eterm* objp;
- Uint size = 0; /* Intentionally Uint due to halfword heap */
+ Uint size = (Uint) *sizep;
DECLARE_ESTACK(s);
+
+ if (!yield_support)
+ yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */
+ else {
+ if (state->reds_left <= 0)
+ return ERTS_IOLIST_YIELD;
+ ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK);
+ init_yield_count = ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED;
+ init_yield_count *= state->reds_left;
+ yield_count = init_yield_count;
+ if (state->estack.start) {
+ /* Restart; restore state... */
+ ESTACK_RESTORE(s, &state->estack);
+ size = (Uint) state->size;
+ obj = state->obj;
+ }
+ }
+
goto L_again;
#define SAFE_ADD(Var, Val) \
@@ -3148,58 +4345,109 @@ int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
obj = ESTACK_POP(s);
L_again:
if (is_list(obj)) {
- L_iter_list:
- objp = list_val(obj);
- /* Head */
- obj = CAR(objp);
- if (is_byte(obj)) {
- size++;
- if (size == 0) {
- goto L_overflow_error;
+ while (1) { /* Tail loop */
+ while (1) { /* Head loop */
+ if (yield_support && --yield_count <= 0)
+ goto L_yield;
+ objp = list_val(obj);
+ /* Head */
+ obj = CAR(objp);
+ if (is_byte(obj)) {
+ size++;
+ if (size == 0) {
+ goto L_overflow_error;
+ }
+ } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+ SAFE_ADD(size, binary_size(obj));
+ } else if (is_list(obj)) {
+ ESTACK_PUSH(s, CDR(objp));
+ continue; /* Head loop */
+ } else if (is_not_nil(obj)) {
+ goto L_type_error;
+ }
+ break;
}
- } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
- SAFE_ADD(size, binary_size(obj));
- } else if (is_list(obj)) {
- ESTACK_PUSH(s, CDR(objp));
- goto L_iter_list; /* on head */
- } else if (is_not_nil(obj)) {
- goto L_type_error;
+ /* Tail */
+ obj = CDR(objp);
+ if (is_list(obj))
+ continue; /* Tail loop */
+ else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+ SAFE_ADD(size, binary_size(obj));
+ } else if (is_not_nil(obj)) {
+ goto L_type_error;
+ }
+ break;
}
- /* Tail */
- obj = CDR(objp);
- if (is_list(obj))
- goto L_iter_list; /* on tail */
- else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+ } else {
+ if (yield_support && --yield_count <= 0)
+ goto L_yield;
+ if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
SAFE_ADD(size, binary_size(obj));
} else if (is_not_nil(obj)) {
goto L_type_error;
}
- } else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
- SAFE_ADD(size, binary_size(obj));
- } else if (is_not_nil(obj)) {
- goto L_type_error;
}
}
#undef SAFE_ADD
- DESTROY_ESTACK(s);
*sizep = (ErlDrvSizeT) size;
- return ERTS_IOLIST_OK;
- L_overflow_error:
+ res = ERTS_IOLIST_OK;
+
+ L_return:
+
DESTROY_ESTACK(s);
- return ERTS_IOLIST_OVERFLOW;
+
+ if (yield_support) {
+ int yc, reds;
+ CLEAR_SAVED_ESTACK(&state->estack);
+ yc = init_yield_count - yield_count;
+ reds = ((yc - 1) / ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED) + 1;
+ BUMP_REDS(state->c_p, reds);
+ state->reds_left -= reds;
+ state->size = (ErlDrvSizeT) size;
+ state->have_size = 1;
+ }
+
+ return res;
+
+ L_overflow_error:
+ res = ERTS_IOLIST_OVERFLOW;
+ size = 0;
+ goto L_return;
L_type_error:
- DESTROY_ESTACK(s);
- return ERTS_IOLIST_TYPE;
+ res = ERTS_IOLIST_TYPE;
+ size = 0;
+ goto L_return;
+
+ L_yield:
+ BUMP_ALL_REDS(state->c_p);
+ state->reds_left = 0;
+ state->size = size;
+ state->obj = obj;
+ ESTACK_SAVE(s, &state->estack);
+ return ERTS_IOLIST_YIELD;
}
-/* return 0 if item is not a non-empty flat list of bytes */
-int
+int erts_iolist_size_yielding(ErtsIOListState *state)
+{
+ ErlDrvSizeT size = state->size;
+ return iolist_size(1, state, state->obj, &size);
+}
+
+int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep)
+{
+ *sizep = 0;
+ return iolist_size(0, NULL, obj, sizep);
+}
+
+/* return 0 if item is not a non-empty flat list of bytes
+ otherwise return the nonzero length of the list */
+Sint
is_string(Eterm list)
{
- int len = 0;
+ Sint len = 0;
while(is_list(list)) {
Eterm* consp = list_val(list);
@@ -3215,145 +4463,6 @@ is_string(Eterm list)
return 0;
}
-#ifdef ERTS_SMP
-
-/*
- * Process and Port timers in smp case
- */
-
-ERTS_SCHED_PREF_PRE_ALLOC_IMPL(ptimer_pre, ErtsSmpPTimer, 1000)
-
-#define ERTS_PTMR_FLGS_ALLCD_SIZE \
- 2
-#define ERTS_PTMR_FLGS_ALLCD_MASK \
- ((((Uint32) 1) << ERTS_PTMR_FLGS_ALLCD_SIZE) - 1)
-
-#define ERTS_PTMR_FLGS_PREALLCD ((Uint32) 1)
-#define ERTS_PTMR_FLGS_SLALLCD ((Uint32) 2)
-#define ERTS_PTMR_FLGS_LLALLCD ((Uint32) 3)
-#define ERTS_PTMR_FLG_CANCELLED (((Uint32) 1) << (ERTS_PTMR_FLGS_ALLCD_SIZE+0))
-
-static void
-init_ptimers(void)
-{
- init_ptimer_pre_alloc();
-}
-
-static ERTS_INLINE void
-free_ptimer(ErtsSmpPTimer *ptimer)
-{
- switch (ptimer->timer.flags & ERTS_PTMR_FLGS_ALLCD_MASK) {
- case ERTS_PTMR_FLGS_PREALLCD:
- (void) ptimer_pre_free(ptimer);
- break;
- case ERTS_PTMR_FLGS_SLALLCD:
- erts_free(ERTS_ALC_T_SL_PTIMER, (void *) ptimer);
- break;
- case ERTS_PTMR_FLGS_LLALLCD:
- erts_free(ERTS_ALC_T_LL_PTIMER, (void *) ptimer);
- break;
- default:
- erl_exit(ERTS_ABORT_EXIT,
- "Internal error: Bad ptimer alloc type\n");
- break;
- }
-}
-
-/* Callback for process timeout cancelled */
-static void
-ptimer_cancelled(ErtsSmpPTimer *ptimer)
-{
- free_ptimer(ptimer);
-}
-
-/* Callback for process timeout */
-static void
-ptimer_timeout(ErtsSmpPTimer *ptimer)
-{
- if (is_internal_pid(ptimer->timer.id)) {
- Process *p;
- p = erts_pid2proc_opt(NULL,
- 0,
- ptimer->timer.id,
- ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS,
- ERTS_P2P_FLG_ALLOW_OTHER_X);
- if (p) {
- if (!ERTS_PROC_IS_EXITING(p)
- && !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) {
- ASSERT(*ptimer->timer.timer_ref == ptimer);
- *ptimer->timer.timer_ref = NULL;
- (*ptimer->timer.timeout_func)(p);
- }
- erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
- }
- }
- else {
- Port *p;
- ASSERT(is_internal_port(ptimer->timer.id));
- p = erts_id2port_sflgs(ptimer->timer.id,
- NULL,
- 0,
- ERTS_PORT_SFLGS_DEAD);
- if (p) {
- if (!(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) {
- ASSERT(*ptimer->timer.timer_ref == ptimer);
- *ptimer->timer.timer_ref = NULL;
- (*ptimer->timer.timeout_func)(p);
- }
- erts_port_release(p);
- }
- }
- free_ptimer(ptimer);
-}
-
-void
-erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
- Eterm id,
- ErlTimeoutProc timeout_func,
- Uint timeout)
-{
- ErtsSmpPTimer *res = ptimer_pre_alloc();
- if (res)
- res->timer.flags = ERTS_PTMR_FLGS_PREALLCD;
- else {
- if (timeout < ERTS_ALC_MIN_LONG_LIVED_TIME) {
- res = erts_alloc(ERTS_ALC_T_SL_PTIMER, sizeof(ErtsSmpPTimer));
- res->timer.flags = ERTS_PTMR_FLGS_SLALLCD;
- }
- else {
- res = erts_alloc(ERTS_ALC_T_LL_PTIMER, sizeof(ErtsSmpPTimer));
- res->timer.flags = ERTS_PTMR_FLGS_LLALLCD;
- }
- }
- res->timer.timeout_func = timeout_func;
- res->timer.timer_ref = timer_ref;
- res->timer.id = id;
- res->timer.tm.active = 0; /* MUST be initalized */
-
- ASSERT(!*timer_ref);
-
- *timer_ref = res;
-
- erts_set_timer(&res->timer.tm,
- (ErlTimeoutProc) ptimer_timeout,
- (ErlCancelProc) ptimer_cancelled,
- (void*) res,
- timeout);
-}
-
-void
-erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer)
-{
- if (ptimer) {
- ASSERT(*ptimer->timer.timer_ref == ptimer);
- *ptimer->timer.timer_ref = NULL;
- ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED;
- erts_cancel_timer(&ptimer->timer.tm);
- }
-}
-
-#endif
-
static int trim_threshold;
static int top_pad;
static int mmap_threshold;
@@ -3363,9 +4472,7 @@ Uint tot_bin_allocated;
void erts_init_utils(void)
{
-#ifdef ERTS_SMP
- init_ptimers();
-#endif
+
}
void erts_init_utils_mem(void)
@@ -3466,6 +4573,294 @@ erts_free_read_env(void *value)
erts_free(ERTS_ALC_T_TMP, value);
}
+
+typedef struct {
+ size_t sz;
+ char *ptr;
+} ErtsEmuArg;
+
+typedef struct {
+ int argc;
+ ErtsEmuArg *arg;
+ size_t no_bytes;
+} ErtsEmuArgs;
+
+ErtsEmuArgs saved_emu_args = {0};
+
+void
+erts_save_emu_args(int argc, char **argv)
+{
+#ifdef DEBUG
+ char *end_ptr;
+#endif
+ char *ptr;
+ int i;
+ size_t arg_sz[100];
+ size_t size;
+
+ ASSERT(!saved_emu_args.argc);
+
+ size = sizeof(ErtsEmuArg)*argc;
+ for (i = 0; i < argc; i++) {
+ size_t sz = sys_strlen(argv[i]);
+ if (i < sizeof(arg_sz)/sizeof(arg_sz[0]))
+ arg_sz[i] = sz;
+ size += sz+1;
+ }
+ ptr = (char *) malloc(size);
+ if (!ptr) {
+ ERTS_INTERNAL_ERROR("malloc failed to allocate memory!");
+ }
+#ifdef DEBUG
+ end_ptr = ptr + size;
+#endif
+ saved_emu_args.arg = (ErtsEmuArg *) ptr;
+ ptr += sizeof(ErtsEmuArg)*argc;
+ saved_emu_args.argc = argc;
+ saved_emu_args.no_bytes = 0;
+ for (i = 0; i < argc; i++) {
+ size_t sz;
+ if (i < sizeof(arg_sz)/sizeof(arg_sz[0]))
+ sz = arg_sz[i];
+ else
+ sz = sys_strlen(argv[i]);
+ saved_emu_args.arg[i].ptr = ptr;
+ saved_emu_args.arg[i].sz = sz;
+ saved_emu_args.no_bytes += sz;
+ ptr += sz+1;
+ sys_strcpy(saved_emu_args.arg[i].ptr, argv[i]);
+ }
+ ASSERT(ptr == end_ptr);
+}
+
+Eterm
+erts_get_emu_args(Process *c_p)
+{
+#ifdef DEBUG
+ Eterm *end_hp;
+#endif
+ int i;
+ Uint hsz;
+ Eterm *hp, res;
+
+ hsz = saved_emu_args.no_bytes*2;
+ hsz += saved_emu_args.argc*2;
+
+ hp = HAlloc(c_p, hsz);
+#ifdef DEBUG
+ end_hp = hp + hsz;
+#endif
+ res = NIL;
+
+ for (i = saved_emu_args.argc-1; i >= 0; i--) {
+ Eterm arg = buf_to_intlist(&hp,
+ saved_emu_args.arg[i].ptr,
+ saved_emu_args.arg[i].sz,
+ NIL);
+ res = CONS(hp, arg, res);
+ hp += 2;
+ }
+
+ ASSERT(hp == end_hp);
+
+ return res;
+}
+
+
+Eterm
+erts_get_ethread_info(Process *c_p)
+{
+ Uint sz, *szp;
+ Eterm res, *hp, **hpp, *end_hp = NULL;
+
+ sz = 0;
+ szp = &sz;
+ hpp = NULL;
+
+ while (1) {
+ Eterm tup, list, name;
+#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \
+ || defined(ETHR_NATIVE_ATOMIC64_IMPL) \
+ || defined(ETHR_NATIVE_DW_ATOMIC_IMPL)
+ char buf[1024];
+ int i;
+ char **str;
+#endif
+
+ res = NIL;
+
+#ifdef ETHR_X86_MEMBAR_H__
+
+ tup = erts_bld_tuple(hpp, szp, 2,
+ erts_bld_string(hpp, szp, "sse2"),
+#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+ erts_bld_string(hpp, szp,
+ (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+ ? "yes" : "no"))
+#else
+ erts_bld_string(hpp, szp, "yes")
+#endif
+ );
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+ tup = erts_bld_tuple(hpp, szp, 2,
+ erts_bld_string(hpp, szp,
+ "x86"
+#ifdef ARCH_64
+ "_64"
+#endif
+ " OOO"),
+ erts_bld_string(hpp, szp,
+#ifdef ETHR_X86_OUT_OF_ORDER
+ "yes"
+#else
+ "no"
+#endif
+ ));
+
+ res = erts_bld_cons(hpp, szp, tup, res);
+#endif
+
+#ifdef ETHR_SPARC_V9_MEMBAR_H__
+
+ tup = erts_bld_tuple(hpp, szp, 2,
+ erts_bld_string(hpp, szp, "Sparc V9"),
+ erts_bld_string(hpp, szp,
+#if defined(ETHR_SPARC_TSO)
+ "TSO"
+#elif defined(ETHR_SPARC_PSO)
+ "PSO"
+#elif defined(ETHR_SPARC_RMO)
+ "RMO"
+#else
+ "undefined"
+#endif
+ ));
+
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+#endif
+
+#ifdef ETHR_PPC_MEMBAR_H__
+
+ tup = erts_bld_tuple(hpp, szp, 2,
+ erts_bld_string(hpp, szp, "lwsync"),
+ erts_bld_string(hpp, szp,
+#if defined(ETHR_PPC_HAVE_LWSYNC)
+ "yes"
+#elif defined(ETHR_PPC_HAVE_NO_LWSYNC)
+ "no"
+#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__)
+ ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"
+#else
+ "undefined"
+#endif
+ ));
+
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+#endif
+
+ tup = erts_bld_tuple(hpp, szp, 2,
+ erts_bld_string(hpp, szp, "Native rw-spinlocks"),
+#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL
+ erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL)
+#else
+ erts_bld_string(hpp, szp, "no")
+#endif
+ );
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+ tup = erts_bld_tuple(hpp, szp, 2,
+ erts_bld_string(hpp, szp, "Native spinlocks"),
+#ifdef ETHR_NATIVE_SPINLOCK_IMPL
+ erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL)
+#else
+ erts_bld_string(hpp, szp, "no")
+#endif
+ );
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+
+ list = NIL;
+#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL
+ if (ethr_have_native_dw_atomic()) {
+ name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL);
+ str = ethr_native_dw_atomic_ops();
+ for (i = 0; str[i]; i++) {
+ erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]);
+ list = erts_bld_cons(hpp, szp,
+ erts_bld_string(hpp, szp, buf),
+ list);
+ }
+ str = ethr_native_su_dw_atomic_ops();
+ for (i = 0; str[i]; i++) {
+ erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]);
+ list = erts_bld_cons(hpp, szp,
+ erts_bld_string(hpp, szp, buf),
+ list);
+ }
+ }
+ else
+#endif
+ name = erts_bld_string(hpp, szp, "no");
+
+ tup = erts_bld_tuple(hpp, szp, 3,
+ erts_bld_string(hpp, szp, "Double word native atomics"),
+ name,
+ list);
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+ list = NIL;
+#ifdef ETHR_NATIVE_ATOMIC64_IMPL
+ name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL);
+ str = ethr_native_atomic64_ops();
+ for (i = 0; str[i]; i++) {
+ erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]);
+ list = erts_bld_cons(hpp, szp,
+ erts_bld_string(hpp, szp, buf),
+ list);
+ }
+#else
+ name = erts_bld_string(hpp, szp, "no");
+#endif
+ tup = erts_bld_tuple(hpp, szp, 3,
+ erts_bld_string(hpp, szp, "64-bit native atomics"),
+ name,
+ list);
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+ list = NIL;
+#ifdef ETHR_NATIVE_ATOMIC32_IMPL
+ name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL);
+ str = ethr_native_atomic32_ops();
+ for (i = 0; str[i]; i++) {
+ erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]);
+ list = erts_bld_cons(hpp, szp,
+ erts_bld_string(hpp, szp, buf),
+ list);
+ }
+#else
+ name = erts_bld_string(hpp, szp, "no");
+#endif
+ tup = erts_bld_tuple(hpp, szp, 3,
+ erts_bld_string(hpp, szp, "32-bit native atomics"),
+ name,
+ list);
+ res = erts_bld_cons(hpp, szp, tup, res);
+
+ if (hpp) {
+ HRelease(c_p, end_hp, *hpp)
+ return res;
+ }
+
+ hp = HAlloc(c_p, sz);
+ end_hp = hp + sz;
+ hpp = &hp;
+ szp = NULL;
+ }
+}
+
/*
* To be used to silence unused result warnings, but do not abuse it.
*/
@@ -3480,19 +4875,7 @@ void erts_silence_warn_unused_result(long unused)
void
erts_interval_init(erts_interval_t *icp)
{
-#ifdef ARCH_64
- erts_atomic_init_nob(&icp->counter.atomic, 0);
-#else
- erts_dw_aint_t dw;
-#ifdef ETHR_SU_DW_NAINT_T__
- dw.dw_sint = 0;
-#else
- dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0;
- dw.sint[ERTS_DW_AINT_LOW_WORD] = 0;
-#endif
- erts_dw_atomic_init_nob(&icp->counter.atomic, &dw);
-
-#endif
+ erts_atomic64_init_nob(&icp->counter.atomic, 0);
#ifdef DEBUG
icp->smp_api = 0;
#endif
@@ -3514,55 +4897,13 @@ erts_smp_interval_init(erts_interval_t *icp)
static ERTS_INLINE Uint64
step_interval_nob(erts_interval_t *icp)
{
-#ifdef ARCH_64
- return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
-#else
- erts_dw_aint_t exp;
-
- erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
- while (1) {
- erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
- new.dw_sint++;
-#else
- new.sint[ERTS_DW_AINT_LOW_WORD]++;
- if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
- new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
- if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
- return erts_interval_dw_aint_to_val__(&new);
-
- }
-#endif
+ return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic);
}
static ERTS_INLINE Uint64
step_interval_relb(erts_interval_t *icp)
{
-#ifdef ARCH_64
- return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic);
-#else
- erts_dw_aint_t exp;
-
- erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
- while (1) {
- erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
- new.dw_sint++;
-#else
- new.sint[ERTS_DW_AINT_LOW_WORD]++;
- if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
- new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
- if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp))
- return erts_interval_dw_aint_to_val__(&new);
-
- }
-#endif
+ return (Uint64) erts_atomic64_inc_read_relb(&icp->counter.atomic);
}
@@ -3570,38 +4911,10 @@ static ERTS_INLINE Uint64
ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
{
Uint64 curr_ic;
-#ifdef ARCH_64
- curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
- if (curr_ic > ic)
- return curr_ic;
- return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
-#else
- erts_dw_aint_t exp;
-
- erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
- curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ curr_ic = (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
if (curr_ic > ic)
return curr_ic;
-
- while (1) {
- erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
- new.dw_sint++;
-#else
- new.sint[ERTS_DW_AINT_LOW_WORD]++;
- if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
- new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
- if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
- return erts_interval_dw_aint_to_val__(&new);
-
- curr_ic = erts_interval_dw_aint_to_val__(&exp);
- if (curr_ic > ic)
- return curr_ic;
- }
-#endif
+ return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic);
}
@@ -3609,38 +4922,10 @@ static ERTS_INLINE Uint64
ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
{
Uint64 curr_ic;
-#ifdef ARCH_64
- curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
- if (curr_ic > ic)
- return curr_ic;
- return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic);
-#else
- erts_dw_aint_t exp;
-
- erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp);
- curr_ic = erts_interval_dw_aint_to_val__(&exp);
+ curr_ic = (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
if (curr_ic > ic)
return curr_ic;
-
- while (1) {
- erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
- new.dw_sint++;
-#else
- new.sint[ERTS_DW_AINT_LOW_WORD]++;
- if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
- new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
- if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp))
- return erts_interval_dw_aint_to_val__(&new);
-
- curr_ic = erts_interval_dw_aint_to_val__(&exp);
- if (curr_ic > ic)
- return curr_ic;
- }
-#endif
+ return (Uint64) erts_atomic64_inc_read_acqb(&icp->counter.atomic);
}
Uint64
@@ -3721,15 +5006,14 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
#endif
}
-
/*
* A millisecond timestamp without time correction where there's no hrtime
* - for tracing on "long" things...
*/
Uint64 erts_timestamp_millis(void)
{
-#ifdef HAVE_GETHRTIME
- return (Uint64) (sys_gethrtime() / 1000000);
+#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
+ return ERTS_MONOTONIC_TO_MSEC(erts_os_monotonic_time());
#else
Uint64 res;
SysTimeval tv;
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-23 21:16:43 +0000