aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/global.h
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/global.h')
-rw-r--r--[-rwxr-xr-x]erts/emulator/beam/global.h719
1 files changed, 578 insertions, 141 deletions
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 012c1c7e6a..1b8595fe57 100755..100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,18 +1,19 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2013. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2014. All Rights Reserved.
*
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*
* %CopyrightEnd%
*/
@@ -41,6 +42,7 @@
#include "error.h"
#include "erl_utils.h"
#include "erl_port.h"
+#include "erl_gc.h"
struct enif_environment_t /* ErlNifEnv */
{
@@ -51,6 +53,7 @@ struct enif_environment_t /* ErlNifEnv */
ErlHeapFragment* heap_frag;
int fpe_was_unmasked;
struct enif_tmp_obj_t* tmp_obj_list;
+ int exception_thrown; /* boolean */
};
extern void erts_pre_nif(struct enif_environment_t*, Process*,
struct erl_module_nif*);
@@ -160,6 +163,7 @@ struct erts_driver_t_ {
void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data); /* Might be NULL */
void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
void (*stop_select)(ErlDrvEvent event, void*); /* Might be NULL */
+ void (*emergency_close)(ErlDrvData drv_data); /* Might be NULL */
};
extern erts_driver_t *driver_list;
@@ -186,11 +190,6 @@ extern void erts_ddll_remove_monitor(Process *p,
extern Eterm erts_ddll_monitor_driver(Process *p,
Eterm description,
ErtsProcLocks plocks);
-/*
- * Max no. of drivers (linked in and dynamically loaded). Each table
- * entry uses 4 bytes.
- */
-#define DRIVER_TAB_SIZE 32
/*
** Just like the driver binary but with initial flags
@@ -232,9 +231,23 @@ typedef struct {
ERTS_INTERNAL_BINARY_FIELDS
SWord orig_size;
void (*destructor)(Binary *);
- char magic_bin_data[1];
+ union {
+ struct {
+ ERTS_BINARY_STRUCT_ALIGNMENT
+ char data[1];
+ } aligned;
+ struct {
+ char data[1];
+ } unaligned;
+ } u;
} ErtsMagicBinary;
+#ifdef ARCH_32
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4
+#else
+#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0
+#endif
+
typedef union {
Binary binary;
ErtsMagicBinary magic_binary;
@@ -254,15 +267,30 @@ typedef union {
#define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \
((ErtsBinary *) (BP))->magic_binary.destructor
#define ERTS_MAGIC_BIN_DATA(BP) \
- ((void *) ((ErtsBinary *) (BP))->magic_binary.magic_bin_data)
-#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \
- ((BP)->orig_size - sizeof(void (*)(Binary *)))
+ ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data)
+#define ERTS_MAGIC_DATA_OFFSET \
+ (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes))
#define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \
- (sizeof(void (*)(Binary *)) + (Sz))
+ (ERTS_MAGIC_DATA_OFFSET + (Sz))
#define ERTS_MAGIC_BIN_SIZE(Sz) \
- (offsetof(ErtsMagicBinary,magic_bin_data) + (Sz))
-#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \
- ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,magic_bin_data)))
+ (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz))
+
+/* On 32-bit arch these macro variants will save memory
+ by not forcing 8-byte alignment for the magic payload.
+*/
+#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \
+ ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data)
+#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \
+ (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes))
+#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \
+ ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET)
+#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \
+ (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz))
+#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \
+ (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz))
+#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \
+ ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data)))
+
#define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary)
#define ErlDrvBinary2Binary(D) ((Binary *) \
@@ -351,8 +379,6 @@ extern Uint display_items; /* no of items to display in traces etc */
extern int erts_backtrace_depth;
extern erts_smp_atomic32_t erts_max_gen_gcs;
-extern int erts_disable_tolerant_timeofday;
-
extern int bif_reductions; /* reductions + fcalls (when doing call_bif) */
extern int stackdump_on_exit;
@@ -375,113 +401,452 @@ extern int stackdump_on_exit;
* DESTROY_ESTACK(Stack)
*/
+typedef struct ErtsEStack_ {
+ Eterm* start;
+ Eterm* sp;
+ Eterm* end;
+ Eterm* edefault;
+ ErtsAlcType_t alloc_type;
+}ErtsEStack;
-void erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end);
-#define ESTK_CONCAT(a,b) a##b
-#define ESTK_SUBSCRIPT(s,i) *((Eterm *)((byte *)ESTK_CONCAT(s,_start) + (i)))
#define DEF_ESTACK_SIZE (16)
-#define DECLARE_ESTACK(s) \
- Eterm ESTK_CONCAT(s,_default_stack)[DEF_ESTACK_SIZE]; \
- Eterm* ESTK_CONCAT(s,_start) = ESTK_CONCAT(s,_default_stack); \
- Eterm* ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start); \
- Eterm* ESTK_CONCAT(s,_end) = ESTK_CONCAT(s,_start) + DEF_ESTACK_SIZE
+void erl_grow_estack(ErtsEStack*, Uint need);
+#define ESTK_CONCAT(a,b) a##b
+#define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack)
+
+#define DECLARE_ESTACK(s) \
+ Eterm ESTK_DEF_STACK(s)[DEF_ESTACK_SIZE]; \
+ ErtsEStack s = { \
+ ESTK_DEF_STACK(s), /* start */ \
+ ESTK_DEF_STACK(s), /* sp */ \
+ ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \
+ ESTK_DEF_STACK(s), /* default */ \
+ ERTS_ALC_T_ESTACK /* alloc_type */ \
+ }
-#define DESTROY_ESTACK(s) \
+#define ESTACK_CHANGE_ALLOCATOR(s,t) \
do { \
- if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) { \
- erts_free(ERTS_ALC_T_ESTACK, ESTK_CONCAT(s,_start)); \
+ if ((s).start != ESTK_DEF_STACK(s)) { \
+ erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \
+ "type of active estack\n"); \
} \
+ (s).alloc_type = (t); \
+ } while (0)
+
+#define DESTROY_ESTACK(s) \
+do { \
+ if ((s).start != ESTK_DEF_STACK(s)) { \
+ erts_free((s).alloc_type, (s).start); \
+ } \
} while(0)
-#define ESTACK_PUSH(s, x) \
-do { \
- if (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_end)) { \
- erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \
- &ESTK_CONCAT(s,_end)); \
- } \
- *ESTK_CONCAT(s,_sp)++ = (x); \
+
+/*
+ * Do not free the stack after this, it may have pointers into what
+ * was saved in 'dst'.
+ */
+#define ESTACK_SAVE(s,dst)\
+do {\
+ if ((s).start == ESTK_DEF_STACK(s)) {\
+ UWord _wsz = ESTACK_COUNT(s);\
+ (dst)->start = erts_alloc((s).alloc_type,\
+ DEF_ESTACK_SIZE * sizeof(Eterm));\
+ memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\
+ (dst)->sp = (dst)->start + _wsz;\
+ (dst)->end = (dst)->start + DEF_ESTACK_SIZE;\
+ (dst)->edefault = NULL;\
+ (dst)->alloc_type = (s).alloc_type;\
+ } else\
+ *(dst) = (s);\
+ } while (0)
+
+#define DESTROY_SAVED_ESTACK(estack)\
+do {\
+ if ((estack)->start) {\
+ erts_free((estack)->alloc_type, (estack)->start);\
+ (estack)->start = NULL;\
+ }\
} while(0)
-#define ESTACK_PUSH2(s, x, y) \
-do { \
- if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 2) { \
- erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \
- &ESTK_CONCAT(s,_end)); \
- } \
- *ESTK_CONCAT(s,_sp)++ = (x); \
- *ESTK_CONCAT(s,_sp)++ = (y); \
+#define CLEAR_SAVED_ESTACK(estack) ((void) ((estack)->start = NULL))
+
+/*
+ * Use on empty stack, only the allocator can be changed before this.
+ * The src stack is reset to NULL.
+ */
+#define ESTACK_RESTORE(s, src) \
+do { \
+ ASSERT((s).start == ESTK_DEF_STACK(s)); \
+ (s) = *(src); /* struct copy */ \
+ (src)->start = NULL; \
+ ASSERT((s).sp >= (s).start); \
+ ASSERT((s).sp <= (s).end); \
+} while (0)
+
+#define ESTACK_IS_STATIC(s) ((s).start == ESTK_DEF_STACK(s))
+
+#define ESTACK_PUSH(s, x) \
+do { \
+ if ((s).sp == (s).end) { \
+ erl_grow_estack(&(s), 1); \
+ } \
+ *(s).sp++ = (x); \
} while(0)
-#define ESTACK_PUSH3(s, x, y, z) \
-do { \
- if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 3) { \
- erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \
- &ESTK_CONCAT(s,_end)); \
- } \
- *ESTK_CONCAT(s,_sp)++ = (x); \
- *ESTK_CONCAT(s,_sp)++ = (y); \
- *ESTK_CONCAT(s,_sp)++ = (z); \
+#define ESTACK_PUSH2(s, x, y) \
+do { \
+ if ((s).sp > (s).end - 2) { \
+ erl_grow_estack(&(s), 2); \
+ } \
+ *(s).sp++ = (x); \
+ *(s).sp++ = (y); \
} while(0)
-#define ESTACK_COUNT(s) (ESTK_CONCAT(s,_sp) - ESTK_CONCAT(s,_start))
+#define ESTACK_PUSH3(s, x, y, z) \
+do { \
+ if ((s).sp > (s).end - 3) { \
+ erl_grow_estack(&s, 3); \
+ } \
+ *(s).sp++ = (x); \
+ *(s).sp++ = (y); \
+ *(s).sp++ = (z); \
+} while(0)
-#define ESTACK_ISEMPTY(s) (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_start))
-#define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp)))
+#define ESTACK_PUSH4(s, E1, E2, E3, E4) \
+do { \
+ if ((s).sp > (s).end - 4) { \
+ erl_grow_estack(&s, 4); \
+ } \
+ *(s).sp++ = (E1); \
+ *(s).sp++ = (E2); \
+ *(s).sp++ = (E3); \
+ *(s).sp++ = (E4); \
+} while(0)
+#define ESTACK_RESERVE(s, push_cnt) \
+do { \
+ if ((s).sp > (s).end - (push_cnt)) { \
+ erl_grow_estack(&(s), (push_cnt)); \
+ } \
+} while(0)
+
+/* Must be preceded by ESTACK_RESERVE */
+#define ESTACK_FAST_PUSH(s, x) \
+do { \
+ ASSERT((s).sp < (s).end); \
+ *s.sp++ = (x); \
+} while(0)
+
+#define ESTACK_COUNT(s) ((s).sp - (s).start)
+#define ESTACK_ISEMPTY(s) ((s).sp == (s).start)
+#define ESTACK_POP(s) (*(--(s).sp))
+
+
+/*
+ * WSTACK: same as ESTACK but with UWord instead of Eterm
+ */
+
+typedef struct ErtsWStack_ {
+ UWord* wstart;
+ UWord* wsp;
+ UWord* wend;
+ UWord* wdefault;
+ ErtsAlcType_t alloc_type;
+}ErtsWStack;
-void erl_grow_wstack(UWord** start, UWord** sp, UWord** end);
-#define WSTK_CONCAT(a,b) a##b
-#define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i)))
#define DEF_WSTACK_SIZE (16)
-#define DECLARE_WSTACK(s) \
- UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE]; \
- UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack); \
- UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start); \
- UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE
+void erl_grow_wstack(ErtsWStack*, Uint need);
+#define WSTK_CONCAT(a,b) a##b
+#define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack)
+
+#define WSTACK_DECLARE(s) \
+ UWord WSTK_DEF_STACK(s)[DEF_WSTACK_SIZE]; \
+ ErtsWStack s = { \
+ WSTK_DEF_STACK(s), /* wstart */ \
+ WSTK_DEF_STACK(s), /* wsp */ \
+ WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \
+ WSTK_DEF_STACK(s), /* wdflt */ \
+ ERTS_ALC_T_ESTACK /* alloc_type */ \
+ }
+#define DECLARE_WSTACK WSTACK_DECLARE
+
+typedef struct ErtsDynamicWStack_ {
+ UWord default_stack[DEF_WSTACK_SIZE];
+ ErtsWStack ws;
+}ErtsDynamicWStack;
+
+#define WSTACK_INIT(dwsp, ALC_TYPE) \
+do { \
+ (dwsp)->ws.wstart = (dwsp)->default_stack; \
+ (dwsp)->ws.wsp = (dwsp)->default_stack; \
+ (dwsp)->ws.wend = (dwsp)->default_stack + DEF_WSTACK_SIZE;\
+ (dwsp)->ws.wdefault = (dwsp)->default_stack; \
+ (dwsp)->ws.alloc_type = ALC_TYPE; \
+} while (0)
-#define DESTROY_WSTACK(s) \
+#define WSTACK_CHANGE_ALLOCATOR(s,t) \
do { \
- if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \
- erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start)); \
+ if (s.wstart != WSTK_DEF_STACK(s)) { \
+ erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \
+ "type of active wstack\n"); \
} \
+ s.alloc_type = (t); \
+ } while (0)
+
+#define WSTACK_DESTROY(s) \
+do { \
+ if (s.wstart != s.wdefault) { \
+ erts_free(s.alloc_type, s.wstart); \
+ } \
} while(0)
+#define DESTROY_WSTACK WSTACK_DESTROY
-#define WSTACK_PUSH(s, x) \
-do { \
- if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) { \
- erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \
- &WSTK_CONCAT(s,_end)); \
- } \
- *WSTK_CONCAT(s,_sp)++ = (x); \
+#define WSTACK_DEBUG(s) \
+ do { \
+ fprintf(stderr, "wstack size = %ld\r\n", s.wsp - s.wstart); \
+ fprintf(stderr, "wstack wstart = %p\r\n", s.wstart); \
+ fprintf(stderr, "wstack wsp = %p\r\n", s.wsp); \
+ } while(0)
+
+/*
+ * Do not free the stack after this, it may have pointers into what
+ * was saved in 'dst'.
+ */
+#define WSTACK_SAVE(s,dst)\
+do {\
+ if (s.wstart == WSTK_DEF_STACK(s)) {\
+ UWord _wsz = WSTACK_COUNT(s);\
+ (dst)->wstart = erts_alloc(s.alloc_type,\
+ DEF_WSTACK_SIZE * sizeof(UWord));\
+ memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\
+ (dst)->wsp = (dst)->wstart + _wsz;\
+ (dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\
+ (dst)->wdefault = NULL;\
+ (dst)->alloc_type = s.alloc_type;\
+ } else\
+ *(dst) = s;\
+ } while (0)
+
+#define DESTROY_SAVED_WSTACK(wstack)\
+do {\
+ if ((wstack)->wstart) {\
+ erts_free((wstack)->alloc_type, (wstack)->wstart);\
+ (wstack)->wstart = NULL;\
+ }\
} while(0)
-#define WSTACK_PUSH2(s, x, y) \
-do { \
- if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) { \
- erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \
- &WSTK_CONCAT(s,_end)); \
- } \
- *WSTK_CONCAT(s,_sp)++ = (x); \
- *WSTK_CONCAT(s,_sp)++ = (y); \
+#define CLEAR_SAVED_WSTACK(wstack) ((void) ((wstack)->wstart = NULL))
+
+/*
+ * Use on empty stack, only the allocator can be changed before this.
+ * The src stack is reset to NULL.
+ */
+#define WSTACK_RESTORE(s, src) \
+do { \
+ ASSERT(s.wstart == WSTK_DEF_STACK(s)); \
+ s = *(src); /* struct copy */ \
+ (src)->wstart = NULL; \
+ ASSERT(s.wsp >= s.wstart); \
+ ASSERT(s.wsp <= s.wend); \
+} while (0)
+
+#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s))
+
+#define WSTACK_PUSH(s, x) \
+do { \
+ if (s.wsp == s.wend) { \
+ erl_grow_wstack(&s, 1); \
+ } \
+ *s.wsp++ = (x); \
+} while(0)
+
+#define WSTACK_PUSH2(s, x, y) \
+do { \
+ if (s.wsp > s.wend - 2) { \
+ erl_grow_wstack(&s, 2); \
+ } \
+ *s.wsp++ = (x); \
+ *s.wsp++ = (y); \
+} while(0)
+
+#define WSTACK_PUSH3(s, x, y, z) \
+do { \
+ if (s.wsp > s.wend - 3) { \
+ erl_grow_wstack(&s, 3); \
+ } \
+ *s.wsp++ = (x); \
+ *s.wsp++ = (y); \
+ *s.wsp++ = (z); \
+} while(0)
+
+#define WSTACK_PUSH4(s, A1, A2, A3, A4) \
+do { \
+ if (s.wsp > s.wend - 4) { \
+ erl_grow_wstack(&s, 4); \
+ } \
+ *s.wsp++ = (A1); \
+ *s.wsp++ = (A2); \
+ *s.wsp++ = (A3); \
+ *s.wsp++ = (A4); \
+} while(0)
+
+#define WSTACK_PUSH5(s, A1, A2, A3, A4, A5) \
+do { \
+ if (s.wsp > s.wend - 5) { \
+ erl_grow_wstack(&s, 5); \
+ } \
+ *s.wsp++ = (A1); \
+ *s.wsp++ = (A2); \
+ *s.wsp++ = (A3); \
+ *s.wsp++ = (A4); \
+ *s.wsp++ = (A5); \
+} while(0)
+
+#define WSTACK_PUSH6(s, A1, A2, A3, A4, A5, A6) \
+do { \
+ if (s.wsp > s.wend - 6) { \
+ erl_grow_wstack(&s, 6); \
+ } \
+ *s.wsp++ = (A1); \
+ *s.wsp++ = (A2); \
+ *s.wsp++ = (A3); \
+ *s.wsp++ = (A4); \
+ *s.wsp++ = (A5); \
+ *s.wsp++ = (A6); \
} while(0)
-#define WSTACK_PUSH3(s, x, y, z) \
+#define WSTACK_RESERVE(s, push_cnt) \
+do { \
+ if (s.wsp > s.wend - (push_cnt)) { \
+ erl_grow_wstack(&s, (push_cnt)); \
+ } \
+} while(0)
+
+/* Must be preceded by WSTACK_RESERVE */
+#define WSTACK_FAST_PUSH(s, x) \
+do { \
+ ASSERT(s.wsp < s.wend); \
+ *s.wsp++ = (x); \
+} while(0)
+
+#define WSTACK_COUNT(s) (s.wsp - s.wstart)
+#define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
+#define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp))
+
+#define WSTACK_ROLLBACK(s, count) (ASSERT(WSTACK_COUNT(s) >= (count)), \
+ s.wsp = s.wstart + (count))
+
+/* PSTACK - Stack of any type.
+ * Usage:
+ * {
+ * #define PSTACK_TYPE MyType
+ * PSTACK_DECLARE(s,16);
+ * MyType *sp = PSTACK_PUSH(s);
+ *
+ * sp->x = ....
+ * sp->y = ....
+ * sp = PSTACK_PUSH(s);
+ * ...
+ * sp = PSTACK_POP(s);
+ * if (PSTACK_IS_EMPTY(s)) {
+ * // sp is invalid when stack is empty after pop
+ * }
+ *
+ * PSTACK_DESTROY(s);
+ * }
+ */
+
+
+typedef struct ErtsPStack_ {
+ byte* pstart;
+ byte* psp;
+ byte* pend;
+ ErtsAlcType_t alloc_type;
+}ErtsPStack;
+
+void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes);
+#define PSTK_CONCAT(a,b) a##b
+#define PSTK_DEF_STACK(s) PSTK_CONCAT(s,_default_pstack)
+
+#define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \
+PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \
+ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \
+ (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \
+ (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\
+ ERTS_ALC_T_ESTACK /* alloc_type */ \
+}
+
+#define PSTACK_CHANGE_ALLOCATOR(s,t) \
do { \
- if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) { \
- erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \
- &WSTK_CONCAT(s,_end)); \
+ if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \
+ erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \
+ "type of active pstack\n"); \
} \
- *WSTK_CONCAT(s,_sp)++ = (x); \
- *WSTK_CONCAT(s,_sp)++ = (y); \
- *WSTK_CONCAT(s,_sp)++ = (z); \
+ s.alloc_type = (t); \
+ } while (0)
+
+#define PSTACK_DESTROY(s) \
+do { \
+ if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \
+ erts_free(s.alloc_type, s.pstart); \
+ } \
} while(0)
-#define WSTACK_COUNT(s) (WSTK_CONCAT(s,_sp) - WSTK_CONCAT(s,_start))
+#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart)
+
+#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart)
+
+#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp))
+
+#define PSTACK_PUSH(s) \
+ (s.psp += sizeof(PSTACK_TYPE), \
+ ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \
+ sizeof(PSTACK_TYPE)) : (void)0), \
+ ((PSTACK_TYPE*) s.psp))
+
+#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE)))
+
+/*
+ * Do not free the stack after this, it may have pointers into what
+ * was saved in 'dst'.
+ */
+#define PSTACK_SAVE(s,dst)\
+do {\
+ if (s.pstart == (byte*)PSTK_DEF_STACK(s)) {\
+ UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\
+ (dst)->pstart = erts_alloc(s.alloc_type,\
+ sizeof(PSTK_DEF_STACK(s)));\
+ sys_memcpy((dst)->pstart, s.pstart, _pbytes);\
+ (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\
+ (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\
+ (dst)->alloc_type = s.alloc_type;\
+ } else\
+ *(dst) = s;\
+ } while (0)
+
+/*
+ * Use on empty stack, only the allocator can be changed before this.
+ * The src stack is reset to NULL.
+ */
+#define PSTACK_RESTORE(s, src) \
+do { \
+ ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \
+ s = *(src); /* struct copy */ \
+ (src)->pstart = NULL; \
+ ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \
+ ASSERT(s.psp < s.pend); \
+} while (0)
+
+#define PSTACK_DESTROY_SAVED(pstack)\
+do {\
+ if ((pstack)->pstart) {\
+ erts_free((pstack)->alloc_type, (pstack)->pstart);\
+ (pstack)->pstart = NULL;\
+ }\
+} while(0)
-#define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start))
-#define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp)))
/* binary.c */
@@ -503,9 +868,6 @@ erts_bld_port_info(Eterm **hpp,
void erts_bif_info_init(void);
/* bif.c */
-Eterm erts_make_ref(Process *);
-Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]);
-void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]);
ERTS_GLB_INLINE Eterm
erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]);
@@ -537,6 +899,10 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg);
Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
+/* beam_bif_load.c */
+Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp);
+
+
/* beam_load.c */
typedef struct {
BeamInstr* current; /* Pointer to: Mod, Name, Arity */
@@ -581,10 +947,13 @@ void print_process_info(int, void *, Process*);
void info(int, void *);
void loaded(int, void *);
+/* erl_arith.c */
+double erts_get_positive_zero_float(void);
+
/* config.c */
-__decl_noreturn void __noreturn erl_exit(int n, char*, ...);
-__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...);
+__decl_noreturn void __noreturn erts_exit(int n, char*, ...);
+__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...);
void erl_error(char*, va_list);
/* copy.c */
@@ -686,29 +1055,6 @@ void MD5Init(MD5_CTX *);
void MD5Update(MD5_CTX *, unsigned char *, unsigned int);
void MD5Final(unsigned char [16], MD5_CTX *);
-/* ggc.c */
-
-
-typedef struct {
- Uint garbage_collections;
- Uint reclaimed;
-} ErtsGCInfo;
-
-void erts_gc_info(ErtsGCInfo *gcip);
-void erts_init_gc(void);
-int erts_garbage_collect(Process*, int, Eterm*, int);
-void erts_garbage_collect_hibernate(Process* p);
-Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(Process* p, Eterm* literals,
- Uint lit_size,
- struct erl_off_heap_header* oh);
-Uint erts_next_heap_size(Uint, Uint);
-Eterm erts_heap_sizes(Process* p);
-
-void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*);
-void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
-void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
-void erts_free_heap_frags(Process* p);
/* io.c */
@@ -724,41 +1070,50 @@ int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_l
void erts_destroy_driver(erts_driver_t *drv);
int erts_save_suspend_process_on_port(Port*, Process*);
Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *);
-void erts_init_io(int, int);
+void erts_init_io(int, int, int);
void erts_raw_port_command(Port*, byte*, Uint);
void driver_report_exit(ErlDrvPort, int);
LineBuf* allocate_linebuf(int);
int async_ready(Port *, void*);
-ErtsPortNames *erts_get_port_names(Eterm);
+ErtsPortNames *erts_get_port_names(Eterm, ErlDrvPort);
void erts_free_port_names(ErtsPortNames *);
Uint erts_port_ioq_size(Port *pp);
-void erts_stale_drv_select(Eterm, ErlDrvEvent, int, int);
+void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int);
Port *erts_get_heart_port(void);
+void erts_emergency_close_ports(void);
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
void erts_lcnt_enable_io_lock_count(int enable);
#endif
+/* driver_tab.c */
+typedef void *(*ErtsStaticNifInitFPtr)(void);
+ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len);
+int erts_is_static_nif(void *handle);
+void erts_init_static_drivers(void);
+
/* erl_drv_thread.c */
void erl_drv_thr_init(void);
/* utils.c */
void erts_cleanup_offheap(ErlOffHeap *offheap);
+Uint64 erts_timestamp_millis(void);
+
Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex);
Eterm store_external_or_ref_in_proc_(Process *, Eterm);
Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm);
#define NC_HEAP_SIZE(NC) \
- (ASSERT_EXPR(is_node_container((NC))), \
+ (ASSERT(is_node_container((NC))), \
IS_CONST((NC)) ? 0 : (thing_arityval(*boxed_val((NC))) + 1))
#define STORE_NC(Hpp, ETpp, NC) \
- (ASSERT_EXPR(is_node_container((NC))), \
+ (ASSERT(is_node_container((NC))), \
IS_CONST((NC)) ? (NC) : store_external_or_ref_((Hpp), (ETpp), (NC)))
#define STORE_NC_IN_PROC(Pp, NC) \
- (ASSERT_EXPR(is_node_container((NC))), \
+ (ASSERT(is_node_container((NC))), \
IS_CONST((NC)) ? (NC) : store_external_or_ref_in_proc_((Pp), (NC)))
/* duplicates from big.h */
@@ -777,6 +1132,12 @@ Sint erts_re_set_loop_limit(Sint limit);
void erts_init_bif_binary(void);
Sint erts_binary_set_loop_limit(Sint limit);
+/* external.c */
+void erts_init_external(void);
+
+/* erl_map.c */
+void erts_init_map(void);
+
/* erl_unicode.c */
void erts_init_unicode(void);
Sint erts_unicode_set_loop_limit(Sint limit);
@@ -794,6 +1155,17 @@ char *erts_convert_filename_to_native(Eterm name, char *statbuf,
ErtsAlcType_t alloc_type,
int allow_empty, int allow_atom,
Sint *used /* out */);
+char *erts_convert_filename_to_encoding(Eterm name, char *statbuf,
+ size_t statbuf_size,
+ ErtsAlcType_t alloc_type,
+ int allow_empty, int allow_atom,
+ int encoding,
+ Sint *used /* out */,
+ Uint extra);
+char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
+ char *statbuf, size_t statbuf_size,
+ ErtsAlcType_t alloc_type, Sint* used,
+ Uint extra_wchars);
Eterm erts_convert_native_to_filename(Process *p, byte *bytes);
Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left,
Uint *num_built, Uint *num_eaten, Eterm tail);
@@ -816,20 +1188,67 @@ struct Sint_buf {
};
char* Sint_to_buf(Sint, struct Sint_buf*);
+#define ERTS_IOLIST_STATE_INITER(C_P, OBJ) \
+ {(C_P), 0, 0, (OBJ), {NULL, NULL, NULL, ERTS_ALC_T_INVALID}, 0, 0}
+
+#define ERTS_IOLIST_STATE_MOVE(TO, FROM) \
+ sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOListState))
+
+#define ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED 8
+
+typedef struct {
+ Process *c_p;
+ ErlDrvSizeT size;
+ Uint offs;
+ Eterm obj;
+ ErtsEStack estack;
+ int reds_left;
+ int have_size;
+} ErtsIOListState;
+
+#define ERTS_IOLIST2BUF_STATE_INITER(C_P, OBJ) \
+ {ERTS_IOLIST_STATE_INITER((C_P), (OBJ)), {NULL, 0, 0, 0}, NULL, 0, NULL, 0}
+
+#define ERTS_IOLIST2BUF_STATE_MOVE(TO, FROM) \
+ sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOList2BufState))
+
+#define ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT 32
+#define ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED 8
+#define ERTS_IOLIST_TO_BUF_BYTES_PER_RED \
+ (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED*ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT)
+
+typedef struct {
+ ErtsIOListState iolist;
+ struct {
+ byte *bptr;
+ size_t size;
+ Uint bitoffs;
+ Uint bitsize;
+ } bcopy;
+ char *buf;
+ ErlDrvSizeT len;
+ Eterm *objp;
+ int offset;
+} ErtsIOList2BufState;
+
#define ERTS_IOLIST_OK 0
#define ERTS_IOLIST_OVERFLOW 1
#define ERTS_IOLIST_TYPE 2
+#define ERTS_IOLIST_YIELD 3
-Eterm buf_to_intlist(Eterm**, char*, size_t, Eterm); /* most callers pass plain char*'s */
+Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */
#define ERTS_IOLIST_TO_BUF_OVERFLOW (~((ErlDrvSizeT) 0))
#define ERTS_IOLIST_TO_BUF_TYPE_ERROR (~((ErlDrvSizeT) 1))
+#define ERTS_IOLIST_TO_BUF_YIELD (~((ErlDrvSizeT) 2))
#define ERTS_IOLIST_TO_BUF_FAILED(R) \
- (((R) & (~((ErlDrvSizeT) 1))) == (~((ErlDrvSizeT) 1)))
+ (((R) & (~((ErlDrvSizeT) 3))) == (~((ErlDrvSizeT) 3)))
#define ERTS_IOLIST_TO_BUF_SUCCEEDED(R) \
(!ERTS_IOLIST_TO_BUF_FAILED((R)))
ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT);
+ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *);
+int erts_iolist_size_yielding(ErtsIOListState *state);
int erts_iolist_size(Eterm, ErlDrvSizeT *);
int is_string(Eterm);
void erl_at_exit(void (*) (void*), void*);
@@ -863,6 +1282,7 @@ Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live);
+Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live);
@@ -901,9 +1321,10 @@ Eterm erts_match_set_lint(Process *p, Eterm matchexpr);
extern void erts_match_set_release_result(Process* p);
enum erts_pam_run_flags {
- ERTS_PAM_TMP_RESULT=0,
- ERTS_PAM_COPY_RESULT=1,
- ERTS_PAM_CONTIGUOUS_TUPLE=2
+ ERTS_PAM_TMP_RESULT=1,
+ ERTS_PAM_COPY_RESULT=2,
+ ERTS_PAM_CONTIGUOUS_TUPLE=4,
+ ERTS_PAM_IGNORE_TRACE_SILENT=8
};
extern Eterm erts_match_set_run(Process *p, Binary *mpsp,
Eterm *args, int num_args,
@@ -925,6 +1346,8 @@ extern erts_driver_t vanilla_driver;
extern erts_driver_t spawn_driver;
extern erts_driver_t fd_driver;
+int erts_beam_jump_table(void);
+
/* Should maybe be placed in erl_message.h, but then we get an include mess. */
ERTS_GLB_INLINE Eterm *
erts_alloc_message_heap_state(Uint size,
@@ -969,12 +1392,13 @@ erts_alloc_message_heap_state(Uint size,
state = erts_smp_atomic32_read_acqb(&receiver->state);
if (statep)
*statep = state;
- if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+ if (state & (ERTS_PSFLG_EXITING
+ | ERTS_PSFLG_PENDING_EXIT))
goto allocate_in_mbuf;
#endif
if (size > (Uint) INT_MAX)
- erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);
+ erts_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);
if (
#if defined(ERTS_SMP)
@@ -989,8 +1413,14 @@ erts_alloc_message_heap_state(Uint size,
state = erts_smp_atomic32_read_nob(&receiver->state);
if (statep)
*statep = state;
- if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))
+ if ((state & (ERTS_PSFLG_EXITING
+ | ERTS_PSFLG_PENDING_EXIT))
+ || (receiver->flags & F_DISABLE_GC)
|| HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
+ /*
+ * The heap is either potentially in an inconsistent
+ * state, or not large enough.
+ */
#ifdef ERTS_SMP
if (locked_main) {
*receiver_locks &= ~ERTS_PROC_LOCK_MAIN;
@@ -1100,6 +1530,13 @@ erts_alloc_message_heap(Uint size,
# define UnUseTmpHeapNoproc(Size) /* Nothing */
#endif /* HEAP_ON_C_STACK */
+ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf);
+ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf);
+ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf);
+ERTS_GLB_INLINE void dtrace_fun_decode(Process *process,
+ Eterm module, Eterm function, int arity,
+ char *process_buf, char *mfa_buf);
+
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
#include "dtrace-wrapper.h"
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-31 22:15:46 +0000