aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/hipe
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/hipe')
-rw-r--r--erts/emulator/hipe/hipe_amd64.c40
-rw-r--r--erts/emulator/hipe/hipe_amd64_bifs.m412
-rw-r--r--erts/emulator/hipe/hipe_arch.h12
-rw-r--r--erts/emulator/hipe/hipe_arm.c193
-rw-r--r--erts/emulator/hipe/hipe_arm_bifs.m44
-rw-r--r--erts/emulator/hipe/hipe_bif0.c1130
-rw-r--r--erts/emulator/hipe/hipe_bif0.h17
-rw-r--r--erts/emulator/hipe/hipe_bif0.tab18
-rw-r--r--erts/emulator/hipe/hipe_bif1.c8
-rw-r--r--erts/emulator/hipe/hipe_bif2.c10
-rw-r--r--erts/emulator/hipe/hipe_bif_list.m438
-rw-r--r--erts/emulator/hipe/hipe_debug.c16
-rw-r--r--erts/emulator/hipe/hipe_gc.c47
-rw-r--r--erts/emulator/hipe/hipe_load.c105
-rw-r--r--erts/emulator/hipe/hipe_load.h48
-rw-r--r--erts/emulator/hipe/hipe_mkliterals.c3
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.c7
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.h8
-rw-r--r--erts/emulator/hipe/hipe_module.c35
-rw-r--r--erts/emulator/hipe/hipe_module.h45
-rw-r--r--erts/emulator/hipe/hipe_native_bif.c35
-rw-r--r--erts/emulator/hipe/hipe_native_bif.h20
-rw-r--r--erts/emulator/hipe/hipe_ppc.c178
-rw-r--r--erts/emulator/hipe/hipe_ppc_bifs.m44
-rw-r--r--erts/emulator/hipe/hipe_risc_gc.h16
-rw-r--r--erts/emulator/hipe/hipe_risc_glue.h18
-rw-r--r--erts/emulator/hipe/hipe_risc_stack.c16
-rw-r--r--erts/emulator/hipe/hipe_sparc.c113
-rw-r--r--erts/emulator/hipe/hipe_sparc_bifs.m44
-rw-r--r--erts/emulator/hipe/hipe_stack.c112
-rw-r--r--erts/emulator/hipe/hipe_stack.h56
-rw-r--r--erts/emulator/hipe/hipe_x86.c118
-rw-r--r--erts/emulator/hipe/hipe_x86_bifs.m412
-rw-r--r--erts/emulator/hipe/hipe_x86_gc.h35
-rw-r--r--erts/emulator/hipe/hipe_x86_glue.h21
-rw-r--r--erts/emulator/hipe/hipe_x86_signal.c2
-rw-r--r--erts/emulator/hipe/hipe_x86_stack.c23
37 files changed, 1303 insertions, 1276 deletions
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 62739d2a78..e3cff4a4ba 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
Sint rel32;
- if (trampoline)
- return -1;
+ ASSERT(trampoline == NULL);
+
rel32 = (Sint)destAddress - (Sint)callAddress - 4;
if ((Sint)(Sint32)rel32 != rel32)
return -1;
@@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
return 0;
}
-#if 0 /* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
- printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
- total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
- if (!atexit_done) {
- atexit_done = 1;
- (void)atexit(alloc_code_stats);
- }
-}
-
-#define ALLOC_CODE_STATS(X) do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X) do{}while(0)
-#endif
-
/*
* Memory allocator for executable code.
*
@@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void)
*/
static void *alloc_code(unsigned int alloc_bytes)
{
- ALLOC_CODE_STATS(++nr_allocs);
- ALLOC_CODE_STATS(total_alloc += alloc_bytes);
-
return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}
@@ -130,6 +104,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code, unsigned int bytes)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
/* Make stub for native code calling exported beam function.
*/
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
@@ -234,6 +213,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index 21739726bb..dca3887564 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -41,11 +41,11 @@ define(HANDLE_GOT_MBUF,`
`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
# define CALL_BIF(F) \
- movq CSYM(F)@GOTPCREL(%rip), %r11; \
+ movq CSYM(nbif_impl_##F)@GOTPCREL(%rip), %r11; \
movq %r11, P_BIF_CALLEE(P); \
call CSYM(hipe_debug_bif_wrapper)
#else
-# define CALL_BIF(F) call CSYM(F)
+# define CALL_BIF(F) call CSYM(nbif_impl_##F)
#endif'
/*
@@ -595,13 +595,9 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu)
#endif /* NO_FPE_SIGNALS */
/*
- * Implement gc_bif_interface_0 as nofail_primop_interface_0.
- */
-define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)')
-
-/*
- * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2,3).
+ * Implement gc_bif_interface_N as standard_bif_interface_N.
*/
+define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)')
define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)')
define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)')
define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)')
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index 6f959815bb..059b8e7f29 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -30,23 +30,31 @@ extern void hipe_patch_load_fe(Uint *address, Uint value);
extern int hipe_patch_insn(void *address, Uint value, Eterm type);
extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline);
-extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
-extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity);
+extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p);
+extern void hipe_free_code(void*, unsigned int);
+extern void *hipe_make_native_stub(void *exp, unsigned int beamArity);
+extern void hipe_free_native_stub(void*);
+
#if defined(__sparc__)
#include "hipe_sparc.h"
+#include "hipe_sparc_asm.h"
#endif
#if defined(__i386__)
#include "hipe_x86.h"
+#include "hipe_x86_asm.h"
#endif
#if defined(__x86_64__)
#include "hipe_amd64.h"
+#include "hipe_amd64_asm.h"
#endif
#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
#include "hipe_ppc.h"
+#include "hipe_ppc_asm.h"
#endif
#if defined(__arm__)
#include "hipe_arm.h"
+#include "hipe_arm_asm.h"
#endif
#if !defined(AEXTERN)
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index f8ef468341..b61939724c 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -25,7 +25,6 @@
#endif
#include "global.h"
#include "erl_binary.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address)
hipe_flush_icache_range(address, 4);
}
-/*
- * Management of 32MB code segments for regular code and trampolines.
- */
-
-#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */
-
-static struct segment {
- unsigned int *base; /* [base,base+32MB[ */
- unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */
- unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */
- /* On ARM we always allocate a trampoline at base+32MB-8 for
- nbif_callemu, so tramp_pos <= base+32MB-8. */
-} curseg;
-
-#define in_area(ptr,start,nbytes) \
- ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
-
-static void *new_code_mapping(void)
-{
- return mmap(0, SEGMENT_NRBYTES,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS,
- -1, 0);
-}
static int check_callees(Eterm callees)
{
@@ -107,126 +82,53 @@ static int check_callees(Eterm callees)
return arity;
}
-static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec)
-{
- unsigned int *base, *address, *tramp_pos, nrfreewords;
- int trampnr;
- Eterm mfa, m, f;
- unsigned int a, *trampoline;
-
- m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */
- tramp_pos = curseg.tramp_pos;
- address = curseg.code_pos;
- nrfreewords = tramp_pos - address;
- if (nrwords > nrfreewords)
- return NULL;
- curseg.code_pos = address + nrwords;
- nrfreewords -= nrwords;
+#define TRAMPOLINE_WORDS 2
- base = curseg.base;
- for (trampnr = 1; trampnr <= nrcallees; ++trampnr) {
- mfa = tuple_val(callees)[trampnr];
- if (is_atom(mfa))
- trampoline = hipe_primop_get_trampoline(mfa);
- else {
- m = tuple_val(mfa)[1];
- f = tuple_val(mfa)[2];
- a = unsigned_val(tuple_val(mfa)[3]);
- trampoline = hipe_mfa_get_trampoline(m, f, a);
- }
- if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
- if (nrfreewords < 2)
- return NULL;
- nrfreewords -= 2;
- tramp_pos = trampoline = tramp_pos - 2;
- trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
- trampoline[1] = 0; /* callee's address */
- hipe_flush_icache_range(trampoline, 2*sizeof(int));
- if (is_atom(mfa))
- hipe_primop_set_trampoline(mfa, trampoline);
- else
- hipe_mfa_set_trampoline(m, f, a, trampoline);
- }
- trampvec[trampnr-1] = trampoline;
+static void generate_trampolines(Uint32* address,
+ int nrcallees, Eterm callees,
+ Uint32** trampvec)
+{
+ Uint32* trampoline = address;
+ int i;
+
+ for (i = 0; i < nrcallees; ++i) {
+ trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
+ trampoline[1] = 0; /* callee's address */
+ trampvec[i] = trampoline;
+ trampoline += TRAMPOLINE_WORDS;
}
- curseg.tramp_pos = tramp_pos;
- return address;
+ hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32));
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
{
- Uint nrwords;
+ Uint code_words;
int nrcallees;
Eterm trampvecbin;
- unsigned int **trampvec;
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
+ Uint32 **trampvec;
+ Uint32 *address;
if (nrbytes & 0x3)
return NULL;
- nrwords = nrbytes >> 2;
+ code_words = nrbytes / sizeof(Uint32);
nrcallees = check_callees(callees);
if (nrcallees < 0)
return NULL;
- trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*));
- trampvec = (unsigned int**)binary_bytes(trampvecbin);
+ trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*));
+ trampvec = (Uint32**)binary_bytes(trampvecbin);
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
- curseg.tramp_pos -= 2;
- curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
- curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
+ address = erts_alloc(ERTS_ALC_T_HIPE_EXEC,
+ (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32));
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
+ generate_trampolines(address + code_words, nrcallees, callees, trampvec);
*trampolines = trampvecbin;
return address;
}
-static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
+void hipe_free_code(void* code, unsigned int bytes)
{
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
- curseg.tramp_pos -= 2;
- curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
- curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
- *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2;
- return address;
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
}
/*
@@ -266,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type)
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
unsigned int *code;
- unsigned int *tramp_callemu;
int callemu_offset;
+ int is_short_jmp;
/*
* Native code calls BEAM via a stub looking as follows:
@@ -277,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
* b nbif_callemu
* .long callee_exp
*
+ * or if nbif_callemu is too far away:
+ *
+ * mov r0, #beamArity
+ * ldr r8, [pc,#0] // callee_exp
+ * ldr pc, [pc,#0] // nbif_callemu
+ * .long callee_exp
+ * .long nbif_callemu
+ *
* I'm using r0 and r8 since they aren't used for
- * parameter passing in native code. The branch to
- * nbif_callemu may need to go via a trampoline.
- * (Trampolines are allowed to modify r12, but they don't.)
+ * parameter passing in native code.
*/
- code = alloc_stub(4, &tramp_callemu);
+ code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32));
if (!code)
return NULL;
callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2;
- if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) {
- callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2;
- if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF))
- abort();
+ is_short_jmp = (callemu_offset >= -0x00800000 &&
+ callemu_offset <= 0x007FFFFF);
+#ifdef DEBUG
+ if (is_short_jmp && (callemu_offset % 3)==0) {
+ is_short_jmp = 0;
}
+#endif
/* mov r0, #beamArity */
code[0] = 0xE3A00000 | (beamArity & 0xFF);
/* ldr r8, [pc,#0] // callee_exp */
code[1] = 0xE59F8000;
- /* b nbif_callemu */
- code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF);
+ if (is_short_jmp) {
+ /* b nbif_callemu */
+ code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF);
+ }
+ else {
+ /* ldr pc, [pc,#0] // nbif_callemu */
+ code[2] = 0xE59FF000;
+ /* .long nbif_callemu */
+ code[4] = (unsigned int)&nbif_callemu;
+ }
/* .long callee_exp */
code[3] = (unsigned int)callee_exp;
- hipe_flush_icache_range(code, 4*sizeof(int));
+ hipe_flush_icache_range(code, 5*sizeof(Uint32));
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA)
{
Uint32 oldI = *address;
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index d7a2fec04a..a9097dabde 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -30,9 +30,9 @@ include(`hipe/hipe_arm_asm.m4')
.arm
`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-# define CALL_BIF(F) ldr r14, =F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper
+# define CALL_BIF(F) ldr r14, =nbif_impl_##F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper
#else
-# define CALL_BIF(F) bl F
+# define CALL_BIF(F) bl nbif_impl_##F
#endif'
define(TEST_GOT_MBUF,`ldr r1, [P, #P_MBUF] /* `TEST_GOT_MBUF' */
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index dcb6c35bfa..9c6ac4bd9c 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -44,6 +44,7 @@
#include "hipe_mode_switch.h"
#include "hipe_native_bif.h"
#include "hipe_bif0.h"
+#include "hipe_load.h"
/* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines
a few constants. #undef them here to avoid warnings. */
#undef F_TIMO
@@ -54,6 +55,7 @@
#define BeamOpCode(Op) ((Uint)BeamOp(Op))
+
int term_to_Sint32(Eterm term, Sint *sp)
{
Sint val;
@@ -374,15 +376,31 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2)
}
/*
+ * BIFs for loading code.
+ */
+
+static HipeLoaderState *get_loader_state(Eterm term)
+{
+ ProcBin *pb;
+
+ if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL;
+
+ pb = (ProcBin*) binary_val(term);
+ return hipe_get_loader_state(pb->val);
+}
+
+
+/*
* Allocate memory and copy machine code to it.
*/
-BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3)
{
Uint nrbytes;
void *bytes;
void *address;
Eterm trampolines;
Eterm *hp;
+ HipeLoaderState *stp;
#ifndef DEBUG
ERTS_DECLARE_DUMMY(Uint bitoffs);
ERTS_DECLARE_DUMMY(Uint bitsize);
@@ -391,7 +409,8 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
Uint bitsize;
#endif
- if (is_not_binary(BIF_ARG_1))
+ if (is_not_binary(BIF_ARG_1) ||
+ (!(stp = get_loader_state(BIF_ARG_3))))
BIF_ERROR(BIF_P, BADARG);
nrbytes = binary_size(BIF_ARG_1);
ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize);
@@ -406,11 +425,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
nrcallees = arityval(tuple_val(BIF_ARG_2)[0]);
else
nrcallees = 0;
+ // XXX: Is there any reason to not just BIF_ERROR, so that the runtime
+ // survives?
erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n",
__func__, (unsigned long)nrbytes, (unsigned long)nrcallees);
}
memcpy(address, bytes, nrbytes);
hipe_flush_icache_range(address, nrbytes);
+ stp->text_segment = address;
+ stp->text_segment_size = nrbytes;
hp = HAlloc(BIF_P, 3);
hp[0] = make_arityval(2);
hp[1] = address_to_term(address, BIF_P);
@@ -423,25 +446,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
/*
* Allocate memory for arbitrary non-Erlang data.
*/
-BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3)
{
- Uint align, nrbytes;
- void *block;
+ Uint align;
+ HipeLoaderState *stp;
if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) ||
+ (!(stp = get_loader_state(BIF_ARG_3))) ||
(align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align)))
BIF_ERROR(BIF_P, BADARG);
- nrbytes = unsigned_val(BIF_ARG_2);
- if (nrbytes == 0)
+
+ if (stp->data_segment_size || stp->data_segment)
+ BIF_ERROR(BIF_P, BADARG);
+
+ stp->data_segment_size = unsigned_val(BIF_ARG_2);
+ if (stp->data_segment_size == 0)
BIF_RET(make_small(0));
- block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes);
- if ((unsigned long)block & (align-1)) {
- fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n",
- __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align);
- erts_free(ERTS_ALC_T_HIPE, block);
+ stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size);
+ if ((unsigned long)stp->data_segment & (align-1)) {
+ fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte "
+ "aligned\r\n",
+ __FUNCTION__, (unsigned long)stp->data_segment_size,
+ stp->data_segment, (unsigned long)align);
+ erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+ stp->data_segment = NULL;
+ stp->data_segment_size = 0;
BIF_ERROR(BIF_P, EXC_NOTSUP);
}
- BIF_RET(address_to_term(block, BIF_P));
+ BIF_RET(address_to_term(stp->data_segment, BIF_P));
}
/*
@@ -537,13 +569,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1)
BIF_RET(val);
}
-struct mfa_t {
+struct hipe_mfa {
Eterm mod;
Eterm fun;
Uint ari;
};
-static int term_to_mfa(Eterm term, struct mfa_t *mfa)
+static int term_to_mfa(Eterm term, struct hipe_mfa *mfa)
{
Eterm mod, fun, a;
Uint ari;
@@ -601,7 +633,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity)
Uint *hipe_bifs_find_pc_from_mfa(Eterm term)
{
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
if (!term_to_mfa(term, &mfa))
return NULL;
@@ -616,12 +648,20 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1)
BIF_RET(address_to_term(pc, BIF_P));
}
+BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1)
+{
+ if (!erts_commit_hipe_patch_load(BIF_ARG_1))
+ BIF_ERROR(BIF_P, BADARG);
+
+ BIF_RET(am_ok);
+}
+
BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
{
Eterm *pc;
void *address;
int is_closure;
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
switch (BIF_ARG_3) {
case am_false:
@@ -644,28 +684,22 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari);
if (pc) {
- hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc);
-#if HIPE
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc);
-#endif
+ DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address);
hipe_set_call_trap(pc, address, is_closure);
BIF_RET(am_true);
-#endif
}
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": no BEAM pc found\r\n");
-#endif
+ DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address);
BIF_RET(am_false);
}
-BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
+BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
{
- struct sdesc *sdesc;
+ struct hipe_sdesc *sdesc;
+ HipeLoaderState* stp;
+
+ stp = get_loader_state(BIF_ARG_2);
+ if (!stp)
+ BIF_ERROR(BIF_P, BADARG);
sdesc = hipe_decode_sdesc(BIF_ARG_1);
if (!sdesc) {
@@ -676,6 +710,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__);
BIF_ERROR(BIF_P, BADARG);
}
+
+ /*
+ * Link into list of sdesc's in same module instance
+ */
+ sdesc->next_in_modi = stp->new_hipe_sdesc;
+ stp->new_hipe_sdesc = sdesc;
+
BIF_RET(NIL);
}
@@ -691,7 +732,7 @@ struct nbif {
};
static struct nbif nbifs[BIF_SIZE] = {
-#define BIF_LIST(MOD,FUN,ARY,CFUN,IX) \
+#define BIF_LIST(MOD,FUN,ARY,BIF,CFUN,IX) \
{ {0,0}, MOD, FUN, ARY, &nbif_##CFUN },
#include "erl_bif_list.h"
#undef BIF_LIST
@@ -778,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3)
struct primop {
HashBucket bucket; /* bucket.hvalue == atom_val(name) */
const void *address;
-#if defined(__arm__)
- void *trampoline;
-#endif
};
static struct primop primops[] = {
@@ -839,29 +877,6 @@ static struct primop *primop_table_get(Eterm name)
return hash_get(&primop_table, &tmpl);
}
-#if defined(__arm__)
-static struct primop *primop_table_put(Eterm name)
-{
- struct primop tmpl;
-
- init_primop_table();
- tmpl.bucket.hvalue = atom_val(name);
- return hash_put(&primop_table, &tmpl);
-}
-
-void *hipe_primop_get_trampoline(Eterm name)
-{
- struct primop *primop = primop_table_get(name);
- return primop ? primop->trampoline : NULL;
-}
-
-void hipe_primop_set_trampoline(Eterm name, void *trampoline)
-{
- struct primop *primop = primop_table_put(name);
- primop->trampoline = trampoline;
-}
-#endif
-
/*
* hipe_bifs_primop_address(Atom) -> address or false
*/
@@ -890,7 +905,8 @@ BIF_RETTYPE hipe_bifs_term_to_word_1(BIF_ALIST_1)
}
/* XXX: this is really a primop, not a BIF */
-BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1)
+/* Called via standard_bif_interface_1 */
+BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1)
{
Eterm res;
Eterm *hp;
@@ -970,7 +986,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2)
atom_buf[0] = '\0';
strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len);
- printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index);
+ printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index);
BIF_ERROR(BIF_P, BADARG);
}
BIF_RET(address_to_term((void *)fe, BIF_P));
@@ -997,30 +1013,35 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
BIF_RET(am_true);
}
+struct hipe_ref_head {
+ struct hipe_ref_head* next;
+ struct hipe_ref_head* prev;
+};
+
/*
- * MFA info hash table:
+ * An exported function called from or implemented by native code
* - maps MFA to native code entry point
- * - the MFAs it calls (refers_to)
- * - the references to it (referred_from)
+ * - all references to it (callers)
* - maps MFA to most recent trampoline [if powerpc or arm]
*/
struct hipe_mfa_info {
+ HashBucket mod2mfa;
struct {
unsigned long hvalue;
struct hipe_mfa_info *next;
} bucket;
Eterm m; /* atom */
Eterm f; /* atom */
- unsigned int a;
+ unsigned int a : sizeof(int)*8 - 1;
+ unsigned int is_stub : 1; /* if beam or not (yet) loaded */
void *remote_address;
- void *local_address;
- Eterm *beam_code;
- Uint orig_beam_op;
- struct hipe_mfa_info_list *refers_to;
- struct ref *referred_from;
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
- void *trampoline;
+ void *new_address;
+ struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */
+ struct hipe_mfa_info* next_in_mod;
+#ifdef DEBUG
+ Export* dbg_export;
#endif
+
};
static struct {
@@ -1038,6 +1059,82 @@ static struct {
erts_smp_rwmtx_t lock;
} hipe_mfa_info_table;
+Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */
+
+static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa)
+{
+ return mfa->mod2mfa.hvalue;
+}
+
+static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa)
+{
+ return tmpl->hvalue != mfa->mod2mfa.hvalue;
+}
+
+static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl)
+{
+ return tmpl; /* hash_put always use mfa itself at template */
+}
+
+static void mod2mfa_free(struct hipe_mfa_info* mfa)
+{
+}
+
+static void mod2mfa_tab_init(void)
+{
+ HashFunctions f;
+ static int init_done = 0;
+
+ if (init_done)
+ return;
+ init_done = 1;
+
+ f.hash = (H_FUN) mod2mfa_hash;
+ f.cmp = (HCMP_FUN) mod2mfa_cmp;
+ f.alloc = (HALLOC_FUN) mod2mfa_alloc;
+ f.free = (HFREE_FUN) mod2mfa_free;
+ f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+ f.meta_free = (HMFREE_FUN) erts_free;
+ f.meta_print = (HMPRINT_FUN) erts_print;
+
+ hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f);
+}
+
+static struct hipe_mfa_info* mod2mfa_get(Module* modp)
+{
+ HashBucket tmpl;
+ tmpl.hvalue = modp->module;
+ return hash_get(&mod2mfa_tab, &tmpl);
+}
+
+static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa)
+{
+ mfa->mod2mfa.hvalue = atom_val(mfa->m);
+ return hash_put(&mod2mfa_tab, mfa);
+}
+
+
+
+/*
+ * An external native call site M:F(...)
+ * to be patched when the callee changes.
+ */
+struct hipe_ref {
+ struct hipe_ref_head head; /* list of refs to same calleee */
+ void *address;
+#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+ void *trampoline;
+#endif
+ unsigned int flags;
+ struct hipe_ref* next_from_modi; /* list of refs from same module instance */
+#if defined(DEBUG)
+ struct hipe_mfa_info* callee;
+ Eterm caller_m, caller_f, caller_a;
+#endif
+};
+#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */
+
+
static inline void hipe_mfa_info_table_init_lock(void)
{
erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
@@ -1063,6 +1160,16 @@ static inline void hipe_mfa_info_table_rwunlock(void)
erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock);
}
+static ERTS_INLINE
+struct hipe_mfa_info* mod2mfa_get_safe(Module* modp)
+{
+ struct hipe_mfa_info* mfa;
+ hipe_mfa_info_table_rlock();
+ mfa = mod2mfa_get(modp);
+ hipe_mfa_info_table_runlock();
+ return mfa;
+}
+
#define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A))
static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size)
@@ -1108,14 +1215,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne
res->m = m;
res->f = f;
res->a = arity;
+ res->is_stub = 0;
res->remote_address = NULL;
- res->local_address = NULL;
- res->beam_code = NULL;
- res->orig_beam_op = 0;
- res->refers_to = NULL;
- res->referred_from = NULL;
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
- res->trampoline = NULL;
+ res->new_address = NULL;
+ res->callers.next = &res->callers;
+ res->callers.prev = &res->callers;
+ res->next_in_mod = NULL;
+#ifdef DEBUG
+ res->dbg_export = NULL;
#endif
return res;
@@ -1133,6 +1240,8 @@ void hipe_mfa_info_table_init(void)
hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size);
hipe_mfa_info_table_init_lock();
+
+ mod2mfa_tab_init();
}
static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity)
@@ -1154,21 +1263,12 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter
return NULL;
}
-#if 0 /* XXX: unused */
-void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity)
-{
- const struct hipe_mfa_info *p;
-
- p = hipe_mfa_info_table_get(m, f, arity);
- return p ? p->address : NULL;
-}
-#endif
-
static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity)
{
unsigned long h;
unsigned int i;
struct hipe_mfa_info *p;
+ struct hipe_mfa_info *first_in_mod;
unsigned int size;
h = HIPE_MFA_HASH(m, f, arity);
@@ -1189,216 +1289,137 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
size = 1 << hipe_mfa_info_table.log2size;
if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */
hipe_mfa_info_table_grow();
- return p;
-}
-static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported)
-{
- struct hipe_mfa_info *p;
+ first_in_mod = mod2mfa_put(p);
+ if (p != first_in_mod) {
+ p->next_in_mod = first_in_mod->next_in_mod;
+ first_in_mod->next_in_mod = p;
+ }
+ else {
+ p->next_in_mod = NULL;
+ }
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(m, f, arity);
- printf(": changing address from %p to %p\r\n", p->local_address, address);
-#endif
- p->local_address = address;
- if (is_exported)
- p->remote_address = address;
- hipe_mfa_info_table_rwunlock();
+ DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p);
+
+ return p;
}
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity)
+static void remove_mfa_info(struct hipe_mfa_info* rm)
{
+ unsigned int i;
struct hipe_mfa_info *p;
- void *trampoline;
-
- hipe_mfa_info_table_rlock();
- p = hipe_mfa_info_table_get_locked(m, f, arity);
- trampoline = p ? p->trampoline : NULL;
- hipe_mfa_info_table_runlock();
- return trampoline;
+ struct hipe_mfa_info **prevp;
+
+ i = rm->bucket.hvalue & hipe_mfa_info_table.mask;
+ prevp = &hipe_mfa_info_table.bucket[i];
+ for (;;) {
+ p = *prevp;
+ ASSERT(p);
+ if (p == rm) {
+ *prevp = p->bucket.next;
+ ASSERT(hipe_mfa_info_table.used > 0);
+ hipe_mfa_info_table.used--;
+ return;
+ }
+ prevp = &p->bucket.next;
+ }
}
-void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline)
+static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address)
{
struct hipe_mfa_info *p;
hipe_mfa_info_table_rwlock();
p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
- p->trampoline = trampoline;
+ DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p);
+ p->new_address = address;
+
hipe_mfa_info_table_rwunlock();
}
-#endif
BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
{
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
void *address;
- int is_exported;
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- address = term_to_address(BIF_ARG_2);
- if (!address)
- BIF_ERROR(BIF_P, BADARG);
- if (BIF_ARG_3 == am_true)
- is_exported = 1;
- else if (BIF_ARG_3 == am_false)
- is_exported = 0;
- else
+ switch (BIF_ARG_3) {
+ case am_true: /* is_exported */
+ if (!term_to_mfa(BIF_ARG_1, &mfa))
+ BIF_ERROR(BIF_P, BADARG);
+ address = term_to_address(BIF_ARG_2);
+ if (!address)
+ BIF_ERROR(BIF_P, BADARG);
+ hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address);
+ break;
+ case am_false:
+ break; /* ignore local functions */
+ default:
BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported);
- BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
-{
- Eterm lst;
- struct mfa_t mfa;
- struct hipe_mfa_info *p;
-
- hipe_mfa_info_table_rwlock();
- lst = BIF_ARG_1;
- while (is_list(lst)) {
- if (!term_to_mfa(CAR(list_val(lst)), &mfa))
- break;
- lst = CDR(list_val(lst));
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p) {
- p->remote_address = NULL;
- p->local_address = NULL;
- if (p->beam_code) {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n",
- p->beam_code, p->orig_beam_op);
-#endif
- p->beam_code[0] = p->orig_beam_op;
- p->beam_code = NULL;
- p->orig_beam_op = 0;
- } else {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": no call trap to remove\r\n");
-#endif
- }
- }
}
- hipe_mfa_info_table_rwunlock();
- if (is_not_nil(lst))
- BIF_ERROR(BIF_P, BADARG);
BIF_RET(NIL);
}
-void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc)
+
+/* Ask if we need to block all threads
+ * while loading/deleting code for this module?
+ */
+int hipe_need_blocking(Module* modp)
{
- Uint orig_beam_op;
struct hipe_mfa_info *p;
- orig_beam_op = pc[0];
- if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) &&
- orig_beam_op != BeamOpCode(op_hipe_trap_call)) {
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari);
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mod, fun, ari);
- printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc);
-#endif
- p->beam_code = pc;
- p->orig_beam_op = orig_beam_op;
- hipe_mfa_info_table_rwunlock();
- } else {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mod, fun, ari);
- printf(": orig op %#lx already saved\r\n", orig_beam_op);
-#endif
+ /* Need to block if we have at least one native caller to this module
+ * or native code to make unaccessible.
+ */
+ hipe_mfa_info_table_rlock();
+ for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
+ ASSERT(!p->new_address);
+ if (p->callers.next != &p->callers || !p->is_stub) {
+ break;
+ }
}
+ hipe_mfa_info_table_runlock();
+ return (p != NULL);
}
-static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
-{
- Export *export_entry;
- void *StubAddress;
-
- ASSERT(is_remote);
-
- export_entry = erts_export_get_or_make_stub(m, f, arity);
- StubAddress = hipe_make_native_stub(export_entry, arity);
- if (!StubAddress)
- erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
- return StubAddress;
-}
-
-static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp)
+static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a)
{
struct hipe_mfa_info *p;
- void *address;
p = hipe_mfa_info_table_get_locked(m, f, a);
- if (p) {
- /* find address, predicting for a runtime apply call */
- address = p->remote_address;
- if (!is_remote)
- address = p->local_address;
- if (address)
- return address;
-
- /* bummer, install stub, checking if one already existed */
- address = p->remote_address;
- if (address)
- return address;
- }
- /* Caller must take the slow path with the write lock held, but allow
- it to avoid some work if it already holds the write lock. */
- if (pp)
- *pp = p;
- return NULL;
-}
-
-static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p)
-{
- void *address;
-
- if (!p)
- p = hipe_mfa_info_table_put_rwlocked(m, f, a);
- address = hipe_make_stub(m, f, a, is_remote);
- /* XXX: how to tell if a BEAM MFA is exported or not? */
- p->remote_address = address;
- return address;
+ return p ? p->remote_address : NULL;
}
-static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a)
{
- struct hipe_mfa_info *p;
- void *address;
+ struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a);
- address = hipe_get_na_try_locked(m, f, a, is_remote, &p);
- if (address)
- return address;
+ if (!p->remote_address) {
+ Export* export_entry = erts_export_get_or_make_stub(m, f, a);
+ void* stubAddress = hipe_make_native_stub(export_entry, a);
+ if (!stubAddress)
+ erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
- address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p);
- return address;
+ p->remote_address = stubAddress;
+ p->is_stub = 1;
+#ifdef DEBUG
+ p->dbg_export = export_entry;
+#endif
+ }
+ return p->remote_address;
}
-static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a)
{
void *address;
hipe_mfa_info_table_rlock();
- address = hipe_get_na_try_locked(m, f, a, is_remote, NULL);
+ address = hipe_get_na_try_locked(m, f, a);
hipe_mfa_info_table_runlock();
if (address)
return address;
hipe_mfa_info_table_rwlock();
- address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL);
+ address = hipe_get_na_slow_rwlocked(m, f, a);
hipe_mfa_info_table_rwunlock();
return address;
}
@@ -1408,11 +1429,12 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a)
{
if (is_not_atom(m) || is_not_atom(f) || a > 255)
return NULL;
- return hipe_get_na_nofail(m, f, a, 1);
+ return hipe_get_na_nofail(m, f, a);
}
/* primop, but called like a BIF for error handling purposes */
-BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
+/* Called via standard_bif_interface_3 */
+BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3)
{
Uint arity;
void *address;
@@ -1420,30 +1442,25 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2))
BIF_ERROR(BIF_P, BADARG);
arity = unsigned_val(BIF_ARG_3); /* no error check */
- address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1);
+ address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity);
BIF_RET((Eterm)address); /* semi-Ok */
}
-BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1)
{
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
void *address;
- int is_remote;
if (!term_to_mfa(BIF_ARG_1, &mfa))
BIF_ERROR(BIF_P, BADARG);
- if (BIF_ARG_2 == am_true)
- is_remote = 1;
- else if (BIF_ARG_2 == am_false)
- is_remote = 0;
- else
- BIF_ERROR(BIF_P, BADARG);
- address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote);
+
+ address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari);
BIF_RET(address_to_term(address, BIF_P));
}
/* primop, but called like a BIF for error handling purposes */
-BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
+/* Called via standard_bif_interface_2 */
+BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2)
{
Eterm hdr, m, f;
void *address;
@@ -1453,14 +1470,14 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
hdr = *boxed_val(BIF_ARG_1);
if (is_export_header(hdr)) {
Export *ep = (Export*)(export_val(BIF_ARG_1)[1]);
- unsigned int actual_arity = ep->code[2];
+ unsigned int actual_arity = ep->info.mfa.arity;
if (actual_arity != BIF_ARG_2)
goto badfun;
- m = ep->code[0];
- f = ep->code[1];
+ m = ep->info.mfa.module;
+ f = ep->info.mfa.function;
} else
goto badfun;
- address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1);
+ address = hipe_get_na_nofail(m, f, BIF_ARG_2);
BIF_RET((Eterm)address);
badfun:
@@ -1471,77 +1488,31 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
{
- struct hipe_mfa_info *mfa;
- long mfa_offset, ra_offset;
- struct hipe_mfa_info **bucket;
- unsigned int i, nrbuckets;
+ const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra);
- if (hipe_is_ra_mode_switch(ra)) {
+ if (!sdesc || sdesc->m_aix == atom_val(am_Empty))
return 0;
- }
- /* Note about locking: the table is only updated from the
- loader, which runs with the rest of the system suspended. */
- /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */
- hipe_mfa_info_table_rlock();
- bucket = hipe_mfa_info_table.bucket;
- nrbuckets = 1 << hipe_mfa_info_table.log2size;
- mfa = NULL;
- mfa_offset = LONG_MAX;
- for (i = 0; i < nrbuckets; ++i) {
- struct hipe_mfa_info *b = bucket[i];
- while (b != NULL) {
- ra_offset = (char*)ra - (char*)b->local_address;
- if (ra_offset > 0 && ra_offset < mfa_offset) {
- mfa_offset = ra_offset;
- mfa = b;
- }
- b = b->bucket.next;
- }
- }
- if (mfa) {
- *m = mfa->m;
- *f = mfa->f;
- *a = mfa->a;
- }
- hipe_mfa_info_table_runlock();
- return mfa ? 1 : 0;
+ *m = make_atom(sdesc->m_aix);
+ *f = make_atom(sdesc->f_aix);
+ *a = sdesc->a;
+ return 1;
}
-/*
- * Patch Reference Handling.
- */
-struct hipe_mfa_info_list {
- struct hipe_mfa_info *mfa;
- struct hipe_mfa_info_list *next;
-};
-
-struct ref {
- struct hipe_mfa_info *caller_mfa;
- void *address;
- void *trampoline;
- unsigned int flags;
- struct ref *next;
-};
-#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */
-#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */
-#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */
-#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */
-/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'})
+/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState})
*/
BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
{
- struct mfa_t callee;
+ struct hipe_mfa callee;
Eterm *tuple;
- struct mfa_t caller;
+ struct hipe_mfa caller;
void *address;
void *trampoline;
unsigned int flags;
struct hipe_mfa_info *callee_mfa;
- struct hipe_mfa_info *caller_mfa;
- struct hipe_mfa_info_list *refers_to;
- struct ref *ref;
+ struct hipe_ref *ref;
+ HipeLoaderState* stp;
if (!term_to_mfa(BIF_ARG_1, &callee))
goto badarg;
@@ -1572,63 +1543,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
if (!trampoline)
goto badarg;
}
- switch (tuple[5]) {
- case am_local:
- break;
- case am_remote:
- flags |= REF_FLAG_IS_REMOTE;
- break;
- default:
- goto badarg;
- }
+ stp = get_loader_state(tuple[5]);
+ if (!stp)
+ goto badarg;
+
hipe_mfa_info_table_rwlock();
callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
- caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari);
-
- refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to));
- refers_to->mfa = callee_mfa;
- refers_to->next = caller_mfa->refers_to;
- caller_mfa->refers_to = refers_to;
- ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref));
- ref->caller_mfa = caller_mfa;
+ ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
ref->address = address;
+#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
ref->trampoline = trampoline;
+#endif
ref->flags = flags;
- ref->next = callee_mfa->referred_from;
- callee_mfa->referred_from = ref;
+
+ /*
+ * Link into list of refs to same callee
+ */
+ ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers);
+ ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers);
+ ref->head.next = callee_mfa->callers.next;
+ ref->head.prev = &callee_mfa->callers;
+ ref->head.next->prev = &ref->head;
+ ref->head.prev->next = &ref->head;
+
+ /*
+ * Link into list of refs from same module instance
+ */
+ ref->next_from_modi = stp->new_hipe_refs;
+ stp->new_hipe_refs = ref;
+
+#if defined(DEBUG)
+ ref->callee = callee_mfa;
+ ref->caller_m = caller.mod;
+ ref->caller_f = caller.fun;
+ ref->caller_a = caller.ari;
+#endif
hipe_mfa_info_table_rwunlock();
- BIF_RET(NIL);
+ DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)",
+ ref, callee.mod, callee.fun, callee.ari, ref->address);
+ DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)",
+ ref, caller.mod, caller.fun, caller.ari, ref->address);
+ BIF_RET(am_ok);
badarg:
BIF_ERROR(BIF_P, BADARG);
}
-/* Given a CalleeMFA, mark each ref to it as pending-redirect.
- * This ensures that remove_refs_from() won't remove them: any
- * removal is instead done at the end of redirect_referred_from().
- */
-BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */
+
+static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me)
{
- struct mfa_t mfa;
- const struct hipe_mfa_info *p;
- struct ref *ref;
+ struct hipe_mfa_info* p;
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p)
- for (ref = p->referred_from; ref != NULL; ref = ref->next)
- ref->flags |= REF_FLAG_PENDING_REDIRECT;
- hipe_mfa_info_table_rwunlock();
- BIF_RET(NIL);
+ p = hash_get(&mod2mfa_tab, unlink_me);
+ ASSERT(p);
+ if (p == unlink_me) {
+ hash_erase(&mod2mfa_tab, p);
+ if (p->next_in_mod)
+ mod2mfa_put(p->next_in_mod);
+ }
+ else {
+ struct hipe_mfa_info** prevp;
+
+ do {
+ prevp = &p->next_in_mod;
+ p = *prevp;
+ ASSERT(p && p->m == unlink_me->m);
+ } while (p != unlink_me);
+
+ *prevp = p->next_in_mod;
+ }
+}
+
+static void purge_mfa(struct hipe_mfa_info* p)
+{
+ ASSERT(p->is_stub);
+ remove_mfa_info(p);
+ hipe_free_native_stub(p->remote_address);
+ erts_free(ERTS_ALC_T_HIPE, p);
}
/* Called by init:restart after unloading all hipe compiled modules
- * to work around bug causing execution of deallocated beam code.
- * Can be removed when delete/purge of native modules works better.
+ * to work around old bug that caused execution of deallocated beam code.
+ * Can be removed now when delete/purge of native modules works better.
* Test: Do init:restart in debug compiled vm with hipe compiled kernel.
*/
static void hipe_purge_all_refs(void)
@@ -1638,126 +1636,249 @@ static void hipe_purge_all_refs(void)
hipe_mfa_info_table_rwlock();
+ ASSERT(hipe_mfa_info_table.used == 0);
bucket = hipe_mfa_info_table.bucket;
nrbuckets = 1 << hipe_mfa_info_table.log2size;
for (i = 0; i < nrbuckets; ++i) {
+ ASSERT(bucket[i] == NULL);
while (bucket[i] != NULL) {
struct hipe_mfa_info* mfa = bucket[i];
bucket[i] = mfa->bucket.next;
- while (mfa->refers_to) {
- struct hipe_mfa_info_list *to = mfa->refers_to;
- mfa->refers_to = to->next;
- erts_free(ERTS_ALC_T_HIPE, to);
- }
- while (mfa->referred_from) {
- struct ref* from = mfa->referred_from;
- mfa->referred_from = from->next;
- erts_free(ERTS_ALC_T_HIPE, from);
- }
- erts_free(ERTS_ALC_T_HIPE, mfa);
+ hash_erase(&mod2mfa_tab, mfa);
+ erts_free(ERTS_ALC_T_HIPE, mfa);
}
}
+ hipe_mfa_info_table.used = 0;
hipe_mfa_info_table_rwunlock();
}
BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
{
- struct mfa_t mfa;
- struct hipe_mfa_info *caller_mfa, *callee_mfa;
- struct hipe_mfa_info_list *refers_to, *tmp_refers_to;
- struct ref **prev, *ref;
-
if (BIF_ARG_1 == am_all) {
hipe_purge_all_refs();
BIF_RET(am_ok);
}
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (caller_mfa) {
- refers_to = caller_mfa->refers_to;
- while (refers_to) {
- callee_mfa = refers_to->mfa;
- prev = &callee_mfa->referred_from;
- ref = *prev;
- while (ref) {
- if (ref->caller_mfa == caller_mfa) {
- if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
- ref->flags |= REF_FLAG_PENDING_REMOVE;
- prev = &ref->next;
- ref = ref->next;
- } else {
- struct ref *tmp = ref;
- ref = ref->next;
- *prev = ref;
- erts_free(ERTS_ALC_T_HIPE, tmp);
- }
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
- }
- tmp_refers_to = refers_to;
- refers_to = refers_to->next;
- erts_free(ERTS_ALC_T_HIPE, tmp_refers_to);
- }
- caller_mfa->refers_to = NULL;
+ ASSERT(!"hipe_bifs_remove_refs_from_1() called");
+ BIF_ERROR(BIF_P, BADARG);
+}
+
+int hipe_purge_need_blocking(Module* modp)
+{
+ /* SVERK: Verify if this is really necessary */
+ if (modp->old.hipe_code) {
+ if (modp->old.hipe_code->first_hipe_ref ||
+ modp->old.hipe_code->first_hipe_sdesc)
+ return 1;
+ }
+ if (!modp->curr.code_hdr) {
+ return mod2mfa_get_safe(modp) != NULL;
+ }
+ return 0;
+}
+
+void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module,
+ int is_blocking)
+{
+ struct hipe_ref* ref = first_ref;
+
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+ while (ref) {
+ struct hipe_ref* free_ref = ref;
+
+ DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+ ref->callee->m, ref->callee->f, ref->callee->a);
+ DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+ ref->caller_m, ref->caller_f, ref->caller_a);
+ ASSERT(ref->caller_m == caller_module);
+
+ /*
+ * Unlink from other refs to same callee
+ */
+ ASSERT(ref->head.next->prev == &ref->head);
+ ASSERT(ref->head.prev->next == &ref->head);
+ ASSERT(ref->head.next != &ref->head);
+ ASSERT(ref->head.prev != &ref->head);
+ ref->head.next->prev = ref->head.prev;
+ ref->head.prev->next = ref->head.next;
+
+ /*
+ * Was this the last ref to that callee?
+ */
+ if (ref->head.next == ref->head.prev) {
+ struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+ if (p->is_stub) {
+ if (!is_blocking)
+ hipe_mfa_info_table_rwlock();
+ unlink_mfa_from_mod(p);
+ purge_mfa(p);
+ if (!is_blocking)
+ hipe_mfa_info_table_rwunlock();
+ }
+ }
+
+ ref = ref->next_from_modi;
+ erts_free(ERTS_ALC_T_HIPE, free_ref);
}
- hipe_mfa_info_table_rwunlock();
- BIF_RET(am_ok);
}
+void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module,
+ int is_blocking)
+{
+ struct hipe_sdesc* sdesc = first_sdesc;
+
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+ ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */
+
+ while (sdesc) {
+ struct hipe_sdesc* free_sdesc = sdesc;
+
+ DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+ ASSERT(make_atom(sdesc->m_aix) == module);
+
+ sdesc = sdesc->next_in_modi;
+ hipe_destruct_sdesc(free_sdesc);
+ }
+}
+
+
+void hipe_purge_module(Module* modp, int is_blocking)
+{
+ ASSERT(modp);
+
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+ DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
+
+ if (modp->old.hipe_code) {
+ /*
+ * Remove all hipe_ref's (external calls) from the old module instance
+ */
+ if (modp->old.hipe_code->first_hipe_ref) {
+ ERTS_SMP_LC_ASSERT(is_blocking);
+
+ hipe_purge_refs(modp->old.hipe_code->first_hipe_ref,
+ make_atom(modp->module), is_blocking);
+ modp->old.hipe_code->first_hipe_ref = NULL;
+ }
-/* redirect_referred_from(CalleeMFA)
- * Redirect all pending-redirect refs in CalleeMFA's referred_from.
- * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from.
+ /*
+ * Remove all hipe_sdesc's for the old module instance
+ */
+ if (modp->old.hipe_code->first_hipe_sdesc) {
+ ERTS_SMP_LC_ASSERT(is_blocking);
+
+ hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc,
+ make_atom(modp->module), is_blocking);
+ modp->old.hipe_code->first_hipe_sdesc = NULL;
+ }
+
+ hipe_free_module(modp->old.hipe_code);
+ modp->old.hipe_code = NULL;
+ }
+
+
+ /*
+ * Remove unreferred hipe_mfa_info's
+ * when all module instances are removed (like in init:restart)
+ */
+ if (is_blocking && modp->curr.code_hdr == NULL) {
+ struct hipe_mfa_info* was_first = mod2mfa_get(modp);
+ struct hipe_mfa_info* is_first = was_first;
+ struct hipe_mfa_info** prevp = &is_first;
+ struct hipe_mfa_info *p;
+
+ if (was_first) {
+ for (p = was_first ; p; p = *prevp) {
+ if (p->callers.next == &p->callers) {
+ *prevp = p->next_in_mod;
+ if (p != was_first)
+ purge_mfa(p);
+ }
+ else
+ prevp = &p->next_in_mod;
+ }
+ if (was_first != is_first) {
+ hash_erase(&mod2mfa_tab, was_first);
+ purge_mfa(was_first);
+ if (is_first)
+ mod2mfa_put(is_first);
+ }
+ }
+ }
+}
+
+
+/*
+ * Redirect all existing native calls to this module
*/
-BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
+void hipe_redirect_to_module(Module* modp)
{
- struct mfa_t mfa;
struct hipe_mfa_info *p;
- struct ref **prev, *ref;
- int is_remote, res;
- void *new_address;
+ struct hipe_ref_head* refh;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+ for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
+ if (p->new_address) {
+ if (p->is_stub) {
+ hipe_free_native_stub(p->remote_address);
+ p->is_stub = 0;
+ }
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address);
+ p->remote_address = p->new_address;
+ p->new_address = NULL;
+#ifdef DEBUG
+ p->dbg_export = NULL;
+#endif
+ }
+ else if (!p->is_stub) {
+ Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a);
+ p->remote_address = hipe_make_native_stub(exp, p->a);
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address);
+ if (!p->remote_address)
+ erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
+ p->is_stub = 1;
+#ifdef DEBUG
+ p->dbg_export = exp;
+#endif
+ }
+ else {
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub");
+ ASSERT(p->remote_address && p->dbg_export);
+ }
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p) {
- prev = &p->referred_from;
- ref = *prev;
- while (ref) {
- if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
- is_remote = ref->flags & REF_FLAG_IS_REMOTE;
- new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote);
- if (ref->flags & REF_FLAG_IS_LOAD_MFA)
- res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa);
- else
- res = hipe_patch_call(ref->address, new_address, ref->trampoline);
- if (res)
- fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__);
- ref->flags &= ~REF_FLAG_PENDING_REDIRECT;
- if (ref->flags & REF_FLAG_PENDING_REMOVE) {
- struct ref *tmp = ref;
- ref = ref->next;
- *prev = ref;
- erts_free(ERTS_ALC_T_HIPE, tmp);
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
+ DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p);
+ for (refh = p->callers.next; refh != &p->callers; refh = refh->next) {
+ struct hipe_ref* ref = (struct hipe_ref*) refh;
+ int res;
+
+ DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)",
+ ref, ref->caller_m, ref->caller_f, ref->caller_a,
+ ref->address, p->remote_address);
+
+ DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a,
+ " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)",
+ ref, p->m,p->f,p->a, ref->address, p->remote_address);
+
+ if (ref->flags & REF_FLAG_IS_LOAD_MFA)
+ res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
+ else {
+#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+ void* trampoline = ref->trampoline;
+#else
+ void* trampoline = NULL;
+#endif
+ res = hipe_patch_call(ref->address, p->remote_address, trampoline);
+ }
+ if (res)
+ fprintf(stderr, "%s: patch failed", __FUNCTION__);
}
+ DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p);
}
- hipe_mfa_info_table_rwunlock();
- BIF_RET(NIL);
}
BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1)
@@ -1811,89 +1932,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value)
}
}
-struct modinfo {
- HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */
- unsigned int code_size;
-};
-
-static Hash modinfo_table;
-
-static HashValue modinfo_hash(void *tmpl)
-{
- Eterm mod = (Eterm)tmpl;
- return atom_val(mod);
-}
-
-static int modinfo_cmp(void *tmpl, void *bucket)
-{
- /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */
- return 0;
-}
-
-static void *modinfo_alloc(void *tmpl)
-{
- struct modinfo *p;
-
- p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p));
- p->code_size = 0;
- return &p->bucket;
-}
-
-static void init_modinfo_table(void)
-{
- HashFunctions f;
- static int init_done = 0;
-
- if (init_done)
- return;
- init_done = 1;
- f.hash = (H_FUN) modinfo_hash;
- f.cmp = (HCMP_FUN) modinfo_cmp;
- f.alloc = (HALLOC_FUN) modinfo_alloc;
- f.free = (HFREE_FUN) NULL;
- f.meta_alloc = (HMALLOC_FUN) erts_alloc;
- f.meta_free = (HMFREE_FUN) erts_free;
- f.meta_print = (HMPRINT_FUN) erts_print;
- hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f);
-}
-
-BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3)
-{
- struct modinfo *p;
- Sint code_size;
-
- init_modinfo_table();
-
- if (is_not_atom(BIF_ARG_1) ||
- is_not_small(BIF_ARG_3) ||
- (code_size = signed_val(BIF_ARG_3)) < 0)
- BIF_ERROR(BIF_P, BADARG);
-
- p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1);
-
- if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */
- p->code_size += code_size;
- else /* whole module */
- p->code_size = code_size;
- BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1)
-{
- struct modinfo *p;
- unsigned int code_size;
-
- init_modinfo_table();
-
- if (is_not_atom(BIF_ARG_1))
- BIF_ERROR(BIF_P, BADARG);
-
- p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1);
-
- code_size = p ? p->code_size : 0;
- BIF_RET(make_small(code_size));
-}
-
BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3)
{
Uint *address, value;
@@ -1929,3 +1967,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3)
BIF_ERROR(BIF_P, BADARG);
BIF_RET(NIL);
}
+
+BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1)
+{
+ Binary *magic;
+ Eterm *hp;
+ Eterm res;
+
+ if (is_not_atom(BIF_ARG_1))
+ BIF_ERROR(BIF_P, BADARG);
+
+ magic = hipe_alloc_loader_state(BIF_ARG_1);
+
+ if (!magic)
+ BIF_ERROR(BIF_P, BADARG);
+
+ hp = HAlloc(BIF_P, PROC_BIN_SIZE);
+ res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic);
+ erts_refc_dec(&magic->refc, 1);
+ BIF_RET(res);
+}
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index c9a8216368..811c3801c1 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -30,19 +30,16 @@ extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa);
extern void hipe_mfa_info_table_init(void);
extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a);
-extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3);
+extern BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3);
extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a);
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a);
-extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline);
-#endif
-#if defined(__arm__)
-extern void *hipe_primop_get_trampoline(Eterm name);
-extern void hipe_primop_set_trampoline(Eterm name, void *trampoline);
-#endif
/* needed in beam_load.c */
-void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc);
+int hipe_need_blocking(Module*);
+int hipe_purge_need_blocking(Module*);
+void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking);
+void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking);
+void hipe_purge_module(Module*, int is_blocking);
+void hipe_redirect_to_module(Module* modp);
/* these are also needed in hipe_amd64.c */
extern void *term_to_address(Eterm);
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 99237aae05..264ea2c34a 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -44,22 +44,20 @@ bif hipe_bifs:ref/1
bif hipe_bifs:ref_get/1
bif hipe_bifs:ref_set/2
-bif hipe_bifs:enter_code/2
-bif hipe_bifs:alloc_data/2
+bif hipe_bifs:enter_code/3
+bif hipe_bifs:alloc_data/3
bif hipe_bifs:constants_size/0
bif hipe_bifs:merge_term/1
bif hipe_bifs:fun_to_address/1
+bif hipe_bifs:commit_patch_load/1
bif hipe_bifs:set_native_address/3
#bif hipe_bifs:address_to_fun/1
bif hipe_bifs:set_funinfo_native_address/3
-bif hipe_bifs:invalidate_funinfo_native_addresses/1
+#bif hipe_bifs:invalidate_funinfo_native_addresses/1
-bif hipe_bifs:update_code_size/3
-bif hipe_bifs:code_size/1
-
-bif hipe_bifs:enter_sdesc/1
+bif hipe_bifs:enter_sdesc/2
bif hipe_bifs:bif_address/3
bif hipe_bifs:primop_address/1
@@ -72,7 +70,7 @@ bif hipe_bifs:term_to_word/1
bif hipe_bifs:get_fe/2
bif hipe_bifs:set_native_address_in_fe/2
-bif hipe_bifs:find_na_or_make_stub/2
+bif hipe_bifs:find_na_or_make_stub/1
bif hipe_bifs:check_crc/1
bif hipe_bifs:system_crc/0
@@ -84,9 +82,9 @@ bif hipe_bifs:patch_insn/3
bif hipe_bifs:patch_call/3
bif hipe_bifs:add_ref/2
-bif hipe_bifs:mark_referred_from/1
bif hipe_bifs:remove_refs_from/1
-bif hipe_bifs:redirect_referred_from/1
+
+bif hipe_bifs:alloc_loader_state/1
# atoms used by add_ref/2
atom call
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 5e127755c6..0c66eb6abe 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -45,7 +45,7 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1)
pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
if (!pc)
BIF_ERROR(BIF_P, BADARG);
- ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+ ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
if (pc[0] == BeamOpCode(op_hipe_trap_call))
BIF_ERROR(BIF_P, BADARG);
if (pc[0] == BeamOpCode(op_hipe_call_count))
@@ -67,7 +67,7 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1)
pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
if (!pc)
BIF_ERROR(BIF_P, BADARG);
- ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+ ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
if (pc[0] != BeamOpCode(op_hipe_call_count))
BIF_RET(am_false);
hcc = (struct hipe_call_count*)pc[-4];
@@ -86,7 +86,7 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1)
pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
if (!pc)
BIF_ERROR(BIF_P, BADARG);
- ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+ ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
if (pc[0] != BeamOpCode(op_hipe_call_count))
BIF_RET(am_false);
hcc = (struct hipe_call_count*)pc[-4];
@@ -102,7 +102,7 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1)
pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1);
if (!pc)
BIF_ERROR(BIF_P, BADARG);
- ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI));
+ ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI));
if (pc[0] != BeamOpCode(op_hipe_call_count))
BIF_RET(am_false);
hcc = (struct hipe_call_count*)pc[-4];
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index dfd34e31d4..e04d3d32d1 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -155,7 +155,7 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0)
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1);
+BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1);
# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
@@ -163,13 +163,13 @@ BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1);
# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
-BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1)
+BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1)
{
- typedef BIF_RETTYPE Bif(BIF_ALIST_1);
- Bif* fp = (Bif*) (BIF_P->hipe.bif_callee);
+ typedef BIF_RETTYPE nBif(NBIF_ALIST_1);
+ nBif* fp = (nBif*) (BIF_P->hipe.bif_callee);
BIF_RETTYPE res;
ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P);
- res = (*fp)(BIF_P, BIF__ARGS);
+ res = (*fp)(NBIF_CALL_ARGS);
ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P);
return res;
}
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index dcf3447af9..f034c4700c 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -71,6 +71,32 @@
****************************************************************/
/*
+ * NOTE:
+ * Beam BIFs have the prototype:
+ * Eterm (*BIF)(Process *c_p, Eterm *regs, UWord *I)
+ * Native BIFs have the prototype:
+ * Eterm (*BIF)(Process *c_p, Eterm *regs)
+ *
+ * Beam BIFs expect 'I' to contain current instruction
+ * pointer when called from beam, and expect 'I' to
+ * contain a pointer to the export entry of the BIF
+ * when called from native code. In order to facilitate
+ * this, beam BIFs are called via wrapper functions
+ * when called from native code. These wrapper functions
+ * are auto-generated (by utils/make_tables) and have
+ * the function names nbif_impl_<BIF>.
+ *
+ * The standard_bif_interface_*() and
+ * gc_bif_interface_*() will add the prefix and
+ * thus call nbif_impl_<cbif_name>. That is, all
+ * functions (true BIFs as well as other c-functions)
+ * called via these interfaces have to be named
+ * nbif_impl_<FUNC>.
+ */
+
+/*
+ * See NOTE above!
+ *
* standard_bif_interface_0(nbif_name, cbif_name)
* standard_bif_interface_1(nbif_name, cbif_name)
* standard_bif_interface_2(nbif_name, cbif_name)
@@ -93,6 +119,8 @@
*/
/*
+ * See NOTE above!
+ *
* gc_bif_interface_0(nbif_name, cbif_name)
* gc_bif_interface_1(nbif_name, cbif_name)
* gc_bif_interface_2(nbif_name, cbif_name)
@@ -153,9 +181,9 @@ standard_bif_interface_0(nbif_ports_0, ports_0)
* BIFs and primops that may do a GC (change heap limit and walk the native stack).
* XXX: erase/1 and put/2 cannot fail
*/
-gc_bif_interface_2(nbif_erts_internal_check_process_code_2, hipe_erts_internal_check_process_code_2)
+gc_bif_interface_1(nbif_erts_internal_check_process_code_1, hipe_erts_internal_check_process_code_1)
gc_bif_interface_1(nbif_erase_1, erase_1)
-gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0)
+gc_bif_interface_1(nbif_erts_internal_garbage_collect_1, erts_internal_garbage_collect_1)
gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc)
gc_bif_interface_2(nbif_put_2, put_2)
@@ -247,7 +275,7 @@ nocons_nofail_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_intege
noproc_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer)
')dnl
-gc_bif_interface_0(nbif_check_get_msg, hipe_check_get_msg)
+nofail_primop_interface_0(nbif_check_get_msg, hipe_check_get_msg)
#`ifdef' NO_FPE_SIGNALS
nocons_nofail_primop_interface_0(nbif_emulate_fpe, hipe_emulate_fpe)
@@ -291,8 +319,8 @@ gc_bif_interface_2(nbif_maps_merge_2, hipe_wrapper_maps_merge_2)
* BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index)
*/
-define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, $4)')
-include(TARGET/`erl_bif_list.h')
+define(BIF_LIST,`standard_bif_interface_$3(nbif_$5, $5)')
+include(TTF_DIR/`erl_bif_list.h')
/*
* Guard BIFs.
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index ace489452f..222a11db3d 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -62,10 +62,12 @@ static void print_beam_pc(BeamInstr *pc)
} else if (pc == &beam_apply[1]) {
printf("normal-process-exit");
} else {
- BeamInstr *mfa = find_function_from_pc(pc);
- if (mfa)
+ ErtsCodeMFA *cmfa = find_function_from_pc(pc);
+ if (cmfa)
erts_printf("%T:%T/%bpu + 0x%bpx",
- mfa[0], mfa[1], mfa[2], pc - &mfa[3]);
+ cmfa->module, cmfa->function,
+ cmfa->arity,
+ pc - erts_codemfa_to_code(cmfa));
else
printf("?");
}
@@ -214,10 +216,10 @@ void hipe_print_pcb(Process *p)
U("seq..clock ", seq_trace_clock);
U("seq..astcnt", seq_trace_lastcnt);
U("seq..token ", seq_trace_token);
- U("intial[0] ", u.initial[0]);
- U("intial[1] ", u.initial[1]);
- U("intial[2] ", u.initial[2]);
- P("current ", current);
+ U("intial.mod ", u.initial.module);
+ U("intial.fun ", u.initial.function);
+ U("intial.ari ", u.initial.arity);
+ U("current ", current);
P("cp ", cp);
P("i ", i);
U("catches ", catches);
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 68c65dea27..cf0435adc9 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
/* known nstack walk state */
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int sdesc_size;
unsigned long ra;
unsigned int i;
@@ -99,7 +99,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
if (IS_MOVED_CONS(val)) {
*nsp_i = ptr[1];
} else if (!erts_is_literal(gval, ptr)) {
- ASSERT(within(ptr, p));
+ ASSERT(erts_dbg_within_proc(ptr, p, NULL));
MOVE_CONS(ptr, val, n_htop, nsp_i);
}
}
@@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
/* known nstack walk state */
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int sdesc_size;
unsigned long ra;
unsigned int i;
@@ -208,7 +208,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
} else if (ErtsInArea(ptr, mature, mature_size)) {
MOVE_BOXED(ptr, val, old_htop, nsp_i);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
- ASSERT(within(ptr, p));
+ ASSERT(erts_dbg_within_proc(ptr, p, NULL));
MOVE_BOXED(ptr, val, n_htop, nsp_i);
}
} else if (is_list(gval)) {
@@ -219,7 +219,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
} else if (ErtsInArea(ptr, mature, mature_size)) {
MOVE_CONS(ptr, val, old_htop, nsp_i);
} else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) {
- ASSERT(within(ptr, p));
+ ASSERT(erts_dbg_within_proc(ptr, p, NULL));
MOVE_CONS(ptr, val, n_htop, nsp_i);
}
}
@@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
/* known nstack walk state */
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
/* arch-specific nstack walk state */
struct nstack_walk_state walk_state;
@@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size)
{
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
/* arch-specific nstack walk state */
struct nstack_walk_state walk_state;
@@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size)
}
return 0;
}
+
+int
+nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size)
+{
+ Eterm *nsp;
+ Eterm *nsp_end;
+ const struct hipe_sdesc *sdesc;
+ /* arch-specific nstack walk state */
+ struct nstack_walk_state walk_state;
+
+ if (!p->hipe.nstack || !nstack_walk_init_check(p))
+ return 0;
+ ASSERT(p->hipe.nsp && p->hipe.nstend);
+ nsp = nstack_walk_nsp_begin(p);
+ nsp_end = nstack_walk_nsp_end(p);
+ sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state);
+
+ /* Check the topmost frame */
+ if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size))
+ return 1;
+
+ while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) {
+ unsigned sdesc_size = nstack_walk_frame_size(sdesc);
+ unsigned long ra = nstack_walk_frame_ra(nsp, sdesc);
+ if (ra == (unsigned long)nbif_stack_trap_ra)
+ ra = (unsigned long)p->hipe.ngra;
+ if (ErtsInArea(ra, seg_start, seg_size))
+ return 1;
+ sdesc = hipe_find_sdesc(ra);
+ nsp = nstack_walk_next_frame(nsp, sdesc_size);
+ }
+ return 0;
+}
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
new file mode 100644
index 0000000000..2998ed87a2
--- /dev/null
+++ b/erts/emulator/hipe/hipe_load.c
@@ -0,0 +1,105 @@
+/*
+ * %CopyrightBegin%
+
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_load.c
+ *
+ * HiPE atomic code loader
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "sys.h"
+#include "global.h"
+#include "erl_binary.h"
+#include "hipe_load.h"
+#include "hipe_bif0.h"
+
+void hipe_free_loader_state(HipeLoaderState *stp)
+{
+ if (stp->module == NIL) return;
+
+ // TODO: Needs to be freed separately. We'd like have a unified executable
+ // code allocator, so postpone this for now.
+ /* if (stp->text_segment) */
+ /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */
+ stp->text_segment = NULL;
+ stp->text_segment_size = 0;
+
+ if (stp->data_segment)
+ erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+ stp->data_segment = NULL;
+ stp->data_segment_size = 0;
+
+ if (stp->new_hipe_refs) {
+ hipe_purge_refs(stp->new_hipe_refs, stp->module, 0);
+ stp->new_hipe_refs = NULL;
+ }
+ if (stp->new_hipe_sdesc) {
+ hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0);
+ stp->new_hipe_sdesc = NULL;
+ }
+
+ stp->module = NIL;
+}
+
+static void
+hipe_loader_state_dtor(Binary* magic)
+{
+ HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
+
+ ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor);
+
+ hipe_free_loader_state(stp);
+}
+
+Binary *hipe_alloc_loader_state(Eterm module)
+{
+ HipeLoaderState *stp;
+ Binary *magic;
+
+ if (is_not_atom(module)) return NULL;
+
+ magic = erts_create_magic_binary(sizeof(HipeLoaderState),
+ hipe_loader_state_dtor);
+ erts_refc_inc(&magic->refc, 1);
+ stp = ERTS_MAGIC_BIN_DATA(magic);
+
+ stp->module = module;
+ stp->text_segment = NULL;
+ stp->text_segment_size = 0;
+ stp->data_segment = NULL;
+ stp->data_segment_size = 0;
+
+ stp->new_hipe_refs = NULL;
+ stp->new_hipe_sdesc = NULL;
+
+ return magic;
+}
+
+HipeLoaderState *
+hipe_get_loader_state(Binary *magic)
+{
+ if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor)
+ return NULL;
+
+ return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic);
+}
diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h
new file mode 100644
index 0000000000..40c8a8aa2a
--- /dev/null
+++ b/erts/emulator/hipe/hipe_load.h
@@ -0,0 +1,48 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_load.h
+ *
+ * HiPE atomic code loader
+ */
+#ifndef HIPE_LOAD_H
+#define HIPE_LOAD_H
+
+#include "global.h"
+
+typedef struct hipe_loader_state {
+ Eterm module; /* Module name, atom */
+
+ void *text_segment;
+ Uint text_segment_size;
+
+ void *data_segment;
+ Uint data_segment_size;
+
+ struct hipe_ref* new_hipe_refs;
+ struct hipe_sdesc* new_hipe_sdesc;
+
+} HipeLoaderState;
+
+extern Binary *hipe_alloc_loader_state(Eterm module);
+extern void hipe_free_loader_state(HipeLoaderState*);
+extern HipeLoaderState *hipe_get_loader_state(Binary *binary);
+
+#endif /* HIPE_LOAD_H */
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index b9d4226705..4573980e1e 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = {
presence or absence of struct erl_fun_thing's "next" field. */
{ 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) },
{ 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) },
-#ifdef HIPE
- { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) },
-#endif
{ 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) },
{ 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) },
{ 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) },
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index ed95045292..0706f8d2c9 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -718,12 +718,9 @@ void hipe_empty_nstack(Process *p)
p->hipe.nstend = NULL;
}
-void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free)
+void hipe_set_closure_stub(ErlFunEntry *fe)
{
- unsigned arity;
-
- arity = fe->arity;
- fe->native_address = (Eterm*) hipe_closure_stub_address(arity);
+ fe->native_address = (Eterm*) hipe_closure_stub_address(fe->arity);
}
Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s)
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index e54b81cf78..334e978307 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -59,12 +59,11 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure);
Process *hipe_mode_switch(Process*, unsigned, Eterm*);
void hipe_inc_nstack(Process *p);
void hipe_empty_nstack(Process *p);
-void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
+void hipe_set_closure_stub(ErlFunEntry *fe);
Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process*);
-ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra);
extern Uint hipe_beam_pc_return[];
extern Uint hipe_beam_pc_throw[];
@@ -113,11 +112,6 @@ ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process *p)
p->stop += 2;
}
-ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra)
-{
- return ra == nbif_return;
-}
-
#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
#endif /* ASM */
diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c
new file mode 100644
index 0000000000..469f077dd2
--- /dev/null
+++ b/erts/emulator/hipe/hipe_module.c
@@ -0,0 +1,35 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "sys.h"
+#include "hipe_arch.h"
+#include "hipe_module.h"
+
+void hipe_free_module(HipeModule *mod)
+{
+ hipe_free_code(mod->text_segment, mod->text_segment_size);
+ if (mod->data_segment) /* Some modules lack data segments */
+ erts_free(ERTS_ALC_T_HIPE, mod->data_segment);
+
+ erts_free(ERTS_ALC_T_HIPE, mod);
+}
diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h
new file mode 100644
index 0000000000..b489f567cb
--- /dev/null
+++ b/erts/emulator/hipe/hipe_module.h
@@ -0,0 +1,45 @@
+/*
+ * %CopyrightBegin%
+
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_module.h
+ *
+ *
+ */
+#ifndef HIPE_MODULE_H
+#define HIPE_MODULE_H
+
+/* Forward-declare type to resolve circular dependency with module.h */
+typedef struct hipe_module HipeModule;
+
+#include "global.h"
+
+struct hipe_module {
+ void *text_segment;
+ Uint text_segment_size;
+ void *data_segment;
+
+ struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */
+ struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */
+};
+
+extern void hipe_free_module(HipeModule *mod);
+
+#endif /* HIPE_MODULE_H */
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 801cef2e56..9a9252e8b8 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -42,8 +42,8 @@
*/
/* for -Wmissing-prototypes :-( */
-extern Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2);
-extern Eterm hipe_show_nstack_1(BIF_ALIST_1);
+extern Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1);
+extern Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1);
/* Used when a BIF can trigger a stack walk. */
static __inline__ void hipe_set_narity(Process *p, unsigned int arity)
@@ -51,22 +51,24 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity)
p->hipe.narity = arity;
}
-Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2)
+/* Called via standard_bif_interface_2 */
+Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1)
{
Eterm ret;
- hipe_set_narity(BIF_P, 2);
- ret = erts_internal_check_process_code_2(BIF_P, BIF__ARGS);
+ hipe_set_narity(BIF_P, 1);
+ ret = nbif_impl_erts_internal_check_process_code_1(NBIF_CALL_ARGS);
hipe_set_narity(BIF_P, 0);
return ret;
}
-Eterm hipe_show_nstack_1(BIF_ALIST_1)
+/* Called via standard_bif_interface_1 */
+Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1)
{
Eterm ret;
hipe_set_narity(BIF_P, 1);
- ret = hipe_bifs_show_nstack_1(BIF_P, BIF__ARGS);
+ ret = nbif_impl_hipe_bifs_show_nstack_1(NBIF_CALL_ARGS);
hipe_set_narity(BIF_P, 0);
return ret;
}
@@ -89,7 +91,7 @@ void hipe_gc(Process *p, Eterm need)
* has begun.
* XXX: BUG: native code should check return status
*/
-BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1)
+BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1)
{
Process* p = BIF_P;
Eterm timeout_value = BIF_ARG_1;
@@ -270,10 +272,10 @@ static struct StackTrace *get_trace_from_exc(Eterm exc)
* This does what the (misnamed) Beam instruction 'raise_ss' does,
* namely, a proper re-throw of an exception that was caught by 'try'.
*/
-
-BIF_RETTYPE hipe_rethrow(BIF_ALIST_2)
+/* Called via standard_bif_interface_2 */
+BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2)
{
- Process* c_p = BIF_P;
+ Process *c_p = BIF_P;
Eterm exc = BIF_ARG_1;
Eterm value = BIF_ARG_2;
@@ -397,7 +399,7 @@ Eterm hipe_bs_utf8_size(Eterm arg)
return make_small(4);
}
-BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3)
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3)
{
Process* p = BIF_P;
Eterm arg = BIF_ARG_1;
@@ -458,7 +460,7 @@ Eterm hipe_bs_put_utf16(Process *p, Eterm arg, byte *base, unsigned int offset,
return new_offset;
}
-BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3)
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3)
{
Process *p = BIF_P;
Eterm arg = BIF_ARG_1;
@@ -467,7 +469,7 @@ BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3)
return hipe_bs_put_utf16(p, arg, base, offset, 0);
}
-BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3)
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3)
{
Process *p = BIF_P;
Eterm arg = BIF_ARG_1;
@@ -485,7 +487,7 @@ static int validate_unicode(Eterm arg)
return 1;
}
-BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1)
+BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1)
{
Process *p = BIF_P;
Eterm arg = BIF_ARG_1;
@@ -503,7 +505,8 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg)
return 1;
}
-BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2)
+/* Called via standard_bif_interface_2 */
+BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2)
{
/* Arguments are Eterm-sized unsigned integers */
Uint dividend = BIF_ARG_1;
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index a02d26087b..38f874888b 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -74,27 +74,27 @@ AEXTERN(void,nbif_select_msg,(Process*));
AEXTERN(Eterm,nbif_cmp_2,(void));
AEXTERN(Eterm,nbif_eq_2,(void));
-BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2);
-BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1);
+BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1);
void hipe_fclearerror_error(Process*);
void hipe_select_msg(Process*);
void hipe_gc(Process*, Eterm);
-BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1);
+BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1);
void hipe_handle_exception(Process*);
-BIF_RETTYPE hipe_rethrow(BIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2);
char *hipe_bs_allocate(int);
Binary *hipe_bs_reallocate(Binary*, int);
int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned);
void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned);
Eterm hipe_bs_utf8_size(Eterm);
-BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3);
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3);
Eterm hipe_bs_utf16_size(Eterm);
-BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3);
-BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3);
-BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1);
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3);
+BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3);
+BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1);
struct erl_bin_match_buffer;
int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm);
-BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2);
#ifdef NO_FPE_SIGNALS
AEXTERN(void,nbif_emulate_fpe,(Process*));
@@ -129,7 +129,7 @@ void hipe_atomic_inc(int*);
void hipe_clear_timeout(Process*);
#endif
-#define BIF_LIST(M,F,A,C,I) AEXTERN(Eterm,nbif_##C,(void));
+#define BIF_LIST(M,F,A,B,C,I) AEXTERN(Eterm,nbif_##C,(void));
#include "erl_bif_list.h"
#undef BIF_LIST
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 9b2048c457..4413748936 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -25,7 +25,6 @@
#endif
#include "global.h"
#include "erl_binary.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes)
asm volatile("sync\n\tisync");
}
-/*
- * Management of 32MB code segments for regular code and trampolines.
- */
-
-#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */
-
-static struct segment {
- unsigned int *base; /* [base,base+32MB[ */
- unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */
- unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */
-} curseg;
-
-#define in_area(ptr,start,nbytes) \
- ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
-
-/* Darwin breakage */
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-static void *new_code_mapping(void)
-{
- return mmap(0, SEGMENT_NRBYTES,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS,
- -1, 0);
-}
-
static int check_callees(Eterm callees)
{
Eterm *tuple;
@@ -119,126 +90,71 @@ static int check_callees(Eterm callees)
return arity;
}
-static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec)
+
+static void generate_trampolines(Uint32* address,
+ int nrcallees, Eterm callees,
+ Uint32** trampvec)
{
- unsigned int *base, *address, *tramp_pos, nrfreewords;
- int trampnr;
+ Uint32* trampoline = address;
+ int i;
- tramp_pos = curseg.tramp_pos;
- address = curseg.code_pos;
- nrfreewords = tramp_pos - address;
- if (nrwords > nrfreewords)
- return NULL;
- curseg.code_pos = address + nrwords;
- nrfreewords -= nrwords;
-
- base = curseg.base;
- for (trampnr = 1; trampnr <= nrcallees; ++trampnr) {
- Eterm mfa = tuple_val(callees)[trampnr];
- Eterm m = tuple_val(mfa)[1];
- Eterm f = tuple_val(mfa)[2];
- unsigned int a = unsigned_val(tuple_val(mfa)[3]);
- unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a);
- if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
+ for (i = 0; i < nrcallees; ++i) {
#if defined(__powerpc64__)
- if (nrfreewords < 7)
- return NULL;
- nrfreewords -= 7;
- tramp_pos = trampoline = tramp_pos - 7;
- trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
- trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
- trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
- trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
- trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
- trampoline[5] = 0x7D6903A6; /* mtctr r11 */
- trampoline[6] = 0x4E800420; /* bctr */
- hipe_flush_icache_range(trampoline, 7*sizeof(int));
+# define TRAMPOLINE_WORDS 7
+ trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
+ trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
+ trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
+ trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
+ trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
+ trampoline[5] = 0x7D6903A6; /* mtctr r11 */
+ trampoline[6] = 0x4E800420; /* bctr */
#else
- if (nrfreewords < 4)
- return NULL;
- nrfreewords -= 4;
- tramp_pos = trampoline = tramp_pos - 4;
- trampoline[0] = 0x39600000; /* addi r11,r0,0 */
- trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
- trampoline[2] = 0x7D6903A6; /* mtctr r11 */
- trampoline[3] = 0x4E800420; /* bctr */
- hipe_flush_icache_range(trampoline, 4*sizeof(int));
+# define TRAMPOLINE_WORDS 4
+ trampoline[0] = 0x39600000; /* addi r11,r0,0 */
+ trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
+ trampoline[2] = 0x7D6903A6; /* mtctr r11 */
+ trampoline[3] = 0x4E800420; /* bctr */
#endif
- hipe_mfa_set_trampoline(m, f, a, trampoline);
- }
- trampvec[trampnr-1] = trampoline;
+ trampvec[i] = trampoline;
+ trampoline += TRAMPOLINE_WORDS;
}
- curseg.tramp_pos = tramp_pos;
- return address;
+ hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32));
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
{
- Uint nrwords;
+ Uint code_words;
int nrcallees;
Eterm trampvecbin;
- unsigned int **trampvec;
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
+ Uint32 **trampvec;
+ Uint32 *address;
if (nrbytes & 0x3)
return NULL;
- nrwords = nrbytes >> 2;
+ code_words = nrbytes / sizeof(Uint32);
nrcallees = check_callees(callees);
if (nrcallees < 0)
return NULL;
- trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*));
- trampvec = (unsigned int**)binary_bytes(trampvecbin);
-
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
+ trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*));
+ trampvec = (Uint32**)binary_bytes(trampvecbin);
+
+ address = erts_alloc(ERTS_ALC_T_HIPE_EXEC,
+ (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32));
+
+ generate_trampolines(address + code_words, nrcallees, callees, trampvec);
*trampolines = trampvecbin;
return address;
}
-static unsigned int *alloc_stub(Uint nrwords)
+void hipe_free_code(void* code, unsigned int bytes)
{
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
- return address;
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
}
static void patch_imm16(Uint32 *address, unsigned int imm16)
@@ -288,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
- unsigned int *code;
+ Uint32 *code;
if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL)
abort();
- code = alloc_stub(7);
+ code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32));
if (!code)
return NULL;
@@ -312,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
/* ba nbif_callemu */
code[6] = 0x48000002 | (unsigned long)&nbif_callemu;
- hipe_flush_icache_range(code, 7*sizeof(int));
+ hipe_flush_icache_range(code, 7*sizeof(Uint32));
return code;
}
@@ -360,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type)
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
- unsigned int *code;
+ Uint32 *code;
/*
* Native code calls BEAM via a stub looking as follows:
@@ -383,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL)
abort();
- code = alloc_stub(4);
+ code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32));
if (!code)
return NULL;
@@ -396,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
/* ba nbif_callemu */
code[3] = 0x48000002 | (unsigned long)&nbif_callemu;
- hipe_flush_icache_range(code, 4*sizeof(int));
+ hipe_flush_icache_range(code, 4*sizeof(Uint32));
return code;
}
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index b540562185..79a8bef77d 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -26,9 +26,9 @@ include(`hipe/hipe_ppc_asm.m4')
#`include' "hipe_literals.h"
`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-# define CALL_BIF(F) STORE_IA(CSYM(F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper)
+# define CALL_BIF(F) STORE_IA(CSYM(nbif_impl_##F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper)
#else
-# define CALL_BIF(F) bl CSYM(F)
+# define CALL_BIF(F) bl CSYM(nbif_impl_##F)
#endif'
.text
diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h
index 09568c140e..f019434f67 100644
--- a/erts/emulator/hipe/hipe_risc_gc.h
+++ b/erts/emulator/hipe/hipe_risc_gc.h
@@ -27,7 +27,7 @@
/* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */
struct nstack_walk_state {
- const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
+ const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
};
static inline int nstack_walk_init_check(const Process *p)
@@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p)
return p->hipe.nsp + nstkarity;
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
{
- const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra);
state->sdesc0 = sdesc;
return sdesc;
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc_ignore_trap(const Process *p,
struct nstack_walk_state *state)
{
unsigned long ra = (unsigned long)p->hipe.nra;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
if (ra == (unsigned long)&nbif_stack_trap_ra)
ra = (unsigned long)p->hipe.ngra;
sdesc = hipe_find_sdesc(ra);
@@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p,
return sdesc;
}
-static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
+static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0)
{
Eterm *nsp = p->hipe.nsp;
p->hipe.nsp = nstack_walk_nsp_begin(p);
@@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp
return nsp >= nsp_end;
}
-static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc)
+static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc)
{
return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc);
}
@@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i)
}
static inline unsigned long
-nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc)
+nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc)
{
return nsp[sdesc_fsize(sdesc)];
}
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 0284265307..1369b392fe 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -66,13 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
static __inline__ void hipe_arch_glue_init(void)
{
- static struct sdesc_with_exnra nbif_return_sdesc = {
- .exnra = (unsigned long)&nbif_fail,
- .sdesc = {
- .bucket = { .hvalue = (unsigned long)&nbif_return },
- .summary = (1<<8),
- },
- };
+ static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+ nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+ nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+ nbif_return_sdesc.sdesc.fsize = 0;
+ nbif_return_sdesc.sdesc.has_exnra = 1;
+ nbif_return_sdesc.sdesc.stk_nargs = 0;
+ nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+ nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+ nbif_return_sdesc.sdesc.a = 0;
+
hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
}
diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index dc98c96b8f..4001bedeb6 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p)
{
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc1;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc1;
+ const struct hipe_sdesc *sdesc;
unsigned long ra;
unsigned long exnra;
unsigned int mask;
@@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p)
#define MINSTACK 128
#define NSKIPFRAMES 4
-void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
+void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc)
{
Eterm *nsp;
Eterm *nsp_end;
@@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
void (*hipe_handle_stack_trap(Process *p))(void)
{
void (*ngra)(void) = p->hipe.ngra;
- const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
hipe_update_stack_trap(p, sdesc);
return ngra;
}
@@ -237,7 +237,7 @@ void hipe_find_handler(Process *p)
unsigned long ra;
unsigned long exnra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
nsp = p->hipe.nsp;
nsp_end = p->hipe.nstend;
@@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
Eterm *nsp_end;
unsigned long ra, prev_ra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
int i;
if (depth < 1)
@@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
ra = (unsigned long)p->hipe.nra;
prev_ra = 0;
i = 0;
- for (;;) {
+ while (nsp < nsp_end) {
if (ra == (unsigned long)nbif_stack_trap_ra)
ra = (unsigned long)p->hipe.ngra;
if (ra != prev_ra) {
@@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
break;
prev_ra = ra;
}
- if (nsp >= nsp_end)
- break;
sdesc = hipe_find_sdesc(ra);
nsp += arity + sdesc_fsize(sdesc);
arity = sdesc_arity(sdesc);
diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c
index 23020f34ee..876b20bb15 100644
--- a/erts/emulator/hipe/hipe_sparc.c
+++ b/erts/emulator/hipe/hipe_sparc.c
@@ -24,7 +24,6 @@
#include "config.h"
#endif
#include "global.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -88,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
Uint32 relDest, newI;
- if (trampoline)
- return -1;
+ ASSERT(trampoline == NULL);
+
relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress);
newI = (1 << 30) | (relDest >> 2);
*(Uint32*)callAddress = newI;
@@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
return 0;
}
-/*
- * Memory allocator for executable code.
- *
- * This is required on x86 because some combinations
- * of Linux kernels and CPU generations default to
- * non-executable memory mappings, causing ordinary
- * malloc() memory to be non-executable.
- */
-static unsigned int code_bytes;
-static char *code_next;
-
-#if 0 /* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
- printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
- total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
- if (!atexit_done) {
- atexit_done = 1;
- (void)atexit(alloc_code_stats);
- }
-}
-
-#define ALLOC_CODE_STATS(X) do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X) do{}while(0)
-#endif
-
-static int morecore(unsigned int alloc_bytes)
-{
- unsigned int map_bytes;
- char *map_hint, *map_start;
-
- /* Page-align the amount to allocate. */
- map_bytes = (alloc_bytes + 4095) & ~4095;
-
- /* Round up small allocations. */
- if (map_bytes < 1024*1024)
- map_bytes = 1024*1024;
- else
- ALLOC_CODE_STATS(++nr_large);
-
- /* Create a new memory mapping, ensuring it is executable
- and in the low 2GB of the address space. Also attempt
- to make it adjacent to the previous mapping. */
- map_hint = code_next + code_bytes;
- if ((unsigned long)map_hint & 4095)
- abort();
- map_start = mmap(map_hint, map_bytes,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS
-#ifdef __x86_64__
- |MAP_32BIT
-#endif
- ,
- -1, 0);
- if (map_start == MAP_FAILED)
- return -1;
-
- ALLOC_CODE_STATS(total_mapped += map_bytes);
-
- /* Merge adjacent mappings, so the trailing portion of the previous
- mapping isn't lost. In practice this is quite successful. */
- if (map_start == map_hint) {
- ALLOC_CODE_STATS(++nr_joins);
- code_bytes += map_bytes;
- } else {
- ALLOC_CODE_STATS(++nr_splits);
- ALLOC_CODE_STATS(total_lost += code_bytes);
- code_next = map_start;
- code_bytes = map_bytes;
- }
-
- ALLOC_CODE_STATS(atexit_alloc_code_stats());
-
- return 0;
-}
-
static void *alloc_code(unsigned int alloc_bytes)
{
- void *res;
-
- /* Align function entries. */
- alloc_bytes = (alloc_bytes + 3) & ~3;
-
- if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
- return NULL;
- ALLOC_CODE_STATS(++nr_allocs);
- ALLOC_CODE_STATS(total_alloc += alloc_bytes);
- res = code_next;
- code_next += alloc_bytes;
- code_bytes -= alloc_bytes;
- return res;
+ return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
@@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code, unsigned int nrbytes)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
unsigned int *code;
@@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index 1389beaa61..14330c2f1c 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -29,9 +29,9 @@ include(`hipe/hipe_sparc_asm.m4')
.align 4
`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-# define CALL_BIF(F) set F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper
+# define CALL_BIF(F) set nbif_impl_##F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper
#else
-# define CALL_BIF(F) call F
+# define CALL_BIF(F) call nbif_impl_##F
#endif'
/*
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index e2e6eb74b1..b80e44bc37 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -43,10 +43,10 @@
*/
struct hipe_sdesc_table hipe_sdesc_table;
-static struct sdesc **alloc_bucket(unsigned int size)
+static struct hipe_sdesc **alloc_bucket(unsigned int size)
{
- unsigned long nbytes = size * sizeof(struct sdesc*);
- struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
+ unsigned long nbytes = size * sizeof(struct hipe_sdesc*);
+ struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
sys_memzero(bucket, nbytes);
return bucket;
}
@@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size)
static void hipe_grow_sdesc_table(void)
{
unsigned int old_size, new_size, new_mask;
- struct sdesc **old_bucket, **new_bucket;
+ struct hipe_sdesc **old_bucket, **new_bucket;
unsigned int i;
old_size = 1 << hipe_sdesc_table.log2size;
@@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void)
new_bucket = alloc_bucket(new_size);
hipe_sdesc_table.bucket = new_bucket;
for (i = 0; i < old_size; ++i) {
- struct sdesc *b = old_bucket[i];
+ struct hipe_sdesc *b = old_bucket[i];
while (b != NULL) {
- struct sdesc *next = b->bucket.next;
+ struct hipe_sdesc *next = b->bucket.next;
unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask;
b->bucket.next = new_bucket[j];
new_bucket[j] = b;
@@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void)
erts_free(ERTS_ALC_T_HIPE, old_bucket);
}
-struct sdesc *hipe_put_sdesc(struct sdesc *sdesc)
+struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc)
{
unsigned long ra;
unsigned int i;
- struct sdesc *chain;
+ struct hipe_sdesc *chain;
unsigned int size;
ra = sdesc->bucket.hvalue;
@@ -102,7 +102,29 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc)
return sdesc;
}
-void hipe_init_sdesc_table(struct sdesc *sdesc)
+void hipe_destruct_sdesc(struct hipe_sdesc *sdesc)
+{
+ unsigned int i;
+ struct hipe_sdesc** prevp;
+ void* free_me;
+
+ i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
+ prevp = &hipe_sdesc_table.bucket[i];
+
+ for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next)
+ ASSERT(*prevp);
+
+ *prevp = sdesc->bucket.next;
+ hipe_sdesc_table.used -= 1;
+
+ if (sdesc->has_exnra)
+ free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc);
+ else
+ free_me = sdesc;
+ erts_free(ERTS_ALC_T_HIPE, free_me);
+}
+
+void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
{
unsigned int log2size, size;
@@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct sdesc *sdesc)
* representation. If different representations are needed in
* the future, this code has to be made target dependent.
*/
-struct sdesc *hipe_decode_sdesc(Eterm arg)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
{
Uint ra, exnra;
Eterm *live;
- Uint fsize, arity, nlive, i, nslots, off;
+ Uint fsize, nargs, stk_nargs, nlive, i, nslots, off;
Uint livebitswords, sdescbytes;
void *p;
- struct sdesc *sdesc;
-
- if (is_not_tuple(arg) ||
- (tuple_val(arg))[0] != make_arityval(6) ||
- term_to_Uint((tuple_val(arg))[1], &ra) == 0 ||
- term_to_Uint((tuple_val(arg))[2], &exnra) == 0 ||
- is_not_small((tuple_val(arg))[3]) ||
- (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 ||
- is_not_small((tuple_val(arg))[4]) ||
- (arity = unsigned_val((tuple_val(arg))[4])) > 255 ||
- is_not_tuple((tuple_val(arg))[5]))
+ struct hipe_sdesc *sdesc;
+ Eterm* mfa_tpl;
+ Eterm* tp;
+
+ if (is_not_tuple(arg))
+ return 0;
+
+ tp = tuple_val(arg);
+ if (tp[0] != make_arityval(6) ||
+ term_to_Uint(tp[1], &ra) == 0 ||
+ term_to_Uint(tp[2], &exnra) == 0 ||
+ is_not_small(tp[3]) ||
+ (fsize = unsigned_val(tp[3])) > 65535 ||
+ is_not_small(tp[4]) ||
+ (stk_nargs = unsigned_val(tp[4])) > 255 ||
+ is_not_tuple(tp[5]) ||
+ is_not_tuple(tp[6]) ||
+ (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) ||
+ is_not_atom(mfa_tpl[1]) ||
+ is_not_atom(mfa_tpl[2]) ||
+ is_not_small(mfa_tpl[3]) ||
+ (nargs = unsigned_val(mfa_tpl[3])) > 255)
return 0;
+
+ if (stk_nargs > nargs)
+ return 0;
+
/* Get tuple with live slots */
- live = tuple_val((tuple_val(arg))[5]) + 1;
+ live = tuple_val(tp[5]) + 1;
/* Get number of live slots */
nlive = arityval(live[-1]);
- /* Calculate size of frame = locals + ra + arguments */
- nslots = fsize + 1 + arity;
+ /* Calculate size of frame = locals + ra + stack arguments */
+ nslots = fsize + 1 + stk_nargs;
/* Check that only valid slots are given. */
for (i = 0; i < nlive; ++i) {
if (is_not_small(live[i]) ||
@@ -155,27 +192,34 @@ struct sdesc *hipe_decode_sdesc(Eterm arg)
}
/* Calculate number of words for the live bitmap. */
- livebitswords = (fsize + arity + 1 + 31) / 32;
+ livebitswords = (fsize + stk_nargs + 1 + 31) / 32;
/* Calculate number of bytes needed for the stack descriptor. */
sdescbytes =
(exnra
- ? offsetof(struct sdesc_with_exnra, sdesc.livebits)
- : offsetof(struct sdesc, livebits))
+ ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits)
+ : offsetof(struct hipe_sdesc, livebits))
+ livebitswords * sizeof(int);
p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes);
/* If we have an exception handler use the
special sdesc_with_exnra structure. */
if (exnra) {
- struct sdesc_with_exnra *sdesc_we = p;
+ struct hipe_sdesc_with_exnra *sdesc_we = p;
sdesc_we->exnra = exnra;
sdesc = &(sdesc_we->sdesc);
} else
sdesc = p;
+ sdesc->m_aix = atom_val(mfa_tpl[1]);
+ sdesc->f_aix = atom_val(mfa_tpl[2]);
+ sdesc->a = nargs;
+
+
/* Initialise head of sdesc. */
sdesc->bucket.next = 0;
sdesc->bucket.hvalue = ra;
- sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity;
+ sdesc->fsize = fsize;
+ sdesc->has_exnra = (exnra ? 1 : 0);
+ sdesc->stk_nargs = stk_nargs;
/* Clear all live-bits */
for (i = 0; i < livebitswords; ++i)
sdesc->livebits[i] = 0;
@@ -184,13 +228,5 @@ struct sdesc *hipe_decode_sdesc(Eterm arg)
off = unsigned_val(live[i]);
sdesc->livebits[off / 32] |= (1 << (off & 31));
}
-#ifdef DEBUG
- {
- Eterm mfa_tpl = tuple_val(arg)[6];
- sdesc->dbg_M = tuple_val(mfa_tpl)[1];
- sdesc->dbg_F = tuple_val(mfa_tpl)[2];
- sdesc->dbg_A = tuple_val(mfa_tpl)[3];
- }
-#endif
return sdesc;
}
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index afa0ed4256..7e30358767 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -30,40 +30,43 @@
#include <stddef.h> /* offsetof() */
-struct sdesc {
+struct hipe_sdesc {
struct {
unsigned long hvalue; /* return address */
- struct sdesc *next; /* hash collision chain */
+ struct hipe_sdesc *next; /* hash collision chain */
} bucket;
- unsigned int summary; /* frame size, exn handler presence flag, arity */
-#ifdef DEBUG
- Eterm dbg_M, dbg_F;
- unsigned dbg_A;
-#endif
- unsigned int livebits[1]; /* size depends on arch & data in summary field */
+ unsigned int fsize : 23; /* frame size */
+ unsigned int has_exnra : 1; /* exn handler presence flag */
+ unsigned int stk_nargs : 8; /* arguments on stack */
+ Uint32 m_aix;
+ Uint32 f_aix;
+ Uint32 a;
+ struct hipe_sdesc* next_in_modi;
+ Uint32 livebits[1]; /* size depends on arch & data in summary field */
};
-struct sdesc_with_exnra {
+struct hipe_sdesc_with_exnra {
unsigned long exnra;
- struct sdesc sdesc;
+ struct hipe_sdesc sdesc;
};
-static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc)
+static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc)
{
- return sdesc->summary >> 9;
+ return sdesc->fsize;
}
-static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc)
+/* Nr of arguments pushed on stack */
+static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc)
{
- return sdesc->summary & 0xFF;
+ return sdesc->stk_nargs;
}
-static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc)
+static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc)
{
- if ((sdesc->summary & (1<<8))) {
+ if (sdesc->has_exnra) {
const char *tmp;
- tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc);
- return ((const struct sdesc_with_exnra*)tmp)->exnra;
+ tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc);
+ return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra;
}
return 0;
}
@@ -72,13 +75,14 @@ struct hipe_sdesc_table {
unsigned int log2size;
unsigned int mask; /* INV: mask == (1 << log2size)-1 */
unsigned int used;
- struct sdesc **bucket;
+ struct hipe_sdesc **bucket;
};
extern struct hipe_sdesc_table hipe_sdesc_table;
-extern struct sdesc *hipe_put_sdesc(struct sdesc*);
-extern void hipe_init_sdesc_table(struct sdesc*);
-extern struct sdesc *hipe_decode_sdesc(Eterm);
+extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
+extern void hipe_destruct_sdesc(struct hipe_sdesc*);
+extern void hipe_init_sdesc_table(struct hipe_sdesc*);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
#define __builtin_expect(x, expected_value) (x)
@@ -86,10 +90,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm);
#define likely(x) __builtin_expect((x),1)
#define unlikely(x) __builtin_expect((x),0)
-static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra)
+static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra)
{
unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
- const struct sdesc *sdesc = hipe_sdesc_table.bucket[i];
+ const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i];
if (likely(sdesc->bucket.hvalue == ra))
return sdesc;
do {
@@ -103,7 +107,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void));
extern void hipe_print_nstack(Process*);
extern void hipe_find_handler(Process*);
extern void (*hipe_handle_stack_trap(Process*))(void);
-extern void hipe_update_stack_trap(Process*, const struct sdesc*);
+extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*);
extern int hipe_fill_stacktrace(Process*, int, Eterm**);
#if 0 && defined(HIPE_NSTACK_GROWS_UP)
@@ -134,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop
extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area,
Uint area_size);
extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size);
+extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size);
+
#endif /* HIPE_STACK_H */
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index 5f6c8c200e..c7e24673ac 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -24,7 +24,6 @@
#include "config.h"
#endif
#include "global.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -62,118 +61,17 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
Uint rel32;
- if (trampoline)
- return -1;
+ ASSERT(trampoline == NULL);
+
rel32 = (Uint)destAddress - (Uint)callAddress - 4;
*(Uint32*)callAddress = rel32;
hipe_flush_icache_word(callAddress);
return 0;
}
-/*
- * Memory allocator for executable code.
- *
- * This is required on x86 because some combinations
- * of Linux kernels and CPU generations default to
- * non-executable memory mappings, causing ordinary
- * malloc() memory to be non-executable.
- */
-static unsigned int code_bytes;
-static char *code_next;
-
-#if 0 /* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
- printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
- total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
- if (!atexit_done) {
- atexit_done = 1;
- (void)atexit(alloc_code_stats);
- }
-}
-
-#define ALLOC_CODE_STATS(X) do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X) do{}while(0)
-#endif
-
-/* FreeBSD 6.1 and Darwin breakage */
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-static int morecore(unsigned int alloc_bytes)
-{
- unsigned int map_bytes;
- char *map_hint, *map_start;
-
- /* Page-align the amount to allocate. */
- map_bytes = (alloc_bytes + 4095) & ~4095;
-
- /* Round up small allocations. */
- if (map_bytes < 1024*1024)
- map_bytes = 1024*1024;
- else
- ALLOC_CODE_STATS(++nr_large);
-
- /* Create a new memory mapping, ensuring it is executable
- and in the low 2GB of the address space. Also attempt
- to make it adjacent to the previous mapping. */
- map_hint = code_next + code_bytes;
- if ((unsigned long)map_hint & 4095)
- abort();
- map_start = mmap(map_hint, map_bytes,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS
-#ifdef __x86_64__
- |MAP_32BIT
-#endif
- ,
- -1, 0);
- if (map_start == MAP_FAILED)
- return -1;
-
- ALLOC_CODE_STATS(total_mapped += map_bytes);
-
- /* Merge adjacent mappings, so the trailing portion of the previous
- mapping isn't lost. In practice this is quite successful. */
- if (map_start == map_hint) {
- ALLOC_CODE_STATS(++nr_joins);
- code_bytes += map_bytes;
- } else {
- ALLOC_CODE_STATS(++nr_splits);
- ALLOC_CODE_STATS(total_lost += code_bytes);
- code_next = map_start;
- code_bytes = map_bytes;
- }
-
- ALLOC_CODE_STATS(atexit_alloc_code_stats());
-
- return 0;
-}
-
static void *alloc_code(unsigned int alloc_bytes)
{
- void *res;
-
- /* Align function entries. */
- alloc_bytes = (alloc_bytes + 3) & ~3;
-
- if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
- return NULL;
- ALLOC_CODE_STATS(++nr_allocs);
- ALLOC_CODE_STATS(total_alloc += alloc_bytes);
- res = code_next;
- code_next += alloc_bytes;
- code_bytes -= alloc_bytes;
- return res;
+ return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
@@ -184,6 +82,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code, unsigned int bytes)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
/*
@@ -264,6 +167,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index c0c149733c..aecf67dc1b 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -32,9 +32,9 @@ include(`hipe/hipe_x86_asm.m4')
#endif'
`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-# define CALL_BIF(F) movl $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper)
+# define CALL_BIF(F) movl $CSYM(nbif_impl_##F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper)
#else
-# define CALL_BIF(F) call CSYM(F)
+# define CALL_BIF(F) call CSYM(nbif_impl_##F)
#endif'
define(TEST_GOT_MBUF,`movl P_MBUF(P), %edx /* `TEST_GOT_MBUF' */
@@ -666,13 +666,9 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu)
#endif /* NO_FPE_SIGNALS */
/*
- * Implement gc_bif_interface_0 as nofail_primop_interface_0.
- */
-define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)')
-
-/*
- * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2,3).
+ * Implement gc_bif_interface_N as standard_bif_interface_N.
*/
+define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)')
define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)')
define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)')
define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)')
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 00fe03d8f9..a703e24b8c 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -30,9 +30,9 @@
struct nstack_walk_state {
#ifdef SKIP_YOUNGEST_FRAME
- const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
+ const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
#else
- struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */
+ struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */
#endif
};
@@ -57,37 +57,36 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p)
#endif
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
{
#ifdef SKIP_YOUNGEST_FRAME
- const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]);
state->sdesc0 = sdesc;
return sdesc;
#else
- unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS;
- if ((int)nstkarity < 0)
- nstkarity = 0;
- state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity;
+ state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */
+ state->sdesc0[0].fsize = 0;
+ state->sdesc0[0].has_exnra = 0;
+ state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+ p->hipe.narity - NR_ARG_REGS);
state->sdesc0[0].livebits[0] = 0;
-# ifdef DEBUG
- state->sdesc0[0].dbg_M = 0;
- state->sdesc0[0].dbg_F = am_undefined;
- state->sdesc0[0].dbg_A = 0;
-# endif
+ state->sdesc0[0].m_aix = 0;
+ state->sdesc0[0].f_aix = atom_val(am_undefined);
+ state->sdesc0[0].a = 0;
/* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */
__asm__ __volatile__("" : : "m"(*state) : "memory");
return &state->sdesc0[0];
#endif
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc_ignore_trap(const Process *p,
struct nstack_walk_state *state)
{
#ifdef SKIP_YOUNGEST_FRAME
unsigned long ra = p->hipe.nsp[0];
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
if (ra == (unsigned long)nbif_stack_trap_ra)
ra = (unsigned long)p->hipe.ngra;
sdesc = hipe_find_sdesc(ra);
@@ -98,7 +97,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p,
#endif
}
-static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
+static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0)
{
#ifdef SKIP_YOUNGEST_FRAME
Eterm *nsp = p->hipe.nsp;
@@ -137,7 +136,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp
return nsp >= nsp_end;
}
-static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc)
+static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc)
{
return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc);
}
@@ -148,7 +147,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i)
}
static inline unsigned long
-nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc)
+nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc)
{
return nsp[sdesc_fsize(sdesc)];
}
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 818d7444e2..de2b061706 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -58,16 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
static __inline__ void hipe_arch_glue_init(void)
{
- static struct sdesc_with_exnra nbif_return_sdesc = {
- .exnra = (unsigned long)nbif_fail,
- .sdesc = {
- .bucket = { .hvalue = (unsigned long)nbif_return },
- .summary = (1<<8),
- #ifdef DEBUG
- .dbg_F = am_return,
- #endif
- },
- };
+ static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+ nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+ nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+ nbif_return_sdesc.sdesc.fsize = 0;
+ nbif_return_sdesc.sdesc.has_exnra = 1;
+ nbif_return_sdesc.sdesc.stk_nargs = 0;
+ nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+ nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+ nbif_return_sdesc.sdesc.a = 0;
+
hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
}
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index 50d08b96d3..1a34ce786c 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -51,7 +51,7 @@
#endif
#include "hipe_signal.h"
-#if __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3)
+#if defined(__GLIBC__) && __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3)
/*
* __libc_sigaction() is the core routine.
* Without libpthread, sigaction() and __sigaction() are both aliases
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index f1559b1451..31582b3a2e 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -52,15 +52,14 @@ void hipe_print_nstack(Process *p)
{
Eterm *nsp;
Eterm *nsp_end;
- struct sdesc sdesc0;
- const struct sdesc *sdesc1;
- const struct sdesc *sdesc;
+ struct hipe_sdesc sdesc0;
+ const struct hipe_sdesc *sdesc1;
+ const struct hipe_sdesc *sdesc;
unsigned long ra;
unsigned long exnra;
unsigned int mask;
unsigned int sdesc_size;
unsigned int i;
- unsigned int nstkarity;
static const char dashes[2*sizeof(long)+5] = {
[0 ... 2*sizeof(long)+3] = '-'
};
@@ -68,10 +67,10 @@ void hipe_print_nstack(Process *p)
nsp = p->hipe.nsp;
nsp_end = p->hipe.nstend;
- nstkarity = p->hipe.narity - NR_ARG_REGS;
- if ((int)nstkarity < 0)
- nstkarity = 0;
- sdesc0.summary = nstkarity;
+ sdesc0.fsize = 0;
+ sdesc0.has_exnra = 0;
+ sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+ p->hipe.narity - NR_ARG_REGS);
sdesc0.livebits[0] = ~1;
sdesc = &sdesc0;
@@ -158,7 +157,7 @@ void hipe_print_nstack(Process *p)
#define MINSTACK 128
#define NSKIPFRAMES 4
-void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
+void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc)
{
Eterm *nsp;
Eterm *nsp_end;
@@ -199,7 +198,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
void (*hipe_handle_stack_trap(Process *p))(void)
{
void (*ngra)(void) = p->hipe.ngra;
- const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
hipe_update_stack_trap(p, sdesc);
return ngra;
}
@@ -220,7 +219,7 @@ void hipe_find_handler(Process *p)
unsigned long ra;
unsigned long exnra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int nstkarity;
nsp = p->hipe.nsp;
@@ -262,7 +261,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
Eterm *nsp_end;
unsigned long ra, prev_ra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int nstkarity;
int i;