aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/hipe
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/hipe')
-rw-r--r--erts/emulator/hipe/hipe_amd64_bifs.m45
-rw-r--r--erts/emulator/hipe/hipe_arm_bifs.m473
-rw-r--r--erts/emulator/hipe/hipe_arm_glue.S41
-rw-r--r--erts/emulator/hipe/hipe_bif0.c172
-rw-r--r--erts/emulator/hipe/hipe_bif0.tab3
-rw-r--r--erts/emulator/hipe/hipe_bif2.c11
-rw-r--r--erts/emulator/hipe/hipe_bif2.tab1
-rw-r--r--erts/emulator/hipe/hipe_bif_list.m424
-rw-r--r--erts/emulator/hipe/hipe_debug.c1
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.c44
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.h3
-rw-r--r--erts/emulator/hipe/hipe_native_bif.c17
-rw-r--r--erts/emulator/hipe/hipe_risc_glue.h8
-rw-r--r--erts/emulator/hipe/hipe_x86_glue.h8
-rw-r--r--erts/emulator/hipe/hipe_x86_signal.c6
15 files changed, 326 insertions, 91 deletions
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index 0de69a617f..a3219c7586 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -39,7 +39,10 @@ define(HANDLE_GOT_MBUF,`
jmp 2b')
`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
-# define CALL_BIF(F) movq $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper)
+# define CALL_BIF(F) \
+ movq CSYM(F)@GOTPCREL(%rip), %r11; \
+ movq %r11, P_BIF_CALLEE(P); \
+ call CSYM(hipe_debug_bif_wrapper)
#else
# define CALL_BIF(F) call CSYM(F)
#endif'
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index bd8bc5ab6b..3ca9a1bcdb 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -25,6 +25,7 @@ include(`hipe/hipe_arm_asm.m4')
.text
.p2align 2
+ .arm
`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
# define CALL_BIF(F) ldr r14, =F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper
@@ -391,7 +392,14 @@ $1:
mov r0, P
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(0)
+#else
QUICK_CALL_RET($2,0)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -407,7 +415,14 @@ $1:
NBIF_ARG(r1,1,0)
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(1)
+#else
QUICK_CALL_RET($2,1)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -424,7 +439,14 @@ $1:
NBIF_ARG(r2,2,1)
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(2)
+#else
QUICK_CALL_RET($2,2)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -442,7 +464,14 @@ $1:
NBIF_ARG(r3,3,2)
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(3)
+#else
QUICK_CALL_RET($2,3)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -466,7 +495,14 @@ $1:
NBIF_ARG(r3,5,2)
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(5)
+#else
QUICK_CALL_RET($2,5)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -488,9 +524,16 @@ define(noproc_primop_interface_0,
#`define' HAVE_$1
.global $1
$1:
- /* XXX: this case is always trivial; how to suppress the branch? */
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(0)
+#else
+ /* XXX: this case is always trivial; how to suppress the branch? */
QUICK_CALL_RET($2,0)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -505,7 +548,14 @@ $1:
NBIF_ARG(r0,1,0)
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(1)
+#else
QUICK_CALL_RET($2,1)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -521,7 +571,14 @@ $1:
NBIF_ARG(r1,2,1)
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(2)
+#else
QUICK_CALL_RET($2,2)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -538,7 +595,14 @@ $1:
NBIF_ARG(r2,3,2)
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(3)
+#else
QUICK_CALL_RET($2,3)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
@@ -558,7 +622,14 @@ $1:
str r4, [sp, #0]
/* Perform a quick save;call;restore;ret sequence. */
+#ifdef __thumb__
+ SAVE_CONTEXT_QUICK
+ bl $2
+ RESTORE_CONTEXT_QUICK
+ NBIF_RET(5)
+#else
QUICK_CALL_RET($2,5)
+#endif
.size $1, .-$1
.type $1, %function
#endif')
diff --git a/erts/emulator/hipe/hipe_arm_glue.S b/erts/emulator/hipe/hipe_arm_glue.S
index 2e2b8604a6..5723afac12 100644
--- a/erts/emulator/hipe/hipe_arm_glue.S
+++ b/erts/emulator/hipe/hipe_arm_glue.S
@@ -25,6 +25,7 @@
.text
.p2align 2
+ .arm
/*
* Enter Erlang from C.
@@ -70,6 +71,7 @@
* Emulated code recursively calls native code.
*/
.global hipe_arm_call_to_native
+ .type hipe_arm_call_to_native, %function
hipe_arm_call_to_native:
ENTER_FROM_C
/* get argument registers */
@@ -85,6 +87,7 @@ hipe_arm_call_to_native:
* This is where native code returns to emulated code.
*/
.global nbif_return
+ .type nbif_return, %function
nbif_return:
str r0, [P, #P_ARG0] /* save retval */
mov r0, #HIPE_MODE_SWITCH_RES_RETURN
@@ -95,6 +98,7 @@ nbif_return:
* Emulated code returns to its native code caller.
*/
.global hipe_arm_return_to_native
+ .type hipe_arm_return_to_native, %function
hipe_arm_return_to_native:
ENTER_FROM_C
/* get return value */
@@ -111,6 +115,7 @@ hipe_arm_return_to_native:
* Emulated code tailcalls native code.
*/
.global hipe_arm_tailcall_to_native
+ .type hipe_arm_tailcall_to_native, %function
hipe_arm_tailcall_to_native:
ENTER_FROM_C
/* get argument registers */
@@ -125,6 +130,7 @@ hipe_arm_tailcall_to_native:
* Emulated code throws an exception to its native code caller.
*/
.global hipe_arm_throw_to_native
+ .type hipe_arm_throw_to_native, %function
hipe_arm_throw_to_native:
ENTER_FROM_C
/* invoke the handler */
@@ -142,6 +148,7 @@ hipe_arm_throw_to_native:
* XXX: Different stubs for different number of register parameters?
*/
.global nbif_callemu
+ .type nbif_callemu, %function
nbif_callemu:
str r8, [P, #P_BEAM_IP]
str r0, [P, #P_ARITY]
@@ -153,6 +160,7 @@ nbif_callemu:
* nbif_apply
*/
.global nbif_apply
+ .type nbif_apply, %function
nbif_apply:
STORE_ARG_REGS
mov r0, #HIPE_MODE_SWITCH_RES_APPLY
@@ -169,6 +177,7 @@ nbif_apply:
*/
#if NR_ARG_REGS >= 6
.global nbif_ccallemu6
+ .type nbif_ccallemu6, %function
nbif_ccallemu6:
str ARG5, [P, #P_ARG5]
#if NR_ARG_REGS > 6
@@ -181,6 +190,7 @@ nbif_ccallemu6:
#if NR_ARG_REGS >= 5
.global nbif_ccallemu5
+ .type nbif_ccallemu5, %function
nbif_ccallemu5:
str ARG4, [P, #P_ARG4]
#if NR_ARG_REGS > 5
@@ -193,6 +203,7 @@ nbif_ccallemu5:
#if NR_ARG_REGS >= 4
.global nbif_ccallemu4
+ .type nbif_ccallemu4, %function
nbif_ccallemu4:
str ARG3, [P, #P_ARG3]
#if NR_ARG_REGS > 4
@@ -205,6 +216,7 @@ nbif_ccallemu4:
#if NR_ARG_REGS >= 3
.global nbif_ccallemu3
+ .type nbif_ccallemu3, %function
nbif_ccallemu3:
str ARG2, [P, #P_ARG2]
#if NR_ARG_REGS > 3
@@ -217,6 +229,7 @@ nbif_ccallemu3:
#if NR_ARG_REGS >= 2
.global nbif_ccallemu2
+ .type nbif_ccallemu2, %function
nbif_ccallemu2:
str ARG1, [P, #P_ARG1]
#if NR_ARG_REGS > 2
@@ -229,6 +242,7 @@ nbif_ccallemu2:
#if NR_ARG_REGS >= 1
.global nbif_ccallemu1
+ .type nbif_ccallemu1, %function
nbif_ccallemu1:
str ARG0, [P, #P_ARG0]
#if NR_ARG_REGS > 1
@@ -240,6 +254,7 @@ nbif_ccallemu1:
#endif
.global nbif_ccallemu0
+ .type nbif_ccallemu0, %function
nbif_ccallemu0:
/* We use r1 not ARG0 here because ARG0 is not
defined when NR_ARG_REGS == 0. */
@@ -254,6 +269,7 @@ nbif_ccallemu0:
* This is where native code suspends.
*/
.global nbif_suspend_0
+ .type nbif_suspend_0, %function
nbif_suspend_0:
mov r0, #HIPE_MODE_SWITCH_RES_SUSPEND
b .suspend_exit
@@ -262,6 +278,7 @@ nbif_suspend_0:
* Suspend from a receive (waiting for a message)
*/
.global nbif_suspend_msg
+ .type nbif_suspend_msg, %function
nbif_suspend_msg:
mov r0, #HIPE_MODE_SWITCH_RES_WAIT
b .suspend_exit
@@ -272,6 +289,7 @@ nbif_suspend_msg:
* else { return 0; }
*/
.global nbif_suspend_msg_timeout
+ .type nbif_suspend_msg_timeout, %function
nbif_suspend_msg_timeout:
ldr r1, [P, #P_FLAGS]
mov r0, #HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT
@@ -286,23 +304,31 @@ nbif_suspend_msg_timeout:
* This is the default exception handler for native code.
*/
.global nbif_fail
+ .type nbif_fail, %function
nbif_fail:
mov r0, #HIPE_MODE_SWITCH_RES_THROW
b .flush_exit /* no need to save RA */
.global nbif_0_gc_after_bif
- .global nbif_1_gc_after_bif
- .global nbif_2_gc_after_bif
- .global nbif_3_gc_after_bif
+ .type nbif_0_gc_after_bif, %function
nbif_0_gc_after_bif:
mov r1, #0
b .gc_after_bif
+
+ .global nbif_1_gc_after_bif
+ .type nbif_1_gc_after_bif, %function
nbif_1_gc_after_bif:
mov r1, #1
b .gc_after_bif
+
+ .global nbif_2_gc_after_bif
+ .type nbif_2_gc_after_bif, %function
nbif_2_gc_after_bif:
mov r1, #2
b .gc_after_bif
+
+ .global nbif_3_gc_after_bif
+ .type nbif_3_gc_after_bif, %function
nbif_3_gc_after_bif:
mov r1, #3
/*FALLTHROUGH*/
@@ -330,18 +356,25 @@ nbif_3_gc_after_bif:
* TEMP_LR contains a copy of LR
*/
.global nbif_0_simple_exception
+ .type nbif_0_simple_exception, %function
nbif_0_simple_exception:
mov r1, #0
b .nbif_simple_exception
+
.global nbif_1_simple_exception
+ .type nbif_1_simple_exception, %function
nbif_1_simple_exception:
mov r1, #1
b .nbif_simple_exception
+
.global nbif_2_simple_exception
+ .type nbif_2_simple_exception, %function
nbif_2_simple_exception:
mov r1, #2
b .nbif_simple_exception
+
.global nbif_3_simple_exception
+ .type nbif_3_simple_exception, %function
nbif_3_simple_exception:
mov r1, #3
/*FALLTHROUGH*/
@@ -385,6 +418,7 @@ nbif_3_simple_exception:
* the gray/white stack boundary
*/
.global nbif_stack_trap_ra
+ .type nbif_stack_trap_ra, %function
nbif_stack_trap_ra: /* a return address, not a function */
# This only handles a single return value.
# If we have more, we need to save them in the PCB.
@@ -401,6 +435,7 @@ nbif_stack_trap_ra: /* a return address, not a function */
* Caller saved its LR in TEMP_LR (== TEMP1) before calling us.
*/
.global hipe_arm_inc_stack
+ .type hipe_arm_inc_stack, %function
hipe_arm_inc_stack:
STORE_ARG_REGS
mov TEMP_ARG0, lr
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index fa99c817f0..8af174170d 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1022,7 +1022,7 @@ BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1)
*/
void hipe_emulate_fpe(Process* p)
{
- if (!finite(p->hipe.float_result)) {
+ if (!isfinite(p->hipe.float_result)) {
p->fp_exception = 1;
}
}
@@ -1101,9 +1101,9 @@ BIF_RETTYPE hipe_bifs_make_fun_3(BIF_ALIST_3)
#endif
/*
- * args: Nativecodeaddress, Module, {Uniq, Index, BeamAddress}
+ * args: Module, {Uniq, Index, BeamAddress}
*/
-BIF_RETTYPE hipe_bifs_make_fe_3(BIF_ALIST_3)
+BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2)
{
Eterm mod;
Uint index;
@@ -1111,20 +1111,15 @@ BIF_RETTYPE hipe_bifs_make_fe_3(BIF_ALIST_3)
void *beam_address;
ErlFunEntry *fe;
Eterm *tp;
- void *native_address;
- native_address = term_to_address(BIF_ARG_1);
- if (!native_address)
+ if (is_not_atom(BIF_ARG_1))
BIF_ERROR(BIF_P, BADARG);
+ mod = BIF_ARG_1;
- if (is_not_atom(BIF_ARG_2))
+ if (is_not_tuple(BIF_ARG_2) ||
+ (arityval(*tuple_val(BIF_ARG_2)) != 3))
BIF_ERROR(BIF_P, BADARG);
- mod = BIF_ARG_2;
-
- if (is_not_tuple(BIF_ARG_3) ||
- (arityval(*tuple_val(BIF_ARG_3)) != 3))
- BIF_ERROR(BIF_P, BADARG);
- tp = tuple_val(BIF_ARG_3);
+ tp = tuple_val(BIF_ARG_2);
if (term_to_Uint(tp[1], &uniq) == 0)
BIF_ERROR(BIF_P, BADARG);
if (term_to_Uint(tp[2], &index) == 0)
@@ -1144,10 +1139,28 @@ BIF_RETTYPE hipe_bifs_make_fe_3(BIF_ALIST_3)
printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index);
BIF_ERROR(BIF_P, BADARG);
}
+ BIF_RET(address_to_term((void *)fe, BIF_P));
+}
+
+/*
+ * args: FE, Nativecodeaddress
+ */
+BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
+{
+ ErlFunEntry *fe;
+ void *native_address;
+
+ fe = (ErlFunEntry *)term_to_address(BIF_ARG_1);
+ if (!fe)
+ BIF_ERROR(BIF_P, BADARG);
+ native_address = term_to_address(BIF_ARG_2);
+ if (!native_address)
+ BIF_ERROR(BIF_P, BADARG);
+
fe->native_address = native_address;
if (erts_refc_dectest(&fe->refc, 0) == 0)
erts_erase_fun_entry(fe);
- BIF_RET(address_to_term((void *)fe, BIF_P));
+ BIF_RET(am_true);
}
#if 0 /* XXX: unused */
@@ -1204,22 +1217,32 @@ static struct {
* they create a new stub for the mfa, which forces locking.
* XXX: Redesign apply et al to avoid those updates.
*/
- erts_smp_mtx_t lock;
+ erts_smp_rwmtx_t lock;
} hipe_mfa_info_table;
static inline void hipe_mfa_info_table_init_lock(void)
{
- erts_smp_mtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
+ erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
+}
+
+static inline void hipe_mfa_info_table_rlock(void)
+{
+ erts_smp_rwmtx_rlock(&hipe_mfa_info_table.lock);
+}
+
+static inline void hipe_mfa_info_table_runlock(void)
+{
+ erts_smp_rwmtx_runlock(&hipe_mfa_info_table.lock);
}
-static inline void hipe_mfa_info_table_lock(void)
+static inline void hipe_mfa_info_table_rwlock(void)
{
- erts_smp_mtx_lock(&hipe_mfa_info_table.lock);
+ erts_smp_rwmtx_rwlock(&hipe_mfa_info_table.lock);
}
-static inline void hipe_mfa_info_table_unlock(void)
+static inline void hipe_mfa_info_table_rwunlock(void)
{
- erts_smp_mtx_unlock(&hipe_mfa_info_table.lock);
+ erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock);
}
#define HIPE_MFA_HASH(M,F,A) ((M) * (F) + (A))
@@ -1320,7 +1343,7 @@ void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity)
}
#endif
-static struct hipe_mfa_info *hipe_mfa_info_table_put_locked(Eterm m, Eterm f, unsigned int arity)
+static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity)
{
unsigned long h;
unsigned int i;
@@ -1349,8 +1372,8 @@ static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address,
{
struct hipe_mfa_info *p;
- hipe_mfa_info_table_lock();
- p = hipe_mfa_info_table_put_locked(m, f, arity);
+ hipe_mfa_info_table_rwlock();
+ p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
#ifdef DEBUG_LINKER
printf("%s: ", __FUNCTION__);
print_mfa(m, f, arity);
@@ -1359,7 +1382,7 @@ static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address,
p->local_address = address;
if (is_exported)
p->remote_address = address;
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
}
#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
@@ -1368,10 +1391,10 @@ void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity)
struct hipe_mfa_info *p;
void *trampoline;
- hipe_mfa_info_table_lock();
- p = hipe_mfa_info_table_put_locked(m, f, arity);
- trampoline = p->trampoline;
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rlock();
+ p = hipe_mfa_info_table_get_locked(m, f, arity);
+ trampoline = p ? p->trampoline : NULL;
+ hipe_mfa_info_table_runlock();
return trampoline;
}
@@ -1379,10 +1402,10 @@ void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampol
{
struct hipe_mfa_info *p;
- hipe_mfa_info_table_lock();
- p = hipe_mfa_info_table_put_locked(m, f, arity);
+ hipe_mfa_info_table_rwlock();
+ p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
p->trampoline = trampoline;
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
}
#endif
@@ -1413,7 +1436,7 @@ BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
struct mfa mfa;
struct hipe_mfa_info *p;
- hipe_mfa_info_table_lock();
+ hipe_mfa_info_table_rwlock();
lst = BIF_ARG_1;
while (is_list(lst)) {
if (!term_to_mfa(CAR(list_val(lst)), &mfa))
@@ -1442,7 +1465,7 @@ BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
}
}
}
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
if (is_not_nil(lst))
BIF_ERROR(BIF_P, BADARG);
BIF_RET(NIL);
@@ -1456,8 +1479,8 @@ void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *p
orig_beam_op = pc[0];
if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) &&
orig_beam_op != BeamOpCode(op_hipe_trap_call)) {
- hipe_mfa_info_table_lock();
- p = hipe_mfa_info_table_put_locked(mod, fun, ari);
+ hipe_mfa_info_table_rwlock();
+ p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari);
#ifdef DEBUG_LINKER
printf("%s: ", __FUNCTION__);
print_mfa(mod, fun, ari);
@@ -1465,7 +1488,7 @@ void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *p
#endif
p->beam_code = pc;
p->orig_beam_op = orig_beam_op;
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
} else {
#ifdef DEBUG_LINKER
printf("%s: ", __FUNCTION__);
@@ -1492,7 +1515,7 @@ static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
return StubAddress;
}
-static void *hipe_get_na_nofail_locked(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp)
{
struct hipe_mfa_info *p;
void *address;
@@ -1510,22 +1533,53 @@ static void *hipe_get_na_nofail_locked(Eterm m, Eterm f, unsigned int a, int is_
address = p->remote_address;
if (address)
return address;
- } else
- p = hipe_mfa_info_table_put_locked(m, f, a);
+ }
+ /* Caller must take the slow path with the write lock held, but allow
+ it to avoid some work if it already holds the write lock. */
+ if (pp)
+ *pp = p;
+ return NULL;
+}
+
+static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p)
+{
+ void *address;
+
+ if (!p)
+ p = hipe_mfa_info_table_put_rwlocked(m, f, a);
address = hipe_make_stub(m, f, a, is_remote);
/* XXX: how to tell if a BEAM MFA is exported or not? */
p->remote_address = address;
return address;
}
+static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote)
+{
+ struct hipe_mfa_info *p;
+ void *address;
+
+ address = hipe_get_na_try_locked(m, f, a, is_remote, &p);
+ if (address)
+ return address;
+
+ address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p);
+ return address;
+}
+
static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote)
{
- void *p;
+ void *address;
- hipe_mfa_info_table_lock();
- p = hipe_get_na_nofail_locked(m, f, a, is_remote);
- hipe_mfa_info_table_unlock();
- return p;
+ hipe_mfa_info_table_rlock();
+ address = hipe_get_na_try_locked(m, f, a, is_remote, NULL);
+ hipe_mfa_info_table_runlock();
+ if (address)
+ return address;
+
+ hipe_mfa_info_table_rwlock();
+ address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL);
+ hipe_mfa_info_table_rwunlock();
+ return address;
}
/* used for apply/3 in hipe_mode_switch */
@@ -1604,7 +1658,7 @@ int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
/* Note about locking: the table is only updated from the
loader, which runs with the rest of the system suspended. */
/* XXX: alas not true; see comment at hipe_mfa_info_table.lock */
- hipe_mfa_info_table_lock();
+ hipe_mfa_info_table_rlock();
bucket = hipe_mfa_info_table.bucket;
nrbuckets = 1 << hipe_mfa_info_table.log2size;
mfa = NULL;
@@ -1625,7 +1679,7 @@ int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
*f = mfa->f;
*a = mfa->a;
}
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_runlock();
return mfa ? 1 : 0;
}
@@ -1702,9 +1756,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
default:
goto badarg;
}
- hipe_mfa_info_table_lock();
- callee_mfa = hipe_mfa_info_table_put_locked(callee.mod, callee.fun, callee.ari);
- caller_mfa = hipe_mfa_info_table_put_locked(caller.mod, caller.fun, caller.ari);
+ hipe_mfa_info_table_rwlock();
+ callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
+ caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari);
refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to));
refers_to->mfa = callee_mfa;
@@ -1718,7 +1772,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
ref->flags = flags;
ref->next = callee_mfa->referred_from;
callee_mfa->referred_from = ref;
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
BIF_RET(NIL);
@@ -1738,12 +1792,12 @@ BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */
if (!term_to_mfa(BIF_ARG_1, &mfa))
BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_lock();
+ hipe_mfa_info_table_rwlock();
p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
if (p)
for (ref = p->referred_from; ref != NULL; ref = ref->next)
ref->flags |= REF_FLAG_PENDING_REDIRECT;
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
BIF_RET(NIL);
}
@@ -1757,7 +1811,7 @@ static void hipe_purge_all_refs(void)
struct hipe_mfa_info **bucket;
unsigned int i, nrbuckets;
- hipe_mfa_info_table_lock();
+ hipe_mfa_info_table_rwlock();
bucket = hipe_mfa_info_table.bucket;
nrbuckets = 1 << hipe_mfa_info_table.log2size;
@@ -1779,7 +1833,7 @@ static void hipe_purge_all_refs(void)
erts_free(ERTS_ALC_T_HIPE, mfa);
}
}
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
}
BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
@@ -1796,7 +1850,7 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
if (!term_to_mfa(BIF_ARG_1, &mfa))
BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_lock();
+ hipe_mfa_info_table_rwlock();
caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
if (caller_mfa) {
refers_to = caller_mfa->refers_to;
@@ -1827,7 +1881,7 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
}
caller_mfa->refers_to = NULL;
}
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
BIF_RET(am_ok);
}
@@ -1846,7 +1900,7 @@ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
if (!term_to_mfa(BIF_ARG_1, &mfa))
BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_lock();
+ hipe_mfa_info_table_rwlock();
p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
if (p) {
prev = &p->referred_from;
@@ -1854,7 +1908,7 @@ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
while (ref) {
if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
is_remote = ref->flags & REF_FLAG_IS_REMOTE;
- new_address = hipe_get_na_nofail_locked(p->m, p->f, p->a, is_remote);
+ new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote);
if (ref->flags & REF_FLAG_IS_LOAD_MFA)
res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa);
else
@@ -1877,7 +1931,7 @@ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
}
}
}
- hipe_mfa_info_table_unlock();
+ hipe_mfa_info_table_rwunlock();
BIF_RET(NIL);
}
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index ce641365e9..2514b1c3a5 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -69,7 +69,8 @@ bif hipe_bifs:atom_to_word/1
bif hipe_bifs:term_to_word/1
#bif hipe_bifs:make_fun/3
-bif hipe_bifs:make_fe/3
+bif hipe_bifs:get_fe/2
+bif hipe_bifs:set_native_address_in_fe/2
#bif hipe_bifs:make_native_stub/2
bif hipe_bifs:find_na_or_make_stub/2
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index e09988e2c5..054911e822 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -31,6 +31,7 @@
#include "erl_process.h"
#include "bif.h"
#include "big.h"
+#include "erl_map.h"
#include "hipe_debug.h"
#include "hipe_mode_switch.h"
#include "hipe_arch.h"
@@ -156,7 +157,8 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0)
BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1);
# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
- if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
+ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
+ __FILE__, __LINE__)
# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
@@ -180,3 +182,10 @@ BIF_RETTYPE hipe_bifs_debug_native_called_2(BIF_ALIST_2)
BIF_RET(am_ok);
}
+/* Stub-BIF for LLVM:
+ * Reloads BP, SP (in llvm unwind label) */
+
+BIF_RETTYPE hipe_bifs_llvm_fix_pinned_regs_0(BIF_ALIST_0)
+{
+ BIF_RET(am_ok);
+}
diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab
index 45a395bf57..1b659cfa90 100644
--- a/erts/emulator/hipe/hipe_bif2.tab
+++ b/erts/emulator/hipe/hipe_bif2.tab
@@ -30,3 +30,4 @@ bif hipe_bifs:in_native/0
bif hipe_bifs:modeswitch_debug_on/0
bif hipe_bifs:modeswitch_debug_off/0
bif hipe_bifs:debug_native_called/2
+bif hipe_bifs:llvm_fix_pinned_regs/0
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 764b8d180c..5f92b6bac4 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2004-2012. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2014. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -151,10 +151,9 @@ standard_bif_interface_0(nbif_ports_0, ports_0)
* BIFs and primops that may do a GC (change heap limit and walk the native stack).
* XXX: erase/1 and put/2 cannot fail
*/
-gc_bif_interface_2(nbif_check_process_code_2, hipe_check_process_code_2)
+gc_bif_interface_2(nbif_erts_internal_check_process_code_2, hipe_erts_internal_check_process_code_2)
gc_bif_interface_1(nbif_erase_1, erase_1)
gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0)
-gc_bif_interface_1(nbif_garbage_collect_1, hipe_garbage_collect_1)
gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc)
gc_bif_interface_2(nbif_put_2, put_2)
@@ -263,7 +262,24 @@ noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc)
* Standard BIFs.
* BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index)
*/
-define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, $4)')
+
+/* BIFs that disable GC while trapping are called via a wrapper
+ * to reserve stack space for the "trap frame".
+ */
+define(CFUN,`ifelse($1,term_to_binary_1,hipe_wrapper_term_to_binary_1,
+ifelse($1,term_to_binary_2,hipe_wrapper_term_to_binary_2,
+ifelse($1,binary_to_term_1,hipe_wrapper_binary_to_term_1,
+ifelse($1,binary_to_term_2,hipe_wrapper_binary_to_term_2,
+ifelse($1,binary_to_list_1,hipe_wrapper_binary_to_list_1,
+ifelse($1,binary_to_list_3,hipe_wrapper_binary_to_list_3,
+ifelse($1,bitstring_to_list_1,hipe_wrapper_bitstring_to_list_1,
+ifelse($1,list_to_binary_1,hipe_wrapper_list_to_binary_1,
+ifelse($1,iolist_to_binary_1,hipe_wrapper_iolist_to_binary_1,
+ifelse($1,binary_list_to_bin_1,hipe_wrapper_binary_list_to_bin_1,
+ifelse($1,list_to_bitstring_1,hipe_wrapper_list_to_bitstring_1,
+$1)))))))))))')
+
+define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, CFUN($4))')
include(TARGET/`erl_bif_list.h')
/*
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index bf25ba82af..32694a8f97 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -36,6 +36,7 @@
#include "beam_load.h"
#include "hipe_mode_switch.h"
#include "hipe_debug.h"
+#include "erl_map.h"
static const char dashes[2*sizeof(long)+5] = {
[0 ... 2*sizeof(long)+3] = '-'
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index adc8793469..1ae1d17b7f 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -2,7 +2,7 @@
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2014. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -37,7 +37,8 @@
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
- if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
+ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \
+ __FILE__, __LINE__)
# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
#else
@@ -184,23 +185,56 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure)
bfun[-4] = (Uint)nfun;
}
-static __inline__ void
-hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
+void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
{
+ if (!hipe_bifcall_from_native_is_recursive(p))
+ return;
+
/* ensure that at least 2 words are available on the BEAM stack */
if ((p->stop - 2) < p->htop) {
- DPRINTF("calling gc to increase BEAM stack size");
+ DPRINTF("calling gc to reserve BEAM stack size");
p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+ ASSERT(!((p->stop - 2) < p->htop));
}
p->stop -= 2;
+ p->stop[0] = NIL;
p->stop[1] = hipe_beam_catch_throw;
+}
+
+static __inline__ void
+hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
+{
+ if (&p->stop[1] < p->hend && p->stop[1] == hipe_beam_catch_throw) {
+ /* Trap frame already reserved */
+ ASSERT(p->stop[0] == NIL);
+ }
+ else {
+ ASSERT(!(p->flags & F_DISABLE_GC));
+ if ((p->stop - 2) < p->htop) {
+ DPRINTF("calling gc to increase BEAM stack size");
+ p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+ ASSERT(!((p->stop - 2) < p->htop));
+ }
+ p->stop -= 2;
+ p->stop[1] = hipe_beam_catch_throw;
+ }
p->stop[0] = make_cp(p->cp);
++p->catches;
p->cp = hipe_beam_pc_return;
}
+void hipe_unreserve_beam_trap_frame(Process *p)
+{
+ if (!hipe_bifcall_from_native_is_recursive(p))
+ return;
+
+ ASSERT(p->stop[0] == NIL && p->stop[1] == hipe_beam_catch_throw);
+ p->stop += 2;
+}
+
static __inline__ void hipe_pop_beam_trap_frame(Process *p)
{
+ ASSERT(p->stop[1] == hipe_beam_catch_throw);
p->cp = cp_val(p->stop[0]);
--p->catches;
p->stop += 2;
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index a3e908a3b3..06721e3c04 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -59,6 +59,9 @@ void hipe_empty_nstack(Process *p);
void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
+void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
+void hipe_unreserve_beam_trap_frame(Process*);
+
extern Uint hipe_beam_pc_return[];
extern Uint hipe_beam_pc_throw[];
extern Uint hipe_beam_pc_resume[];
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 1f76268934..7d343dd91e 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -41,8 +41,7 @@
*/
/* for -Wmissing-prototypes :-( */
-extern Eterm hipe_check_process_code_2(BIF_ALIST_2);
-extern Eterm hipe_garbage_collect_1(BIF_ALIST_1);
+extern Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2);
extern Eterm hipe_show_nstack_1(BIF_ALIST_1);
/* Used when a BIF can trigger a stack walk. */
@@ -51,22 +50,12 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity)
p->hipe.narity = arity;
}
-Eterm hipe_check_process_code_2(BIF_ALIST_2)
+Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2)
{
Eterm ret;
hipe_set_narity(BIF_P, 2);
- ret = check_process_code_2(BIF_P, BIF__ARGS);
- hipe_set_narity(BIF_P, 0);
- return ret;
-}
-
-Eterm hipe_garbage_collect_1(BIF_ALIST_1)
-{
- Eterm ret;
-
- hipe_set_narity(BIF_P, 1);
- ret = garbage_collect_1(BIF_P, BIF__ARGS);
+ ret = erts_internal_check_process_code_2(BIF_P, BIF__ARGS);
hipe_set_narity(BIF_P, 0);
return ret;
}
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index cc2671c016..dbb7086dae 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -214,6 +214,14 @@ hipe_trap_from_native_is_recursive(Process *p)
return 0;
}
+/* Native called BIF. Is it a recursive call?
+ i.e should we return back to native when BIF is done? */
+static __inline__ int
+hipe_bifcall_from_native_is_recursive(Process *p)
+{
+ return (p->hipe.nra != (void(*)(void))&nbif_return);
+}
+
/* Native makes a call which needs to unload the parameters.
This differs from hipe_call_from_native_is_recursive() in
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 63ad250d60..4b6e495b9a 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -207,6 +207,14 @@ hipe_trap_from_native_is_recursive(Process *p)
return 0;
}
+/* Native called BIF. Is it a recursive call?
+ i.e should we return back to native when BIF is done? */
+static __inline__ int
+hipe_bifcall_from_native_is_recursive(Process *p)
+{
+ return (*p->hipe.nsp != (Eterm)nbif_return);
+}
+
/* Native makes a call which needs to unload the parameters.
This differs from hipe_call_from_native_is_recursive() in
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index 8f997aafab..f5668013e2 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -2,7 +2,7 @@
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2013. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2014. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -304,7 +304,9 @@ static void hipe_sigaltstack(void *ss_sp)
*/
void hipe_thread_signal_init(void)
{
- hipe_sigaltstack(erts_alloc(ERTS_ALC_T_HIPE, SIGSTKSZ));
+ /* Stack don't really need to be cache aligned.
+ We use it to suppress false leak report from valgrind */
+ hipe_sigaltstack(erts_alloc_permanent_cache_aligned(ERTS_ALC_T_HIPE, SIGSTKSZ));
}
#endif
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-15 11:39:41 +0000