aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/hipe
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/hipe')
-rw-r--r--erts/emulator/hipe/hipe_amd64.c117
-rw-r--r--erts/emulator/hipe/hipe_bif0.c9
-rw-r--r--erts/emulator/hipe/hipe_bif0.tab1
-rw-r--r--erts/emulator/hipe/hipe_bif_list.m41
-rw-r--r--erts/emulator/hipe/hipe_native_bif.c26
-rw-r--r--erts/emulator/hipe/hipe_native_bif.h2
-rw-r--r--erts/emulator/hipe/hipe_primops.h1
7 files changed, 132 insertions, 25 deletions
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index e3cff4a4ba..f23f341e6d 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -28,6 +28,7 @@
#include "error.h"
#include "bif.h"
#include "big.h" /* term_to_Sint() */
+#include "erl_binary.h"
#include "hipe_arch.h"
#include "hipe_bif0.h"
@@ -38,6 +39,8 @@
#undef ERL_FUN_SIZE
#include "hipe_literals.h"
+static void patch_trampoline(void *trampoline, void *destAddress);
+
const Uint sse2_fnegate_mask[2] = {0x8000000000000000,0};
void hipe_patch_load_fe(Uint64 *address, Uint64 value)
@@ -52,9 +55,9 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
switch (type) {
case am_closure:
case am_constant:
+ case am_c_const:
*(Uint64*)address = value;
break;
- case am_c_const:
case am_atom:
/* check that value fits in an unsigned imm32 */
/* XXX: are we sure it's not really a signed imm32? */
@@ -71,14 +74,18 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
- Sint rel32;
+ Sint64 destOffset = (Sint64)destAddress - (Sint64)callAddress - 4;
- ASSERT(trampoline == NULL);
+ if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) {
+ destOffset = (Sint64)trampoline - (Sint64)callAddress - 4;
- rel32 = (Sint)destAddress - (Sint)callAddress - 4;
- if ((Sint)(Sint32)rel32 != rel32)
- return -1;
- *(Uint32*)callAddress = (Uint32)rel32;
+ if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L))
+ return -1;
+
+ patch_trampoline(trampoline, destAddress);
+ }
+
+ *(Uint32*)callAddress = (Uint32)destOffset;
hipe_flush_icache_word(callAddress);
return 0;
}
@@ -96,12 +103,80 @@ static void *alloc_code(unsigned int alloc_bytes)
return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}
+static int check_callees(Eterm callees)
+{
+ Eterm *tuple;
+ Uint arity;
+ Uint i;
+
+ if (is_not_tuple(callees))
+ return -1;
+ tuple = tuple_val(callees);
+ arity = arityval(tuple[0]);
+ for (i = 1; i <= arity; ++i) {
+ Eterm mfa = tuple[i];
+ if (is_atom(mfa))
+ continue;
+ if (is_not_tuple(mfa) ||
+ tuple_val(mfa)[0] != make_arityval(3) ||
+ is_not_atom(tuple_val(mfa)[1]) ||
+ is_not_atom(tuple_val(mfa)[2]) ||
+ is_not_small(tuple_val(mfa)[3]) ||
+ unsigned_val(tuple_val(mfa)[3]) > 255)
+ return -1;
+ }
+ return arity;
+}
+
+#define TRAMPOLINE_BYTES 12
+
+static void generate_trampolines(unsigned char *address,
+ int nrcallees, Eterm callees,
+ unsigned char **trampvec)
+{
+ unsigned char *trampoline = address;
+ int i;
+
+ for(i = 0; i < nrcallees; ++i) {
+ trampoline[0] = 0x48; /* movabsq $..., %rax; */
+ trampoline[1] = 0xb8;
+ *(void**)(trampoline+2) = NULL; /* callee's address */
+ trampoline[10] = 0xff; /* jmpq *%rax */
+ trampoline[11] = 0xe0;
+ trampvec[i] = trampoline;
+ trampoline += TRAMPOLINE_BYTES;
+ }
+ hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_BYTES);
+}
+
+static void patch_trampoline(void *trampoline, void *destAddress)
+{
+ unsigned char *tp = (unsigned char*) trampoline;
+
+ ASSERT(tp[0] == 0x48 && tp[1] == 0xb8);
+
+ *(void**)(tp+2) = destAddress; /* callee's address */
+ hipe_flush_icache_word(tp+2);
+}
+
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
{
- if (is_not_nil(callees))
+ int nrcallees;
+ Eterm trampvecbin;
+ unsigned char **trampvec;
+ unsigned char *address;
+
+ nrcallees = check_callees(callees);
+ if (nrcallees < 0)
return NULL;
- *trampolines = NIL;
- return alloc_code(nrbytes);
+
+ trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned char*));
+ trampvec = (unsigned char **)binary_bytes(trampvecbin);
+
+ address = alloc_code(nrbytes + nrcallees*TRAMPOLINE_BYTES);
+ generate_trampolines(address + nrbytes, nrcallees, callees, trampvec);
+ *trampolines = trampvecbin;
+ return address;
}
void hipe_free_code(void* code, unsigned int bytes)
@@ -129,10 +204,9 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
*/
unsigned int codeSize;
unsigned char *code, *codep;
- unsigned int callEmuOffset;
- codeSize = /* 23, 26, 29, or 32 bytes */
- 23 + /* 23 when all offsets are 8-bit */
+ codeSize = /* 30, 33, 36, or 39 bytes */
+ 30 + /* 30 when all offsets are 8-bit */
(P_CALLEE_EXP >= 128 ? 3 : 0) +
((P_CALLEE_EXP + 4) >= 128 ? 3 : 0) +
(P_ARITY >= 128 ? 3 : 0);
@@ -197,14 +271,15 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
codep[0] = beamArity;
codep += 1;
- /* jmp callemu; 5 bytes */
- callEmuOffset = (unsigned char*)nbif_callemu - (code + codeSize);
- codep[0] = 0xe9;
- codep[1] = callEmuOffset & 0xFF;
- codep[2] = (callEmuOffset >> 8) & 0xFF;
- codep[3] = (callEmuOffset >> 16) & 0xFF;
- codep[4] = (callEmuOffset >> 24) & 0xFF;
- codep += 5;
+ /* jmp callemu; 12 bytes */
+ codep[0] = 0x48;
+ codep[1] = 0xb8;
+ codep += 2;
+ *(Uint64*)codep = (Uint64)nbif_callemu;
+ codep += 8;
+ codep[0] = 0xff;
+ codep[1] = 0xe0;
+ codep += 2;
ASSERT(codep == code + codeSize);
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 380031bf13..a8be64e08d 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1112,7 +1112,7 @@ static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa)
struct hipe_ref {
struct hipe_ref_head head; /* list of refs to same calleee */
void *address;
-#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
void *trampoline;
#endif
unsigned int flags;
@@ -1543,7 +1543,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
ref = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(struct hipe_ref));
ref->address = address;
-#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
ref->trampoline = trampoline;
#endif
ref->flags = flags;
@@ -1771,7 +1771,8 @@ void hipe_redirect_to_module(Module* modp)
struct hipe_mfa_info *p;
struct hipe_ref_head* refh;
- ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
+ ERTS_LC_ASSERT(erts_thr_progress_is_blocking() ||
+ erts_is_multi_scheduling_blocked());
for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
if (p->new_address) {
@@ -1818,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp)
if (ref->flags & REF_FLAG_IS_LOAD_MFA)
res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
else {
-#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
void* trampoline = ref->trampoline;
#else
void* trampoline = NULL;
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 0380e8c795..6728e20123 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -109,6 +109,7 @@ atom suspend_0
atom gc_1
atom hipe_apply
atom rethrow
+atom raw_raise
atom find_na_or_make_stub
atom nonclosure_address
atom atomic_inc
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 0562d676ae..33b3cc1ee5 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -220,6 +220,7 @@ standard_bif_interface_1(nbif_bnot_1, bnot_1)
standard_bif_interface_1(nbif_set_timeout, hipe_set_timeout)
standard_bif_interface_1(nbif_conv_big_to_float, hipe_conv_big_to_float)
standard_bif_interface_2(nbif_rethrow, hipe_rethrow)
+standard_bif_interface_3(nbif_raw_raise, hipe_raw_raise)
standard_bif_interface_3(nbif_find_na_or_make_stub, hipe_find_na_or_make_stub)
standard_bif_interface_2(nbif_nonclosure_address, hipe_nonclosure_address)
nocons_nofail_primop_interface_0(nbif_fclearerror_error, hipe_fclearerror_error)
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index ac1480d2ed..498b43ac6b 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -314,6 +314,32 @@ BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2)
}
}
+/* Called via standard_bif_interface_3 */
+BIF_RETTYPE nbif_impl_hipe_raw_raise(NBIF_ALIST_3)
+{
+ Process *c_p = BIF_P;
+ Eterm class = BIF_ARG_1;
+ Eterm value = BIF_ARG_2;
+ Eterm stacktrace = BIF_ARG_3;
+ Eterm reason;
+
+ if (class == am_error) {
+ c_p->fvalue = value;
+ reason = EXC_ERROR;
+ } else if (class == am_exit) {
+ c_p->fvalue = value;
+ reason = EXC_EXIT;
+ } else if (class == am_throw) {
+ c_p->fvalue = value;
+ reason = EXC_THROWN;
+ } else {
+ return am_badarg;
+ }
+ reason &= ~EXF_SAVETRACE;
+ c_p->ftrace = stacktrace;
+ BIF_ERROR(c_p, reason);
+}
+
/*
* Support for compiled binary syntax operations.
*/
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index 5711594b1e..ba42b126be 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -36,6 +36,7 @@ AEXTERN(int,nbif_suspend_msg,(void));
AEXTERN(int,nbif_suspend_msg_timeout,(void));
AEXTERN(Eterm,nbif_rethrow,(Process*, Eterm, Eterm));
+AEXTERN(Eterm,nbif_raw_raise,(Process*, Eterm, Eterm, Eterm));
AEXTERN(Eterm,nbif_set_timeout,(Process*, Eterm));
AEXTERN(Eterm,nbif_gc_1,(void));
@@ -82,6 +83,7 @@ void hipe_gc(Process*, Eterm);
BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1);
void hipe_handle_exception(Process*);
BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2);
+BIF_RETTYPE nbif_impl_hipe_raw_raise(NBIF_ALIST_3);
char *hipe_bs_allocate(int);
Binary *hipe_bs_reallocate(Binary*, int);
int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned);
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index d6fd10bdff..c5f10672f3 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -46,6 +46,7 @@ PRIMOP_LIST(am_clear_timeout, &nbif_clear_timeout)
PRIMOP_LIST(am_select_msg, &nbif_select_msg)
PRIMOP_LIST(am_set_timeout, &nbif_set_timeout)
PRIMOP_LIST(am_rethrow, &nbif_rethrow)
+PRIMOP_LIST(am_raw_raise, &nbif_raw_raise)
PRIMOP_LIST(am_bs_get_integer_2, &nbif_bs_get_integer_2)
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-24 16:53:11 +0000