From d8e69ab77f32ce1503125fa464d6db3ba1c904d0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 14:38:09 +0200 Subject: erts: Refactor rename structs hipe_mfa and hipe_ref --- erts/emulator/hipe/hipe_bif0.c | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 3336fded7a..b6d0abb2d1 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -537,13 +537,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1) BIF_RET(val); } -struct mfa_t { +struct hipe_mfa { Eterm mod; Eterm fun; Uint ari; }; -static int term_to_mfa(Eterm term, struct mfa_t *mfa) +static int term_to_mfa(Eterm term, struct hipe_mfa *mfa) { Eterm mod, fun, a; Uint ari; @@ -601,7 +601,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) Uint *hipe_bifs_find_pc_from_mfa(Eterm term) { - struct mfa_t mfa; + struct hipe_mfa mfa; if (!term_to_mfa(term, &mfa)) return NULL; @@ -621,7 +621,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) Eterm *pc; void *address; int is_closure; - struct mfa_t mfa; + struct hipe_mfa mfa; switch (BIF_ARG_3) { case am_false: @@ -1017,7 +1017,7 @@ struct hipe_mfa_info { Eterm *beam_code; Uint orig_beam_op; struct hipe_mfa_info_list *refers_to; - struct ref *referred_from; + struct hipe_ref *referred_from; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) void *trampoline; #endif @@ -1235,7 +1235,7 @@ void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampol BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; int is_exported; @@ -1257,7 +1257,7 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) { Eterm lst; - struct mfa_t mfa; + struct hipe_mfa mfa; struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); @@ -1426,7 +1426,7 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; int is_remote; @@ -1512,12 +1512,12 @@ struct hipe_mfa_info_list { struct hipe_mfa_info_list *next; }; -struct ref { +struct hipe_ref { struct hipe_mfa_info *caller_mfa; void *address; void *trampoline; unsigned int flags; - struct ref *next; + struct hipe_ref *next; }; #define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ #define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ @@ -1528,16 +1528,16 @@ struct ref { */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { - struct mfa_t callee; + struct hipe_mfa callee; Eterm *tuple; - struct mfa_t caller; + struct hipe_mfa caller; void *address; void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; struct hipe_mfa_info *caller_mfa; struct hipe_mfa_info_list *refers_to; - struct ref *ref; + struct hipe_ref *ref; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1607,9 +1607,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) */ BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ { - struct mfa_t mfa; + struct hipe_mfa mfa; const struct hipe_mfa_info *p; - struct ref *ref; + struct hipe_ref *ref; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); @@ -1647,7 +1647,7 @@ static void hipe_purge_all_refs(void) erts_free(ERTS_ALC_T_HIPE, to); } while (mfa->referred_from) { - struct ref* from = mfa->referred_from; + struct hipe_ref* from = mfa->referred_from; mfa->referred_from = from->next; erts_free(ERTS_ALC_T_HIPE, from); } @@ -1659,10 +1659,10 @@ static void hipe_purge_all_refs(void) BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; struct hipe_mfa_info *caller_mfa, *callee_mfa; struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct ref **prev, *ref; + struct hipe_ref **prev, *ref; if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); @@ -1686,7 +1686,7 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) prev = &ref->next; ref = ref->next; } else { - struct ref *tmp = ref; + struct hipe_ref *tmp = ref; ref = ref->next; *prev = ref; erts_free(ERTS_ALC_T_HIPE, tmp); @@ -1713,9 +1713,9 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) */ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; struct hipe_mfa_info *p; - struct ref **prev, *ref; + struct hipe_ref **prev, *ref; int is_remote, res; void *new_address; @@ -1738,7 +1738,7 @@ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); ref->flags &= ~REF_FLAG_PENDING_REDIRECT; if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct ref *tmp = ref; + struct hipe_ref *tmp = ref; ref = ref->next; *prev = ref; erts_free(ERTS_ALC_T_HIPE, tmp); -- cgit v1.2.3 From 3bffb5797c9498e2294b65ba24abec5842655a11 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 15:04:32 +0200 Subject: erts: Refactor rename struct hipe_sdesc --- erts/emulator/hipe/hipe_bif0.c | 2 +- erts/emulator/hipe/hipe_gc.c | 8 ++++---- erts/emulator/hipe/hipe_risc_gc.h | 16 ++++++++-------- erts/emulator/hipe/hipe_risc_glue.h | 2 +- erts/emulator/hipe/hipe_risc_stack.c | 12 ++++++------ erts/emulator/hipe/hipe_stack.c | 28 ++++++++++++++-------------- erts/emulator/hipe/hipe_stack.h | 32 ++++++++++++++++---------------- erts/emulator/hipe/hipe_x86_gc.h | 18 +++++++++--------- erts/emulator/hipe/hipe_x86_glue.h | 2 +- erts/emulator/hipe/hipe_x86_stack.c | 14 +++++++------- 10 files changed, 67 insertions(+), 67 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index b6d0abb2d1..95c2d73971 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -665,7 +665,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) { - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 68c65dea27..be665ccdb9 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h index 09568c140e..f019434f67 100644 --- a/erts/emulator/hipe/hipe_risc_gc.h +++ b/erts/emulator/hipe/hipe_risc_gc.h @@ -27,7 +27,7 @@ /* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */ struct nstack_walk_state { - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ }; static inline int nstack_walk_init_check(const Process *p) @@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) return p->hipe.nsp + nstkarity; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); state->sdesc0 = sdesc; return sdesc; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { unsigned long ra = (unsigned long)p->hipe.nra; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)&nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, return sdesc; } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { Eterm *nsp = p->hipe.nsp; p->hipe.nsp = nstack_walk_nsp_begin(p); @@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 0284265307..3d84731cbe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,7 +66,7 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { + static struct hipe_sdesc_with_exnra nbif_return_sdesc = { .exnra = (unsigned long)&nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)&nbif_return }, diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index dc98c96b8f..7528a3ab00 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -237,7 +237,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; @@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; int i; if (depth < 1) diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index e2e6eb74b1..a26132431b 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -43,10 +43,10 @@ */ struct hipe_sdesc_table hipe_sdesc_table; -static struct sdesc **alloc_bucket(unsigned int size) +static struct hipe_sdesc **alloc_bucket(unsigned int size) { - unsigned long nbytes = size * sizeof(struct sdesc*); - struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); + unsigned long nbytes = size * sizeof(struct hipe_sdesc*); + struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); sys_memzero(bucket, nbytes); return bucket; } @@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size) static void hipe_grow_sdesc_table(void) { unsigned int old_size, new_size, new_mask; - struct sdesc **old_bucket, **new_bucket; + struct hipe_sdesc **old_bucket, **new_bucket; unsigned int i; old_size = 1 << hipe_sdesc_table.log2size; @@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void) new_bucket = alloc_bucket(new_size); hipe_sdesc_table.bucket = new_bucket; for (i = 0; i < old_size; ++i) { - struct sdesc *b = old_bucket[i]; + struct hipe_sdesc *b = old_bucket[i]; while (b != NULL) { - struct sdesc *next = b->bucket.next; + struct hipe_sdesc *next = b->bucket.next; unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask; b->bucket.next = new_bucket[j]; new_bucket[j] = b; @@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void) erts_free(ERTS_ALC_T_HIPE, old_bucket); } -struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) +struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) { unsigned long ra; unsigned int i; - struct sdesc *chain; + struct hipe_sdesc *chain; unsigned int size; ra = sdesc->bucket.hvalue; @@ -102,7 +102,7 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) return sdesc; } -void hipe_init_sdesc_table(struct sdesc *sdesc) +void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,14 +121,14 @@ void hipe_init_sdesc_table(struct sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; Uint fsize, arity, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; if (is_not_tuple(arg) || (tuple_val(arg))[0] != make_arityval(6) || @@ -159,14 +159,14 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra - ? offsetof(struct sdesc_with_exnra, sdesc.livebits) - : offsetof(struct sdesc, livebits)) + ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits) + : offsetof(struct hipe_sdesc, livebits)) + livebitswords * sizeof(int); p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes); /* If we have an exception handler use the special sdesc_with_exnra structure. */ if (exnra) { - struct sdesc_with_exnra *sdesc_we = p; + struct hipe_sdesc_with_exnra *sdesc_we = p; sdesc_we->exnra = exnra; sdesc = &(sdesc_we->sdesc); } else diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index afa0ed4256..e9d05bf99c 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -30,10 +30,10 @@ #include /* offsetof() */ -struct sdesc { +struct hipe_sdesc { struct { unsigned long hvalue; /* return address */ - struct sdesc *next; /* hash collision chain */ + struct hipe_sdesc *next; /* hash collision chain */ } bucket; unsigned int summary; /* frame size, exn handler presence flag, arity */ #ifdef DEBUG @@ -43,27 +43,27 @@ struct sdesc { unsigned int livebits[1]; /* size depends on arch & data in summary field */ }; -struct sdesc_with_exnra { +struct hipe_sdesc_with_exnra { unsigned long exnra; - struct sdesc sdesc; + struct hipe_sdesc sdesc; }; -static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { return sdesc->summary >> 9; } -static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { return sdesc->summary & 0xFF; } -static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc) +static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { if ((sdesc->summary & (1<<8))) { const char *tmp; - tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc); - return ((const struct sdesc_with_exnra*)tmp)->exnra; + tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); + return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; } return 0; } @@ -72,13 +72,13 @@ struct hipe_sdesc_table { unsigned int log2size; unsigned int mask; /* INV: mask == (1 << log2size)-1 */ unsigned int used; - struct sdesc **bucket; + struct hipe_sdesc **bucket; }; extern struct hipe_sdesc_table hipe_sdesc_table; -extern struct sdesc *hipe_put_sdesc(struct sdesc*); -extern void hipe_init_sdesc_table(struct sdesc*); -extern struct sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_init_sdesc_table(struct hipe_sdesc*); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) @@ -86,10 +86,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm); #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) -static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra) +static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra) { unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; - const struct sdesc *sdesc = hipe_sdesc_table.bucket[i]; + const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i]; if (likely(sdesc->bucket.hvalue == ra)) return sdesc; do { @@ -103,7 +103,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void)); extern void hipe_print_nstack(Process*); extern void hipe_find_handler(Process*); extern void (*hipe_handle_stack_trap(Process*))(void); -extern void hipe_update_stack_trap(Process*, const struct sdesc*); +extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*); extern int hipe_fill_stacktrace(Process*, int, Eterm**); #if 0 && defined(HIPE_NSTACK_GROWS_UP) diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 00fe03d8f9..c025259871 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -30,9 +30,9 @@ struct nstack_walk_state { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ #else - struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ + struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ #endif }; @@ -57,11 +57,11 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); + const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); state->sdesc0 = sdesc; return sdesc; #else @@ -81,13 +81,13 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME unsigned long ra = p->hipe.nsp[0]; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -98,7 +98,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, #endif } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { #ifdef SKIP_YOUNGEST_FRAME Eterm *nsp = p->hipe.nsp; @@ -137,7 +137,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -148,7 +148,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 818d7444e2..d0c8eec2bc 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,7 +58,7 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { + static struct hipe_sdesc_with_exnra nbif_return_sdesc = { .exnra = (unsigned long)nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)nbif_return }, diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index f1559b1451..6975cc4669 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -52,9 +52,9 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - struct sdesc sdesc0; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + struct hipe_sdesc sdesc0; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -158,7 +158,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -199,7 +199,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -220,7 +220,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; nsp = p->hipe.nsp; @@ -262,7 +262,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; int i; -- cgit v1.2.3 From 9cb85a8cd1e02263f34c57aac68d16a2f1c03180 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 4 Oct 2016 12:26:54 +0200 Subject: erts: Refactor hipe_sdesc.summary into bit fields --- erts/emulator/hipe/hipe_risc_glue.h | 4 +++- erts/emulator/hipe/hipe_stack.c | 4 +++- erts/emulator/hipe/hipe_stack.h | 10 ++++++---- erts/emulator/hipe/hipe_x86_gc.h | 4 +++- erts/emulator/hipe/hipe_x86_glue.h | 4 +++- erts/emulator/hipe/hipe_x86_stack.c | 4 +++- 6 files changed, 21 insertions(+), 9 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 3d84731cbe..09804e3016 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -70,7 +70,9 @@ static __inline__ void hipe_arch_glue_init(void) .exnra = (unsigned long)&nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)&nbif_return }, - .summary = (1<<8), + .fsize = 0, + .has_exnra = 1, + .arity = 0 }, }; hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index a26132431b..17bef0718c 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -175,7 +175,9 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; - sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity; + sdesc->fsize = fsize; + sdesc->has_exnra = (exnra ? 1 : 0); + sdesc->arity = arity; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index e9d05bf99c..81f2e53dbc 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -35,7 +35,9 @@ struct hipe_sdesc { unsigned long hvalue; /* return address */ struct hipe_sdesc *next; /* hash collision chain */ } bucket; - unsigned int summary; /* frame size, exn handler presence flag, arity */ + unsigned int fsize : 23; /* frame size */ + unsigned int has_exnra : 1; /* exn handler presence flag */ + unsigned int arity : 8; #ifdef DEBUG Eterm dbg_M, dbg_F; unsigned dbg_A; @@ -50,17 +52,17 @@ struct hipe_sdesc_with_exnra { static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { - return sdesc->summary >> 9; + return sdesc->fsize; } static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->summary & 0xFF; + return sdesc->arity; } static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { - if ((sdesc->summary & (1<<8))) { + if (sdesc->has_exnra) { const char *tmp; tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index c025259871..7802076f70 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -68,7 +68,9 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; if ((int)nstkarity < 0) nstkarity = 0; - state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity; + state->sdesc0[0].fsize = 0; + state->sdesc0[0].has_exnra = 0; + state->sdesc0[0].arity = nstkarity; state->sdesc0[0].livebits[0] = 0; # ifdef DEBUG state->sdesc0[0].dbg_M = 0; diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index d0c8eec2bc..0e9fcd61f1 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -62,7 +62,9 @@ static __inline__ void hipe_arch_glue_init(void) .exnra = (unsigned long)nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)nbif_return }, - .summary = (1<<8), + .fsize = 0, + .has_exnra = 1, + .arity = 0, #ifdef DEBUG .dbg_F = am_return, #endif diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index 6975cc4669..9426b166e2 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -71,7 +71,9 @@ void hipe_print_nstack(Process *p) nstkarity = p->hipe.narity - NR_ARG_REGS; if ((int)nstkarity < 0) nstkarity = 0; - sdesc0.summary = nstkarity; + sdesc0.fsize = 0; + sdesc0.has_exnra = 0; + sdesc0.arity = nstkarity; sdesc0.livebits[0] = ~1; sdesc = &sdesc0; -- cgit v1.2.3 From 65737b70b9b71d53147e4155f8974b0fad01e1f6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 8 May 2016 15:54:33 +0200 Subject: erts: Refactor module_start_staging with a copy_module() function. --- erts/emulator/beam/module.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 4f36377450..d4f6e17c56 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -181,6 +181,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0; static int entries_at_start_staging = 0; +static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) +{ + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; +} + void module_start_staging(void) { IndexTable* src = &module_tables[erts_active_code_ix()]; @@ -199,10 +206,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) erts_index_lookup(dst, i); ASSERT(src_mod->module == dst_mod->module); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } /* @@ -213,10 +217,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); -- cgit v1.2.3 From 099c60de4033d7b397d4b3fb47f183b52fcba855 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 21:21:10 +0200 Subject: erts: Improve hipe load/upgrade/purge machinery A step toward better integration of hipe load and purge Highlights: * code_server no longer needs to call hipe_unified_loader:post_beam_load/1 Instead new internal function hipe_redirect_to_module() is called by loading BIFs to patch native call sites if needed. * hipe_purge_module() is called by erts_internal:purge_module/2 to purge any native code. * struct hipe_mfa_info redesigned and only used for exported functions that are called from or implemented by native code. A list of native call sites (struct hipe_ref) are kept for each hipe_mfa_info. * struct hipe_sdesc used by hipe_find_mfa_from_ra() to build native stack traces. --- erts/emulator/beam/beam_bif_load.c | 91 +++-- erts/emulator/beam/beam_emu.c | 1 + erts/emulator/beam/beam_load.c | 94 ++++- erts/emulator/beam/beam_load.h | 20 + erts/emulator/beam/export.c | 8 + erts/emulator/beam/module.c | 69 +++- erts/emulator/beam/module.h | 15 + erts/emulator/hipe/hipe_amd64.c | 15 + erts/emulator/hipe/hipe_arch.h | 10 +- erts/emulator/hipe/hipe_arm.c | 10 + erts/emulator/hipe/hipe_bif0.c | 764 +++++++++++++++++++----------------- erts/emulator/hipe/hipe_bif0.h | 5 +- erts/emulator/hipe/hipe_bif0.tab | 6 +- erts/emulator/hipe/hipe_ppc.c | 10 + erts/emulator/hipe/hipe_risc_glue.h | 20 +- erts/emulator/hipe/hipe_stack.c | 90 +++-- erts/emulator/hipe/hipe_stack.h | 18 +- erts/emulator/hipe/hipe_x86.c | 14 + erts/emulator/hipe/hipe_x86_gc.h | 14 +- erts/emulator/hipe/hipe_x86_glue.h | 23 +- erts/emulator/hipe/hipe_x86_stack.c | 7 +- erts/emulator/test/code_SUITE.erl | 22 +- 22 files changed, 813 insertions(+), 513 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 153fa205ed..237513095a 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,12 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_bif0.h" +# define IF_HIPE(X) (X) +#else +# define IF_HIPE(X) (0) +#endif #ifdef HIPE # include "hipe_stack.h" @@ -169,6 +175,11 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) if (res == BIF_ARG_1) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); +#ifdef HIPE + if (!modp) + modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + hipe_redirect_to_module(modp); +#endif } else { erts_abort_staging_code_ix(); @@ -241,7 +252,7 @@ struct m { Uint exception; }; -static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); #ifdef ERTS_SMP static void smp_code_ix_commiter(void*); @@ -377,8 +388,9 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || - erts_is_default_trace_enabled()) { - /* tracing involved, fallback with thread blocking */ + erts_is_default_trace_enabled() || + IF_HIPE(hipe_need_blocking(p[i].modp))) { + /* tracing or hipe need thread blocking */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -436,32 +448,36 @@ finish_loading_1(BIF_ALIST_1) } done: - return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1); } static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, - struct m* loaded, int nloaded) + struct m* mods, int nmods, int free_mods) { #ifdef ERTS_SMP if (is_blocking || !commit) #endif { if (commit) { + int i; erts_end_staging_code_ix(); erts_commit_staging_code_ix(); - if (loaded) { - int i; - for (i=0; i < nloaded; i++) { - set_default_trace_pattern(loaded[i].module); + + for (i=0; i < nmods; i++) { + if (mods[i].modp->curr.code_hdr) { + set_default_trace_pattern(mods[i].module); } + #ifdef HIPE + hipe_redirect_to_module(mods[i].modp); + #endif } } else { erts_abort_staging_code_ix(); } - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { erts_smp_thr_progress_unblock(); @@ -474,8 +490,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, else { ASSERT(is_value(res)); - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } erts_end_staging_code_ix(); /* @@ -653,8 +669,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else { if (modp->curr.num_breakpoints > 0 || - modp->curr.num_traced_exports > 0) { - /* we have tracing, retry single threaded */ + modp->curr.num_traced_exports > 0 || + IF_HIPE(hipe_need_blocking(modp))) { + /* tracing or hipe need to go single threaded */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -668,7 +685,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) success = 1; } } - return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); + { + struct m mod; + Eterm retval; + mod.module = BIF_ARG_1; + mod.modp = modp; + retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); + return retval; + } } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) @@ -809,6 +833,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); + #ifdef HIPE + hipe_redirect_to_module(modp); + #endif } else if (BIF_ARG_2 == am_false) { int i; @@ -1619,15 +1646,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Unload any NIF library */ - if (modp->old.nif != NULL) { + if (modp->old.nif != NULL + || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } } /* @@ -1646,7 +1676,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); - +#ifdef HIPE + hipe_purge_module(modp); +#endif ERTS_BIF_PREP_RET(ret, am_true); } @@ -1719,6 +1751,8 @@ delete_code(Module* modp) (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); + DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2], + "export trace cleared, code_ix=%d", code_ix); erts_clear_export_break(modp, ep->code+3); } else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler @@ -1727,17 +1761,16 @@ delete_code(Module* modp) ep->addressv[code_ix] = ep->code+3; ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; + DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2], + "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref); modp->old = modp->curr; - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - modp->curr.nif = NULL; - + erts_module_instance_init(&modp->curr); } @@ -1751,9 +1784,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code_hdr && modp->old.code_hdr) { - return am_not_purged; - } else if (!modp->old.code_hdr) { /* Make the current version old. */ + if (modp->curr.code_hdr) { + if (modp->old.code_hdr) { + return am_not_purged; + } + /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 0ba06058a5..ccb0f786ec 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6105,6 +6105,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA(fi[0], fi[1], fi[2], "call_error_handler"); /* * Search for the error_handler module. */ diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index d69b18e22f..9b206f9a23 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -482,7 +482,8 @@ static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); + BeamCodeHeader* code, Uint size, + void* hipe_code_start, UWord hipe_code_size); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -842,9 +843,7 @@ erts_finish_loading(Binary* magic, Process* c_p, erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(struct erl_module_instance)); inst_p = mod_tab_p->on_load; - inst_p->nif = 0; - inst_p->num_breakpoints = 0; - inst_p->num_traced_exports = 0; + erts_module_instance_init(inst_p); } inst_p->code_hdr = stp->hdr; @@ -1094,7 +1093,8 @@ loader_state_dtor(Binary* magic) static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code_hdr, Uint size) + BeamCodeHeader* code_hdr, Uint size, + void* hipe_code_start, UWord hipe_code_size) { Module* modp; Eterm retval; @@ -1117,6 +1117,17 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ +#if defined(HIPE) + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs); + modp->curr.first_hipe_ref = modp->new_hipe_refs; + modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc; + modp->curr.hipe_code_start = hipe_code_start; + modp->new_hipe_refs = NULL; + modp->new_hipe_sdesc = NULL; +# ifdef DEBUG + modp->curr.hipe_code_size = hipe_code_size; +# endif +#endif /* * Update ranges (used for finding a function from a PC value). @@ -4773,9 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } ep = erts_export_put(stp->module, stp->export[i].function, stp->export[i].arity); - if (!on_load) { - ep->addressv[erts_staging_code_ix()] = address; - } else { + if (on_load) { /* * on_load: Don't make any of the exported functions * callable yet. Keep any function in the current @@ -4783,6 +4792,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) */ ep->code[4] = (BeamInstr) address; } + else + ep->addressv[erts_staging_code_ix()] = address; } /* @@ -5989,17 +6000,12 @@ code_module_md5_1(BIF_ALIST_1) static BeamInstr* make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { + DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", &fp[5]); fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); fp[1] = native; fp[2] = mod; fp[3] = func; fp[4] = arity; -#ifdef HIPE - if (native) { - fp[5] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); - } -#endif fp[5] = OpCode; return fp + WORDS_PER_FUNCTION; } @@ -6087,6 +6093,8 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) if (stp->export[i].function == function && stp->export[i].arity == arity) { Export* ep = erts_export_put(mod, function, arity); ep->addressv[erts_staging_code_ix()] = fp+5; + DBG_TRACE_MFA(mod,function,arity,"set beam stub at %p in export at %p (code_ix=%d)", + fp+5, ep, erts_staging_code_ix()); return; } } @@ -6282,6 +6290,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) byte* temp_alloc = NULL; byte* bytes; Uint size; + UWord hipe_code_start = NULL, hipe_code_size = 0; /* * Must initialize stp->lambdas here because the error handling code @@ -6297,7 +6306,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(3)) { + if (tp[0] != make_arityval(5)) { goto error; } Funcs = tp[1]; @@ -6314,6 +6323,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } size = binary_size(Beam); +#ifdef HIPE + if (!term_to_Uint(tp[4], &hipe_code_start)) + goto error; +# ifdef DEBUG + if (!term_to_Uint(tp[5], &hipe_code_size)) + goto error; +# endif +#endif + /* * Scan the Beam binary and read the interesting sections. */ @@ -6476,7 +6494,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size); + code_hdr, code_size, + (void*)hipe_code_start, hipe_code_size); if (rval != NIL) { goto error; } @@ -6517,3 +6536,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp) } } +#ifdef ENABLE_DBG_TRACE_MFA + +#define MFA_MAX 10 +Eterm dbg_trace_m[MFA_MAX]; +Eterm dbg_trace_f[MFA_MAX]; +Uint dbg_trace_a[MFA_MAX]; +unsigned int dbg_trace_ix = 0; + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a) +{ + unsigned i = dbg_trace_ix++; + ASSERT(i < MFA_MAX); + dbg_trace_m[i] = am_atom_put(m, strlen(m)); + dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_a[i] = a; +} + +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a) +{ + unsigned int i; + for (i = 0; i < dbg_trace_ix; ++i) { + if (m == dbg_trace_m[i] && + (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) { + + return i+1; + } + } + return 0; +} + +void dbg_vtrace_mfa(unsigned ix, const char* format, ...) +{ + va_list arglist; + va_start(arglist, format); + ASSERT(--ix < MFA_MAX); + erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ", + dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]); + + erts_vfprintf(stderr, format, arglist); + va_end(arglist); +} + +#endif /* ENABLE_DBG_TRACE_MFA */ diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 6be4031822..5f0f34fcdd 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -151,4 +151,24 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) +#ifdef DEBUG +# define ENABLE_DBG_TRACE_MFA +#endif + +#ifdef ENABLE_DBG_TRACE_MFA + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a); +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a); +void dbg_vtrace_mfa(unsigned ix, const char* format, ...); +#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\ + unsigned ix;\ + if ((ix=dbg_is_traced_mfa(M,F,A))) \ + dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\ + }while(0) + +#else +# define dbg_set_traced_mfa(M,F,A) +# define DBG_TRACE_MFA(M,F,A,FMT, ...) +#endif /* ENABLE_DBG_TRACE_MFA */ + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2a19211987..9da7fae1dc 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -145,6 +145,9 @@ export_alloc(struct export_entry* tmpl_e) blob->entryv[ix].ep = &blob->exp; } ix = 0; + + DBG_TRACE_MFA(obj->code[0], obj->code[1], obj->code[2], + "export allocation at %p", obj); } else { /* Existing entry in another table, use free entry in blob */ blob = entry_to_blob(tmpl_e); @@ -163,9 +166,14 @@ export_free(struct export_entry* obj) obj->slot.index = -1; for (i=0; i < ERTS_NUM_CODE_IX; i++) { if (blob->entryv[i].slot.index >= 0) { + DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2], + "export entry slot %u freed for %p", + (obj - blob->entryv), &blob->exp); return; } } + DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2], + "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index d4f6e17c56..93ce8fcf9f 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "module.h" +#include "beam_catches.h" #ifdef DEBUG # define IF_DEBUG(x) x @@ -67,6 +68,23 @@ static int module_cmp(Module* tmpl, Module* obj) return tmpl->module != obj->module; } +void erts_module_instance_init(struct erl_module_instance* modi) +{ + modi->code_hdr = 0; + modi->code_length = 0; + modi->catches = BEAM_CATCHES_NIL; + modi->nif = NULL; + modi->num_breakpoints = 0; + modi->num_traced_exports = 0; +#ifdef HIPE + modi->first_hipe_ref = NULL; + modi->first_hipe_sdesc = NULL; + modi->hipe_code_start = NULL; +# ifdef DEBUG + modi->hipe_code_size = 0; +# endif +#endif +} static Module* module_alloc(Module* tmpl) { @@ -74,18 +92,16 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code_hdr = 0; - obj->old.code_hdr = 0; - obj->curr.code_length = 0; - obj->old.code_length = 0; obj->slot.index = -1; - obj->curr.nif = NULL; - obj->old.nif = NULL; - obj->curr.num_breakpoints = 0; - obj->old.num_breakpoints = 0; - obj->curr.num_traced_exports = 0; - obj->old.num_traced_exports = 0; + erts_module_instance_init(&obj->curr); + erts_module_instance_init(&obj->old); obj->on_load = 0; +#ifdef HIPE + obj->first_hipe_mfa = NULL; + obj->new_hipe_refs = NULL; + obj->new_hipe_sdesc = NULL; +#endif + DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -139,19 +155,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix) } } -Module* -erts_put_module(Eterm mod) + +static Module* put_module(Eterm mod, IndexTable* mod_tab) { Module e; - IndexTable* mod_tab; int oldsz, newsz; Module* res; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_has_code_write_permission()); - - mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); @@ -160,6 +171,24 @@ erts_put_module(Eterm mod) return res; } +Module* +erts_put_module(Eterm mod) +{ + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_has_code_write_permission()); + + return put_module(mod, &module_tables[erts_staging_code_ix()]); +} + +Module* +erts_put_active_module(Eterm mod) +{ + ASSERT(is_atom(mod)); + //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + return put_module(mod, &module_tables[erts_active_code_ix()]); +} + Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); @@ -186,6 +215,11 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) dst_mod->curr = src_mod->curr; dst_mod->old = src_mod->old; dst_mod->on_load = src_mod->on_load; +#ifdef HIPE + dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa; + dst_mod->new_hipe_refs = src_mod->new_hipe_refs; + dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc; +#endif } void module_start_staging(void) @@ -217,6 +251,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1c1afc8461..694583597b 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -30,6 +30,14 @@ struct erl_module_instance { struct erl_module_nif* nif; int num_breakpoints; int num_traced_exports; +#ifdef HIPE + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ + void* hipe_code_start; +# ifdef DEBUG + UWord hipe_code_size; +# endif +#endif }; typedef struct erl_module { @@ -40,10 +48,17 @@ typedef struct erl_module { struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ struct erl_module_instance* on_load; +#ifdef HIPE + struct hipe_mfa_info* first_hipe_mfa; + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; +#endif } Module; +void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); +Module* erts_put_active_module(Eterm mod); /* only while blocked */ void init_module_table(void); void module_start_staging(void); diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 62739d2a78..df53f4db30 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -130,6 +130,13 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code) +{ + ALLOC_CODE_STATS(--nr_allocs); + /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + /* Make stub for native code calling exported beam function. */ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -234,6 +241,14 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + ALLOC_CODE_STATS(++nr_allocs); + /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index 6f959815bb..df38a80069 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -31,22 +31,30 @@ extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity); +extern void hipe_free_code(void*); +extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); +extern void hipe_free_native_stub(void*); + #if defined(__sparc__) #include "hipe_sparc.h" +#include "hipe_sparc_asm.h" #endif #if defined(__i386__) #include "hipe_x86.h" +#include "hipe_x86_asm.h" #endif #if defined(__x86_64__) #include "hipe_amd64.h" +#include "hipe_amd64_asm.h" #endif #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc.h" +#include "hipe_ppc_asm.h" #endif #if defined(__arm__) #include "hipe_arm.h" +#include "hipe_arm_asm.h" #endif #if !defined(AEXTERN) diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index f8ef468341..64e35b62d8 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -198,6 +198,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return address; } +void hipe_free_code(void* code, unsigned int bytes) +{ + /*SVERK: Leaking code memory */ +} + static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) { unsigned int *address; @@ -229,6 +234,11 @@ static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) return address; } +void hipe_free_native_stub(void* stub) +{ + /*SVERK: Leaking code stub */ +} + /* * ARMv5's support for 32-bit immediates is effectively non-existent. * Hence, every 32-bit immediate is stored in memory and loaded via diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 95c2d73971..8be6dc9aa9 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -54,6 +54,7 @@ #define BeamOpCode(Op) ((Uint)BeamOp(Op)) + int term_to_Sint32(Eterm term, Sint *sp) { Sint val; @@ -644,30 +645,21 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); if (pc) { - hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc); -#if HIPE -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address); hipe_set_call_trap(pc, address, is_closure); BIF_RET(am_true); -#endif } -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no BEAM pc found\r\n"); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); BIF_RET(am_false); } BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) { struct hipe_sdesc *sdesc; + Module* modp; + int do_commit; - sdesc = hipe_decode_sdesc(BIF_ARG_1); + sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit); if (!sdesc) { fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); @@ -676,6 +668,21 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); } + + /* + * Link into list of sdesc's in same module instance + */ + modp = erts_put_active_module(make_atom(sdesc->m_aix)); + ASSERT(modp); + if (do_commit) { /* Direct "hipe-patching" of early loaded module */ + sdesc->next_in_modi = modp->curr.first_hipe_sdesc; + modp->curr.first_hipe_sdesc = sdesc; + } + else { /* Normal module loading/upgrade */ + sdesc->next_in_modi = modp->new_hipe_sdesc; + modp->new_hipe_sdesc = sdesc; + } + BIF_RET(NIL); } @@ -970,7 +977,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2) atom_buf[0] = '\0'; strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len); - printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index); + printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index); BIF_ERROR(BIF_P, BADARG); } BIF_RET(address_to_term((void *)fe, BIF_P)); @@ -997,11 +1004,15 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_RET(am_true); } +struct hipe_ref_head { + struct hipe_ref_head* next; + struct hipe_ref_head* prev; +}; + /* - * MFA info hash table: + * An exported function called from or implemented by native code * - maps MFA to native code entry point - * - the MFAs it calls (refers_to) - * - the references to it (referred_from) + * - all references to it (callers) * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { @@ -1011,16 +1022,19 @@ struct hipe_mfa_info { } bucket; Eterm m; /* atom */ Eterm f; /* atom */ - unsigned int a; + unsigned int a : sizeof(int)*8 - 1; + unsigned int is_stub : 1; /* if beam or not (yet) loaded */ void *remote_address; - void *local_address; - Eterm *beam_code; - Uint orig_beam_op; - struct hipe_mfa_info_list *refers_to; - struct hipe_ref *referred_from; + void *new_address; + struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ + struct hipe_mfa_info* next_in_mod; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) void *trampoline; #endif +#ifdef DEBUG + Export* dbg_export; +#endif + }; static struct { @@ -1038,6 +1052,25 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; + +/* + * An external native call site M:F(...) + * to be patched when the callee changes. + */ +struct hipe_ref { + struct hipe_ref_head head; /* list of refs to same calleee */ + void *address; + void *trampoline; + unsigned int flags; + struct hipe_ref* next_from_modi; /* list of refs from same module instance */ +#if defined(DEBUG) + struct hipe_mfa_info* callee; + Eterm caller_m, caller_f, caller_a; +#endif +}; +#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ + + static inline void hipe_mfa_info_table_init_lock(void) { erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); @@ -1108,15 +1141,18 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->m = m; res->f = f; res->a = arity; + res->is_stub = 0; res->remote_address = NULL; - res->local_address = NULL; - res->beam_code = NULL; - res->orig_beam_op = 0; - res->refers_to = NULL; - res->referred_from = NULL; + res->new_address = NULL; + res->callers.next = &res->callers; + res->callers.prev = &res->callers; + res->next_in_mod = NULL; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) res->trampoline = NULL; #endif +#ifdef DEBUG + res->dbg_export = NULL; +#endif return res; } @@ -1154,22 +1190,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter return NULL; } -#if 0 /* XXX: unused */ -void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity) -{ - const struct hipe_mfa_info *p; - - p = hipe_mfa_info_table_get(m, f, arity); - return p ? p->address : NULL; -} -#endif - static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity) { unsigned long h; unsigned int i; struct hipe_mfa_info *p; unsigned int size; + Module* modp; h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; @@ -1189,23 +1216,47 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, size = 1 << hipe_mfa_info_table.log2size; if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); + + modp = erts_put_active_module(m); + ASSERT(modp); + p->next_in_mod = modp->first_hipe_mfa; + modp->first_hipe_mfa = p; + + DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); + return p; } -static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported) +static void remove_mfa_info(struct hipe_mfa_info* rm) +{ + unsigned int i; + struct hipe_mfa_info *p; + struct hipe_mfa_info **prevp; + + i = rm->bucket.hvalue & hipe_mfa_info_table.mask; + prevp = &hipe_mfa_info_table.bucket[i]; + for (;;) { + p = *prevp; + ASSERT(p); + if (p == rm) { + *prevp = p->bucket.next; + ASSERT(hipe_mfa_info_table.used > 0); + hipe_mfa_info_table.used--; + return; + } + prevp = &p->bucket.next; + } +} + +static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) { struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); p = hipe_mfa_info_table_put_rwlocked(m, f, arity); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(m, f, arity); - printf(": changing address from %p to %p\r\n", p->local_address, address); -#endif - p->local_address = address; - if (is_exported) - p->remote_address = address; + DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p); + p->new_address = address; + hipe_mfa_info_table_rwunlock(); } @@ -1237,168 +1288,85 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { struct hipe_mfa mfa; void *address; - int is_exported; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - address = term_to_address(BIF_ARG_2); - if (!address) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_3 == am_true) - is_exported = 1; - else if (BIF_ARG_3 == am_false) - is_exported = 0; - else + switch (BIF_ARG_3) { + case am_true: /* is_exported */ + if (!term_to_mfa(BIF_ARG_1, &mfa)) + BIF_ERROR(BIF_P, BADARG); + address = term_to_address(BIF_ARG_2); + if (!address) + BIF_ERROR(BIF_P, BADARG); + hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address); + break; + case am_false: + break; /* ignore local functions */ + default: BIF_ERROR(BIF_P, BADARG); - hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported); - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) -{ - Eterm lst; - struct hipe_mfa mfa; - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - lst = BIF_ARG_1; - while (is_list(lst)) { - if (!term_to_mfa(CAR(list_val(lst)), &mfa)) - break; - lst = CDR(list_val(lst)); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - p->remote_address = NULL; - p->local_address = NULL; - if (p->beam_code) { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n", - p->beam_code, p->orig_beam_op); -#endif - p->beam_code[0] = p->orig_beam_op; - p->beam_code = NULL; - p->orig_beam_op = 0; - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no call trap to remove\r\n"); -#endif - } - } } - hipe_mfa_info_table_rwunlock(); - if (is_not_nil(lst)) - BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } -void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc) + +/* Ask if we need to block all threads + * while loading/deleting (beam) code for this module? + */ +int hipe_need_blocking(Module* modp) { - Uint orig_beam_op; struct hipe_mfa_info *p; - orig_beam_op = pc[0]; - if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) && - orig_beam_op != BeamOpCode(op_hipe_trap_call)) { - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc); -#endif - p->beam_code = pc; - p->orig_beam_op = orig_beam_op; - hipe_mfa_info_table_rwunlock(); - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": orig op %#lx already saved\r\n", orig_beam_op); -#endif + /* Need to block if we have at least one native caller to this module + * or native code to make unaccessible. + */ + hipe_mfa_info_table_rlock(); + for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + ASSERT(!p->new_address); + if (p->callers.next != &p->callers || !p->is_stub) { + break; + } } + hipe_mfa_info_table_runlock(); + return (p != NULL); } -static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) -{ - Export *export_entry; - void *StubAddress; - - ASSERT(is_remote); - - export_entry = erts_export_get_or_make_stub(m, f, arity); - StubAddress = hipe_make_native_stub(export_entry, arity); - if (!StubAddress) - erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - return StubAddress; -} - -static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp) +static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a) { struct hipe_mfa_info *p; - void *address; p = hipe_mfa_info_table_get_locked(m, f, a); - if (p) { - /* find address, predicting for a runtime apply call */ - address = p->remote_address; - if (!is_remote) - address = p->local_address; - if (address) - return address; - - /* bummer, install stub, checking if one already existed */ - address = p->remote_address; - if (address) - return address; - } - /* Caller must take the slow path with the write lock held, but allow - it to avoid some work if it already holds the write lock. */ - if (pp) - *pp = p; - return NULL; -} - -static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p) -{ - void *address; - - if (!p) - p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_make_stub(m, f, a, is_remote); - /* XXX: how to tell if a BEAM MFA is exported or not? */ - p->remote_address = address; - return address; + return p ? p->remote_address : NULL; } -static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a) { - struct hipe_mfa_info *p; - void *address; + struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_get_na_try_locked(m, f, a, is_remote, &p); - if (address) - return address; + if (!p->remote_address) { + Export* export_entry = erts_export_get_or_make_stub(m, f, a); + void* stubAddress = hipe_make_native_stub(export_entry, a); + if (!stubAddress) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p); - return address; + p->remote_address = stubAddress; + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = export_entry; +#endif + } + return p->remote_address; } -static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a) { void *address; hipe_mfa_info_table_rlock(); - address = hipe_get_na_try_locked(m, f, a, is_remote, NULL); + address = hipe_get_na_try_locked(m, f, a); hipe_mfa_info_table_runlock(); if (address) return address; hipe_mfa_info_table_rwlock(); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL); + address = hipe_get_na_slow_rwlocked(m, f, a); hipe_mfa_info_table_rwunlock(); return address; } @@ -1408,7 +1376,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) { if (is_not_atom(m) || is_not_atom(f) || a > 255) return NULL; - return hipe_get_na_nofail(m, f, a, 1); + return hipe_get_na_nofail(m, f, a); } /* primop, but called like a BIF for error handling purposes */ @@ -1420,25 +1388,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); arity = unsigned_val(BIF_ARG_3); /* no error check */ - address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1); + address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity); BIF_RET((Eterm)address); /* semi-Ok */ } -BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) { struct hipe_mfa mfa; void *address; - int is_remote; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_2 == am_true) - is_remote = 1; - else if (BIF_ARG_2 == am_false) - is_remote = 0; - else - BIF_ERROR(BIF_P, BADARG); - address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote); + + address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari); BIF_RET(address_to_term(address, BIF_P)); } @@ -1460,7 +1422,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) f = ep->code[1]; } else goto badfun; - address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1); + address = hipe_get_na_nofail(m, f, BIF_ARG_2); BIF_RET((Eterm)address); badfun: @@ -1471,60 +1433,19 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) { - struct hipe_mfa_info *mfa; - long mfa_offset, ra_offset; - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra); - /* Note about locking: the table is only updated from the - loader, which runs with the rest of the system suspended. */ - /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */ - hipe_mfa_info_table_rlock(); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - mfa = NULL; - mfa_offset = LONG_MAX; - for (i = 0; i < nrbuckets; ++i) { - struct hipe_mfa_info *b = bucket[i]; - while (b != NULL) { - ra_offset = (char*)ra - (char*)b->local_address; - if (ra_offset > 0 && ra_offset < mfa_offset) { - mfa_offset = ra_offset; - mfa = b; - } - b = b->bucket.next; - } - } - if (mfa) { - *m = mfa->m; - *f = mfa->f; - *a = mfa->a; - } - hipe_mfa_info_table_runlock(); - return mfa ? 1 : 0; -} + if (!sdesc || sdesc->m_aix == atom_val(am_Empty)) + return 0; -/* - * Patch Reference Handling. - */ -struct hipe_mfa_info_list { - struct hipe_mfa_info *mfa; - struct hipe_mfa_info_list *next; -}; + *m = make_atom(sdesc->m_aix); + *f = make_atom(sdesc->f_aix); + *a = sdesc->a; + return 1; +} -struct hipe_ref { - struct hipe_mfa_info *caller_mfa; - void *address; - void *trampoline; - unsigned int flags; - struct hipe_ref *next; -}; -#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ -#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ -#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */ -#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */ -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { @@ -1535,9 +1456,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; - struct hipe_mfa_info *caller_mfa; - struct hipe_mfa_info_list *refers_to; struct hipe_ref *ref; + Module* modp; + int do_commit; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1569,62 +1490,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) goto badarg; } switch (tuple[5]) { - case am_local: - break; - case am_remote: - flags |= REF_FLAG_IS_REMOTE; - break; - default: - goto badarg; + case am_true: do_commit = 1; break; + case am_false: do_commit = 0; break; + default: goto badarg; } hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); - caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari); - - refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to)); - refers_to->mfa = callee_mfa; - refers_to->next = caller_mfa->refers_to; - caller_mfa->refers_to = refers_to; - ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref)); - ref->caller_mfa = caller_mfa; + ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; ref->trampoline = trampoline; ref->flags = flags; - ref->next = callee_mfa->referred_from; - callee_mfa->referred_from = ref; + + /* + * Link into list of refs to same callee + */ + ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers); + ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers); + ref->head.next = callee_mfa->callers.next; + ref->head.prev = &callee_mfa->callers; + ref->head.next->prev = &ref->head; + ref->head.prev->next = &ref->head; + + /* + * Link into list of refs from same module instance + */ + modp = erts_put_active_module(caller.mod); + ASSERT(modp); + if (do_commit) { /* Direct "hipe-patching" of early loaded module */ + ref->next_from_modi = modp->curr.first_hipe_ref; + modp->curr.first_hipe_ref = ref; + } + else { /* Normal module loading/upgrade */ + ref->next_from_modi = modp->new_hipe_refs; + modp->new_hipe_refs = ref; + } + +#if defined(DEBUG) + ref->callee = callee_mfa; + ref->caller_m = caller.mod; + ref->caller_f = caller.fun; + ref->caller_a = caller.ari; +#endif hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d", + ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d", + ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit); BIF_RET(NIL); badarg: BIF_ERROR(BIF_P, BADARG); } -/* Given a CalleeMFA, mark each ref to it as pending-redirect. - * This ensures that remove_refs_from() won't remove them: any - * removal is instead done at the end of redirect_referred_from(). - */ -BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ + +static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - struct hipe_mfa mfa; - const struct hipe_mfa_info *p; - struct hipe_ref *ref; + Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix()); + struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; + struct hipe_mfa_info* p; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) - for (ref = p->referred_from; ref != NULL; ref = ref->next) - ref->flags |= REF_FLAG_PENDING_REDIRECT; - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + ASSERT(modp); + for (;;) { + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + if (p == unlink_me) { + *prevp = p->next_in_mod; + break; + } + prevp = &p->next_in_mod; + } +} + +static void purge_mfa(struct hipe_mfa_info* p) +{ + ASSERT(p->is_stub); + remove_mfa_info(p); + hipe_free_native_stub(p->remote_address); + erts_free(ERTS_ALC_T_HIPE, p); } /* Called by init:restart after unloading all hipe compiled modules - * to work around bug causing execution of deallocated beam code. - * Can be removed when delete/purge of native modules works better. + * to work around old bug that caused execution of deallocated beam code. + * Can be removed now when delete/purge of native modules works better. * Test: Do init:restart in debug compiled vm with hipe compiled kernel. */ static void hipe_purge_all_refs(void) @@ -1634,126 +1583,205 @@ static void hipe_purge_all_refs(void) hipe_mfa_info_table_rwlock(); + ASSERT(hipe_mfa_info_table.used == 0); bucket = hipe_mfa_info_table.bucket; nrbuckets = 1 << hipe_mfa_info_table.log2size; for (i = 0; i < nrbuckets; ++i) { + ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { + Module* modp; struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - while (mfa->refers_to) { - struct hipe_mfa_info_list *to = mfa->refers_to; - mfa->refers_to = to->next; - erts_free(ERTS_ALC_T_HIPE, to); - } - while (mfa->referred_from) { - struct hipe_ref* from = mfa->referred_from; - mfa->referred_from = from->next; - erts_free(ERTS_ALC_T_HIPE, from); - } + ASSERT(mfa->callers.next == &mfa->callers); + + modp = erts_get_module(mfa->m, erts_active_code_ix()); + if (modp) { + /* unsafe write to active module */ + modp->first_hipe_mfa = NULL; + } erts_free(ERTS_ALC_T_HIPE, mfa); } } + hipe_mfa_info_table.used = 0; hipe_mfa_info_table_rwunlock(); } BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct hipe_mfa mfa; - struct hipe_mfa_info *caller_mfa, *callee_mfa; - struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct hipe_ref **prev, *ref; - if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); BIF_RET(am_ok); } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (caller_mfa) { - refers_to = caller_mfa->refers_to; - while (refers_to) { - callee_mfa = refers_to->mfa; - prev = &callee_mfa->referred_from; - ref = *prev; - while (ref) { - if (ref->caller_mfa == caller_mfa) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - ref->flags |= REF_FLAG_PENDING_REMOVE; - prev = &ref->next; - ref = ref->next; - } else { - struct hipe_ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } - } else { - prev = &ref->next; - ref = ref->next; - } - } - tmp_refers_to = refers_to; - refers_to = refers_to->next; - erts_free(ERTS_ALC_T_HIPE, tmp_refers_to); - } - caller_mfa->refers_to = NULL; + ASSERT(!"hipe_bifs_remove_refs_from_1() called"); + BIF_ERROR(BIF_P, BADARG); +} + +int hipe_purge_need_blocking(Module* modp) +{ + /* SVERK: Verify if this is really necessary */ + return (modp->old.first_hipe_ref || + modp->old.first_hipe_sdesc || + (!modp->curr.code_hdr && modp->first_hipe_mfa)); +} + +void hipe_purge_module(Module* modp) +{ + struct hipe_ref* ref; + struct hipe_sdesc* sdesc; + + ASSERT(modp); + + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); + + /* + * Remove all hipe_ref's (external calls) from the old module instance + */ + ref = modp->old.first_hipe_ref; + + while (ref) { + struct hipe_ref* free_ref = ref; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == make_atom(modp->module)); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + unlink_mfa_from_mod(p); + purge_mfa(p); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); + } + modp->old.first_hipe_ref = NULL; + + /* + * Remove all hipe_sdesc's for the old module instance + */ + sdesc = modp->old.first_hipe_sdesc; + + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(sdesc->m_aix == modp->module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } + modp->old.first_hipe_sdesc = NULL; + + /* + * Remove unreferred hipe_mfa_info's + */ + if (modp->curr.code_hdr == NULL) { + struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; + struct hipe_mfa_info* p = *prevp; + ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking()); + for (; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + } + if (modp->old.hipe_code_start) { +#ifdef DEBUG + sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size); +#endif + hipe_free_code(modp->old.hipe_code_start); + modp->old.hipe_code_start = NULL; } - hipe_mfa_info_table_rwunlock(); - BIF_RET(am_ok); } -/* redirect_referred_from(CalleeMFA) - * Redirect all pending-redirect refs in CalleeMFA's referred_from. - * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from. +/* + * Redirect all existing native calls to this module */ -BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) +void hipe_redirect_to_module(Module* modp) { - struct hipe_mfa mfa; struct hipe_mfa_info *p; - struct hipe_ref **prev, *ref; - int is_remote, res; - void *new_address; + struct hipe_ref_head* refh; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + if (p->new_address) { + if (p->is_stub) { + hipe_free_native_stub(p->remote_address); + p->is_stub = 0; + } + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address); + p->remote_address = p->new_address; + p->new_address = NULL; +#ifdef DEBUG + p->dbg_export = NULL; +#endif + } + else if (!p->is_stub) { + Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a); + p->remote_address = hipe_make_native_stub(exp, p->a); + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address); + if (!p->remote_address) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = exp; +#endif + } + else { + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub"); + ASSERT(p->remote_address && p->dbg_export); + } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - prev = &p->referred_from; - ref = *prev; - while (ref) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - is_remote = ref->flags & REF_FLAG_IS_REMOTE; - new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote); - if (ref->flags & REF_FLAG_IS_LOAD_MFA) - res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, new_address, ref->trampoline); - if (res) - fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); - ref->flags &= ~REF_FLAG_PENDING_REDIRECT; - if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct hipe_ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } else { - prev = &ref->next; - ref = ref->next; - } - } else { - prev = &ref->next; - ref = ref->next; - } + DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p); + for (refh = p->callers.next; refh != &p->callers; refh = refh->next) { + struct hipe_ref* ref = (struct hipe_ref*) refh; + int res; + + DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)", + ref, ref->caller_m, ref->caller_f, ref->caller_a, + ref->address, p->remote_address); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, + " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)", + ref, p->m,p->f,p->a, ref->address, p->remote_address); + + if (ref->flags & REF_FLAG_IS_LOAD_MFA) + res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); + else + res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline); + if (res) + fprintf(stderr, "%s: patch failed", __FUNCTION__); } + DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); } BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index c9a8216368..1914405e2d 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -42,7 +42,10 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); #endif /* needed in beam_load.c */ -void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc); +int hipe_need_blocking(Module*); +int hipe_purge_need_blocking(Module*); +void hipe_purge_module(Module*); +void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ extern void *term_to_address(Eterm); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 99237aae05..eae84383df 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -54,7 +54,7 @@ bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 bif hipe_bifs:set_funinfo_native_address/3 -bif hipe_bifs:invalidate_funinfo_native_addresses/1 +#bif hipe_bifs:invalidate_funinfo_native_addresses/1 bif hipe_bifs:update_code_size/3 bif hipe_bifs:code_size/1 @@ -72,7 +72,7 @@ bif hipe_bifs:term_to_word/1 bif hipe_bifs:get_fe/2 bif hipe_bifs:set_native_address_in_fe/2 -bif hipe_bifs:find_na_or_make_stub/2 +bif hipe_bifs:find_na_or_make_stub/1 bif hipe_bifs:check_crc/1 bif hipe_bifs:system_crc/0 @@ -84,9 +84,7 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:mark_referred_from/1 bif hipe_bifs:remove_refs_from/1 -bif hipe_bifs:redirect_referred_from/1 # atoms used by add_ref/2 atom call diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index 9b2048c457..a1a6e2ad02 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -214,6 +214,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return address; } +void hipe_free_code(void* code, unsigned int bytes) +{ + /*SVERK: Leaking code memory */ +} + static unsigned int *alloc_stub(Uint nrwords) { unsigned int *address; @@ -241,6 +246,11 @@ static unsigned int *alloc_stub(Uint nrwords) return address; } +void hipe_free_native_stub(void* stub) +{ + /*SVERK: Leaking code stubs */ +} + static void patch_imm16(Uint32 *address, unsigned int imm16) { unsigned int insn = *address; diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 09804e3016..1369b392fe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,15 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct hipe_sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)&nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)&nbif_return }, - .fsize = 0, - .has_exnra = 1, - .arity = 0 - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index 17bef0718c..3d082668a6 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -102,6 +102,28 @@ struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) return sdesc; } +void hipe_destruct_sdesc(struct hipe_sdesc *sdesc) +{ + unsigned int i; + struct hipe_sdesc** prevp; + void* free_me; + + i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; + prevp = &hipe_sdesc_table.bucket[i]; + + for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next) + ASSERT(*prevp); + + *prevp = sdesc->bucket.next; + hipe_sdesc_table.used -= 1; + + if (sdesc->has_exnra) + free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc); + else + free_me = sdesc; + erts_free(ERTS_ALC_T_HIPE, free_me); +} + void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) { Uint ra, exnra; Eterm *live; - Uint fsize, arity, nlive, i, nslots, off; + Uint fsize, nargs, stk_nargs, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; struct hipe_sdesc *sdesc; - - if (is_not_tuple(arg) || - (tuple_val(arg))[0] != make_arityval(6) || - term_to_Uint((tuple_val(arg))[1], &ra) == 0 || - term_to_Uint((tuple_val(arg))[2], &exnra) == 0 || - is_not_small((tuple_val(arg))[3]) || - (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 || - is_not_small((tuple_val(arg))[4]) || - (arity = unsigned_val((tuple_val(arg))[4])) > 255 || - is_not_tuple((tuple_val(arg))[5])) + Eterm* mfa_tpl; + Eterm* tp; + + if (is_not_tuple(arg)) + return 0; + + tp = tuple_val(arg); + if (tp[0] != make_arityval(7) || + term_to_Uint(tp[1], &ra) == 0 || + term_to_Uint(tp[2], &exnra) == 0 || + is_not_small(tp[3]) || + (fsize = unsigned_val(tp[3])) > 65535 || + is_not_small(tp[4]) || + (stk_nargs = unsigned_val(tp[4])) > 255 || + is_not_tuple(tp[5]) || + is_not_tuple(tp[6]) || + (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) || + is_not_atom(mfa_tpl[1]) || + is_not_atom(mfa_tpl[2]) || + is_not_small(mfa_tpl[3]) || + (nargs = unsigned_val(mfa_tpl[3])) > 255) return 0; + + if (stk_nargs > nargs) + return 0; + /* Get tuple with live slots */ - live = tuple_val((tuple_val(arg))[5]) + 1; + live = tuple_val(tp[5]) + 1; /* Get number of live slots */ nlive = arityval(live[-1]); - /* Calculate size of frame = locals + ra + arguments */ - nslots = fsize + 1 + arity; + /* Calculate size of frame = locals + ra + stack arguments */ + nslots = fsize + 1 + stk_nargs; /* Check that only valid slots are given. */ for (i = 0; i < nlive; ++i) { if (is_not_small(live[i]) || @@ -154,8 +191,14 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) return 0; } + switch(tp[7]) { + case am_true: *do_commitp = 1; break; + case am_false: *do_commitp = 0; break; + default: return 0; + } + /* Calculate number of words for the live bitmap. */ - livebitswords = (fsize + arity + 1 + 31) / 32; + livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra @@ -172,12 +215,17 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) } else sdesc = p; + sdesc->m_aix = atom_val(mfa_tpl[1]); + sdesc->f_aix = atom_val(mfa_tpl[2]); + sdesc->a = nargs; + + /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; sdesc->fsize = fsize; sdesc->has_exnra = (exnra ? 1 : 0); - sdesc->arity = arity; + sdesc->stk_nargs = stk_nargs; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; @@ -186,13 +234,5 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) off = unsigned_val(live[i]); sdesc->livebits[off / 32] |= (1 << (off & 31)); } -#ifdef DEBUG - { - Eterm mfa_tpl = tuple_val(arg)[6]; - sdesc->dbg_M = tuple_val(mfa_tpl)[1]; - sdesc->dbg_F = tuple_val(mfa_tpl)[2]; - sdesc->dbg_A = tuple_val(mfa_tpl)[3]; - } -#endif return sdesc; } diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 81f2e53dbc..1863b0db8c 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -37,12 +37,12 @@ struct hipe_sdesc { } bucket; unsigned int fsize : 23; /* frame size */ unsigned int has_exnra : 1; /* exn handler presence flag */ - unsigned int arity : 8; -#ifdef DEBUG - Eterm dbg_M, dbg_F; - unsigned dbg_A; -#endif - unsigned int livebits[1]; /* size depends on arch & data in summary field */ + unsigned int stk_nargs : 8; /* arguments on stack */ + Uint32 m_aix; + Uint32 f_aix; + Uint32 a; + struct hipe_sdesc* next_in_modi; + Uint32 livebits[1]; /* size depends on arch & data in summary field */ }; struct hipe_sdesc_with_exnra { @@ -55,9 +55,10 @@ static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) return sdesc->fsize; } +/* Nr of arguments pushed on stack */ static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->arity; + return sdesc->stk_nargs; } static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) @@ -79,8 +80,9 @@ struct hipe_sdesc_table { extern struct hipe_sdesc_table hipe_sdesc_table; extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_destruct_sdesc(struct hipe_sdesc*); extern void hipe_init_sdesc_table(struct hipe_sdesc*); -extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 5f6c8c200e..8a3ba8bc7f 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -184,6 +184,15 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + /* SVERK: Leak !!! + ALLOC_CODE_STATS(--nr_allocs); + ALLOC_CODE_STATS(total_alloc -= bytes); + + erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/ +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { /* @@ -264,6 +273,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + /* SVERK: leak leak drip drop */ +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 7802076f70..e92e6ea718 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -65,18 +65,14 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) state->sdesc0 = sdesc; return sdesc; #else - unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; state->sdesc0[0].fsize = 0; state->sdesc0[0].has_exnra = 0; - state->sdesc0[0].arity = nstkarity; + state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); state->sdesc0[0].livebits[0] = 0; -# ifdef DEBUG - state->sdesc0[0].dbg_M = 0; - state->sdesc0[0].dbg_F = am_undefined; - state->sdesc0[0].dbg_A = 0; -# endif + state->sdesc0[0].m_aix = 0; + state->sdesc0[0].f_aix = atom_val(am_undefined); + state->sdesc0[0].a = 0; /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */ __asm__ __volatile__("" : : "m"(*state) : "memory"); return &state->sdesc0[0]; diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 0e9fcd61f1..de2b061706 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,18 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct hipe_sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)nbif_return }, - .fsize = 0, - .has_exnra = 1, - .arity = 0, - #ifdef DEBUG - .dbg_F = am_return, - #endif - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index 9426b166e2..31582b3a2e 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -60,7 +60,6 @@ void hipe_print_nstack(Process *p) unsigned int mask; unsigned int sdesc_size; unsigned int i; - unsigned int nstkarity; static const char dashes[2*sizeof(long)+5] = { [0 ... 2*sizeof(long)+3] = '-' }; @@ -68,12 +67,10 @@ void hipe_print_nstack(Process *p) nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; - nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; sdesc0.fsize = 0; sdesc0.has_exnra = 0; - sdesc0.arity = nstkarity; + sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); sdesc0.livebits[0] = ~1; sdesc = &sdesc0; diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 8427bb134d..b76ba6427f 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -445,62 +445,64 @@ module_md5_ok(Code) -> make_stub(Config) when is_list(Config) -> catch erlang:purge_module(my_code_test), MD5 = erlang:md5(<<>>), + Arg3 = {[], [], MD5, 0, 0}, Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "my_code_test"), {ok,my_code_test,Code} = compile:file(File, [binary]), - my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}), + my_code_test = code:make_stub_module(my_code_test, Code, Arg3), true = erlang:delete_module(my_code_test), true = erlang:purge_module(my_code_test), my_code_test = code:make_stub_module(my_code_test, make_unaligned_sub_binary(Code), - {[],[],MD5}), + Arg3), true = erlang:delete_module(my_code_test), true = erlang:purge_module(my_code_test), my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code), - {[],[],MD5}), + Arg3), true = erlang:delete_module(my_code_test), true = erlang:purge_module(my_code_test), %% Should fail. {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})), + (catch code:make_stub_module(my_code_test, <<"bad">>, Arg3)), {'EXIT',{badarg,_}} = (catch code:make_stub_module(my_code_test, bit_sized_binary(Code), - {[],[],MD5})), + Arg3)), {'EXIT',{badarg,_}} = (catch code:make_stub_module(my_code_test_with_wrong_name, - Code, {[],[],MD5})), + Code, Arg3)), ok. make_stub_many_funs(Config) when is_list(Config) -> catch erlang:purge_module(many_funs), MD5 = erlang:md5(<<>>), + Arg3 = {[], [], MD5, 0, 0}, Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "many_funs"), {ok,many_funs,Code} = compile:file(File, [binary]), - many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}), + many_funs = code:make_stub_module(many_funs, Code, Arg3), true = erlang:delete_module(many_funs), true = erlang:purge_module(many_funs), many_funs = code:make_stub_module(many_funs, make_unaligned_sub_binary(Code), - {[],[],MD5}), + Arg3), true = erlang:delete_module(many_funs), true = erlang:purge_module(many_funs), %% Should fail. {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})), + (catch code:make_stub_module(many_funs, <<"bad">>, Arg3)), {'EXIT',{badarg,_}} = (catch code:make_stub_module(many_funs, bit_sized_binary(Code), - {[],[],MD5})), + Arg3)), ok. constant_pools(Config) when is_list(Config) -> -- cgit v1.2.3 From 4d96d297044274c46deda1adfc567297449a9ba9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 19:23:27 +0200 Subject: erts: Remove unused hipe_bifs:code_size and hipe_bifs:update_code_size --- erts/emulator/hipe/hipe_bif0.c | 83 ---------------------------------------- erts/emulator/hipe/hipe_bif0.tab | 3 -- 2 files changed, 86 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 8be6dc9aa9..6f9bfe2f40 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1835,89 +1835,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value) } } -struct modinfo { - HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */ - unsigned int code_size; -}; - -static Hash modinfo_table; - -static HashValue modinfo_hash(void *tmpl) -{ - Eterm mod = (Eterm)tmpl; - return atom_val(mod); -} - -static int modinfo_cmp(void *tmpl, void *bucket) -{ - /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */ - return 0; -} - -static void *modinfo_alloc(void *tmpl) -{ - struct modinfo *p; - - p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p)); - p->code_size = 0; - return &p->bucket; -} - -static void init_modinfo_table(void) -{ - HashFunctions f; - static int init_done = 0; - - if (init_done) - return; - init_done = 1; - f.hash = (H_FUN) modinfo_hash; - f.cmp = (HCMP_FUN) modinfo_cmp; - f.alloc = (HALLOC_FUN) modinfo_alloc; - f.free = (HFREE_FUN) NULL; - f.meta_alloc = (HMALLOC_FUN) erts_alloc; - f.meta_free = (HMFREE_FUN) erts_free; - f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f); -} - -BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3) -{ - struct modinfo *p; - Sint code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1) || - is_not_small(BIF_ARG_3) || - (code_size = signed_val(BIF_ARG_3)) < 0) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1); - - if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */ - p->code_size += code_size; - else /* whole module */ - p->code_size = code_size; - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1) -{ - struct modinfo *p; - unsigned int code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1); - - code_size = p ? p->code_size : 0; - BIF_RET(make_small(code_size)); -} - BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3) { Uint *address, value; diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index eae84383df..ff9c1d06fb 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -56,9 +56,6 @@ bif hipe_bifs:set_native_address/3 bif hipe_bifs:set_funinfo_native_address/3 #bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:update_code_size/3 -bif hipe_bifs:code_size/1 - bif hipe_bifs:enter_sdesc/1 bif hipe_bifs:bif_address/3 -- cgit v1.2.3 From 39072836944d00c288beebfd98b14593f9609006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Thu, 19 May 2016 17:38:54 +0200 Subject: Add a loader state for HiPE code loading Just like the BEAM loader state (as returned by erlang:prepare_loading/2), the HiPE loader state is contained in a magic binary. Eventually, we will separate HiPE loading into a prepare and a finalise phase, like the BEAM loader, where the prepare phase will be implemented by hipe_unified_loader and the finalise phase be implemented in C by hipe_load.c and beam_load.c, making prepare side-effect free and finalise atomic. The finalise phase will be exposed through the erlang:finish_loading/1 API, just like the BEAM loader, as this will allow HiPE and BEAM modules to be mixed in the same atomic "commit". The usage of a loader state makes it easier to keep track of all resources allocated during loading, and will not only make it easy to prevent leaks when hipe_unified_loader crashes, but also paves the way for proper, leak-free, unloading of HiPE modules. --- erts/emulator/Makefile.in | 2 + erts/emulator/beam/beam_bif_load.c | 14 +++-- erts/emulator/beam/beam_load.c | 69 +++++++++++++++---------- erts/emulator/beam/module.c | 5 +- erts/emulator/beam/module.h | 9 ++-- erts/emulator/hipe/hipe_bif0.c | 88 +++++++++++++++++++++++++------- erts/emulator/hipe/hipe_bif0.tab | 6 ++- erts/emulator/hipe/hipe_load.c | 101 +++++++++++++++++++++++++++++++++++++ erts/emulator/hipe/hipe_load.h | 45 +++++++++++++++++ erts/emulator/hipe/hipe_module.c | 37 ++++++++++++++ erts/emulator/hipe/hipe_module.h | 42 +++++++++++++++ 11 files changed, 361 insertions(+), 57 deletions(-) create mode 100644 erts/emulator/hipe/hipe_load.c create mode 100644 erts/emulator/hipe/hipe_load.h create mode 100644 erts/emulator/hipe/hipe_module.c create mode 100644 erts/emulator/hipe/hipe_module.h (limited to 'erts/emulator') diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 21bcbbab27..fa4de5805e 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -884,7 +884,9 @@ HIPE_OBJS= \ $(OBJDIR)/hipe_bif2.o \ $(OBJDIR)/hipe_debug.o \ $(OBJDIR)/hipe_gc.o \ + $(OBJDIR)/hipe_load.o \ $(OBJDIR)/hipe_mode_switch.o \ + $(OBJDIR)/hipe_module.o \ $(OBJDIR)/hipe_native_bif.o \ $(OBJDIR)/hipe_stack.o $(HIPE_ARCH_OBJS) ifdef HIPE_ENABLED diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 237513095a..3302de0787 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -150,7 +150,13 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { Module* modp; - Eterm res; + Eterm res, mod; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) || + is_not_atom(mod = erts_module_for_prepared_code + (((ProcBin*)binary_val(BIF_ARG_1))->val))) { + BIF_ERROR(BIF_P, BADARG); + } if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], @@ -160,7 +166,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { ASSERT(modp->curr.code_hdr != NULL); @@ -172,12 +178,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - if (res == BIF_ARG_1) { + if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); #ifdef HIPE if (!modp) - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); hipe_redirect_to_module(modp); #endif } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 9b206f9a23..7d29f393e5 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -44,6 +44,7 @@ #include "hipe_bif0.h" #include "hipe_mode_switch.h" #include "hipe_arch.h" +#include "hipe_load.h" #endif ErlDrvBinary* erts_gzinflate_buffer(char*, int); @@ -482,8 +483,8 @@ static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size, - void* hipe_code_start, UWord hipe_code_size); + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -942,6 +943,13 @@ erts_module_for_prepared_code(Binary* magic) LoaderState* stp; if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { +#ifdef HIPE + HipeLoaderState *hipe_stp; + if ((hipe_stp = hipe_get_loader_state(magic)) + && hipe_stp->text_segment != 0) { + return hipe_stp->module; + } +#endif return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1094,7 +1102,7 @@ static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, Uint size, - void* hipe_code_start, UWord hipe_code_size) + HipeModule *hipe_code) { Module* modp; Eterm retval; @@ -1121,12 +1129,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs); modp->curr.first_hipe_ref = modp->new_hipe_refs; modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc; - modp->curr.hipe_code_start = hipe_code_start; modp->new_hipe_refs = NULL; modp->new_hipe_sdesc = NULL; -# ifdef DEBUG - modp->curr.hipe_code_size = hipe_code_size; -# endif + modp->curr.hipe_code = hipe_code; #endif /* @@ -6271,10 +6276,13 @@ patch_funentries(Eterm Patchlist) */ Eterm -erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) +erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { Binary* magic; + Binary* hipe_magic; LoaderState* stp; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; BeamInstr Funcs; BeamInstr Patchlist; Eterm MD5Bin; @@ -6290,7 +6298,6 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) byte* temp_alloc = NULL; byte* bytes; Uint size; - UWord hipe_code_start = NULL, hipe_code_size = 0; /* * Must initialize stp->lambdas here because the error handling code @@ -6298,15 +6305,19 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ magic = erts_alloc_loader_state(); stp = ERTS_MAGIC_BIN_DATA(magic); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); - if (is_not_atom(Mod)) { + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { goto error; } if (is_not_tuple(Info)) { goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(5)) { + if (tp[0] != make_arityval(3)) { goto error; } Funcs = tp[1]; @@ -6323,20 +6334,11 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } size = binary_size(Beam); -#ifdef HIPE - if (!term_to_Uint(tp[4], &hipe_code_start)) - goto error; -# ifdef DEBUG - if (!term_to_Uint(tp[5], &hipe_code_size)) - goto error; -# endif -#endif - /* * Scan the Beam binary and read the interesting sections. */ - stp->module = Mod; + stp->module = hipe_stp->module; stp->group_leader = p->group_leader; stp->num_functions = n; if (!init_iff_file(stp, bytes, size)) { @@ -6450,7 +6452,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) #else op = (Eterm) BeamOpCode(op_move_return_n); #endif - fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); + fp = make_stub(fp, hipe_stp->module, func, arity, (Uint)native_address, + op); } /* @@ -6489,13 +6492,19 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) erts_free_aligned_binary_bytes(tmp); } + /* + * Initialise HiPE module + */ + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + /* * Insert the module in the module table. */ - rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size, - (void*)hipe_code_start, hipe_code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module, + code_hdr, code_size, hipe_code); if (rval != NIL) { goto error; } @@ -6511,12 +6520,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } if (patch_funentries(Patchlist)) { + Eterm mod = hipe_stp->module; + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - return Mod; + hipe_free_loader_state(hipe_magic); + + return mod; } error: + erts_free(ERTS_ALC_T_HIPE, hipe_code); erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); BIF_ERROR(p, BADARG); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 93ce8fcf9f..5c56ad20df 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -79,10 +79,7 @@ void erts_module_instance_init(struct erl_module_instance* modi) #ifdef HIPE modi->first_hipe_ref = NULL; modi->first_hipe_sdesc = NULL; - modi->hipe_code_start = NULL; -# ifdef DEBUG - modi->hipe_code_size = 0; -# endif + modi->hipe_code = NULL; #endif } diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 694583597b..c3adaf12fc 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -23,6 +23,10 @@ #include "index.h" +#ifdef HIPE +#include "hipe_module.h" +#endif + struct erl_module_instance { BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ @@ -33,10 +37,7 @@ struct erl_module_instance { #ifdef HIPE struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ - void* hipe_code_start; -# ifdef DEBUG - UWord hipe_code_size; -# endif + HipeModule *hipe_code; #endif }; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 6f9bfe2f40..daa198d695 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -44,6 +44,7 @@ #include "hipe_mode_switch.h" #include "hipe_native_bif.h" #include "hipe_bif0.h" +#include "hipe_load.h" /* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines a few constants. #undef them here to avoid warnings. */ #undef F_TIMO @@ -374,16 +375,32 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2) BIF_RET(BIF_ARG_1); } +/* + * BIFs for loading code. + */ + +static HipeLoaderState *get_loader_state(Eterm term) +{ + ProcBin *pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL; + + pb = (ProcBin*) binary_val(term); + return hipe_get_loader_state(pb->val); +} + + /* * Allocate memory and copy machine code to it. */ -BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) { Uint nrbytes; void *bytes; void *address; Eterm trampolines; Eterm *hp; + HipeLoaderState *stp; #ifndef DEBUG ERTS_DECLARE_DUMMY(Uint bitoffs); ERTS_DECLARE_DUMMY(Uint bitsize); @@ -392,13 +409,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) Uint bitsize; #endif - if (is_not_binary(BIF_ARG_1)) + if (is_not_binary(BIF_ARG_1) || + (!(stp = get_loader_state(BIF_ARG_3)))) BIF_ERROR(BIF_P, BADARG); nrbytes = binary_size(BIF_ARG_1); ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); ASSERT(bitoffs == 0); ASSERT(bitsize == 0); trampolines = NIL; + // XXX: Trampolines are not tracked and freed address = hipe_alloc_code(nrbytes, BIF_ARG_2, &trampolines, BIF_P); if (!address) { Uint nrcallees; @@ -407,11 +426,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) nrcallees = arityval(tuple_val(BIF_ARG_2)[0]); else nrcallees = 0; + // XXX: Is there any reason to not just BIF_ERROR, so that the runtime + // survives? erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", __func__, (unsigned long)nrbytes, (unsigned long)nrcallees); } memcpy(address, bytes, nrbytes); hipe_flush_icache_range(address, nrbytes); + stp->text_segment = address; + stp->text_segment_size = nrbytes; hp = HAlloc(BIF_P, 3); hp[0] = make_arityval(2); hp[1] = address_to_term(address, BIF_P); @@ -424,25 +447,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) /* * Allocate memory for arbitrary non-Erlang data. */ -BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3) { - Uint align, nrbytes; - void *block; + Uint align; + HipeLoaderState *stp; if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) || + (!(stp = get_loader_state(BIF_ARG_3))) || (align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align))) BIF_ERROR(BIF_P, BADARG); - nrbytes = unsigned_val(BIF_ARG_2); - if (nrbytes == 0) + + if (stp->data_segment_size || stp->data_segment) + BIF_ERROR(BIF_P, BADARG); + + stp->data_segment_size = unsigned_val(BIF_ARG_2); + if (stp->data_segment_size == 0) BIF_RET(make_small(0)); - block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); - if ((unsigned long)block & (align-1)) { - fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", - __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); - erts_free(ERTS_ALC_T_HIPE, block); + stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size); + if ((unsigned long)stp->data_segment & (align-1)) { + fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte " + "aligned\r\n", + __FUNCTION__, (unsigned long)stp->data_segment_size, + stp->data_segment, (unsigned long)align); + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; BIF_ERROR(BIF_P, EXC_NOTSUP); } - BIF_RET(address_to_term(block, BIF_P)); + BIF_RET(address_to_term(stp->data_segment, BIF_P)); } /* @@ -1711,12 +1743,14 @@ void hipe_purge_module(Module* modp) prevp = &p->next_in_mod; } } - if (modp->old.hipe_code_start) { + if (modp->old.hipe_code && modp->old.hipe_code->text_segment) { #ifdef DEBUG - sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size); + sys_memset(modp->old.hipe_code->text_segment, 0xfe, + modp->old.hipe_code->text_segment_size); #endif - hipe_free_code(modp->old.hipe_code_start); - modp->old.hipe_code_start = NULL; + hipe_free_code(modp->old.hipe_code->text_segment); + modp->old.hipe_code->text_segment = NULL; + modp->old.hipe_code->text_segment_size = 0; } } @@ -1870,3 +1904,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3) BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } + +BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1) +{ + Binary *magic; + Eterm *hp; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + magic = hipe_alloc_loader_state(BIF_ARG_1); + + if (!magic) + BIF_ERROR(BIF_P, BADARG); + + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->refc, 1); + BIF_RET(res); +} diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index ff9c1d06fb..1e4bde5ef2 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -44,8 +44,8 @@ bif hipe_bifs:ref/1 bif hipe_bifs:ref_get/1 bif hipe_bifs:ref_set/2 -bif hipe_bifs:enter_code/2 -bif hipe_bifs:alloc_data/2 +bif hipe_bifs:enter_code/3 +bif hipe_bifs:alloc_data/3 bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 @@ -83,6 +83,8 @@ bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 bif hipe_bifs:remove_refs_from/1 +bif hipe_bifs:alloc_loader_state/1 + # atoms used by add_ref/2 atom call atom load_mfa diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c new file mode 100644 index 0000000000..5bce3f1aee --- /dev/null +++ b/erts/emulator/hipe/hipe_load.c @@ -0,0 +1,101 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.c + * + * HiPE atomic code loader + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "global.h" +#include "erl_binary.h" +#include "hipe_load.h" + +/* + * This destructor function can safely be called multiple times. + */ +static void +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + if (stp->module == NIL) return; + + erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n", + stp->module); + + // TODO: Needs to be freed separately. We'd like have a unified executable + // code allocator, so postpone this for now. + /* if (stp->text_segment) */ + /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */ + stp->text_segment = NULL; + stp->text_segment_size = 0; + + if (stp->data_segment) + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; + + stp->module = NIL; +} + +Binary *hipe_alloc_loader_state(Eterm module) +{ + HipeLoaderState *stp; + Binary *magic; + + if (is_not_atom(module)) return NULL; + + erts_fprintf(stderr, "Creating HiPE loader state for module %T\n", module); + + magic = erts_create_magic_binary(sizeof(HipeLoaderState), + hipe_loader_state_dtor); + erts_refc_inc(&magic->refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); + + stp->module = module; + stp->text_segment = NULL; + stp->text_segment_size = 0; + stp->data_segment = NULL; + stp->data_segment_size = 0; + + return magic; +} + +void hipe_free_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return; + + /* Why does BEAM do a refc_dec here? What is holding that reference? */ + hipe_loader_state_dtor(magic); +} + +HipeLoaderState * +hipe_get_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return NULL; + + return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic); +} diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h new file mode 100644 index 0000000000..9e24e915ed --- /dev/null +++ b/erts/emulator/hipe/hipe_load.h @@ -0,0 +1,45 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.h + * + * HiPE atomic code loader + */ +#ifndef HIPE_LOAD_H +#define HIPE_LOAD_H + +#include "global.h" + +typedef struct hipe_loader_state { + Eterm module; /* Module name, atom */ + + void *text_segment; + Uint text_segment_size; + + void *data_segment; + Uint data_segment_size; + +} HipeLoaderState; + +extern Binary *hipe_alloc_loader_state(Eterm module); +extern void hipe_free_loader_state(Binary *binary); +extern HipeLoaderState *hipe_get_loader_state(Binary *binary); + +#endif /* HIPE_LOAD_H */ diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c new file mode 100644 index 0000000000..1553bf9087 --- /dev/null +++ b/erts/emulator/hipe/hipe_module.c @@ -0,0 +1,37 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "hipe_module.h" + +void hipe_free_module(HipeModule *mod) +{ +#ifdef DEBUG + sys_memzero(mod->text_segment, mod->text_segment_size); +#endif + /* XXX: erts_free(ERTS_ALC_T_HIPE, mod->text_segment); */ + if (mod->data_segment) /* Some modules lack data segments */ + erts_free(ERTS_ALC_T_HIPE, mod->data_segment); + + erts_free(ERTS_ALC_T_HIPE, mod); +} diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h new file mode 100644 index 0000000000..1e1302fc0a --- /dev/null +++ b/erts/emulator/hipe/hipe_module.h @@ -0,0 +1,42 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_module.h + * + * + */ +#ifndef HIPE_MODULE_H +#define HIPE_MODULE_H + +/* Forward-declare type to resolve circular dependency with module.h */ +typedef struct hipe_module HipeModule; + +#include "global.h" + +struct hipe_module { + void *text_segment; + Uint text_segment_size; + void *data_segment; +}; + +extern void hipe_free_module(HipeModule *mod); + +#endif /* HIPE_MODULE_H */ -- cgit v1.2.3 From 5e879fb1d4a9273f0a6183da7509dcdbc24f9653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Fri, 16 Sep 2016 19:33:38 +0200 Subject: erts: Check hipe stack in check_process_code This is part of commit 1bd508921dd93086b05e7d0038b816b36c421d86. I did not include the fun-checking as we have a new purge strategy for funs in OTP 20. That remains to be solved some other way for hipe. --- erts/emulator/beam/beam_bif_load.c | 26 ++++++++++++++++- erts/emulator/hipe/hipe_gc.c | 33 ++++++++++++++++++++++ erts/emulator/hipe/hipe_stack.h | 2 ++ erts/emulator/hipe/hipe_x86_gc.h | 1 + erts/emulator/test/hipe_SUITE.erl | 57 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 116 insertions(+), 3 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 3302de0787..ec1db8d9c2 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1069,6 +1069,8 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) BeamInstr* start; char* mod_start; Uint mod_size; + void *nat_start = NULL; + Uint nat_size = 0; Eterm* sp; *redsp += 1; @@ -1099,6 +1101,20 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) } } +#ifdef HIPE + /* + * Check all continuation pointers stored on the native stack if the module + * has native code. + */ + if (modp->old.hipe_code) { + nat_start = modp->old.hipe_code->text_segment; + nat_size = modp->old.hipe_code->text_segment_size; + if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) { + return am_true; + } + } +#endif + /* * Check all continuation pointers stored in stackdump * and clear exception stackdump if there is a pointer @@ -1115,8 +1131,16 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) rp->ftrace = NIL; } else { int i; + char *area_start = mod_start; + Uint area_size = mod_size; +#ifdef HIPE + if (rp->freason & EXF_NATIVE) { + area_start = nat_start; + area_size = nat_size; + } +#endif for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + if (ErtsInArea(s->trace[i], area_start, area_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index be665ccdb9..e6ce7ce628 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) } return 0; } + +int +nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size) +{ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + if (!p->hipe.nstack || !nstack_walk_init_check(p)) + return 0; + ASSERT(p->hipe.nsp && p->hipe.nstend); + nsp = nstack_walk_nsp_begin(p); + nsp_end = nstack_walk_nsp_end(p); + sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state); + + /* Check the topmost frame */ + if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size)) + return 1; + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned long ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + if (ErtsInArea(ra, seg_start, seg_size)) + return 1; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return 0; +} diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 1863b0db8c..537755fefa 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -138,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area, Uint area_size); extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size); +extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size); + #endif /* HIPE_STACK_H */ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index e92e6ea718..a703e24b8c 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -65,6 +65,7 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) state->sdesc0 = sdesc; return sdesc; #else + state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */ state->sdesc0[0].fsize = 0; state->sdesc0[0].has_exnra = 0; state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl index a556b4ddc0..0b44dd7fb7 100644 --- a/erts/emulator/test/hipe_SUITE.erl +++ b/erts/emulator/test/hipe_SUITE.erl @@ -19,12 +19,17 @@ %% -module(hipe_SUITE). --export([all/0, t_copy_literals/1]). +-export([all/0 + ,t_copy_literals/1 + ,t_purge/1 + ]). all() -> case erlang:system_info(hipe_architecture) of undefined -> {skip, "HiPE is disabled"}; - _ -> [t_copy_literals] + _ -> [t_copy_literals + ,t_purge + ] end. t_copy_literals(doc) -> @@ -65,3 +70,51 @@ t_copy_literals(Config) when is_list(Config) -> true = erlang:delete_module(ref_cell), true = erlang:purge_module(ref_cell), ok. + +t_purge(doc) -> "Checks that native code is properly found and purged"; +t_purge(Config) when is_list(Config) -> + Data = proplists:get_value(data_dir, Config), + Priv = proplists:get_value(priv_dir, Config), + SrcFile = filename:join(Data, "ref_cell"), + BeamFile = filename:join(Priv, "ref_cell"), + {ok,ref_cell} = c:c(SrcFile, [{outdir,Priv},native]), + true = code:is_module_native(ref_cell), + + PA = ref_cell:start_link(), + + %% Unload, PA should still be running + true = erlang:delete_module(ref_cell), + %% Can't use ref_cel:call/2, it's in old code! + call(PA, {put_res_of, fun()-> hej end}), + hej = call(PA, get), + + %% Load same module again + code:load_abs(BeamFile), + true = code:is_module_native(ref_cell), + PB = ref_cell:start_link(), + + %% Purge old code, PA should be killed, PB should survive + unlink(PA), + ARef = monitor(process, PA), + true = erlang:purge_module(ref_cell), + receive {'DOWN', ARef, process, PA, killed} -> ok + after 1 -> ct:fail("PA was not killed") + end, + + %% Unload, PB should still be running + true = erlang:delete_module(ref_cell), + call(PB, {put_res_of, fun()-> svejs end}), + svejs = call(PB, get), + + unlink(PB), + BRef = monitor(process, PB), + true = erlang:purge_module(ref_cell), + receive {'DOWN', BRef, process, PB, killed} -> ok + after 1 -> ct:fail("PB was not killed") + end, + + ok. + +call(Pid, Call) -> + Pid ! {Call, self()}, + receive {Pid, Res} -> Res end. -- cgit v1.2.3 From a82be01198c1f3714f29f0a0e3ce3ed6d6bce0a6 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 16 Sep 2016 19:39:24 +0200 Subject: erts: Fill freed EXEC memory with illegal instructions on DEBUG to get a nice SIGILL crash. For x86 and x86_64 only. --- erts/emulator/beam/erl_alloc.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 3c2c9def3b..d96b90fd55 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -4129,12 +4129,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *dptr; Uint size; + int free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); - sys_memset((void *) dptr, n, size + FENCE_SZ); +#ifdef ERTS_ALC_A_EXEC +# if defined(__i386__) || defined(__x86_64__) + if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) { + free_pattern = 0x0f; /* Illegal instruction */ + } +# endif +#endif + sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); (*real_af->free)(n, real_af->extra, dptr); -- cgit v1.2.3 From e36c8d49359db1622ad381f705c54d460c4fb5e9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 16 Sep 2016 20:01:11 +0200 Subject: erts: Use hipe_free_module --- erts/emulator/hipe/hipe_amd64.c | 5 +++-- erts/emulator/hipe/hipe_arch.h | 4 ++-- erts/emulator/hipe/hipe_bif0.c | 11 +++-------- erts/emulator/hipe/hipe_module.c | 6 ++---- 4 files changed, 10 insertions(+), 16 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index df53f4db30..96b8c22265 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -130,10 +130,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } -void hipe_free_code(void* code) +void hipe_free_code(void* code, unsigned int bytes) { ALLOC_CODE_STATS(--nr_allocs); - /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + ALLOC_CODE_STATS(total_alloc -= bytes); + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index df38a80069..059b8e7f29 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -30,8 +30,8 @@ extern void hipe_patch_load_fe(Uint *address, Uint value); extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); -extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void hipe_free_code(void*); +extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p); +extern void hipe_free_code(void*, unsigned int); extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); extern void hipe_free_native_stub(void*); diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index daa198d695..46d4378f21 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1743,14 +1743,9 @@ void hipe_purge_module(Module* modp) prevp = &p->next_in_mod; } } - if (modp->old.hipe_code && modp->old.hipe_code->text_segment) { -#ifdef DEBUG - sys_memset(modp->old.hipe_code->text_segment, 0xfe, - modp->old.hipe_code->text_segment_size); -#endif - hipe_free_code(modp->old.hipe_code->text_segment); - modp->old.hipe_code->text_segment = NULL; - modp->old.hipe_code->text_segment_size = 0; + if (modp->old.hipe_code) { + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; } } diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c index 1553bf9087..469f077dd2 100644 --- a/erts/emulator/hipe/hipe_module.c +++ b/erts/emulator/hipe/hipe_module.c @@ -22,14 +22,12 @@ #include "config.h" #endif #include "sys.h" +#include "hipe_arch.h" #include "hipe_module.h" void hipe_free_module(HipeModule *mod) { -#ifdef DEBUG - sys_memzero(mod->text_segment, mod->text_segment_size); -#endif - /* XXX: erts_free(ERTS_ALC_T_HIPE, mod->text_segment); */ + hipe_free_code(mod->text_segment, mod->text_segment_size); if (mod->data_segment) /* Some modules lack data segments */ erts_free(ERTS_ALC_T_HIPE, mod->data_segment); -- cgit v1.2.3 From 966098ceb9dd9d18e9bcd37cd06b96045903e320 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 20 Sep 2016 16:16:50 +0200 Subject: erts: Move new hipe ref and sdesc lists to loader state --- erts/emulator/beam/beam_bif_load.c | 1 - erts/emulator/beam/beam_load.c | 11 +-- erts/emulator/beam/module.c | 6 -- erts/emulator/beam/module.h | 4 - erts/emulator/hipe/hipe_bif0.c | 150 +++++++++++++++++++++---------------- erts/emulator/hipe/hipe_bif0.tab | 2 +- erts/emulator/hipe/hipe_load.c | 3 + erts/emulator/hipe/hipe_load.h | 3 + erts/emulator/hipe/hipe_module.h | 3 + 9 files changed, 100 insertions(+), 83 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index ec1db8d9c2..f5d1ca7e54 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1798,7 +1798,6 @@ delete_code(Module* modp) ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); - DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref); modp->old = modp->curr; erts_module_instance_init(&modp->curr); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 7d29f393e5..0eb390bf4c 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1126,11 +1126,8 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ #if defined(HIPE) - DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs); - modp->curr.first_hipe_ref = modp->new_hipe_refs; - modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc; - modp->new_hipe_refs = NULL; - modp->new_hipe_sdesc = NULL; + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " + "first_hipe_ref = %p", hipe_code->first_hipe_ref); modp->curr.hipe_code = hipe_code; #endif @@ -6498,6 +6495,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) hipe_code->text_segment = hipe_stp->text_segment; hipe_code->text_segment_size = hipe_stp->text_segment_size; hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; /* * Insert the module in the module table. @@ -6524,6 +6523,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) /* Prevent code from being freed */ hipe_stp->text_segment = 0; hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 5c56ad20df..e9b18ba525 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -77,8 +77,6 @@ void erts_module_instance_init(struct erl_module_instance* modi) modi->num_breakpoints = 0; modi->num_traced_exports = 0; #ifdef HIPE - modi->first_hipe_ref = NULL; - modi->first_hipe_sdesc = NULL; modi->hipe_code = NULL; #endif } @@ -95,8 +93,6 @@ static Module* module_alloc(Module* tmpl) obj->on_load = 0; #ifdef HIPE obj->first_hipe_mfa = NULL; - obj->new_hipe_refs = NULL; - obj->new_hipe_sdesc = NULL; #endif DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; @@ -214,8 +210,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) dst_mod->on_load = src_mod->on_load; #ifdef HIPE dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa; - dst_mod->new_hipe_refs = src_mod->new_hipe_refs; - dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc; #endif } diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index c3adaf12fc..9de610ccf7 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -35,8 +35,6 @@ struct erl_module_instance { int num_breakpoints; int num_traced_exports; #ifdef HIPE - struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ - struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ HipeModule *hipe_code; #endif }; @@ -51,8 +49,6 @@ typedef struct erl_module { struct erl_module_instance* on_load; #ifdef HIPE struct hipe_mfa_info* first_hipe_mfa; - struct hipe_ref* new_hipe_refs; - struct hipe_sdesc* new_hipe_sdesc; #endif } Module; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 46d4378f21..da895e4e8b 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -685,12 +685,17 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) BIF_RET(am_false); } -BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) +BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { struct hipe_sdesc *sdesc; + HipeLoaderState* stp; Module* modp; int do_commit; + stp = get_loader_state(BIF_ARG_2); + if (!stp) + BIF_ERROR(BIF_P, BADARG); + sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit); if (!sdesc) { fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__); @@ -707,12 +712,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) modp = erts_put_active_module(make_atom(sdesc->m_aix)); ASSERT(modp); if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - sdesc->next_in_modi = modp->curr.first_hipe_sdesc; - modp->curr.first_hipe_sdesc = sdesc; + ASSERT(modp->curr.hipe_code); + sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc; + modp->curr.hipe_code->first_hipe_sdesc = sdesc; } else { /* Normal module loading/upgrade */ - sdesc->next_in_modi = modp->new_hipe_sdesc; - modp->new_hipe_sdesc = sdesc; + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; } BIF_RET(NIL); @@ -1491,13 +1497,14 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) struct hipe_ref *ref; Module* modp; int do_commit; + HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; if (is_not_tuple(BIF_ARG_2)) goto badarg; tuple = tuple_val(BIF_ARG_2); - if (tuple[0] != make_arityval(5)) + if (tuple[0] != make_arityval(6)) goto badarg; if (!term_to_mfa(tuple[1], &caller)) goto badarg; @@ -1526,6 +1533,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) case am_false: do_commit = 0; break; default: goto badarg; } + stp = get_loader_state(tuple[6]); + if (!stp) + goto badarg; + hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); @@ -1550,12 +1561,13 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) modp = erts_put_active_module(caller.mod); ASSERT(modp); if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - ref->next_from_modi = modp->curr.first_hipe_ref; - modp->curr.first_hipe_ref = ref; + ASSERT(modp->curr.hipe_code); + ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref; + modp->curr.hipe_code->first_hipe_ref = ref; } else { /* Normal module loading/upgrade */ - ref->next_from_modi = modp->new_hipe_refs; - modp->new_hipe_refs = ref; + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; } #if defined(DEBUG) @@ -1653,9 +1665,12 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) int hipe_purge_need_blocking(Module* modp) { /* SVERK: Verify if this is really necessary */ - return (modp->old.first_hipe_ref || - modp->old.first_hipe_sdesc || - (!modp->curr.code_hdr && modp->first_hipe_mfa)); + if (modp->old.hipe_code) { + if (modp->old.hipe_code->first_hipe_ref || + modp->old.hipe_code->first_hipe_sdesc) + return 1; + } + return !modp->curr.code_hdr && modp->first_hipe_mfa; } void hipe_purge_module(Module* modp) @@ -1667,68 +1682,75 @@ void hipe_purge_module(Module* modp) DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); - /* - * Remove all hipe_ref's (external calls) from the old module instance - */ - ref = modp->old.first_hipe_ref; - - while (ref) { - struct hipe_ref* free_ref = ref; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, - ref->callee->m, ref->callee->f, ref->callee->a); - DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, - ref->caller_m, ref->caller_f, ref->caller_a); - ASSERT(ref->caller_m == make_atom(modp->module)); - + if (modp->old.hipe_code) { /* - * Unlink from other refs to same callee + * Remove all hipe_ref's (external calls) from the old module instance */ - ASSERT(ref->head.next->prev == &ref->head); - ASSERT(ref->head.prev->next == &ref->head); - ASSERT(ref->head.next != &ref->head); - ASSERT(ref->head.prev != &ref->head); - ref->head.next->prev = ref->head.prev; - ref->head.prev->next = ref->head.next; + ref = modp->old.hipe_code->first_hipe_ref; + + while (ref) { + struct hipe_ref* free_ref = ref; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == make_atom(modp->module)); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + unlink_mfa_from_mod(p); + purge_mfa(p); + } + } - /* - * Was this the last ref to that callee? - */ - if (ref->head.next == ref->head.prev) { - struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); - if (p->is_stub) { - unlink_mfa_from_mod(p); - purge_mfa(p); - } + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); } + modp->old.hipe_code->first_hipe_ref = NULL; - ref = ref->next_from_modi; - erts_free(ERTS_ALC_T_HIPE, free_ref); - } - modp->old.first_hipe_ref = NULL; + /* + * Remove all hipe_sdesc's for the old module instance + */ + sdesc = modp->old.hipe_code->first_hipe_sdesc; - /* - * Remove all hipe_sdesc's for the old module instance - */ - sdesc = modp->old.first_hipe_sdesc; + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; - while (sdesc) { - struct hipe_sdesc* free_sdesc = sdesc; + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(sdesc->m_aix == modp->module); - DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); - ASSERT(sdesc->m_aix == modp->module); + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } + modp->old.hipe_code->first_hipe_sdesc = NULL; - sdesc = sdesc->next_in_modi; - hipe_destruct_sdesc(free_sdesc); + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; } - modp->old.first_hipe_sdesc = NULL; + /* - * Remove unreferred hipe_mfa_info's + * Remove unreferred hipe_mfa_info's + * when all module instances are removed (like in init:restart) */ if (modp->curr.code_hdr == NULL) { struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; @@ -1743,10 +1765,6 @@ void hipe_purge_module(Module* modp) prevp = &p->next_in_mod; } } - if (modp->old.hipe_code) { - hipe_free_module(modp->old.hipe_code); - modp->old.hipe_code = NULL; - } } diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 1e4bde5ef2..8475b8ce76 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -56,7 +56,7 @@ bif hipe_bifs:set_native_address/3 bif hipe_bifs:set_funinfo_native_address/3 #bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:enter_sdesc/1 +bif hipe_bifs:enter_sdesc/2 bif hipe_bifs:bif_address/3 bif hipe_bifs:primop_address/1 diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index 5bce3f1aee..cebbbafdd3 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -79,6 +79,9 @@ Binary *hipe_alloc_loader_state(Eterm module) stp->data_segment = NULL; stp->data_segment_size = 0; + stp->new_hipe_refs = NULL; + stp->new_hipe_sdesc = NULL; + return magic; } diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h index 9e24e915ed..17dd3c3c05 100644 --- a/erts/emulator/hipe/hipe_load.h +++ b/erts/emulator/hipe/hipe_load.h @@ -36,6 +36,9 @@ typedef struct hipe_loader_state { void *data_segment; Uint data_segment_size; + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; + } HipeLoaderState; extern Binary *hipe_alloc_loader_state(Eterm module); diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h index 1e1302fc0a..b489f567cb 100644 --- a/erts/emulator/hipe/hipe_module.h +++ b/erts/emulator/hipe/hipe_module.h @@ -35,6 +35,9 @@ struct hipe_module { void *text_segment; Uint text_segment_size; void *data_segment; + + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ }; extern void hipe_free_module(HipeModule *mod); -- cgit v1.2.3 From e77d7a8417368617c4c228af9556a7f6a8f3e84c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 28 Sep 2016 20:55:40 +0200 Subject: erts: Fix early hipe patch loading by introducing hipe_bifs:commit_patch_load/1 that creates the HipeModule. --- erts/emulator/beam/beam_load.c | 39 ++++++++++++++++++++++++++ erts/emulator/beam/global.h | 1 + erts/emulator/hipe/hipe_bif0.c | 59 ++++++++++++++-------------------------- erts/emulator/hipe/hipe_bif0.tab | 1 + erts/emulator/hipe/hipe_load.c | 5 ++++ erts/emulator/hipe/hipe_stack.c | 10 ++----- erts/emulator/hipe/hipe_stack.h | 2 +- 7 files changed, 69 insertions(+), 48 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0eb390bf4c..4a833689e3 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6540,6 +6540,45 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) BIF_ERROR(p, BADARG); } +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) +{ + Binary* hipe_magic; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; + Module* modp; + + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { + return 0; + } + + modp = erts_get_module(hipe_stp->module, erts_active_code_ix()); + if (!modp) + return 0; + + /* + * Initialise HiPE module + */ + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + modp->curr.hipe_code = hipe_code; + + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + + return 1; +} + #undef WORDS_PER_FUNCTION static int safe_mul(UWord a, UWord b, UWord* resp) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 93996e8b41..29a0bc8d0b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1048,6 +1048,7 @@ void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin); /* beam_ranges.c */ void erts_init_ranges(void); diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index da895e4e8b..67508e6f5d 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -649,6 +649,14 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1) BIF_RET(address_to_term(pc, BIF_P)); } +BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) +{ + if (!erts_commit_hipe_patch_load(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(am_ok); +} + BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) { Eterm *pc; @@ -689,14 +697,12 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { struct hipe_sdesc *sdesc; HipeLoaderState* stp; - Module* modp; - int do_commit; stp = get_loader_state(BIF_ARG_2); if (!stp) BIF_ERROR(BIF_P, BADARG); - sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit); + sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); @@ -709,17 +715,8 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) /* * Link into list of sdesc's in same module instance */ - modp = erts_put_active_module(make_atom(sdesc->m_aix)); - ASSERT(modp); - if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - ASSERT(modp->curr.hipe_code); - sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc; - modp->curr.hipe_code->first_hipe_sdesc = sdesc; - } - else { /* Normal module loading/upgrade */ - sdesc->next_in_modi = stp->new_hipe_sdesc; - stp->new_hipe_sdesc = sdesc; - } + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; BIF_RET(NIL); } @@ -1483,7 +1480,7 @@ int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) } -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { @@ -1495,8 +1492,6 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) unsigned int flags; struct hipe_mfa_info *callee_mfa; struct hipe_ref *ref; - Module* modp; - int do_commit; HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) @@ -1504,7 +1499,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (is_not_tuple(BIF_ARG_2)) goto badarg; tuple = tuple_val(BIF_ARG_2); - if (tuple[0] != make_arityval(6)) + if (tuple[0] != make_arityval(5)) goto badarg; if (!term_to_mfa(tuple[1], &caller)) goto badarg; @@ -1528,12 +1523,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (!trampoline) goto badarg; } - switch (tuple[5]) { - case am_true: do_commit = 1; break; - case am_false: do_commit = 0; break; - default: goto badarg; - } - stp = get_loader_state(tuple[6]); + stp = get_loader_state(tuple[5]); if (!stp) goto badarg; @@ -1558,17 +1548,8 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) /* * Link into list of refs from same module instance */ - modp = erts_put_active_module(caller.mod); - ASSERT(modp); - if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - ASSERT(modp->curr.hipe_code); - ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref; - modp->curr.hipe_code->first_hipe_ref = ref; - } - else { /* Normal module loading/upgrade */ - ref->next_from_modi = stp->new_hipe_refs; - stp->new_hipe_refs = ref; - } + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; #if defined(DEBUG) ref->callee = callee_mfa; @@ -1578,10 +1559,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) #endif hipe_mfa_info_table_rwunlock(); - DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d", - ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit); - DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d", - ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)", + ref, callee.mod, callee.fun, callee.ari, ref->address); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)", + ref, caller.mod, caller.fun, caller.ari, ref->address); BIF_RET(NIL); badarg: diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 8475b8ce76..264ea2c34a 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -50,6 +50,7 @@ bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 bif hipe_bifs:fun_to_address/1 +bif hipe_bifs:commit_patch_load/1 bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index cebbbafdd3..48f0cab293 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -57,6 +57,11 @@ hipe_loader_state_dtor(Binary* magic) stp->data_segment_size = 0; stp->module = NIL; + + ASSERT(!stp->new_hipe_refs && !stp->new_hipe_sdesc); + /* ToDO: Purge lists 'new_hipe_refs' and 'new_hipe_sdesc' + * if this is a failed load. + */ } Binary *hipe_alloc_loader_state(Eterm module) diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index 3d082668a6..b80e44bc37 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -143,7 +143,7 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; @@ -158,7 +158,7 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) return 0; tp = tuple_val(arg); - if (tp[0] != make_arityval(7) || + if (tp[0] != make_arityval(6) || term_to_Uint(tp[1], &ra) == 0 || term_to_Uint(tp[2], &exnra) == 0 || is_not_small(tp[3]) || @@ -191,12 +191,6 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) return 0; } - switch(tp[7]) { - case am_true: *do_commitp = 1; break; - case am_false: *do_commitp = 0; break; - default: return 0; - } - /* Calculate number of words for the live bitmap. */ livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 537755fefa..7e30358767 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -82,7 +82,7 @@ extern struct hipe_sdesc_table hipe_sdesc_table; extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); extern void hipe_destruct_sdesc(struct hipe_sdesc*); extern void hipe_init_sdesc_table(struct hipe_sdesc*); -extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) -- cgit v1.2.3 From 870d49f0f8d383e62ea231700089933c4cf8fe2a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 29 Sep 2016 17:38:54 +0200 Subject: hipe,erts: Remove cached fun 'native_address' Did not work with purge and made worse by new purge strategy. Did yield terrible performance when fun thing is created *before* fun code is loaded. Like when receiving not yet loaded fun from other node. The cached 'native_address' in ErlFunThing will not be updated leading to mode switch and error_handler being called for every call to the fun from native code. --- erts/emulator/beam/beam_emu.c | 3 --- erts/emulator/beam/erl_fun.h | 3 --- erts/emulator/beam/external.c | 4 ---- erts/emulator/hipe/hipe_mkliterals.c | 3 --- 4 files changed, 13 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ccb0f786ec..3fa4fb5b3f 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6667,9 +6667,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) funp->fe = fe; funp->num_free = num_free; funp->creator = p->common.id; -#ifdef HIPE - funp->native_address = fe->native_address; -#endif funp->arity = (int)fe->address[-1] - num_free; for (i = 0; i < num_free; i++) { *hp++ = reg[i]; diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 73c3e19c1c..6ba055d92c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -57,9 +57,6 @@ typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ ErlFunEntry* fe; /* Pointer to fun entry. */ struct erl_off_heap_header* next; -#ifdef HIPE - UWord* native_address; /* Native code for the fun. */ -#endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ /* -- The following may be compound Erlang terms ---------------------- */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index beed847578..ca79f83184 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3749,7 +3749,6 @@ dec_term_atom_common: if (funp->fe->native_address == NULL) { hipe_set_closure_stub(funp->fe, num_free); } - funp->native_address = funp->fe->native_address; #endif hp = factory->hp; @@ -3821,9 +3820,6 @@ dec_term_atom_common: funp->fe = erts_put_fun_entry(module, old_uniq, old_index); funp->arity = funp->fe->address[-1] - num_free; -#ifdef HIPE - funp->native_address = funp->fe->native_address; -#endif hp = factory->hp; /* Environment */ diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index b9d4226705..4573980e1e 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = { presence or absence of struct erl_fun_thing's "next" field. */ { 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) }, { 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) }, -#ifdef HIPE - { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) }, -#endif { 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) }, { 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) }, { 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) }, -- cgit v1.2.3 From 1e8588e8c310a628fb0215ef15dc3cc29f98c336 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 2 Oct 2016 16:17:21 +0200 Subject: erts: Refactor hipe_loader_state_dtor into a true destructor that is only called once. Basically switch hipe_free_loader_state and hipe_loader_state_dtor. --- erts/emulator/beam/beam_load.c | 2 +- erts/emulator/hipe/hipe_load.c | 26 +++++++++++--------------- erts/emulator/hipe/hipe_load.h | 2 +- 3 files changed, 13 insertions(+), 17 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 4a833689e3..813788f66c 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -6528,7 +6528,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - hipe_free_loader_state(hipe_magic); + hipe_free_loader_state(hipe_stp); return mod; } diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index 48f0cab293..6261857d8f 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -32,13 +32,8 @@ #include "erl_binary.h" #include "hipe_load.h" -/* - * This destructor function can safely be called multiple times. - */ -static void -hipe_loader_state_dtor(Binary* magic) +void hipe_free_loader_state(HipeLoaderState *stp) { - HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); if (stp->module == NIL) return; erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n", @@ -64,6 +59,16 @@ hipe_loader_state_dtor(Binary* magic) */ } +static void +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor); + + hipe_free_loader_state(stp); +} + Binary *hipe_alloc_loader_state(Eterm module) { HipeLoaderState *stp; @@ -90,15 +95,6 @@ Binary *hipe_alloc_loader_state(Eterm module) return magic; } -void hipe_free_loader_state(Binary *magic) -{ - if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) - return; - - /* Why does BEAM do a refc_dec here? What is holding that reference? */ - hipe_loader_state_dtor(magic); -} - HipeLoaderState * hipe_get_loader_state(Binary *magic) { diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h index 17dd3c3c05..40c8a8aa2a 100644 --- a/erts/emulator/hipe/hipe_load.h +++ b/erts/emulator/hipe/hipe_load.h @@ -42,7 +42,7 @@ typedef struct hipe_loader_state { } HipeLoaderState; extern Binary *hipe_alloc_loader_state(Eterm module); -extern void hipe_free_loader_state(Binary *binary); +extern void hipe_free_loader_state(HipeLoaderState*); extern HipeLoaderState *hipe_get_loader_state(Binary *binary); #endif /* HIPE_LOAD_H */ -- cgit v1.2.3 From 663360ea2bb7766ad2015fff7f3f8a8f53c3eef2 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 2 Oct 2016 22:49:59 +0200 Subject: erts: Refactor out hipe_purge_refs/sdesc from hipe_purge_module. --- erts/emulator/hipe/hipe_bif0.c | 119 +++++++++++++++++++++++------------------ erts/emulator/hipe/hipe_bif0.h | 2 + 2 files changed, 68 insertions(+), 53 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 67508e6f5d..58eee1693f 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1654,11 +1654,66 @@ int hipe_purge_need_blocking(Module* modp) return !modp->curr.code_hdr && modp->first_hipe_mfa; } -void hipe_purge_module(Module* modp) +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) { - struct hipe_ref* ref; - struct hipe_sdesc* sdesc; + struct hipe_ref* ref = first_ref; + + while (ref) { + struct hipe_ref* free_ref = ref; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == caller_module); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + unlink_mfa_from_mod(p); + purge_mfa(p); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); + } +} +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module) +{ + struct hipe_sdesc* sdesc = first_sdesc; + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(make_atom(sdesc->m_aix) == module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } +} + + +void hipe_purge_module(Module* modp) +{ ASSERT(modp); DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); @@ -1667,62 +1722,20 @@ void hipe_purge_module(Module* modp) /* * Remove all hipe_ref's (external calls) from the old module instance */ - ref = modp->old.hipe_code->first_hipe_ref; - - while (ref) { - struct hipe_ref* free_ref = ref; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, - ref->callee->m, ref->callee->f, ref->callee->a); - DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, - ref->caller_m, ref->caller_f, ref->caller_a); - ASSERT(ref->caller_m == make_atom(modp->module)); - - /* - * Unlink from other refs to same callee - */ - ASSERT(ref->head.next->prev == &ref->head); - ASSERT(ref->head.prev->next == &ref->head); - ASSERT(ref->head.next != &ref->head); - ASSERT(ref->head.prev != &ref->head); - ref->head.next->prev = ref->head.prev; - ref->head.prev->next = ref->head.next; - - /* - * Was this the last ref to that callee? - */ - if (ref->head.next == ref->head.prev) { - struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); - if (p->is_stub) { - unlink_mfa_from_mod(p); - purge_mfa(p); - } - } - - ref = ref->next_from_modi; - erts_free(ERTS_ALC_T_HIPE, free_ref); + if (modp->old.hipe_code->first_hipe_ref) { + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, + make_atom(modp->module)); + modp->old.hipe_code->first_hipe_ref = NULL; } - modp->old.hipe_code->first_hipe_ref = NULL; /* * Remove all hipe_sdesc's for the old module instance */ - sdesc = modp->old.hipe_code->first_hipe_sdesc; - - while (sdesc) { - struct hipe_sdesc* free_sdesc = sdesc; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); - ASSERT(sdesc->m_aix == modp->module); - - sdesc = sdesc->next_in_modi; - hipe_destruct_sdesc(free_sdesc); + if (modp->old.hipe_code->first_hipe_sdesc) { + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, + make_atom(modp->module)); + modp->old.hipe_code->first_hipe_sdesc = NULL; } - modp->old.hipe_code->first_hipe_sdesc = NULL; hipe_free_module(modp->old.hipe_code); modp->old.hipe_code = NULL; diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 1914405e2d..2b7c01b975 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -44,6 +44,8 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); /* needed in beam_load.c */ int hipe_need_blocking(Module*); int hipe_purge_need_blocking(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm); void hipe_purge_module(Module*); void hipe_redirect_to_module(Module* modp); -- cgit v1.2.3 From d9e8fa3d13be54246297ac9776e835ce0bdcd362 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 2 Oct 2016 22:51:49 +0200 Subject: erts: Free hipe_refs and hipe_sdesc of a failed load --- erts/emulator/hipe/hipe_load.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index 6261857d8f..c05dd886a4 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -31,6 +31,7 @@ #include "global.h" #include "erl_binary.h" #include "hipe_load.h" +#include "hipe_bif0.h" void hipe_free_loader_state(HipeLoaderState *stp) { @@ -51,12 +52,16 @@ void hipe_free_loader_state(HipeLoaderState *stp) stp->data_segment = NULL; stp->data_segment_size = 0; - stp->module = NIL; + if (stp->new_hipe_refs) { + hipe_purge_refs(stp->new_hipe_refs, stp->module); + stp->new_hipe_refs = NULL; + } + if (stp->new_hipe_sdesc) { + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module); + stp->new_hipe_sdesc = NULL; + } - ASSERT(!stp->new_hipe_refs && !stp->new_hipe_sdesc); - /* ToDO: Purge lists 'new_hipe_refs' and 'new_hipe_sdesc' - * if this is a failed load. - */ + stp->module = NIL; } static void -- cgit v1.2.3 From d347e91735cce9ace9c376ba4913fcf688da22f8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 3 Oct 2016 19:22:08 +0200 Subject: erts: Remove debug printout for hipe loader state creation and destruction. --- erts/emulator/hipe/hipe_load.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index c05dd886a4..fcbcf1a6a4 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -37,9 +37,6 @@ void hipe_free_loader_state(HipeLoaderState *stp) { if (stp->module == NIL) return; - erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n", - stp->module); - // TODO: Needs to be freed separately. We'd like have a unified executable // code allocator, so postpone this for now. /* if (stp->text_segment) */ @@ -81,8 +78,6 @@ Binary *hipe_alloc_loader_state(Eterm module) if (is_not_atom(module)) return NULL; - erts_fprintf(stderr, "Creating HiPE loader state for module %T\n", module); - magic = erts_create_magic_binary(sizeof(HipeLoaderState), hipe_loader_state_dtor); erts_refc_inc(&magic->refc, 1); -- cgit v1.2.3 From 6eee26d55f32113e3ea4575e5500f2fc3c312fff Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 4 Oct 2016 21:46:53 +0200 Subject: erts: Enable exec_alloc for all hipe architectures For non-amd64 it's a "normal" allocator with a wrapper around mseg_alloc to call mprotect(PROT_EXEC). --- erts/emulator/beam/erl_alloc.c | 16 +++++++++---- erts/emulator/beam/erl_alloc_util.c | 48 ++++++++++++++++++++++++++++++++++++- erts/emulator/beam/erl_alloc_util.h | 6 +++++ erts/emulator/sys/common/erl_mmap.c | 4 ++-- erts/emulator/sys/common/erl_mmap.h | 5 ++-- erts/emulator/sys/common/erl_mseg.c | 2 +- 6 files changed, 70 insertions(+), 11 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index d96b90fd55..56aa78a08a 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->init.util.rmbcmt = 0; ip->init.util.acul = 0; +# ifdef ERTS_HAVE_EXEC_MMAPPER ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; ip->init.util.mseg_mmapper = &erts_exec_mmapper; +# else + ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc; +# endif } #endif /* ERTS_ALC_A_EXEC */ @@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'X': if (has_prefix("scs", argv[i]+3)) { -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER init->mseg.exec_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); @@ -2852,7 +2858,7 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); #endif @@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc) #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) terms[length++] = ERTS_MAKE_AM("literal_mmap"); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER terms[length++] = ERTS_MAKE_AM("exec_mmap"); #endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair) # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) struct erts_mmap_info_struct mmap_info_literal; # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER struct erts_mmap_info_struct mmap_info_exec; # endif #endif @@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair) erts_bld_atom(hpp,szp,"literal_mmap"), ainfo); # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); ainfo = (air->only_sz ? NIL : diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 2995f2f822..77be48035d 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -/* Used by literal allocator that has its own mmapper (super carrier) */ +/* For allocators that have their own mmapper (super carrier), + * like literal_alloc and exec_alloc on amd64 + */ void* erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { @@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ +#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) + +/* + * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + */ +void* +erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res = erts_alcu_mseg_alloc(allctr, size_p, flags); + + if (res) { + int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void* +erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + + if (seg && old_size) { + int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) { + int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void +erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +{ + int r = mprotect(seg, size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + erts_alcu_mseg_dealloc(allctr, seg, size, flags); +} +#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ + #endif /* HAVE_ERTS_MSEG */ void* diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f50f09907a..2958bdf8d1 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif + +# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif #endif /* HAVE_ERTS_MSEG */ void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 7bbb406f29..3ab03971a5 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -21,6 +21,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_process.h" #include "erl_smp.h" @@ -358,12 +359,11 @@ char* erts_literals_start; UWord erts_literals_size; #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER ErtsMemMapper erts_exec_mmapper; #endif - #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ do { \ mm->size.supercarrier.used.total += (SZ); \ diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index fa51b663fa..92e9eb9e41 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -159,9 +159,10 @@ Eterm erts_mmap_info_options(ErtsMemMapper*, extern ErtsMemMapper erts_dflt_mmapper; # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) extern ErtsMemMapper erts_literal_mmapper; -# endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_ALC_A_EXEC +# define ERTS_HAVE_EXEC_MMAPPER extern ErtsMemMapper erts_exec_mmapper; +# endif # endif #endif /* ERTS_WANT_MEM_MAPPERS */ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index f3306a888c..e76082177e 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1414,7 +1414,7 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER /* Initialize erts_exec_mapper *FIRST*, to increase probability * of getting low memory for HiPE AMD64's small code model. */ -- cgit v1.2.3 From d9149ef5e2591d34ba781a08b1ae3e1823f7a13a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 5 Oct 2016 17:54:15 +0200 Subject: erts: Fix old leak of hipe code on x86 32-bit --- erts/emulator/hipe/hipe_x86.c | 112 ++---------------------------------------- 1 file changed, 3 insertions(+), 109 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 8a3ba8bc7f..b091bee6fb 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -70,110 +69,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -/* FreeBSD 6.1 and Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -186,11 +84,7 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * void hipe_free_code(void* code, unsigned int bytes) { - /* SVERK: Leak !!! - ALLOC_CODE_STATS(--nr_allocs); - ALLOC_CODE_STATS(total_alloc -= bytes); - - erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -275,7 +169,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) void hipe_free_native_stub(void* stub) { - /* SVERK: leak leak drip drop */ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } void hipe_arch_print_pcb(struct hipe_process_state *p) -- cgit v1.2.3 From e048ebf4bddf8434d0f4402863c0f6313a741df0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 5 Oct 2016 19:48:01 +0200 Subject: erts: Fix old leak of sparc hipe code --- erts/emulator/hipe/hipe_sparc.c | 109 ++++------------------------------------ 1 file changed, 11 insertions(+), 98 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 23020f34ee..75bfb4fef1 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int nrbytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; @@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ -- cgit v1.2.3 From 25b28611ea06d3e2d3708d20e53118ec24ae8321 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 5 Oct 2016 19:09:28 +0200 Subject: erts: Fix old leak for arm hipe code * Use erts_alloc(ERTS_ALC_T_HIPE_EXEC,_) * Each module has its own trampolines * Stubs do not use trampolines, they do their own long jumps. --- erts/emulator/hipe/hipe_arm.c | 201 ++++++++++++----------------------------- erts/emulator/hipe/hipe_bif0.c | 1 - 2 files changed, 57 insertions(+), 145 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index 64e35b62d8..b61939724c 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address) hipe_flush_icache_range(address, 4); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ - /* On ARM we always allocate a trampoline at base+32MB-8 for - nbif_callemu, so tramp_pos <= base+32MB-8. */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} static int check_callees(Eterm callees) { @@ -107,136 +82,53 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) -{ - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; - Eterm mfa, m, f; - unsigned int a, *trampoline; +#define TRAMPOLINE_WORDS 2 - m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */ - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - mfa = tuple_val(callees)[trampnr]; - if (is_atom(mfa)) - trampoline = hipe_primop_get_trampoline(mfa); - else { - m = tuple_val(mfa)[1]; - f = tuple_val(mfa)[2]; - a = unsigned_val(tuple_val(mfa)[3]); - trampoline = hipe_mfa_get_trampoline(m, f, a); - } - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { - if (nrfreewords < 2) - return NULL; - nrfreewords -= 2; - tramp_pos = trampoline = tramp_pos - 2; - trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - trampoline[1] = 0; /* callee's address */ - hipe_flush_icache_range(trampoline, 2*sizeof(int)); - if (is_atom(mfa)) - hipe_primop_set_trampoline(mfa, trampoline); - else - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) +{ + Uint32* trampoline = address; + int i; + + for (i = 0; i < nrcallees; ++i) { + trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ + trampoline[1] = 0; /* callee's address */ + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } void hipe_free_code(void* code, unsigned int bytes) { - /*SVERK: Leaking code memory */ -} - -static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) -{ - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2; - return address; -} - -void hipe_free_native_stub(void* stub) -{ - /*SVERK: Leaking code stub */ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } /* @@ -276,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; - unsigned int *tramp_callemu; int callemu_offset; + int is_short_jmp; /* * Native code calls BEAM via a stub looking as follows: @@ -287,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) * b nbif_callemu * .long callee_exp * + * or if nbif_callemu is too far away: + * + * mov r0, #beamArity + * ldr r8, [pc,#0] // callee_exp + * ldr pc, [pc,#0] // nbif_callemu + * .long callee_exp + * .long nbif_callemu + * * I'm using r0 and r8 since they aren't used for - * parameter passing in native code. The branch to - * nbif_callemu may need to go via a trampoline. - * (Trampolines are allowed to modify r12, but they don't.) + * parameter passing in native code. */ - code = alloc_stub(4, &tramp_callemu); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32)); if (!code) return NULL; callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) { - callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) - abort(); + is_short_jmp = (callemu_offset >= -0x00800000 && + callemu_offset <= 0x007FFFFF); +#ifdef DEBUG + if (is_short_jmp && (callemu_offset % 3)==0) { + is_short_jmp = 0; } +#endif /* mov r0, #beamArity */ code[0] = 0xE3A00000 | (beamArity & 0xFF); /* ldr r8, [pc,#0] // callee_exp */ code[1] = 0xE59F8000; - /* b nbif_callemu */ - code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + if (is_short_jmp) { + /* b nbif_callemu */ + code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + } + else { + /* ldr pc, [pc,#0] // nbif_callemu */ + code[2] = 0xE59FF000; + /* .long nbif_callemu */ + code[4] = (unsigned int)&nbif_callemu; + } /* .long callee_exp */ code[3] = (unsigned int)callee_exp; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 5*sizeof(Uint32)); return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA) { Uint32 oldI = *address; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 58eee1693f..f9a657124a 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -417,7 +417,6 @@ BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) ASSERT(bitoffs == 0); ASSERT(bitsize == 0); trampolines = NIL; - // XXX: Trampolines are not tracked and freed address = hipe_alloc_code(nrbytes, BIF_ARG_2, &trampolines, BIF_P); if (!address) { Uint nrcallees; -- cgit v1.2.3 From 65fbc0464cce8266024129fcf10ef0906f907eb4 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 12 Oct 2016 19:00:53 +0200 Subject: erts: Fix old leak for ppc hipe code * Use erts_alloc(ERTS_ALC_T_HIPE_EXEC,_) * Each module has its own trampolines --- erts/emulator/hipe/hipe_ppc.c | 178 ++++++++++-------------------------------- 1 file changed, 42 insertions(+), 136 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index a1a6e2ad02..4413748936 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes) asm volatile("sync\n\tisync"); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -/* Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} - static int check_callees(Eterm callees) { Eterm *tuple; @@ -119,136 +90,71 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) + +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) { - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; + Uint32* trampoline = address; + int i; - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - Eterm mfa = tuple_val(callees)[trampnr]; - Eterm m = tuple_val(mfa)[1]; - Eterm f = tuple_val(mfa)[2]; - unsigned int a = unsigned_val(tuple_val(mfa)[3]); - unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a); - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { + for (i = 0; i < nrcallees; ++i) { #if defined(__powerpc64__) - if (nrfreewords < 7) - return NULL; - nrfreewords -= 7; - tramp_pos = trampoline = tramp_pos - 7; - trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ - trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ - trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ - trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[5] = 0x7D6903A6; /* mtctr r11 */ - trampoline[6] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 7*sizeof(int)); +# define TRAMPOLINE_WORDS 7 + trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ + trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ + trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ + trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[5] = 0x7D6903A6; /* mtctr r11 */ + trampoline[6] = 0x4E800420; /* bctr */ #else - if (nrfreewords < 4) - return NULL; - nrfreewords -= 4; - tramp_pos = trampoline = tramp_pos - 4; - trampoline[0] = 0x39600000; /* addi r11,r0,0 */ - trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ - trampoline[2] = 0x7D6903A6; /* mtctr r11 */ - trampoline[3] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 4*sizeof(int)); +# define TRAMPOLINE_WORDS 4 + trampoline[0] = 0x39600000; /* addi r11,r0,0 */ + trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ + trampoline[2] = 0x7D6903A6; /* mtctr r11 */ + trampoline[3] = 0x4E800420; /* bctr */ #endif - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); + + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); + + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } void hipe_free_code(void* code, unsigned int bytes) { - /*SVERK: Leaking code memory */ -} - -static unsigned int *alloc_stub(Uint nrwords) -{ - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } void hipe_free_native_stub(void* stub) { - /*SVERK: Leaking code stubs */ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } static void patch_imm16(Uint32 *address, unsigned int imm16) @@ -298,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(7); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32)); if (!code) return NULL; @@ -322,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[6] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 7*sizeof(int)); + hipe_flush_icache_range(code, 7*sizeof(Uint32)); return code; } @@ -370,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; /* * Native code calls BEAM via a stub looking as follows: @@ -393,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(4); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32)); if (!code) return NULL; @@ -406,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[3] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 4*sizeof(Uint32)); return code; } -- cgit v1.2.3 From 6cd1469d0c8f9af602332b20772134c1241f6429 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 6 Oct 2016 20:09:21 +0200 Subject: erts: Fix bug in stack walk on risc Symptom: Got ra==NULL at nsp==nsp_end when running basic_SUITE:basic_arith on arm why has this not been detected before? --- erts/emulator/hipe/hipe_risc_stack.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index 7528a3ab00..4001bedeb6 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) ra = (unsigned long)p->hipe.nra; prev_ra = 0; i = 0; - for (;;) { + while (nsp < nsp_end) { if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; if (ra != prev_ra) { @@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) break; prev_ra = ra; } - if (nsp >= nsp_end) - break; sdesc = hipe_find_sdesc(ra); nsp += arity + sdesc_fsize(sdesc); arity = sdesc_arity(sdesc); -- cgit v1.2.3 From deed6619dee9db9c0347b25987f3a91d51424079 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 7 Oct 2016 16:13:55 +0200 Subject: erts: Let code:make_stub_module raise 'notsup' if hipe is disabled. Makes the code simpler to just ifdef away a lot of hipe stuff. --- erts/emulator/beam/beam_bif_load.c | 10 +++++++--- erts/emulator/beam/beam_load.c | 29 +++++++++-------------------- 2 files changed, 16 insertions(+), 23 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index f5d1ca7e54..0a4d89d51b 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -149,6 +149,9 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { +#if !defined(HIPE) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else Module* modp; Eterm res, mod; @@ -181,11 +184,9 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); -#ifdef HIPE if (!modp) modp = erts_get_module(mod, erts_active_code_ix()); hipe_redirect_to_module(modp); -#endif } else { erts_abort_staging_code_ix(); @@ -194,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; +#endif } BIF_RETTYPE @@ -1069,9 +1071,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) BeamInstr* start; char* mod_start; Uint mod_size; + Eterm* sp; +#ifdef HIPE void *nat_start = NULL; Uint nat_size = 0; - Eterm* sp; +#endif *redsp += 1; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 813788f66c..4833bcc6e9 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -481,10 +481,12 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, Uint size, HipeModule *hipe_code); +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -539,8 +541,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); static Eterm md5_of_module(Process* p, BeamCodeHeader*); static Eterm has_native(BeamCodeHeader*); static Eterm native_addresses(Process* p, BeamCodeHeader*); -int patch_funentries(Eterm Patchlist); -int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -1098,6 +1098,7 @@ loader_state_dtor(Binary* magic) ASSERT(stp->genop_blocks == 0); } +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, @@ -1125,11 +1126,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ -#if defined(HIPE) DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " "first_hipe_ref = %p", hipe_code->first_hipe_ref); modp->curr.hipe_code = hipe_code; -#endif /* * Update ranges (used for finding a function from a PC value). @@ -1138,6 +1137,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size) @@ -5997,6 +5997,7 @@ code_module_md5_1(BIF_ALIST_1) return res; } +#ifdef HIPE #define WORDS_PER_FUNCTION 6 static BeamInstr* @@ -6075,9 +6076,7 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) Eterm mod = fp[2]; Eterm function = fp[3]; int arity = fp[4]; -#ifdef HIPE Lambda* lp; -#endif if (is_bif(mod, function, arity)) { fp[1] = 0; @@ -6106,7 +6105,6 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) * Search the lambda table to find out which. */ -#ifdef HIPE n = stp->num_lambdas; for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; @@ -6115,7 +6113,6 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) fe->address = &(fp[5]); } } -#endif return; } @@ -6124,10 +6121,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) [{Adr, Patchtyppe} | Addresses] and the address of a fun_entry. */ -int +static int patch(Eterm Addresses, Uint fe) { -#ifdef HIPE Eterm* listp; Eterm tuple; Eterm* tp; @@ -6163,15 +6159,13 @@ patch(Eterm Addresses, Uint fe) } -#endif return 1; } -int +static int patch_funentries(Eterm Patchlist) { -#ifdef HIPE while (!is_nil(Patchlist)) { Eterm Info; Eterm MFA; @@ -6260,18 +6254,15 @@ patch_funentries(Eterm Patchlist) return 0; } -#endif return 1; /* Signal that all went well */ } - /* * Do a dummy load of a module. No threaded code will be loaded. * Used for loading native code. * Will also patch all references to fun_entries to point to * the new fun_entries created. */ - Eterm erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { @@ -6444,11 +6435,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) * as the body until we know what kind of trap we should put there. */ code_hdr->functions[i] = fp; -#ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ -#else - op = (Eterm) BeamOpCode(op_move_return_n); -#endif fp = make_stub(fp, hipe_stp->module, func, arity, (Uint)native_address, op); } @@ -6580,6 +6567,8 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) } #undef WORDS_PER_FUNCTION +#endif /* HIPE */ + static int safe_mul(UWord a, UWord b, UWord* resp) { -- cgit v1.2.3 From 9aa58e5e93af074d1e5e54848c29d40d826de361 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 7 Oct 2016 16:18:07 +0200 Subject: erts: Remove code_SUITE:make_stub and make_stub_many_funs Hard to unit test now when it takes a magic HipeLoaderState as argument. All hipe testing should be enough exercise for code:make_stub_module. --- erts/emulator/test/code_SUITE.erl | 69 ++------------------------------------- 1 file changed, 3 insertions(+), 66 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index b76ba6427f..465ddfa7a8 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -23,8 +23,8 @@ versions/1,new_binary_types/1, call_purged_fun_code_gone/1, call_purged_fun_code_reload/1, call_purged_fun_code_there/1, multi_proc_purge/1, t_check_old_code/1, - external_fun/1,get_chunk/1,module_md5/1,make_stub/1, - make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1, + external_fun/1,get_chunk/1,module_md5/1, + constant_pools/1,constant_refc_binaries/1, false_dependency/1,coverage/1,fun_confusion/1, t_copy_literals/1, t_copy_literals_frags/1]). @@ -37,7 +37,7 @@ all() -> [versions, new_binary_types, call_purged_fun_code_gone, call_purged_fun_code_reload, call_purged_fun_code_there, multi_proc_purge, t_check_old_code, external_fun, get_chunk, - module_md5, make_stub, make_stub_many_funs, + module_md5, constant_pools, constant_refc_binaries, false_dependency, coverage, fun_confusion, t_copy_literals, t_copy_literals_frags]. @@ -442,69 +442,6 @@ module_md5_ok(Code) -> end. -make_stub(Config) when is_list(Config) -> - catch erlang:purge_module(my_code_test), - MD5 = erlang:md5(<<>>), - Arg3 = {[], [], MD5, 0, 0}, - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - {ok,my_code_test,Code} = compile:file(File, [binary]), - - my_code_test = code:make_stub_module(my_code_test, Code, Arg3), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, - make_unaligned_sub_binary(Code), - Arg3), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code), - Arg3), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, <<"bad">>, Arg3)), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, - bit_sized_binary(Code), - Arg3)), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test_with_wrong_name, - Code, Arg3)), - ok. - -make_stub_many_funs(Config) when is_list(Config) -> - catch erlang:purge_module(many_funs), - MD5 = erlang:md5(<<>>), - Arg3 = {[], [], MD5, 0, 0}, - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "many_funs"), - {ok,many_funs,Code} = compile:file(File, [binary]), - - many_funs = code:make_stub_module(many_funs, Code, Arg3), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - many_funs = code:make_stub_module(many_funs, - make_unaligned_sub_binary(Code), - Arg3), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, <<"bad">>, Arg3)), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, - bit_sized_binary(Code), - Arg3)), - ok. - constant_pools(Config) when is_list(Config) -> Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "literals"), -- cgit v1.2.3 From df9e09fc1f938f9902052dcb896b42558ee9779d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 7 Oct 2016 15:54:33 +0200 Subject: erts: Remove dead alloc stats in hipe_amd64.c erlang:system_info({allocator, exec_alloc}) have stats if we need it. --- erts/emulator/hipe/hipe_amd64.c | 32 -------------------------------- 1 file changed, 32 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 96b8c22265..3173afad0a 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - /* * Memory allocator for executable code. * @@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void) */ static void *alloc_code(unsigned int alloc_bytes) { - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } @@ -132,9 +106,6 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * void hipe_free_code(void* code, unsigned int bytes) { - ALLOC_CODE_STATS(--nr_allocs); - ALLOC_CODE_STATS(total_alloc -= bytes); - erts_free(ERTS_ALC_T_HIPE_EXEC, code); } @@ -244,9 +215,6 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) void hipe_free_native_stub(void* stub) { - ALLOC_CODE_STATS(++nr_allocs); - /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ - erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } -- cgit v1.2.3 From 8bb80fe76f5b4debe981e42ba70260c5a12b250f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 14 Oct 2016 14:43:22 +0200 Subject: erts: Cleanup hipe trampoline code --- erts/emulator/hipe/hipe_amd64.c | 4 +-- erts/emulator/hipe/hipe_bif0.c | 70 +++++++---------------------------------- erts/emulator/hipe/hipe_bif0.h | 8 ----- erts/emulator/hipe/hipe_sparc.c | 4 +-- erts/emulator/hipe/hipe_x86.c | 4 +-- 5 files changed, 18 insertions(+), 72 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 3173afad0a..e3cff4a4ba 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Sint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Sint)destAddress - (Sint)callAddress - 4; if ((Sint)(Sint32)rel32 != rel32) return -1; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index f9a657124a..0402a4de98 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -819,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3) struct primop { HashBucket bucket; /* bucket.hvalue == atom_val(name) */ const void *address; -#if defined(__arm__) - void *trampoline; -#endif }; static struct primop primops[] = { @@ -880,29 +877,6 @@ static struct primop *primop_table_get(Eterm name) return hash_get(&primop_table, &tmpl); } -#if defined(__arm__) -static struct primop *primop_table_put(Eterm name) -{ - struct primop tmpl; - - init_primop_table(); - tmpl.bucket.hvalue = atom_val(name); - return hash_put(&primop_table, &tmpl); -} - -void *hipe_primop_get_trampoline(Eterm name) -{ - struct primop *primop = primop_table_get(name); - return primop ? primop->trampoline : NULL; -} - -void hipe_primop_set_trampoline(Eterm name, void *trampoline) -{ - struct primop *primop = primop_table_put(name); - primop->trampoline = trampoline; -} -#endif - /* * hipe_bifs_primop_address(Atom) -> address or false */ @@ -1062,9 +1036,6 @@ struct hipe_mfa_info { void *new_address; struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ struct hipe_mfa_info* next_in_mod; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - void *trampoline; -#endif #ifdef DEBUG Export* dbg_export; #endif @@ -1094,7 +1065,9 @@ static struct { struct hipe_ref { struct hipe_ref_head head; /* list of refs to same calleee */ void *address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void *trampoline; +#endif unsigned int flags; struct hipe_ref* next_from_modi; /* list of refs from same module instance */ #if defined(DEBUG) @@ -1181,9 +1154,6 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->callers.next = &res->callers; res->callers.prev = &res->callers; res->next_in_mod = NULL; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - res->trampoline = NULL; -#endif #ifdef DEBUG res->dbg_export = NULL; #endif @@ -1294,30 +1264,6 @@ static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) hipe_mfa_info_table_rwunlock(); } -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity) -{ - struct hipe_mfa_info *p; - void *trampoline; - - hipe_mfa_info_table_rlock(); - p = hipe_mfa_info_table_get_locked(m, f, arity); - trampoline = p ? p->trampoline : NULL; - hipe_mfa_info_table_runlock(); - return trampoline; -} - -void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline) -{ - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(m, f, arity); - p->trampoline = trampoline; - hipe_mfa_info_table_rwunlock(); -} -#endif - BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { struct hipe_mfa mfa; @@ -1531,7 +1477,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; +#endif ref->flags = flags; /* @@ -1815,8 +1763,14 @@ void hipe_redirect_to_module(Module* modp) if (ref->flags & REF_FLAG_IS_LOAD_MFA) res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline); + else { +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void* trampoline = ref->trampoline; +#else + void* trampoline = NULL; +#endif + res = hipe_patch_call(ref->address, p->remote_address, trampoline); + } if (res) fprintf(stderr, "%s: patch failed", __FUNCTION__); } diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 2b7c01b975..02f0d6c823 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -32,14 +32,6 @@ extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a); -extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline); -#endif -#if defined(__arm__) -extern void *hipe_primop_get_trampoline(Eterm name); -extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); -#endif /* needed in beam_load.c */ int hipe_need_blocking(Module*); diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 75bfb4fef1..876b20bb15 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -87,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint32 relDest, newI; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress); newI = (1 << 30) | (relDest >> 2); *(Uint32*)callAddress = newI; diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index b091bee6fb..c7e24673ac 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -61,8 +61,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Uint)destAddress - (Uint)callAddress - 4; *(Uint32*)callAddress = rel32; hipe_flush_icache_word(callAddress); -- cgit v1.2.3 From 7d5021bc9e1150b5d839438756851b3175f3ba85 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 12 Oct 2016 16:56:41 +0200 Subject: erts: Disable DBG_TRACE_MFA for debug build --- erts/emulator/beam/beam_load.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 5f0f34fcdd..43cf6597df 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -151,10 +151,16 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) -#ifdef DEBUG -# define ENABLE_DBG_TRACE_MFA -#endif +/* + * MFA event debug "tracing" usage: + * + * #define ENABLE_DBG_TRACE_MFA + * call dbg_set_traced_mfa("mymod","myfunc",arity) + * for the function(s) to trace, in some init function. + * + * Run and get stderr printouts when interesting things happen to your MFA. + */ #ifdef ENABLE_DBG_TRACE_MFA void dbg_set_traced_mfa(const char* m, const char* f, Uint a); -- cgit v1.2.3 From 3582e6f420d84ddac64b55cb13100f1ae7f31b08 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 13 Oct 2016 20:59:58 +0200 Subject: erts: Replace unsafe Module.first_hipe_ref with hash table mod2mfa_tab --- erts/emulator/beam/beam_bif_load.c | 2 +- erts/emulator/beam/module.c | 7 +- erts/emulator/beam/module.h | 3 - erts/emulator/hipe/hipe_bif0.c | 196 ++++++++++++++++++++++++++++--------- erts/emulator/hipe/hipe_bif0.h | 6 +- erts/emulator/hipe/hipe_load.c | 4 +- 6 files changed, 155 insertions(+), 63 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 0a4d89d51b..c31abbb84f 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1711,7 +1711,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); #ifdef HIPE - hipe_purge_module(modp); + hipe_purge_module(modp, is_blocking); #endif ERTS_BIF_PREP_RET(ret, am_true); } diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index e9b18ba525..3e622c0f3a 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -91,9 +91,6 @@ static Module* module_alloc(Module* tmpl) erts_module_instance_init(&obj->curr); erts_module_instance_init(&obj->old); obj->on_load = 0; -#ifdef HIPE - obj->first_hipe_mfa = NULL; -#endif DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -128,6 +125,7 @@ void init_module_table(void) erts_smp_atomic_init_nob(&tot_module_bytes, 0); } + Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { @@ -208,9 +206,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) dst_mod->curr = src_mod->curr; dst_mod->old = src_mod->old; dst_mod->on_load = src_mod->on_load; -#ifdef HIPE - dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa; -#endif } void module_start_staging(void) diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 9de610ccf7..1d6d9dd463 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -47,9 +47,6 @@ typedef struct erl_module { struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ struct erl_module_instance* on_load; -#ifdef HIPE - struct hipe_mfa_info* first_hipe_mfa; -#endif } Module; void erts_module_instance_init(struct erl_module_instance* modi); diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 0402a4de98..4c9757e8d3 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1024,6 +1024,7 @@ struct hipe_ref_head { * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { + HashBucket mod2mfa; struct { unsigned long hvalue; struct hipe_mfa_info *next; @@ -1057,6 +1058,61 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; +Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */ + +static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa) +{ + return mfa->mod2mfa.hvalue; +} + +static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa) +{ + return tmpl->hvalue != mfa->mod2mfa.hvalue; +} + +static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl) +{ + return tmpl; /* hash_put always use mfa itself at template */ +} + +static void mod2mfa_free(struct hipe_mfa_info* mfa) +{ +} + +static void mod2mfa_tab_init(void) +{ + HashFunctions f; + static int init_done = 0; + + if (init_done) + return; + init_done = 1; + + f.hash = (H_FUN) mod2mfa_hash; + f.cmp = (HCMP_FUN) mod2mfa_cmp; + f.alloc = (HALLOC_FUN) mod2mfa_alloc; + f.free = (HFREE_FUN) mod2mfa_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; + + hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f); +} + +static struct hipe_mfa_info* mod2mfa_get(Module* modp) +{ + HashBucket tmpl; + tmpl.hvalue = modp->module; + return hash_get(&mod2mfa_tab, &tmpl); +} + +static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) +{ + mfa->mod2mfa.hvalue = atom_val(mfa->m); + return hash_put(&mod2mfa_tab, mfa); +} + + /* * An external native call site M:F(...) @@ -1103,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } +static ERTS_INLINE +struct hipe_mfa_info* mod2mfa_get_safe(Module* modp) +{ + struct hipe_mfa_info* mfa; + hipe_mfa_info_table_rlock(); + mfa = mod2mfa_get(modp); + hipe_mfa_info_table_runlock(); + return mfa; +} + #define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) @@ -1173,6 +1239,8 @@ void hipe_mfa_info_table_init(void) hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size); hipe_mfa_info_table_init_lock(); + + mod2mfa_tab_init(); } static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity) @@ -1199,8 +1267,8 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned long h; unsigned int i; struct hipe_mfa_info *p; + struct hipe_mfa_info *first_in_mod; unsigned int size; - Module* modp; h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; @@ -1221,10 +1289,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); - modp = erts_put_active_module(m); - ASSERT(modp); - p->next_in_mod = modp->first_hipe_mfa; - modp->first_hipe_mfa = p; + first_in_mod = mod2mfa_put(p); + if (p != first_in_mod) { + p->next_in_mod = first_in_mod->next_in_mod; + first_in_mod->next_in_mod = p; + } + else { + p->next_in_mod = NULL; + } DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); @@ -1288,7 +1360,7 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) /* Ask if we need to block all threads - * while loading/deleting (beam) code for this module? + * while loading/deleting code for this module? */ int hipe_need_blocking(Module* modp) { @@ -1298,7 +1370,7 @@ int hipe_need_blocking(Module* modp) * or native code to make unaccessible. */ hipe_mfa_info_table_rlock(); - for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { ASSERT(!p->new_address); if (p->callers.next != &p->callers || !p->is_stub) { break; @@ -1519,19 +1591,25 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix()); - struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; struct hipe_mfa_info* p; - ASSERT(modp); - for (;;) { - p = *prevp; - ASSERT(p && p->m == unlink_me->m); - if (p == unlink_me) { - *prevp = p->next_in_mod; - break; - } - prevp = &p->next_in_mod; + p = hash_get(&mod2mfa_tab, unlink_me); + ASSERT(p); + if (p == unlink_me) { + hash_erase(&mod2mfa_tab, p); + if (p->next_in_mod) + mod2mfa_put(p->next_in_mod); + } + else { + struct hipe_mfa_info** prevp; + + do { + prevp = &p->next_in_mod; + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + } while (p != unlink_me); + + *prevp = p->next_in_mod; } } @@ -1561,18 +1639,11 @@ static void hipe_purge_all_refs(void) for (i = 0; i < nrbuckets; ++i) { ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { - Module* modp; struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - ASSERT(mfa->callers.next == &mfa->callers); - - modp = erts_get_module(mfa->m, erts_active_code_ix()); - if (modp) { - /* unsafe write to active module */ - modp->first_hipe_mfa = NULL; - } - erts_free(ERTS_ALC_T_HIPE, mfa); + hash_erase(&mod2mfa_tab, mfa); + erts_free(ERTS_ALC_T_HIPE, mfa); } } hipe_mfa_info_table.used = 0; @@ -1598,18 +1669,22 @@ int hipe_purge_need_blocking(Module* modp) modp->old.hipe_code->first_hipe_sdesc) return 1; } - return !modp->curr.code_hdr && modp->first_hipe_mfa; + if (!modp->curr.code_hdr) { + return mod2mfa_get_safe(modp) != NULL; + } + return 0; } -void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module, + int is_blocking) { struct hipe_ref* ref = first_ref; + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + while (ref) { struct hipe_ref* free_ref = ref; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, ref->callee->m, ref->callee->f, ref->callee->a); DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, @@ -1632,8 +1707,12 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) if (ref->head.next == ref->head.prev) { struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); if (p->is_stub) { + if (!is_blocking) + hipe_mfa_info_table_rwlock(); unlink_mfa_from_mod(p); purge_mfa(p); + if (!is_blocking) + hipe_mfa_info_table_rwunlock(); } } @@ -1642,14 +1721,18 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) } } -void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module) +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module, + int is_blocking) { struct hipe_sdesc* sdesc = first_sdesc; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ + while (sdesc) { struct hipe_sdesc* free_sdesc = sdesc; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); ASSERT(make_atom(sdesc->m_aix) == module); @@ -1659,10 +1742,12 @@ void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module) } -void hipe_purge_module(Module* modp) +void hipe_purge_module(Module* modp, int is_blocking) { ASSERT(modp); + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); if (modp->old.hipe_code) { @@ -1670,8 +1755,10 @@ void hipe_purge_module(Module* modp) * Remove all hipe_ref's (external calls) from the old module instance */ if (modp->old.hipe_code->first_hipe_ref) { + ERTS_SMP_LC_ASSERT(is_blocking); + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, - make_atom(modp->module)); + make_atom(modp->module), is_blocking); modp->old.hipe_code->first_hipe_ref = NULL; } @@ -1679,8 +1766,10 @@ void hipe_purge_module(Module* modp) * Remove all hipe_sdesc's for the old module instance */ if (modp->old.hipe_code->first_hipe_sdesc) { + ERTS_SMP_LC_ASSERT(is_blocking); + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, - make_atom(modp->module)); + make_atom(modp->module), is_blocking); modp->old.hipe_code->first_hipe_sdesc = NULL; } @@ -1693,17 +1782,28 @@ void hipe_purge_module(Module* modp) * Remove unreferred hipe_mfa_info's * when all module instances are removed (like in init:restart) */ - if (modp->curr.code_hdr == NULL) { - struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; - struct hipe_mfa_info* p = *prevp; - ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking()); - for (; p; p = *prevp) { - if (p->callers.next == &p->callers) { - *prevp = p->next_in_mod; - purge_mfa(p); + if (is_blocking && modp->curr.code_hdr == NULL) { + struct hipe_mfa_info* was_first = mod2mfa_get(modp); + struct hipe_mfa_info* is_first = was_first; + struct hipe_mfa_info** prevp = &is_first; + struct hipe_mfa_info *p; + + if (was_first) { + for (p = was_first ; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + if (p != was_first) + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + if (was_first != is_first) { + hash_erase(&mod2mfa_tab, was_first); + purge_mfa(was_first); + if (is_first) + mod2mfa_put(is_first); } - else - prevp = &p->next_in_mod; } } } @@ -1719,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp) ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { if (p->new_address) { if (p->is_stub) { hipe_free_native_stub(p->remote_address); diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 02f0d6c823..4a59bacc6e 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -36,9 +36,9 @@ extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned in /* needed in beam_load.c */ int hipe_need_blocking(Module*); int hipe_purge_need_blocking(Module*); -void hipe_purge_refs(struct hipe_ref*, Eterm); -void hipe_purge_sdescs(struct hipe_sdesc*, Eterm); -void hipe_purge_module(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking); +void hipe_purge_module(Module*, int is_blocking); void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index fcbcf1a6a4..2998ed87a2 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -50,11 +50,11 @@ void hipe_free_loader_state(HipeLoaderState *stp) stp->data_segment_size = 0; if (stp->new_hipe_refs) { - hipe_purge_refs(stp->new_hipe_refs, stp->module); + hipe_purge_refs(stp->new_hipe_refs, stp->module, 0); stp->new_hipe_refs = NULL; } if (stp->new_hipe_sdesc) { - hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module); + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0); stp->new_hipe_sdesc = NULL; } -- cgit v1.2.3 From a28149b2500a4db9b3c56b168dcd18effca87a3a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 17 Oct 2016 11:50:01 +0200 Subject: erts: Cleanup dead code --- erts/emulator/beam/module.c | 9 --------- erts/emulator/beam/module.h | 1 - 2 files changed, 10 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 3e622c0f3a..705dd1d43f 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -171,15 +171,6 @@ erts_put_module(Eterm mod) return put_module(mod, &module_tables[erts_staging_code_ix()]); } -Module* -erts_put_active_module(Eterm mod) -{ - ASSERT(is_atom(mod)); - //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - return put_module(mod, &module_tables[erts_active_code_ix()]); -} - Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1d6d9dd463..8f44e11f3d 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -52,7 +52,6 @@ typedef struct erl_module { void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); -Module* erts_put_active_module(Eterm mod); /* only while blocked */ void init_module_table(void); void module_start_staging(void); -- cgit v1.2.3