From d8e69ab77f32ce1503125fa464d6db3ba1c904d0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 14:38:09 +0200 Subject: erts: Refactor rename structs hipe_mfa and hipe_ref --- erts/emulator/hipe/hipe_bif0.c | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 3336fded7a..b6d0abb2d1 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -537,13 +537,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1) BIF_RET(val); } -struct mfa_t { +struct hipe_mfa { Eterm mod; Eterm fun; Uint ari; }; -static int term_to_mfa(Eterm term, struct mfa_t *mfa) +static int term_to_mfa(Eterm term, struct hipe_mfa *mfa) { Eterm mod, fun, a; Uint ari; @@ -601,7 +601,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) Uint *hipe_bifs_find_pc_from_mfa(Eterm term) { - struct mfa_t mfa; + struct hipe_mfa mfa; if (!term_to_mfa(term, &mfa)) return NULL; @@ -621,7 +621,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) Eterm *pc; void *address; int is_closure; - struct mfa_t mfa; + struct hipe_mfa mfa; switch (BIF_ARG_3) { case am_false: @@ -1017,7 +1017,7 @@ struct hipe_mfa_info { Eterm *beam_code; Uint orig_beam_op; struct hipe_mfa_info_list *refers_to; - struct ref *referred_from; + struct hipe_ref *referred_from; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) void *trampoline; #endif @@ -1235,7 +1235,7 @@ void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampol BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; int is_exported; @@ -1257,7 +1257,7 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) { Eterm lst; - struct mfa_t mfa; + struct hipe_mfa mfa; struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); @@ -1426,7 +1426,7 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; int is_remote; @@ -1512,12 +1512,12 @@ struct hipe_mfa_info_list { struct hipe_mfa_info_list *next; }; -struct ref { +struct hipe_ref { struct hipe_mfa_info *caller_mfa; void *address; void *trampoline; unsigned int flags; - struct ref *next; + struct hipe_ref *next; }; #define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ #define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ @@ -1528,16 +1528,16 @@ struct ref { */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { - struct mfa_t callee; + struct hipe_mfa callee; Eterm *tuple; - struct mfa_t caller; + struct hipe_mfa caller; void *address; void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; struct hipe_mfa_info *caller_mfa; struct hipe_mfa_info_list *refers_to; - struct ref *ref; + struct hipe_ref *ref; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1607,9 +1607,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) */ BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ { - struct mfa_t mfa; + struct hipe_mfa mfa; const struct hipe_mfa_info *p; - struct ref *ref; + struct hipe_ref *ref; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); @@ -1647,7 +1647,7 @@ static void hipe_purge_all_refs(void) erts_free(ERTS_ALC_T_HIPE, to); } while (mfa->referred_from) { - struct ref* from = mfa->referred_from; + struct hipe_ref* from = mfa->referred_from; mfa->referred_from = from->next; erts_free(ERTS_ALC_T_HIPE, from); } @@ -1659,10 +1659,10 @@ static void hipe_purge_all_refs(void) BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; struct hipe_mfa_info *caller_mfa, *callee_mfa; struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct ref **prev, *ref; + struct hipe_ref **prev, *ref; if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); @@ -1686,7 +1686,7 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) prev = &ref->next; ref = ref->next; } else { - struct ref *tmp = ref; + struct hipe_ref *tmp = ref; ref = ref->next; *prev = ref; erts_free(ERTS_ALC_T_HIPE, tmp); @@ -1713,9 +1713,9 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) */ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; struct hipe_mfa_info *p; - struct ref **prev, *ref; + struct hipe_ref **prev, *ref; int is_remote, res; void *new_address; @@ -1738,7 +1738,7 @@ BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); ref->flags &= ~REF_FLAG_PENDING_REDIRECT; if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct ref *tmp = ref; + struct hipe_ref *tmp = ref; ref = ref->next; *prev = ref; erts_free(ERTS_ALC_T_HIPE, tmp); -- cgit v1.2.3 From 3bffb5797c9498e2294b65ba24abec5842655a11 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 15:04:32 +0200 Subject: erts: Refactor rename struct hipe_sdesc --- erts/emulator/hipe/hipe_bif0.c | 2 +- erts/emulator/hipe/hipe_gc.c | 8 ++++---- erts/emulator/hipe/hipe_risc_gc.h | 16 ++++++++-------- erts/emulator/hipe/hipe_risc_glue.h | 2 +- erts/emulator/hipe/hipe_risc_stack.c | 12 ++++++------ erts/emulator/hipe/hipe_stack.c | 28 ++++++++++++++-------------- erts/emulator/hipe/hipe_stack.h | 32 ++++++++++++++++---------------- erts/emulator/hipe/hipe_x86_gc.h | 18 +++++++++--------- erts/emulator/hipe/hipe_x86_glue.h | 2 +- erts/emulator/hipe/hipe_x86_stack.c | 14 +++++++------- 10 files changed, 67 insertions(+), 67 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index b6d0abb2d1..95c2d73971 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -665,7 +665,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) { - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 68c65dea27..be665ccdb9 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h index 09568c140e..f019434f67 100644 --- a/erts/emulator/hipe/hipe_risc_gc.h +++ b/erts/emulator/hipe/hipe_risc_gc.h @@ -27,7 +27,7 @@ /* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */ struct nstack_walk_state { - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ }; static inline int nstack_walk_init_check(const Process *p) @@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) return p->hipe.nsp + nstkarity; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); state->sdesc0 = sdesc; return sdesc; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { unsigned long ra = (unsigned long)p->hipe.nra; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)&nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, return sdesc; } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { Eterm *nsp = p->hipe.nsp; p->hipe.nsp = nstack_walk_nsp_begin(p); @@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 0284265307..3d84731cbe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,7 +66,7 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { + static struct hipe_sdesc_with_exnra nbif_return_sdesc = { .exnra = (unsigned long)&nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)&nbif_return }, diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index dc98c96b8f..7528a3ab00 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -237,7 +237,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; @@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; int i; if (depth < 1) diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index e2e6eb74b1..a26132431b 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -43,10 +43,10 @@ */ struct hipe_sdesc_table hipe_sdesc_table; -static struct sdesc **alloc_bucket(unsigned int size) +static struct hipe_sdesc **alloc_bucket(unsigned int size) { - unsigned long nbytes = size * sizeof(struct sdesc*); - struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); + unsigned long nbytes = size * sizeof(struct hipe_sdesc*); + struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); sys_memzero(bucket, nbytes); return bucket; } @@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size) static void hipe_grow_sdesc_table(void) { unsigned int old_size, new_size, new_mask; - struct sdesc **old_bucket, **new_bucket; + struct hipe_sdesc **old_bucket, **new_bucket; unsigned int i; old_size = 1 << hipe_sdesc_table.log2size; @@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void) new_bucket = alloc_bucket(new_size); hipe_sdesc_table.bucket = new_bucket; for (i = 0; i < old_size; ++i) { - struct sdesc *b = old_bucket[i]; + struct hipe_sdesc *b = old_bucket[i]; while (b != NULL) { - struct sdesc *next = b->bucket.next; + struct hipe_sdesc *next = b->bucket.next; unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask; b->bucket.next = new_bucket[j]; new_bucket[j] = b; @@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void) erts_free(ERTS_ALC_T_HIPE, old_bucket); } -struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) +struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) { unsigned long ra; unsigned int i; - struct sdesc *chain; + struct hipe_sdesc *chain; unsigned int size; ra = sdesc->bucket.hvalue; @@ -102,7 +102,7 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) return sdesc; } -void hipe_init_sdesc_table(struct sdesc *sdesc) +void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,14 +121,14 @@ void hipe_init_sdesc_table(struct sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; Uint fsize, arity, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; if (is_not_tuple(arg) || (tuple_val(arg))[0] != make_arityval(6) || @@ -159,14 +159,14 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra - ? offsetof(struct sdesc_with_exnra, sdesc.livebits) - : offsetof(struct sdesc, livebits)) + ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits) + : offsetof(struct hipe_sdesc, livebits)) + livebitswords * sizeof(int); p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes); /* If we have an exception handler use the special sdesc_with_exnra structure. */ if (exnra) { - struct sdesc_with_exnra *sdesc_we = p; + struct hipe_sdesc_with_exnra *sdesc_we = p; sdesc_we->exnra = exnra; sdesc = &(sdesc_we->sdesc); } else diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index afa0ed4256..e9d05bf99c 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -30,10 +30,10 @@ #include /* offsetof() */ -struct sdesc { +struct hipe_sdesc { struct { unsigned long hvalue; /* return address */ - struct sdesc *next; /* hash collision chain */ + struct hipe_sdesc *next; /* hash collision chain */ } bucket; unsigned int summary; /* frame size, exn handler presence flag, arity */ #ifdef DEBUG @@ -43,27 +43,27 @@ struct sdesc { unsigned int livebits[1]; /* size depends on arch & data in summary field */ }; -struct sdesc_with_exnra { +struct hipe_sdesc_with_exnra { unsigned long exnra; - struct sdesc sdesc; + struct hipe_sdesc sdesc; }; -static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { return sdesc->summary >> 9; } -static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { return sdesc->summary & 0xFF; } -static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc) +static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { if ((sdesc->summary & (1<<8))) { const char *tmp; - tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc); - return ((const struct sdesc_with_exnra*)tmp)->exnra; + tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); + return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; } return 0; } @@ -72,13 +72,13 @@ struct hipe_sdesc_table { unsigned int log2size; unsigned int mask; /* INV: mask == (1 << log2size)-1 */ unsigned int used; - struct sdesc **bucket; + struct hipe_sdesc **bucket; }; extern struct hipe_sdesc_table hipe_sdesc_table; -extern struct sdesc *hipe_put_sdesc(struct sdesc*); -extern void hipe_init_sdesc_table(struct sdesc*); -extern struct sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_init_sdesc_table(struct hipe_sdesc*); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) @@ -86,10 +86,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm); #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) -static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra) +static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra) { unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; - const struct sdesc *sdesc = hipe_sdesc_table.bucket[i]; + const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i]; if (likely(sdesc->bucket.hvalue == ra)) return sdesc; do { @@ -103,7 +103,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void)); extern void hipe_print_nstack(Process*); extern void hipe_find_handler(Process*); extern void (*hipe_handle_stack_trap(Process*))(void); -extern void hipe_update_stack_trap(Process*, const struct sdesc*); +extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*); extern int hipe_fill_stacktrace(Process*, int, Eterm**); #if 0 && defined(HIPE_NSTACK_GROWS_UP) diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 00fe03d8f9..c025259871 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -30,9 +30,9 @@ struct nstack_walk_state { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ #else - struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ + struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ #endif }; @@ -57,11 +57,11 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); + const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); state->sdesc0 = sdesc; return sdesc; #else @@ -81,13 +81,13 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME unsigned long ra = p->hipe.nsp[0]; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -98,7 +98,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, #endif } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { #ifdef SKIP_YOUNGEST_FRAME Eterm *nsp = p->hipe.nsp; @@ -137,7 +137,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -148,7 +148,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 818d7444e2..d0c8eec2bc 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,7 +58,7 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { + static struct hipe_sdesc_with_exnra nbif_return_sdesc = { .exnra = (unsigned long)nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)nbif_return }, diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index f1559b1451..6975cc4669 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -52,9 +52,9 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - struct sdesc sdesc0; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + struct hipe_sdesc sdesc0; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -158,7 +158,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -199,7 +199,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -220,7 +220,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; nsp = p->hipe.nsp; @@ -262,7 +262,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; int i; -- cgit v1.2.3 From 9cb85a8cd1e02263f34c57aac68d16a2f1c03180 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 4 Oct 2016 12:26:54 +0200 Subject: erts: Refactor hipe_sdesc.summary into bit fields --- erts/emulator/hipe/hipe_risc_glue.h | 4 +++- erts/emulator/hipe/hipe_stack.c | 4 +++- erts/emulator/hipe/hipe_stack.h | 10 ++++++---- erts/emulator/hipe/hipe_x86_gc.h | 4 +++- erts/emulator/hipe/hipe_x86_glue.h | 4 +++- erts/emulator/hipe/hipe_x86_stack.c | 4 +++- 6 files changed, 21 insertions(+), 9 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 3d84731cbe..09804e3016 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -70,7 +70,9 @@ static __inline__ void hipe_arch_glue_init(void) .exnra = (unsigned long)&nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)&nbif_return }, - .summary = (1<<8), + .fsize = 0, + .has_exnra = 1, + .arity = 0 }, }; hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index a26132431b..17bef0718c 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -175,7 +175,9 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; - sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity; + sdesc->fsize = fsize; + sdesc->has_exnra = (exnra ? 1 : 0); + sdesc->arity = arity; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index e9d05bf99c..81f2e53dbc 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -35,7 +35,9 @@ struct hipe_sdesc { unsigned long hvalue; /* return address */ struct hipe_sdesc *next; /* hash collision chain */ } bucket; - unsigned int summary; /* frame size, exn handler presence flag, arity */ + unsigned int fsize : 23; /* frame size */ + unsigned int has_exnra : 1; /* exn handler presence flag */ + unsigned int arity : 8; #ifdef DEBUG Eterm dbg_M, dbg_F; unsigned dbg_A; @@ -50,17 +52,17 @@ struct hipe_sdesc_with_exnra { static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { - return sdesc->summary >> 9; + return sdesc->fsize; } static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->summary & 0xFF; + return sdesc->arity; } static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { - if ((sdesc->summary & (1<<8))) { + if (sdesc->has_exnra) { const char *tmp; tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index c025259871..7802076f70 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -68,7 +68,9 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; if ((int)nstkarity < 0) nstkarity = 0; - state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity; + state->sdesc0[0].fsize = 0; + state->sdesc0[0].has_exnra = 0; + state->sdesc0[0].arity = nstkarity; state->sdesc0[0].livebits[0] = 0; # ifdef DEBUG state->sdesc0[0].dbg_M = 0; diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index d0c8eec2bc..0e9fcd61f1 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -62,7 +62,9 @@ static __inline__ void hipe_arch_glue_init(void) .exnra = (unsigned long)nbif_fail, .sdesc = { .bucket = { .hvalue = (unsigned long)nbif_return }, - .summary = (1<<8), + .fsize = 0, + .has_exnra = 1, + .arity = 0, #ifdef DEBUG .dbg_F = am_return, #endif diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index 6975cc4669..9426b166e2 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -71,7 +71,9 @@ void hipe_print_nstack(Process *p) nstkarity = p->hipe.narity - NR_ARG_REGS; if ((int)nstkarity < 0) nstkarity = 0; - sdesc0.summary = nstkarity; + sdesc0.fsize = 0; + sdesc0.has_exnra = 0; + sdesc0.arity = nstkarity; sdesc0.livebits[0] = ~1; sdesc = &sdesc0; -- cgit v1.2.3 From 099c60de4033d7b397d4b3fb47f183b52fcba855 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 21:21:10 +0200 Subject: erts: Improve hipe load/upgrade/purge machinery A step toward better integration of hipe load and purge Highlights: * code_server no longer needs to call hipe_unified_loader:post_beam_load/1 Instead new internal function hipe_redirect_to_module() is called by loading BIFs to patch native call sites if needed. * hipe_purge_module() is called by erts_internal:purge_module/2 to purge any native code. * struct hipe_mfa_info redesigned and only used for exported functions that are called from or implemented by native code. A list of native call sites (struct hipe_ref) are kept for each hipe_mfa_info. * struct hipe_sdesc used by hipe_find_mfa_from_ra() to build native stack traces. --- erts/emulator/hipe/hipe_amd64.c | 15 + erts/emulator/hipe/hipe_arch.h | 10 +- erts/emulator/hipe/hipe_arm.c | 10 + erts/emulator/hipe/hipe_bif0.c | 764 +++++++++++++++++++----------------- erts/emulator/hipe/hipe_bif0.h | 5 +- erts/emulator/hipe/hipe_bif0.tab | 6 +- erts/emulator/hipe/hipe_ppc.c | 10 + erts/emulator/hipe/hipe_risc_glue.h | 20 +- erts/emulator/hipe/hipe_stack.c | 90 +++-- erts/emulator/hipe/hipe_stack.h | 18 +- erts/emulator/hipe/hipe_x86.c | 14 + erts/emulator/hipe/hipe_x86_gc.h | 14 +- erts/emulator/hipe/hipe_x86_glue.h | 23 +- erts/emulator/hipe/hipe_x86_stack.c | 7 +- 14 files changed, 564 insertions(+), 442 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 62739d2a78..df53f4db30 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -130,6 +130,13 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code) +{ + ALLOC_CODE_STATS(--nr_allocs); + /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + /* Make stub for native code calling exported beam function. */ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -234,6 +241,14 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + ALLOC_CODE_STATS(++nr_allocs); + /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index 6f959815bb..df38a80069 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -31,22 +31,30 @@ extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity); +extern void hipe_free_code(void*); +extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); +extern void hipe_free_native_stub(void*); + #if defined(__sparc__) #include "hipe_sparc.h" +#include "hipe_sparc_asm.h" #endif #if defined(__i386__) #include "hipe_x86.h" +#include "hipe_x86_asm.h" #endif #if defined(__x86_64__) #include "hipe_amd64.h" +#include "hipe_amd64_asm.h" #endif #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc.h" +#include "hipe_ppc_asm.h" #endif #if defined(__arm__) #include "hipe_arm.h" +#include "hipe_arm_asm.h" #endif #if !defined(AEXTERN) diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index f8ef468341..64e35b62d8 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -198,6 +198,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return address; } +void hipe_free_code(void* code, unsigned int bytes) +{ + /*SVERK: Leaking code memory */ +} + static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) { unsigned int *address; @@ -229,6 +234,11 @@ static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) return address; } +void hipe_free_native_stub(void* stub) +{ + /*SVERK: Leaking code stub */ +} + /* * ARMv5's support for 32-bit immediates is effectively non-existent. * Hence, every 32-bit immediate is stored in memory and loaded via diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 95c2d73971..8be6dc9aa9 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -54,6 +54,7 @@ #define BeamOpCode(Op) ((Uint)BeamOp(Op)) + int term_to_Sint32(Eterm term, Sint *sp) { Sint val; @@ -644,30 +645,21 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); if (pc) { - hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc); -#if HIPE -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address); hipe_set_call_trap(pc, address, is_closure); BIF_RET(am_true); -#endif } -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no BEAM pc found\r\n"); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); BIF_RET(am_false); } BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) { struct hipe_sdesc *sdesc; + Module* modp; + int do_commit; - sdesc = hipe_decode_sdesc(BIF_ARG_1); + sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit); if (!sdesc) { fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); @@ -676,6 +668,21 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); } + + /* + * Link into list of sdesc's in same module instance + */ + modp = erts_put_active_module(make_atom(sdesc->m_aix)); + ASSERT(modp); + if (do_commit) { /* Direct "hipe-patching" of early loaded module */ + sdesc->next_in_modi = modp->curr.first_hipe_sdesc; + modp->curr.first_hipe_sdesc = sdesc; + } + else { /* Normal module loading/upgrade */ + sdesc->next_in_modi = modp->new_hipe_sdesc; + modp->new_hipe_sdesc = sdesc; + } + BIF_RET(NIL); } @@ -970,7 +977,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2) atom_buf[0] = '\0'; strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len); - printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index); + printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index); BIF_ERROR(BIF_P, BADARG); } BIF_RET(address_to_term((void *)fe, BIF_P)); @@ -997,11 +1004,15 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_RET(am_true); } +struct hipe_ref_head { + struct hipe_ref_head* next; + struct hipe_ref_head* prev; +}; + /* - * MFA info hash table: + * An exported function called from or implemented by native code * - maps MFA to native code entry point - * - the MFAs it calls (refers_to) - * - the references to it (referred_from) + * - all references to it (callers) * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { @@ -1011,16 +1022,19 @@ struct hipe_mfa_info { } bucket; Eterm m; /* atom */ Eterm f; /* atom */ - unsigned int a; + unsigned int a : sizeof(int)*8 - 1; + unsigned int is_stub : 1; /* if beam or not (yet) loaded */ void *remote_address; - void *local_address; - Eterm *beam_code; - Uint orig_beam_op; - struct hipe_mfa_info_list *refers_to; - struct hipe_ref *referred_from; + void *new_address; + struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ + struct hipe_mfa_info* next_in_mod; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) void *trampoline; #endif +#ifdef DEBUG + Export* dbg_export; +#endif + }; static struct { @@ -1038,6 +1052,25 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; + +/* + * An external native call site M:F(...) + * to be patched when the callee changes. + */ +struct hipe_ref { + struct hipe_ref_head head; /* list of refs to same calleee */ + void *address; + void *trampoline; + unsigned int flags; + struct hipe_ref* next_from_modi; /* list of refs from same module instance */ +#if defined(DEBUG) + struct hipe_mfa_info* callee; + Eterm caller_m, caller_f, caller_a; +#endif +}; +#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ + + static inline void hipe_mfa_info_table_init_lock(void) { erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); @@ -1108,15 +1141,18 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->m = m; res->f = f; res->a = arity; + res->is_stub = 0; res->remote_address = NULL; - res->local_address = NULL; - res->beam_code = NULL; - res->orig_beam_op = 0; - res->refers_to = NULL; - res->referred_from = NULL; + res->new_address = NULL; + res->callers.next = &res->callers; + res->callers.prev = &res->callers; + res->next_in_mod = NULL; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) res->trampoline = NULL; #endif +#ifdef DEBUG + res->dbg_export = NULL; +#endif return res; } @@ -1154,22 +1190,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter return NULL; } -#if 0 /* XXX: unused */ -void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity) -{ - const struct hipe_mfa_info *p; - - p = hipe_mfa_info_table_get(m, f, arity); - return p ? p->address : NULL; -} -#endif - static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity) { unsigned long h; unsigned int i; struct hipe_mfa_info *p; unsigned int size; + Module* modp; h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; @@ -1189,23 +1216,47 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, size = 1 << hipe_mfa_info_table.log2size; if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); + + modp = erts_put_active_module(m); + ASSERT(modp); + p->next_in_mod = modp->first_hipe_mfa; + modp->first_hipe_mfa = p; + + DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); + return p; } -static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported) +static void remove_mfa_info(struct hipe_mfa_info* rm) +{ + unsigned int i; + struct hipe_mfa_info *p; + struct hipe_mfa_info **prevp; + + i = rm->bucket.hvalue & hipe_mfa_info_table.mask; + prevp = &hipe_mfa_info_table.bucket[i]; + for (;;) { + p = *prevp; + ASSERT(p); + if (p == rm) { + *prevp = p->bucket.next; + ASSERT(hipe_mfa_info_table.used > 0); + hipe_mfa_info_table.used--; + return; + } + prevp = &p->bucket.next; + } +} + +static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) { struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); p = hipe_mfa_info_table_put_rwlocked(m, f, arity); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(m, f, arity); - printf(": changing address from %p to %p\r\n", p->local_address, address); -#endif - p->local_address = address; - if (is_exported) - p->remote_address = address; + DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p); + p->new_address = address; + hipe_mfa_info_table_rwunlock(); } @@ -1237,168 +1288,85 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { struct hipe_mfa mfa; void *address; - int is_exported; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - address = term_to_address(BIF_ARG_2); - if (!address) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_3 == am_true) - is_exported = 1; - else if (BIF_ARG_3 == am_false) - is_exported = 0; - else + switch (BIF_ARG_3) { + case am_true: /* is_exported */ + if (!term_to_mfa(BIF_ARG_1, &mfa)) + BIF_ERROR(BIF_P, BADARG); + address = term_to_address(BIF_ARG_2); + if (!address) + BIF_ERROR(BIF_P, BADARG); + hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address); + break; + case am_false: + break; /* ignore local functions */ + default: BIF_ERROR(BIF_P, BADARG); - hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported); - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) -{ - Eterm lst; - struct hipe_mfa mfa; - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - lst = BIF_ARG_1; - while (is_list(lst)) { - if (!term_to_mfa(CAR(list_val(lst)), &mfa)) - break; - lst = CDR(list_val(lst)); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - p->remote_address = NULL; - p->local_address = NULL; - if (p->beam_code) { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n", - p->beam_code, p->orig_beam_op); -#endif - p->beam_code[0] = p->orig_beam_op; - p->beam_code = NULL; - p->orig_beam_op = 0; - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no call trap to remove\r\n"); -#endif - } - } } - hipe_mfa_info_table_rwunlock(); - if (is_not_nil(lst)) - BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } -void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc) + +/* Ask if we need to block all threads + * while loading/deleting (beam) code for this module? + */ +int hipe_need_blocking(Module* modp) { - Uint orig_beam_op; struct hipe_mfa_info *p; - orig_beam_op = pc[0]; - if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) && - orig_beam_op != BeamOpCode(op_hipe_trap_call)) { - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc); -#endif - p->beam_code = pc; - p->orig_beam_op = orig_beam_op; - hipe_mfa_info_table_rwunlock(); - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": orig op %#lx already saved\r\n", orig_beam_op); -#endif + /* Need to block if we have at least one native caller to this module + * or native code to make unaccessible. + */ + hipe_mfa_info_table_rlock(); + for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + ASSERT(!p->new_address); + if (p->callers.next != &p->callers || !p->is_stub) { + break; + } } + hipe_mfa_info_table_runlock(); + return (p != NULL); } -static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) -{ - Export *export_entry; - void *StubAddress; - - ASSERT(is_remote); - - export_entry = erts_export_get_or_make_stub(m, f, arity); - StubAddress = hipe_make_native_stub(export_entry, arity); - if (!StubAddress) - erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - return StubAddress; -} - -static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp) +static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a) { struct hipe_mfa_info *p; - void *address; p = hipe_mfa_info_table_get_locked(m, f, a); - if (p) { - /* find address, predicting for a runtime apply call */ - address = p->remote_address; - if (!is_remote) - address = p->local_address; - if (address) - return address; - - /* bummer, install stub, checking if one already existed */ - address = p->remote_address; - if (address) - return address; - } - /* Caller must take the slow path with the write lock held, but allow - it to avoid some work if it already holds the write lock. */ - if (pp) - *pp = p; - return NULL; -} - -static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p) -{ - void *address; - - if (!p) - p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_make_stub(m, f, a, is_remote); - /* XXX: how to tell if a BEAM MFA is exported or not? */ - p->remote_address = address; - return address; + return p ? p->remote_address : NULL; } -static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a) { - struct hipe_mfa_info *p; - void *address; + struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_get_na_try_locked(m, f, a, is_remote, &p); - if (address) - return address; + if (!p->remote_address) { + Export* export_entry = erts_export_get_or_make_stub(m, f, a); + void* stubAddress = hipe_make_native_stub(export_entry, a); + if (!stubAddress) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p); - return address; + p->remote_address = stubAddress; + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = export_entry; +#endif + } + return p->remote_address; } -static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a) { void *address; hipe_mfa_info_table_rlock(); - address = hipe_get_na_try_locked(m, f, a, is_remote, NULL); + address = hipe_get_na_try_locked(m, f, a); hipe_mfa_info_table_runlock(); if (address) return address; hipe_mfa_info_table_rwlock(); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL); + address = hipe_get_na_slow_rwlocked(m, f, a); hipe_mfa_info_table_rwunlock(); return address; } @@ -1408,7 +1376,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) { if (is_not_atom(m) || is_not_atom(f) || a > 255) return NULL; - return hipe_get_na_nofail(m, f, a, 1); + return hipe_get_na_nofail(m, f, a); } /* primop, but called like a BIF for error handling purposes */ @@ -1420,25 +1388,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); arity = unsigned_val(BIF_ARG_3); /* no error check */ - address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1); + address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity); BIF_RET((Eterm)address); /* semi-Ok */ } -BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) { struct hipe_mfa mfa; void *address; - int is_remote; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_2 == am_true) - is_remote = 1; - else if (BIF_ARG_2 == am_false) - is_remote = 0; - else - BIF_ERROR(BIF_P, BADARG); - address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote); + + address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari); BIF_RET(address_to_term(address, BIF_P)); } @@ -1460,7 +1422,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) f = ep->code[1]; } else goto badfun; - address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1); + address = hipe_get_na_nofail(m, f, BIF_ARG_2); BIF_RET((Eterm)address); badfun: @@ -1471,60 +1433,19 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) { - struct hipe_mfa_info *mfa; - long mfa_offset, ra_offset; - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra); - /* Note about locking: the table is only updated from the - loader, which runs with the rest of the system suspended. */ - /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */ - hipe_mfa_info_table_rlock(); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - mfa = NULL; - mfa_offset = LONG_MAX; - for (i = 0; i < nrbuckets; ++i) { - struct hipe_mfa_info *b = bucket[i]; - while (b != NULL) { - ra_offset = (char*)ra - (char*)b->local_address; - if (ra_offset > 0 && ra_offset < mfa_offset) { - mfa_offset = ra_offset; - mfa = b; - } - b = b->bucket.next; - } - } - if (mfa) { - *m = mfa->m; - *f = mfa->f; - *a = mfa->a; - } - hipe_mfa_info_table_runlock(); - return mfa ? 1 : 0; -} + if (!sdesc || sdesc->m_aix == atom_val(am_Empty)) + return 0; -/* - * Patch Reference Handling. - */ -struct hipe_mfa_info_list { - struct hipe_mfa_info *mfa; - struct hipe_mfa_info_list *next; -}; + *m = make_atom(sdesc->m_aix); + *f = make_atom(sdesc->f_aix); + *a = sdesc->a; + return 1; +} -struct hipe_ref { - struct hipe_mfa_info *caller_mfa; - void *address; - void *trampoline; - unsigned int flags; - struct hipe_ref *next; -}; -#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ -#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ -#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */ -#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */ -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { @@ -1535,9 +1456,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; - struct hipe_mfa_info *caller_mfa; - struct hipe_mfa_info_list *refers_to; struct hipe_ref *ref; + Module* modp; + int do_commit; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1569,62 +1490,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) goto badarg; } switch (tuple[5]) { - case am_local: - break; - case am_remote: - flags |= REF_FLAG_IS_REMOTE; - break; - default: - goto badarg; + case am_true: do_commit = 1; break; + case am_false: do_commit = 0; break; + default: goto badarg; } hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); - caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari); - - refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to)); - refers_to->mfa = callee_mfa; - refers_to->next = caller_mfa->refers_to; - caller_mfa->refers_to = refers_to; - ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref)); - ref->caller_mfa = caller_mfa; + ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; ref->trampoline = trampoline; ref->flags = flags; - ref->next = callee_mfa->referred_from; - callee_mfa->referred_from = ref; + + /* + * Link into list of refs to same callee + */ + ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers); + ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers); + ref->head.next = callee_mfa->callers.next; + ref->head.prev = &callee_mfa->callers; + ref->head.next->prev = &ref->head; + ref->head.prev->next = &ref->head; + + /* + * Link into list of refs from same module instance + */ + modp = erts_put_active_module(caller.mod); + ASSERT(modp); + if (do_commit) { /* Direct "hipe-patching" of early loaded module */ + ref->next_from_modi = modp->curr.first_hipe_ref; + modp->curr.first_hipe_ref = ref; + } + else { /* Normal module loading/upgrade */ + ref->next_from_modi = modp->new_hipe_refs; + modp->new_hipe_refs = ref; + } + +#if defined(DEBUG) + ref->callee = callee_mfa; + ref->caller_m = caller.mod; + ref->caller_f = caller.fun; + ref->caller_a = caller.ari; +#endif hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d", + ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d", + ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit); BIF_RET(NIL); badarg: BIF_ERROR(BIF_P, BADARG); } -/* Given a CalleeMFA, mark each ref to it as pending-redirect. - * This ensures that remove_refs_from() won't remove them: any - * removal is instead done at the end of redirect_referred_from(). - */ -BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ + +static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - struct hipe_mfa mfa; - const struct hipe_mfa_info *p; - struct hipe_ref *ref; + Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix()); + struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; + struct hipe_mfa_info* p; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) - for (ref = p->referred_from; ref != NULL; ref = ref->next) - ref->flags |= REF_FLAG_PENDING_REDIRECT; - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + ASSERT(modp); + for (;;) { + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + if (p == unlink_me) { + *prevp = p->next_in_mod; + break; + } + prevp = &p->next_in_mod; + } +} + +static void purge_mfa(struct hipe_mfa_info* p) +{ + ASSERT(p->is_stub); + remove_mfa_info(p); + hipe_free_native_stub(p->remote_address); + erts_free(ERTS_ALC_T_HIPE, p); } /* Called by init:restart after unloading all hipe compiled modules - * to work around bug causing execution of deallocated beam code. - * Can be removed when delete/purge of native modules works better. + * to work around old bug that caused execution of deallocated beam code. + * Can be removed now when delete/purge of native modules works better. * Test: Do init:restart in debug compiled vm with hipe compiled kernel. */ static void hipe_purge_all_refs(void) @@ -1634,126 +1583,205 @@ static void hipe_purge_all_refs(void) hipe_mfa_info_table_rwlock(); + ASSERT(hipe_mfa_info_table.used == 0); bucket = hipe_mfa_info_table.bucket; nrbuckets = 1 << hipe_mfa_info_table.log2size; for (i = 0; i < nrbuckets; ++i) { + ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { + Module* modp; struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - while (mfa->refers_to) { - struct hipe_mfa_info_list *to = mfa->refers_to; - mfa->refers_to = to->next; - erts_free(ERTS_ALC_T_HIPE, to); - } - while (mfa->referred_from) { - struct hipe_ref* from = mfa->referred_from; - mfa->referred_from = from->next; - erts_free(ERTS_ALC_T_HIPE, from); - } + ASSERT(mfa->callers.next == &mfa->callers); + + modp = erts_get_module(mfa->m, erts_active_code_ix()); + if (modp) { + /* unsafe write to active module */ + modp->first_hipe_mfa = NULL; + } erts_free(ERTS_ALC_T_HIPE, mfa); } } + hipe_mfa_info_table.used = 0; hipe_mfa_info_table_rwunlock(); } BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct hipe_mfa mfa; - struct hipe_mfa_info *caller_mfa, *callee_mfa; - struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct hipe_ref **prev, *ref; - if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); BIF_RET(am_ok); } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (caller_mfa) { - refers_to = caller_mfa->refers_to; - while (refers_to) { - callee_mfa = refers_to->mfa; - prev = &callee_mfa->referred_from; - ref = *prev; - while (ref) { - if (ref->caller_mfa == caller_mfa) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - ref->flags |= REF_FLAG_PENDING_REMOVE; - prev = &ref->next; - ref = ref->next; - } else { - struct hipe_ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } - } else { - prev = &ref->next; - ref = ref->next; - } - } - tmp_refers_to = refers_to; - refers_to = refers_to->next; - erts_free(ERTS_ALC_T_HIPE, tmp_refers_to); - } - caller_mfa->refers_to = NULL; + ASSERT(!"hipe_bifs_remove_refs_from_1() called"); + BIF_ERROR(BIF_P, BADARG); +} + +int hipe_purge_need_blocking(Module* modp) +{ + /* SVERK: Verify if this is really necessary */ + return (modp->old.first_hipe_ref || + modp->old.first_hipe_sdesc || + (!modp->curr.code_hdr && modp->first_hipe_mfa)); +} + +void hipe_purge_module(Module* modp) +{ + struct hipe_ref* ref; + struct hipe_sdesc* sdesc; + + ASSERT(modp); + + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); + + /* + * Remove all hipe_ref's (external calls) from the old module instance + */ + ref = modp->old.first_hipe_ref; + + while (ref) { + struct hipe_ref* free_ref = ref; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == make_atom(modp->module)); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + unlink_mfa_from_mod(p); + purge_mfa(p); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); + } + modp->old.first_hipe_ref = NULL; + + /* + * Remove all hipe_sdesc's for the old module instance + */ + sdesc = modp->old.first_hipe_sdesc; + + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(sdesc->m_aix == modp->module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } + modp->old.first_hipe_sdesc = NULL; + + /* + * Remove unreferred hipe_mfa_info's + */ + if (modp->curr.code_hdr == NULL) { + struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; + struct hipe_mfa_info* p = *prevp; + ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking()); + for (; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + } + if (modp->old.hipe_code_start) { +#ifdef DEBUG + sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size); +#endif + hipe_free_code(modp->old.hipe_code_start); + modp->old.hipe_code_start = NULL; } - hipe_mfa_info_table_rwunlock(); - BIF_RET(am_ok); } -/* redirect_referred_from(CalleeMFA) - * Redirect all pending-redirect refs in CalleeMFA's referred_from. - * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from. +/* + * Redirect all existing native calls to this module */ -BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) +void hipe_redirect_to_module(Module* modp) { - struct hipe_mfa mfa; struct hipe_mfa_info *p; - struct hipe_ref **prev, *ref; - int is_remote, res; - void *new_address; + struct hipe_ref_head* refh; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + if (p->new_address) { + if (p->is_stub) { + hipe_free_native_stub(p->remote_address); + p->is_stub = 0; + } + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address); + p->remote_address = p->new_address; + p->new_address = NULL; +#ifdef DEBUG + p->dbg_export = NULL; +#endif + } + else if (!p->is_stub) { + Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a); + p->remote_address = hipe_make_native_stub(exp, p->a); + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address); + if (!p->remote_address) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = exp; +#endif + } + else { + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub"); + ASSERT(p->remote_address && p->dbg_export); + } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - prev = &p->referred_from; - ref = *prev; - while (ref) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - is_remote = ref->flags & REF_FLAG_IS_REMOTE; - new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote); - if (ref->flags & REF_FLAG_IS_LOAD_MFA) - res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, new_address, ref->trampoline); - if (res) - fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); - ref->flags &= ~REF_FLAG_PENDING_REDIRECT; - if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct hipe_ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } else { - prev = &ref->next; - ref = ref->next; - } - } else { - prev = &ref->next; - ref = ref->next; - } + DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p); + for (refh = p->callers.next; refh != &p->callers; refh = refh->next) { + struct hipe_ref* ref = (struct hipe_ref*) refh; + int res; + + DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)", + ref, ref->caller_m, ref->caller_f, ref->caller_a, + ref->address, p->remote_address); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, + " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)", + ref, p->m,p->f,p->a, ref->address, p->remote_address); + + if (ref->flags & REF_FLAG_IS_LOAD_MFA) + res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); + else + res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline); + if (res) + fprintf(stderr, "%s: patch failed", __FUNCTION__); } + DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); } BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index c9a8216368..1914405e2d 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -42,7 +42,10 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); #endif /* needed in beam_load.c */ -void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc); +int hipe_need_blocking(Module*); +int hipe_purge_need_blocking(Module*); +void hipe_purge_module(Module*); +void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ extern void *term_to_address(Eterm); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 99237aae05..eae84383df 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -54,7 +54,7 @@ bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 bif hipe_bifs:set_funinfo_native_address/3 -bif hipe_bifs:invalidate_funinfo_native_addresses/1 +#bif hipe_bifs:invalidate_funinfo_native_addresses/1 bif hipe_bifs:update_code_size/3 bif hipe_bifs:code_size/1 @@ -72,7 +72,7 @@ bif hipe_bifs:term_to_word/1 bif hipe_bifs:get_fe/2 bif hipe_bifs:set_native_address_in_fe/2 -bif hipe_bifs:find_na_or_make_stub/2 +bif hipe_bifs:find_na_or_make_stub/1 bif hipe_bifs:check_crc/1 bif hipe_bifs:system_crc/0 @@ -84,9 +84,7 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:mark_referred_from/1 bif hipe_bifs:remove_refs_from/1 -bif hipe_bifs:redirect_referred_from/1 # atoms used by add_ref/2 atom call diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index 9b2048c457..a1a6e2ad02 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -214,6 +214,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return address; } +void hipe_free_code(void* code, unsigned int bytes) +{ + /*SVERK: Leaking code memory */ +} + static unsigned int *alloc_stub(Uint nrwords) { unsigned int *address; @@ -241,6 +246,11 @@ static unsigned int *alloc_stub(Uint nrwords) return address; } +void hipe_free_native_stub(void* stub) +{ + /*SVERK: Leaking code stubs */ +} + static void patch_imm16(Uint32 *address, unsigned int imm16) { unsigned int insn = *address; diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 09804e3016..1369b392fe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,15 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct hipe_sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)&nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)&nbif_return }, - .fsize = 0, - .has_exnra = 1, - .arity = 0 - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index 17bef0718c..3d082668a6 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -102,6 +102,28 @@ struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) return sdesc; } +void hipe_destruct_sdesc(struct hipe_sdesc *sdesc) +{ + unsigned int i; + struct hipe_sdesc** prevp; + void* free_me; + + i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; + prevp = &hipe_sdesc_table.bucket[i]; + + for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next) + ASSERT(*prevp); + + *prevp = sdesc->bucket.next; + hipe_sdesc_table.used -= 1; + + if (sdesc->has_exnra) + free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc); + else + free_me = sdesc; + erts_free(ERTS_ALC_T_HIPE, free_me); +} + void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) { Uint ra, exnra; Eterm *live; - Uint fsize, arity, nlive, i, nslots, off; + Uint fsize, nargs, stk_nargs, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; struct hipe_sdesc *sdesc; - - if (is_not_tuple(arg) || - (tuple_val(arg))[0] != make_arityval(6) || - term_to_Uint((tuple_val(arg))[1], &ra) == 0 || - term_to_Uint((tuple_val(arg))[2], &exnra) == 0 || - is_not_small((tuple_val(arg))[3]) || - (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 || - is_not_small((tuple_val(arg))[4]) || - (arity = unsigned_val((tuple_val(arg))[4])) > 255 || - is_not_tuple((tuple_val(arg))[5])) + Eterm* mfa_tpl; + Eterm* tp; + + if (is_not_tuple(arg)) + return 0; + + tp = tuple_val(arg); + if (tp[0] != make_arityval(7) || + term_to_Uint(tp[1], &ra) == 0 || + term_to_Uint(tp[2], &exnra) == 0 || + is_not_small(tp[3]) || + (fsize = unsigned_val(tp[3])) > 65535 || + is_not_small(tp[4]) || + (stk_nargs = unsigned_val(tp[4])) > 255 || + is_not_tuple(tp[5]) || + is_not_tuple(tp[6]) || + (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) || + is_not_atom(mfa_tpl[1]) || + is_not_atom(mfa_tpl[2]) || + is_not_small(mfa_tpl[3]) || + (nargs = unsigned_val(mfa_tpl[3])) > 255) return 0; + + if (stk_nargs > nargs) + return 0; + /* Get tuple with live slots */ - live = tuple_val((tuple_val(arg))[5]) + 1; + live = tuple_val(tp[5]) + 1; /* Get number of live slots */ nlive = arityval(live[-1]); - /* Calculate size of frame = locals + ra + arguments */ - nslots = fsize + 1 + arity; + /* Calculate size of frame = locals + ra + stack arguments */ + nslots = fsize + 1 + stk_nargs; /* Check that only valid slots are given. */ for (i = 0; i < nlive; ++i) { if (is_not_small(live[i]) || @@ -154,8 +191,14 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) return 0; } + switch(tp[7]) { + case am_true: *do_commitp = 1; break; + case am_false: *do_commitp = 0; break; + default: return 0; + } + /* Calculate number of words for the live bitmap. */ - livebitswords = (fsize + arity + 1 + 31) / 32; + livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra @@ -172,12 +215,17 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) } else sdesc = p; + sdesc->m_aix = atom_val(mfa_tpl[1]); + sdesc->f_aix = atom_val(mfa_tpl[2]); + sdesc->a = nargs; + + /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; sdesc->fsize = fsize; sdesc->has_exnra = (exnra ? 1 : 0); - sdesc->arity = arity; + sdesc->stk_nargs = stk_nargs; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; @@ -186,13 +234,5 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) off = unsigned_val(live[i]); sdesc->livebits[off / 32] |= (1 << (off & 31)); } -#ifdef DEBUG - { - Eterm mfa_tpl = tuple_val(arg)[6]; - sdesc->dbg_M = tuple_val(mfa_tpl)[1]; - sdesc->dbg_F = tuple_val(mfa_tpl)[2]; - sdesc->dbg_A = tuple_val(mfa_tpl)[3]; - } -#endif return sdesc; } diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 81f2e53dbc..1863b0db8c 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -37,12 +37,12 @@ struct hipe_sdesc { } bucket; unsigned int fsize : 23; /* frame size */ unsigned int has_exnra : 1; /* exn handler presence flag */ - unsigned int arity : 8; -#ifdef DEBUG - Eterm dbg_M, dbg_F; - unsigned dbg_A; -#endif - unsigned int livebits[1]; /* size depends on arch & data in summary field */ + unsigned int stk_nargs : 8; /* arguments on stack */ + Uint32 m_aix; + Uint32 f_aix; + Uint32 a; + struct hipe_sdesc* next_in_modi; + Uint32 livebits[1]; /* size depends on arch & data in summary field */ }; struct hipe_sdesc_with_exnra { @@ -55,9 +55,10 @@ static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) return sdesc->fsize; } +/* Nr of arguments pushed on stack */ static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->arity; + return sdesc->stk_nargs; } static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) @@ -79,8 +80,9 @@ struct hipe_sdesc_table { extern struct hipe_sdesc_table hipe_sdesc_table; extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_destruct_sdesc(struct hipe_sdesc*); extern void hipe_init_sdesc_table(struct hipe_sdesc*); -extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 5f6c8c200e..8a3ba8bc7f 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -184,6 +184,15 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + /* SVERK: Leak !!! + ALLOC_CODE_STATS(--nr_allocs); + ALLOC_CODE_STATS(total_alloc -= bytes); + + erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/ +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { /* @@ -264,6 +273,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + /* SVERK: leak leak drip drop */ +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 7802076f70..e92e6ea718 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -65,18 +65,14 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) state->sdesc0 = sdesc; return sdesc; #else - unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; state->sdesc0[0].fsize = 0; state->sdesc0[0].has_exnra = 0; - state->sdesc0[0].arity = nstkarity; + state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); state->sdesc0[0].livebits[0] = 0; -# ifdef DEBUG - state->sdesc0[0].dbg_M = 0; - state->sdesc0[0].dbg_F = am_undefined; - state->sdesc0[0].dbg_A = 0; -# endif + state->sdesc0[0].m_aix = 0; + state->sdesc0[0].f_aix = atom_val(am_undefined); + state->sdesc0[0].a = 0; /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */ __asm__ __volatile__("" : : "m"(*state) : "memory"); return &state->sdesc0[0]; diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 0e9fcd61f1..de2b061706 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,18 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct hipe_sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)nbif_return }, - .fsize = 0, - .has_exnra = 1, - .arity = 0, - #ifdef DEBUG - .dbg_F = am_return, - #endif - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index 9426b166e2..31582b3a2e 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -60,7 +60,6 @@ void hipe_print_nstack(Process *p) unsigned int mask; unsigned int sdesc_size; unsigned int i; - unsigned int nstkarity; static const char dashes[2*sizeof(long)+5] = { [0 ... 2*sizeof(long)+3] = '-' }; @@ -68,12 +67,10 @@ void hipe_print_nstack(Process *p) nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; - nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; sdesc0.fsize = 0; sdesc0.has_exnra = 0; - sdesc0.arity = nstkarity; + sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); sdesc0.livebits[0] = ~1; sdesc = &sdesc0; -- cgit v1.2.3 From 4d96d297044274c46deda1adfc567297449a9ba9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Sep 2016 19:23:27 +0200 Subject: erts: Remove unused hipe_bifs:code_size and hipe_bifs:update_code_size --- erts/emulator/hipe/hipe_bif0.c | 83 ---------------------------------------- erts/emulator/hipe/hipe_bif0.tab | 3 -- 2 files changed, 86 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 8be6dc9aa9..6f9bfe2f40 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1835,89 +1835,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value) } } -struct modinfo { - HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */ - unsigned int code_size; -}; - -static Hash modinfo_table; - -static HashValue modinfo_hash(void *tmpl) -{ - Eterm mod = (Eterm)tmpl; - return atom_val(mod); -} - -static int modinfo_cmp(void *tmpl, void *bucket) -{ - /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */ - return 0; -} - -static void *modinfo_alloc(void *tmpl) -{ - struct modinfo *p; - - p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p)); - p->code_size = 0; - return &p->bucket; -} - -static void init_modinfo_table(void) -{ - HashFunctions f; - static int init_done = 0; - - if (init_done) - return; - init_done = 1; - f.hash = (H_FUN) modinfo_hash; - f.cmp = (HCMP_FUN) modinfo_cmp; - f.alloc = (HALLOC_FUN) modinfo_alloc; - f.free = (HFREE_FUN) NULL; - f.meta_alloc = (HMALLOC_FUN) erts_alloc; - f.meta_free = (HMFREE_FUN) erts_free; - f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f); -} - -BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3) -{ - struct modinfo *p; - Sint code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1) || - is_not_small(BIF_ARG_3) || - (code_size = signed_val(BIF_ARG_3)) < 0) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1); - - if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */ - p->code_size += code_size; - else /* whole module */ - p->code_size = code_size; - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1) -{ - struct modinfo *p; - unsigned int code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1); - - code_size = p ? p->code_size : 0; - BIF_RET(make_small(code_size)); -} - BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3) { Uint *address, value; diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index eae84383df..ff9c1d06fb 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -56,9 +56,6 @@ bif hipe_bifs:set_native_address/3 bif hipe_bifs:set_funinfo_native_address/3 #bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:update_code_size/3 -bif hipe_bifs:code_size/1 - bif hipe_bifs:enter_sdesc/1 bif hipe_bifs:bif_address/3 -- cgit v1.2.3 From 39072836944d00c288beebfd98b14593f9609006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Thu, 19 May 2016 17:38:54 +0200 Subject: Add a loader state for HiPE code loading Just like the BEAM loader state (as returned by erlang:prepare_loading/2), the HiPE loader state is contained in a magic binary. Eventually, we will separate HiPE loading into a prepare and a finalise phase, like the BEAM loader, where the prepare phase will be implemented by hipe_unified_loader and the finalise phase be implemented in C by hipe_load.c and beam_load.c, making prepare side-effect free and finalise atomic. The finalise phase will be exposed through the erlang:finish_loading/1 API, just like the BEAM loader, as this will allow HiPE and BEAM modules to be mixed in the same atomic "commit". The usage of a loader state makes it easier to keep track of all resources allocated during loading, and will not only make it easy to prevent leaks when hipe_unified_loader crashes, but also paves the way for proper, leak-free, unloading of HiPE modules. --- erts/emulator/hipe/hipe_bif0.c | 88 +++++++++++++++++++++++++++------- erts/emulator/hipe/hipe_bif0.tab | 6 ++- erts/emulator/hipe/hipe_load.c | 101 +++++++++++++++++++++++++++++++++++++++ erts/emulator/hipe/hipe_load.h | 45 +++++++++++++++++ erts/emulator/hipe/hipe_module.c | 37 ++++++++++++++ erts/emulator/hipe/hipe_module.h | 42 ++++++++++++++++ 6 files changed, 300 insertions(+), 19 deletions(-) create mode 100644 erts/emulator/hipe/hipe_load.c create mode 100644 erts/emulator/hipe/hipe_load.h create mode 100644 erts/emulator/hipe/hipe_module.c create mode 100644 erts/emulator/hipe/hipe_module.h (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 6f9bfe2f40..daa198d695 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -44,6 +44,7 @@ #include "hipe_mode_switch.h" #include "hipe_native_bif.h" #include "hipe_bif0.h" +#include "hipe_load.h" /* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines a few constants. #undef them here to avoid warnings. */ #undef F_TIMO @@ -374,16 +375,32 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2) BIF_RET(BIF_ARG_1); } +/* + * BIFs for loading code. + */ + +static HipeLoaderState *get_loader_state(Eterm term) +{ + ProcBin *pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL; + + pb = (ProcBin*) binary_val(term); + return hipe_get_loader_state(pb->val); +} + + /* * Allocate memory and copy machine code to it. */ -BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) { Uint nrbytes; void *bytes; void *address; Eterm trampolines; Eterm *hp; + HipeLoaderState *stp; #ifndef DEBUG ERTS_DECLARE_DUMMY(Uint bitoffs); ERTS_DECLARE_DUMMY(Uint bitsize); @@ -392,13 +409,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) Uint bitsize; #endif - if (is_not_binary(BIF_ARG_1)) + if (is_not_binary(BIF_ARG_1) || + (!(stp = get_loader_state(BIF_ARG_3)))) BIF_ERROR(BIF_P, BADARG); nrbytes = binary_size(BIF_ARG_1); ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); ASSERT(bitoffs == 0); ASSERT(bitsize == 0); trampolines = NIL; + // XXX: Trampolines are not tracked and freed address = hipe_alloc_code(nrbytes, BIF_ARG_2, &trampolines, BIF_P); if (!address) { Uint nrcallees; @@ -407,11 +426,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) nrcallees = arityval(tuple_val(BIF_ARG_2)[0]); else nrcallees = 0; + // XXX: Is there any reason to not just BIF_ERROR, so that the runtime + // survives? erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", __func__, (unsigned long)nrbytes, (unsigned long)nrcallees); } memcpy(address, bytes, nrbytes); hipe_flush_icache_range(address, nrbytes); + stp->text_segment = address; + stp->text_segment_size = nrbytes; hp = HAlloc(BIF_P, 3); hp[0] = make_arityval(2); hp[1] = address_to_term(address, BIF_P); @@ -424,25 +447,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) /* * Allocate memory for arbitrary non-Erlang data. */ -BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3) { - Uint align, nrbytes; - void *block; + Uint align; + HipeLoaderState *stp; if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) || + (!(stp = get_loader_state(BIF_ARG_3))) || (align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align))) BIF_ERROR(BIF_P, BADARG); - nrbytes = unsigned_val(BIF_ARG_2); - if (nrbytes == 0) + + if (stp->data_segment_size || stp->data_segment) + BIF_ERROR(BIF_P, BADARG); + + stp->data_segment_size = unsigned_val(BIF_ARG_2); + if (stp->data_segment_size == 0) BIF_RET(make_small(0)); - block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); - if ((unsigned long)block & (align-1)) { - fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", - __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); - erts_free(ERTS_ALC_T_HIPE, block); + stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size); + if ((unsigned long)stp->data_segment & (align-1)) { + fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte " + "aligned\r\n", + __FUNCTION__, (unsigned long)stp->data_segment_size, + stp->data_segment, (unsigned long)align); + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; BIF_ERROR(BIF_P, EXC_NOTSUP); } - BIF_RET(address_to_term(block, BIF_P)); + BIF_RET(address_to_term(stp->data_segment, BIF_P)); } /* @@ -1711,12 +1743,14 @@ void hipe_purge_module(Module* modp) prevp = &p->next_in_mod; } } - if (modp->old.hipe_code_start) { + if (modp->old.hipe_code && modp->old.hipe_code->text_segment) { #ifdef DEBUG - sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size); + sys_memset(modp->old.hipe_code->text_segment, 0xfe, + modp->old.hipe_code->text_segment_size); #endif - hipe_free_code(modp->old.hipe_code_start); - modp->old.hipe_code_start = NULL; + hipe_free_code(modp->old.hipe_code->text_segment); + modp->old.hipe_code->text_segment = NULL; + modp->old.hipe_code->text_segment_size = 0; } } @@ -1870,3 +1904,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3) BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } + +BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1) +{ + Binary *magic; + Eterm *hp; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + magic = hipe_alloc_loader_state(BIF_ARG_1); + + if (!magic) + BIF_ERROR(BIF_P, BADARG); + + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->refc, 1); + BIF_RET(res); +} diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index ff9c1d06fb..1e4bde5ef2 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -44,8 +44,8 @@ bif hipe_bifs:ref/1 bif hipe_bifs:ref_get/1 bif hipe_bifs:ref_set/2 -bif hipe_bifs:enter_code/2 -bif hipe_bifs:alloc_data/2 +bif hipe_bifs:enter_code/3 +bif hipe_bifs:alloc_data/3 bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 @@ -83,6 +83,8 @@ bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 bif hipe_bifs:remove_refs_from/1 +bif hipe_bifs:alloc_loader_state/1 + # atoms used by add_ref/2 atom call atom load_mfa diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c new file mode 100644 index 0000000000..5bce3f1aee --- /dev/null +++ b/erts/emulator/hipe/hipe_load.c @@ -0,0 +1,101 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.c + * + * HiPE atomic code loader + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "global.h" +#include "erl_binary.h" +#include "hipe_load.h" + +/* + * This destructor function can safely be called multiple times. + */ +static void +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + if (stp->module == NIL) return; + + erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n", + stp->module); + + // TODO: Needs to be freed separately. We'd like have a unified executable + // code allocator, so postpone this for now. + /* if (stp->text_segment) */ + /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */ + stp->text_segment = NULL; + stp->text_segment_size = 0; + + if (stp->data_segment) + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; + + stp->module = NIL; +} + +Binary *hipe_alloc_loader_state(Eterm module) +{ + HipeLoaderState *stp; + Binary *magic; + + if (is_not_atom(module)) return NULL; + + erts_fprintf(stderr, "Creating HiPE loader state for module %T\n", module); + + magic = erts_create_magic_binary(sizeof(HipeLoaderState), + hipe_loader_state_dtor); + erts_refc_inc(&magic->refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); + + stp->module = module; + stp->text_segment = NULL; + stp->text_segment_size = 0; + stp->data_segment = NULL; + stp->data_segment_size = 0; + + return magic; +} + +void hipe_free_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return; + + /* Why does BEAM do a refc_dec here? What is holding that reference? */ + hipe_loader_state_dtor(magic); +} + +HipeLoaderState * +hipe_get_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return NULL; + + return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic); +} diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h new file mode 100644 index 0000000000..9e24e915ed --- /dev/null +++ b/erts/emulator/hipe/hipe_load.h @@ -0,0 +1,45 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.h + * + * HiPE atomic code loader + */ +#ifndef HIPE_LOAD_H +#define HIPE_LOAD_H + +#include "global.h" + +typedef struct hipe_loader_state { + Eterm module; /* Module name, atom */ + + void *text_segment; + Uint text_segment_size; + + void *data_segment; + Uint data_segment_size; + +} HipeLoaderState; + +extern Binary *hipe_alloc_loader_state(Eterm module); +extern void hipe_free_loader_state(Binary *binary); +extern HipeLoaderState *hipe_get_loader_state(Binary *binary); + +#endif /* HIPE_LOAD_H */ diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c new file mode 100644 index 0000000000..1553bf9087 --- /dev/null +++ b/erts/emulator/hipe/hipe_module.c @@ -0,0 +1,37 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "hipe_module.h" + +void hipe_free_module(HipeModule *mod) +{ +#ifdef DEBUG + sys_memzero(mod->text_segment, mod->text_segment_size); +#endif + /* XXX: erts_free(ERTS_ALC_T_HIPE, mod->text_segment); */ + if (mod->data_segment) /* Some modules lack data segments */ + erts_free(ERTS_ALC_T_HIPE, mod->data_segment); + + erts_free(ERTS_ALC_T_HIPE, mod); +} diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h new file mode 100644 index 0000000000..1e1302fc0a --- /dev/null +++ b/erts/emulator/hipe/hipe_module.h @@ -0,0 +1,42 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_module.h + * + * + */ +#ifndef HIPE_MODULE_H +#define HIPE_MODULE_H + +/* Forward-declare type to resolve circular dependency with module.h */ +typedef struct hipe_module HipeModule; + +#include "global.h" + +struct hipe_module { + void *text_segment; + Uint text_segment_size; + void *data_segment; +}; + +extern void hipe_free_module(HipeModule *mod); + +#endif /* HIPE_MODULE_H */ -- cgit v1.2.3 From 5e879fb1d4a9273f0a6183da7509dcdbc24f9653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20L=C3=A5ng?= Date: Fri, 16 Sep 2016 19:33:38 +0200 Subject: erts: Check hipe stack in check_process_code This is part of commit 1bd508921dd93086b05e7d0038b816b36c421d86. I did not include the fun-checking as we have a new purge strategy for funs in OTP 20. That remains to be solved some other way for hipe. --- erts/emulator/hipe/hipe_gc.c | 33 +++++++++++++++++++++++++++++++++ erts/emulator/hipe/hipe_stack.h | 2 ++ erts/emulator/hipe/hipe_x86_gc.h | 1 + 3 files changed, 36 insertions(+) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index be665ccdb9..e6ce7ce628 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) } return 0; } + +int +nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size) +{ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + if (!p->hipe.nstack || !nstack_walk_init_check(p)) + return 0; + ASSERT(p->hipe.nsp && p->hipe.nstend); + nsp = nstack_walk_nsp_begin(p); + nsp_end = nstack_walk_nsp_end(p); + sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state); + + /* Check the topmost frame */ + if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size)) + return 1; + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned long ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + if (ErtsInArea(ra, seg_start, seg_size)) + return 1; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return 0; +} diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 1863b0db8c..537755fefa 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -138,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area, Uint area_size); extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size); +extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size); + #endif /* HIPE_STACK_H */ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index e92e6ea718..a703e24b8c 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -65,6 +65,7 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) state->sdesc0 = sdesc; return sdesc; #else + state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */ state->sdesc0[0].fsize = 0; state->sdesc0[0].has_exnra = 0; state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : -- cgit v1.2.3 From e36c8d49359db1622ad381f705c54d460c4fb5e9 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 16 Sep 2016 20:01:11 +0200 Subject: erts: Use hipe_free_module --- erts/emulator/hipe/hipe_amd64.c | 5 +++-- erts/emulator/hipe/hipe_arch.h | 4 ++-- erts/emulator/hipe/hipe_bif0.c | 11 +++-------- erts/emulator/hipe/hipe_module.c | 6 ++---- 4 files changed, 10 insertions(+), 16 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index df53f4db30..96b8c22265 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -130,10 +130,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } -void hipe_free_code(void* code) +void hipe_free_code(void* code, unsigned int bytes) { ALLOC_CODE_STATS(--nr_allocs); - /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + ALLOC_CODE_STATS(total_alloc -= bytes); + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index df38a80069..059b8e7f29 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -30,8 +30,8 @@ extern void hipe_patch_load_fe(Uint *address, Uint value); extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); -extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void hipe_free_code(void*); +extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p); +extern void hipe_free_code(void*, unsigned int); extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); extern void hipe_free_native_stub(void*); diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index daa198d695..46d4378f21 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1743,14 +1743,9 @@ void hipe_purge_module(Module* modp) prevp = &p->next_in_mod; } } - if (modp->old.hipe_code && modp->old.hipe_code->text_segment) { -#ifdef DEBUG - sys_memset(modp->old.hipe_code->text_segment, 0xfe, - modp->old.hipe_code->text_segment_size); -#endif - hipe_free_code(modp->old.hipe_code->text_segment); - modp->old.hipe_code->text_segment = NULL; - modp->old.hipe_code->text_segment_size = 0; + if (modp->old.hipe_code) { + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; } } diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c index 1553bf9087..469f077dd2 100644 --- a/erts/emulator/hipe/hipe_module.c +++ b/erts/emulator/hipe/hipe_module.c @@ -22,14 +22,12 @@ #include "config.h" #endif #include "sys.h" +#include "hipe_arch.h" #include "hipe_module.h" void hipe_free_module(HipeModule *mod) { -#ifdef DEBUG - sys_memzero(mod->text_segment, mod->text_segment_size); -#endif - /* XXX: erts_free(ERTS_ALC_T_HIPE, mod->text_segment); */ + hipe_free_code(mod->text_segment, mod->text_segment_size); if (mod->data_segment) /* Some modules lack data segments */ erts_free(ERTS_ALC_T_HIPE, mod->data_segment); -- cgit v1.2.3 From 966098ceb9dd9d18e9bcd37cd06b96045903e320 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 20 Sep 2016 16:16:50 +0200 Subject: erts: Move new hipe ref and sdesc lists to loader state --- erts/emulator/hipe/hipe_bif0.c | 150 ++++++++++++++++++++++----------------- erts/emulator/hipe/hipe_bif0.tab | 2 +- erts/emulator/hipe/hipe_load.c | 3 + erts/emulator/hipe/hipe_load.h | 3 + erts/emulator/hipe/hipe_module.h | 3 + 5 files changed, 94 insertions(+), 67 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 46d4378f21..da895e4e8b 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -685,12 +685,17 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) BIF_RET(am_false); } -BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) +BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { struct hipe_sdesc *sdesc; + HipeLoaderState* stp; Module* modp; int do_commit; + stp = get_loader_state(BIF_ARG_2); + if (!stp) + BIF_ERROR(BIF_P, BADARG); + sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit); if (!sdesc) { fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__); @@ -707,12 +712,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) modp = erts_put_active_module(make_atom(sdesc->m_aix)); ASSERT(modp); if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - sdesc->next_in_modi = modp->curr.first_hipe_sdesc; - modp->curr.first_hipe_sdesc = sdesc; + ASSERT(modp->curr.hipe_code); + sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc; + modp->curr.hipe_code->first_hipe_sdesc = sdesc; } else { /* Normal module loading/upgrade */ - sdesc->next_in_modi = modp->new_hipe_sdesc; - modp->new_hipe_sdesc = sdesc; + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; } BIF_RET(NIL); @@ -1491,13 +1497,14 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) struct hipe_ref *ref; Module* modp; int do_commit; + HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; if (is_not_tuple(BIF_ARG_2)) goto badarg; tuple = tuple_val(BIF_ARG_2); - if (tuple[0] != make_arityval(5)) + if (tuple[0] != make_arityval(6)) goto badarg; if (!term_to_mfa(tuple[1], &caller)) goto badarg; @@ -1526,6 +1533,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) case am_false: do_commit = 0; break; default: goto badarg; } + stp = get_loader_state(tuple[6]); + if (!stp) + goto badarg; + hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); @@ -1550,12 +1561,13 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) modp = erts_put_active_module(caller.mod); ASSERT(modp); if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - ref->next_from_modi = modp->curr.first_hipe_ref; - modp->curr.first_hipe_ref = ref; + ASSERT(modp->curr.hipe_code); + ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref; + modp->curr.hipe_code->first_hipe_ref = ref; } else { /* Normal module loading/upgrade */ - ref->next_from_modi = modp->new_hipe_refs; - modp->new_hipe_refs = ref; + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; } #if defined(DEBUG) @@ -1653,9 +1665,12 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) int hipe_purge_need_blocking(Module* modp) { /* SVERK: Verify if this is really necessary */ - return (modp->old.first_hipe_ref || - modp->old.first_hipe_sdesc || - (!modp->curr.code_hdr && modp->first_hipe_mfa)); + if (modp->old.hipe_code) { + if (modp->old.hipe_code->first_hipe_ref || + modp->old.hipe_code->first_hipe_sdesc) + return 1; + } + return !modp->curr.code_hdr && modp->first_hipe_mfa; } void hipe_purge_module(Module* modp) @@ -1667,68 +1682,75 @@ void hipe_purge_module(Module* modp) DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); - /* - * Remove all hipe_ref's (external calls) from the old module instance - */ - ref = modp->old.first_hipe_ref; - - while (ref) { - struct hipe_ref* free_ref = ref; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, - ref->callee->m, ref->callee->f, ref->callee->a); - DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, - ref->caller_m, ref->caller_f, ref->caller_a); - ASSERT(ref->caller_m == make_atom(modp->module)); - + if (modp->old.hipe_code) { /* - * Unlink from other refs to same callee + * Remove all hipe_ref's (external calls) from the old module instance */ - ASSERT(ref->head.next->prev == &ref->head); - ASSERT(ref->head.prev->next == &ref->head); - ASSERT(ref->head.next != &ref->head); - ASSERT(ref->head.prev != &ref->head); - ref->head.next->prev = ref->head.prev; - ref->head.prev->next = ref->head.next; + ref = modp->old.hipe_code->first_hipe_ref; + + while (ref) { + struct hipe_ref* free_ref = ref; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == make_atom(modp->module)); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + unlink_mfa_from_mod(p); + purge_mfa(p); + } + } - /* - * Was this the last ref to that callee? - */ - if (ref->head.next == ref->head.prev) { - struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); - if (p->is_stub) { - unlink_mfa_from_mod(p); - purge_mfa(p); - } + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); } + modp->old.hipe_code->first_hipe_ref = NULL; - ref = ref->next_from_modi; - erts_free(ERTS_ALC_T_HIPE, free_ref); - } - modp->old.first_hipe_ref = NULL; + /* + * Remove all hipe_sdesc's for the old module instance + */ + sdesc = modp->old.hipe_code->first_hipe_sdesc; - /* - * Remove all hipe_sdesc's for the old module instance - */ - sdesc = modp->old.first_hipe_sdesc; + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; - while (sdesc) { - struct hipe_sdesc* free_sdesc = sdesc; + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(sdesc->m_aix == modp->module); - DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); - ASSERT(sdesc->m_aix == modp->module); + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } + modp->old.hipe_code->first_hipe_sdesc = NULL; - sdesc = sdesc->next_in_modi; - hipe_destruct_sdesc(free_sdesc); + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; } - modp->old.first_hipe_sdesc = NULL; + /* - * Remove unreferred hipe_mfa_info's + * Remove unreferred hipe_mfa_info's + * when all module instances are removed (like in init:restart) */ if (modp->curr.code_hdr == NULL) { struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; @@ -1743,10 +1765,6 @@ void hipe_purge_module(Module* modp) prevp = &p->next_in_mod; } } - if (modp->old.hipe_code) { - hipe_free_module(modp->old.hipe_code); - modp->old.hipe_code = NULL; - } } diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 1e4bde5ef2..8475b8ce76 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -56,7 +56,7 @@ bif hipe_bifs:set_native_address/3 bif hipe_bifs:set_funinfo_native_address/3 #bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:enter_sdesc/1 +bif hipe_bifs:enter_sdesc/2 bif hipe_bifs:bif_address/3 bif hipe_bifs:primop_address/1 diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index 5bce3f1aee..cebbbafdd3 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -79,6 +79,9 @@ Binary *hipe_alloc_loader_state(Eterm module) stp->data_segment = NULL; stp->data_segment_size = 0; + stp->new_hipe_refs = NULL; + stp->new_hipe_sdesc = NULL; + return magic; } diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h index 9e24e915ed..17dd3c3c05 100644 --- a/erts/emulator/hipe/hipe_load.h +++ b/erts/emulator/hipe/hipe_load.h @@ -36,6 +36,9 @@ typedef struct hipe_loader_state { void *data_segment; Uint data_segment_size; + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; + } HipeLoaderState; extern Binary *hipe_alloc_loader_state(Eterm module); diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h index 1e1302fc0a..b489f567cb 100644 --- a/erts/emulator/hipe/hipe_module.h +++ b/erts/emulator/hipe/hipe_module.h @@ -35,6 +35,9 @@ struct hipe_module { void *text_segment; Uint text_segment_size; void *data_segment; + + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ }; extern void hipe_free_module(HipeModule *mod); -- cgit v1.2.3 From e77d7a8417368617c4c228af9556a7f6a8f3e84c Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 28 Sep 2016 20:55:40 +0200 Subject: erts: Fix early hipe patch loading by introducing hipe_bifs:commit_patch_load/1 that creates the HipeModule. --- erts/emulator/hipe/hipe_bif0.c | 59 ++++++++++++++-------------------------- erts/emulator/hipe/hipe_bif0.tab | 1 + erts/emulator/hipe/hipe_load.c | 5 ++++ erts/emulator/hipe/hipe_stack.c | 10 ++----- erts/emulator/hipe/hipe_stack.h | 2 +- 5 files changed, 29 insertions(+), 48 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index da895e4e8b..67508e6f5d 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -649,6 +649,14 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1) BIF_RET(address_to_term(pc, BIF_P)); } +BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) +{ + if (!erts_commit_hipe_patch_load(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(am_ok); +} + BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) { Eterm *pc; @@ -689,14 +697,12 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { struct hipe_sdesc *sdesc; HipeLoaderState* stp; - Module* modp; - int do_commit; stp = get_loader_state(BIF_ARG_2); if (!stp) BIF_ERROR(BIF_P, BADARG); - sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit); + sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); @@ -709,17 +715,8 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) /* * Link into list of sdesc's in same module instance */ - modp = erts_put_active_module(make_atom(sdesc->m_aix)); - ASSERT(modp); - if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - ASSERT(modp->curr.hipe_code); - sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc; - modp->curr.hipe_code->first_hipe_sdesc = sdesc; - } - else { /* Normal module loading/upgrade */ - sdesc->next_in_modi = stp->new_hipe_sdesc; - stp->new_hipe_sdesc = sdesc; - } + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; BIF_RET(NIL); } @@ -1483,7 +1480,7 @@ int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) } -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { @@ -1495,8 +1492,6 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) unsigned int flags; struct hipe_mfa_info *callee_mfa; struct hipe_ref *ref; - Module* modp; - int do_commit; HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) @@ -1504,7 +1499,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (is_not_tuple(BIF_ARG_2)) goto badarg; tuple = tuple_val(BIF_ARG_2); - if (tuple[0] != make_arityval(6)) + if (tuple[0] != make_arityval(5)) goto badarg; if (!term_to_mfa(tuple[1], &caller)) goto badarg; @@ -1528,12 +1523,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (!trampoline) goto badarg; } - switch (tuple[5]) { - case am_true: do_commit = 1; break; - case am_false: do_commit = 0; break; - default: goto badarg; - } - stp = get_loader_state(tuple[6]); + stp = get_loader_state(tuple[5]); if (!stp) goto badarg; @@ -1558,17 +1548,8 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) /* * Link into list of refs from same module instance */ - modp = erts_put_active_module(caller.mod); - ASSERT(modp); - if (do_commit) { /* Direct "hipe-patching" of early loaded module */ - ASSERT(modp->curr.hipe_code); - ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref; - modp->curr.hipe_code->first_hipe_ref = ref; - } - else { /* Normal module loading/upgrade */ - ref->next_from_modi = stp->new_hipe_refs; - stp->new_hipe_refs = ref; - } + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; #if defined(DEBUG) ref->callee = callee_mfa; @@ -1578,10 +1559,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) #endif hipe_mfa_info_table_rwunlock(); - DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d", - ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit); - DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d", - ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)", + ref, callee.mod, callee.fun, callee.ari, ref->address); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)", + ref, caller.mod, caller.fun, caller.ari, ref->address); BIF_RET(NIL); badarg: diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 8475b8ce76..264ea2c34a 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -50,6 +50,7 @@ bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 bif hipe_bifs:fun_to_address/1 +bif hipe_bifs:commit_patch_load/1 bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index cebbbafdd3..48f0cab293 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -57,6 +57,11 @@ hipe_loader_state_dtor(Binary* magic) stp->data_segment_size = 0; stp->module = NIL; + + ASSERT(!stp->new_hipe_refs && !stp->new_hipe_sdesc); + /* ToDO: Purge lists 'new_hipe_refs' and 'new_hipe_sdesc' + * if this is a failed load. + */ } Binary *hipe_alloc_loader_state(Eterm module) diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index 3d082668a6..b80e44bc37 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -143,7 +143,7 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; @@ -158,7 +158,7 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) return 0; tp = tuple_val(arg); - if (tp[0] != make_arityval(7) || + if (tp[0] != make_arityval(6) || term_to_Uint(tp[1], &ra) == 0 || term_to_Uint(tp[2], &exnra) == 0 || is_not_small(tp[3]) || @@ -191,12 +191,6 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) return 0; } - switch(tp[7]) { - case am_true: *do_commitp = 1; break; - case am_false: *do_commitp = 0; break; - default: return 0; - } - /* Calculate number of words for the live bitmap. */ livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 537755fefa..7e30358767 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -82,7 +82,7 @@ extern struct hipe_sdesc_table hipe_sdesc_table; extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); extern void hipe_destruct_sdesc(struct hipe_sdesc*); extern void hipe_init_sdesc_table(struct hipe_sdesc*); -extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) -- cgit v1.2.3 From 870d49f0f8d383e62ea231700089933c4cf8fe2a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 29 Sep 2016 17:38:54 +0200 Subject: hipe,erts: Remove cached fun 'native_address' Did not work with purge and made worse by new purge strategy. Did yield terrible performance when fun thing is created *before* fun code is loaded. Like when receiving not yet loaded fun from other node. The cached 'native_address' in ErlFunThing will not be updated leading to mode switch and error_handler being called for every call to the fun from native code. --- erts/emulator/hipe/hipe_mkliterals.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index b9d4226705..4573980e1e 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = { presence or absence of struct erl_fun_thing's "next" field. */ { 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) }, { 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) }, -#ifdef HIPE - { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) }, -#endif { 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) }, { 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) }, { 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) }, -- cgit v1.2.3 From 1e8588e8c310a628fb0215ef15dc3cc29f98c336 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 2 Oct 2016 16:17:21 +0200 Subject: erts: Refactor hipe_loader_state_dtor into a true destructor that is only called once. Basically switch hipe_free_loader_state and hipe_loader_state_dtor. --- erts/emulator/hipe/hipe_load.c | 26 +++++++++++--------------- erts/emulator/hipe/hipe_load.h | 2 +- 2 files changed, 12 insertions(+), 16 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index 48f0cab293..6261857d8f 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -32,13 +32,8 @@ #include "erl_binary.h" #include "hipe_load.h" -/* - * This destructor function can safely be called multiple times. - */ -static void -hipe_loader_state_dtor(Binary* magic) +void hipe_free_loader_state(HipeLoaderState *stp) { - HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); if (stp->module == NIL) return; erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n", @@ -64,6 +59,16 @@ hipe_loader_state_dtor(Binary* magic) */ } +static void +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor); + + hipe_free_loader_state(stp); +} + Binary *hipe_alloc_loader_state(Eterm module) { HipeLoaderState *stp; @@ -90,15 +95,6 @@ Binary *hipe_alloc_loader_state(Eterm module) return magic; } -void hipe_free_loader_state(Binary *magic) -{ - if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) - return; - - /* Why does BEAM do a refc_dec here? What is holding that reference? */ - hipe_loader_state_dtor(magic); -} - HipeLoaderState * hipe_get_loader_state(Binary *magic) { diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h index 17dd3c3c05..40c8a8aa2a 100644 --- a/erts/emulator/hipe/hipe_load.h +++ b/erts/emulator/hipe/hipe_load.h @@ -42,7 +42,7 @@ typedef struct hipe_loader_state { } HipeLoaderState; extern Binary *hipe_alloc_loader_state(Eterm module); -extern void hipe_free_loader_state(Binary *binary); +extern void hipe_free_loader_state(HipeLoaderState*); extern HipeLoaderState *hipe_get_loader_state(Binary *binary); #endif /* HIPE_LOAD_H */ -- cgit v1.2.3 From 663360ea2bb7766ad2015fff7f3f8a8f53c3eef2 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 2 Oct 2016 22:49:59 +0200 Subject: erts: Refactor out hipe_purge_refs/sdesc from hipe_purge_module. --- erts/emulator/hipe/hipe_bif0.c | 119 +++++++++++++++++++++++------------------ erts/emulator/hipe/hipe_bif0.h | 2 + 2 files changed, 68 insertions(+), 53 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 67508e6f5d..58eee1693f 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1654,11 +1654,66 @@ int hipe_purge_need_blocking(Module* modp) return !modp->curr.code_hdr && modp->first_hipe_mfa; } -void hipe_purge_module(Module* modp) +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) { - struct hipe_ref* ref; - struct hipe_sdesc* sdesc; + struct hipe_ref* ref = first_ref; + + while (ref) { + struct hipe_ref* free_ref = ref; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == caller_module); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + unlink_mfa_from_mod(p); + purge_mfa(p); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); + } +} +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module) +{ + struct hipe_sdesc* sdesc = first_sdesc; + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(make_atom(sdesc->m_aix) == module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } +} + + +void hipe_purge_module(Module* modp) +{ ASSERT(modp); DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); @@ -1667,62 +1722,20 @@ void hipe_purge_module(Module* modp) /* * Remove all hipe_ref's (external calls) from the old module instance */ - ref = modp->old.hipe_code->first_hipe_ref; - - while (ref) { - struct hipe_ref* free_ref = ref; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, - ref->callee->m, ref->callee->f, ref->callee->a); - DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, - ref->caller_m, ref->caller_f, ref->caller_a); - ASSERT(ref->caller_m == make_atom(modp->module)); - - /* - * Unlink from other refs to same callee - */ - ASSERT(ref->head.next->prev == &ref->head); - ASSERT(ref->head.prev->next == &ref->head); - ASSERT(ref->head.next != &ref->head); - ASSERT(ref->head.prev != &ref->head); - ref->head.next->prev = ref->head.prev; - ref->head.prev->next = ref->head.next; - - /* - * Was this the last ref to that callee? - */ - if (ref->head.next == ref->head.prev) { - struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); - if (p->is_stub) { - unlink_mfa_from_mod(p); - purge_mfa(p); - } - } - - ref = ref->next_from_modi; - erts_free(ERTS_ALC_T_HIPE, free_ref); + if (modp->old.hipe_code->first_hipe_ref) { + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, + make_atom(modp->module)); + modp->old.hipe_code->first_hipe_ref = NULL; } - modp->old.hipe_code->first_hipe_ref = NULL; /* * Remove all hipe_sdesc's for the old module instance */ - sdesc = modp->old.hipe_code->first_hipe_sdesc; - - while (sdesc) { - struct hipe_sdesc* free_sdesc = sdesc; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); - ASSERT(sdesc->m_aix == modp->module); - - sdesc = sdesc->next_in_modi; - hipe_destruct_sdesc(free_sdesc); + if (modp->old.hipe_code->first_hipe_sdesc) { + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, + make_atom(modp->module)); + modp->old.hipe_code->first_hipe_sdesc = NULL; } - modp->old.hipe_code->first_hipe_sdesc = NULL; hipe_free_module(modp->old.hipe_code); modp->old.hipe_code = NULL; diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 1914405e2d..2b7c01b975 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -44,6 +44,8 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); /* needed in beam_load.c */ int hipe_need_blocking(Module*); int hipe_purge_need_blocking(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm); void hipe_purge_module(Module*); void hipe_redirect_to_module(Module* modp); -- cgit v1.2.3 From d9e8fa3d13be54246297ac9776e835ce0bdcd362 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Sun, 2 Oct 2016 22:51:49 +0200 Subject: erts: Free hipe_refs and hipe_sdesc of a failed load --- erts/emulator/hipe/hipe_load.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index 6261857d8f..c05dd886a4 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -31,6 +31,7 @@ #include "global.h" #include "erl_binary.h" #include "hipe_load.h" +#include "hipe_bif0.h" void hipe_free_loader_state(HipeLoaderState *stp) { @@ -51,12 +52,16 @@ void hipe_free_loader_state(HipeLoaderState *stp) stp->data_segment = NULL; stp->data_segment_size = 0; - stp->module = NIL; + if (stp->new_hipe_refs) { + hipe_purge_refs(stp->new_hipe_refs, stp->module); + stp->new_hipe_refs = NULL; + } + if (stp->new_hipe_sdesc) { + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module); + stp->new_hipe_sdesc = NULL; + } - ASSERT(!stp->new_hipe_refs && !stp->new_hipe_sdesc); - /* ToDO: Purge lists 'new_hipe_refs' and 'new_hipe_sdesc' - * if this is a failed load. - */ + stp->module = NIL; } static void -- cgit v1.2.3 From d347e91735cce9ace9c376ba4913fcf688da22f8 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Mon, 3 Oct 2016 19:22:08 +0200 Subject: erts: Remove debug printout for hipe loader state creation and destruction. --- erts/emulator/hipe/hipe_load.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index c05dd886a4..fcbcf1a6a4 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -37,9 +37,6 @@ void hipe_free_loader_state(HipeLoaderState *stp) { if (stp->module == NIL) return; - erts_fprintf(stderr, "Destroying HiPE loader state for module %T\n", - stp->module); - // TODO: Needs to be freed separately. We'd like have a unified executable // code allocator, so postpone this for now. /* if (stp->text_segment) */ @@ -81,8 +78,6 @@ Binary *hipe_alloc_loader_state(Eterm module) if (is_not_atom(module)) return NULL; - erts_fprintf(stderr, "Creating HiPE loader state for module %T\n", module); - magic = erts_create_magic_binary(sizeof(HipeLoaderState), hipe_loader_state_dtor); erts_refc_inc(&magic->refc, 1); -- cgit v1.2.3 From d9149ef5e2591d34ba781a08b1ae3e1823f7a13a Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 5 Oct 2016 17:54:15 +0200 Subject: erts: Fix old leak of hipe code on x86 32-bit --- erts/emulator/hipe/hipe_x86.c | 112 ++---------------------------------------- 1 file changed, 3 insertions(+), 109 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 8a3ba8bc7f..b091bee6fb 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -70,110 +69,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -/* FreeBSD 6.1 and Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -186,11 +84,7 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * void hipe_free_code(void* code, unsigned int bytes) { - /* SVERK: Leak !!! - ALLOC_CODE_STATS(--nr_allocs); - ALLOC_CODE_STATS(total_alloc -= bytes); - - erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -275,7 +169,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) void hipe_free_native_stub(void* stub) { - /* SVERK: leak leak drip drop */ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } void hipe_arch_print_pcb(struct hipe_process_state *p) -- cgit v1.2.3 From e048ebf4bddf8434d0f4402863c0f6313a741df0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 5 Oct 2016 19:48:01 +0200 Subject: erts: Fix old leak of sparc hipe code --- erts/emulator/hipe/hipe_sparc.c | 109 ++++------------------------------------ 1 file changed, 11 insertions(+), 98 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 23020f34ee..75bfb4fef1 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int nrbytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; @@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ -- cgit v1.2.3 From 25b28611ea06d3e2d3708d20e53118ec24ae8321 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 5 Oct 2016 19:09:28 +0200 Subject: erts: Fix old leak for arm hipe code * Use erts_alloc(ERTS_ALC_T_HIPE_EXEC,_) * Each module has its own trampolines * Stubs do not use trampolines, they do their own long jumps. --- erts/emulator/hipe/hipe_arm.c | 201 ++++++++++++----------------------------- erts/emulator/hipe/hipe_bif0.c | 1 - 2 files changed, 57 insertions(+), 145 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index 64e35b62d8..b61939724c 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address) hipe_flush_icache_range(address, 4); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ - /* On ARM we always allocate a trampoline at base+32MB-8 for - nbif_callemu, so tramp_pos <= base+32MB-8. */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} static int check_callees(Eterm callees) { @@ -107,136 +82,53 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) -{ - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; - Eterm mfa, m, f; - unsigned int a, *trampoline; +#define TRAMPOLINE_WORDS 2 - m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */ - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - mfa = tuple_val(callees)[trampnr]; - if (is_atom(mfa)) - trampoline = hipe_primop_get_trampoline(mfa); - else { - m = tuple_val(mfa)[1]; - f = tuple_val(mfa)[2]; - a = unsigned_val(tuple_val(mfa)[3]); - trampoline = hipe_mfa_get_trampoline(m, f, a); - } - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { - if (nrfreewords < 2) - return NULL; - nrfreewords -= 2; - tramp_pos = trampoline = tramp_pos - 2; - trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - trampoline[1] = 0; /* callee's address */ - hipe_flush_icache_range(trampoline, 2*sizeof(int)); - if (is_atom(mfa)) - hipe_primop_set_trampoline(mfa, trampoline); - else - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) +{ + Uint32* trampoline = address; + int i; + + for (i = 0; i < nrcallees; ++i) { + trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ + trampoline[1] = 0; /* callee's address */ + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } void hipe_free_code(void* code, unsigned int bytes) { - /*SVERK: Leaking code memory */ -} - -static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) -{ - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2; - return address; -} - -void hipe_free_native_stub(void* stub) -{ - /*SVERK: Leaking code stub */ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } /* @@ -276,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; - unsigned int *tramp_callemu; int callemu_offset; + int is_short_jmp; /* * Native code calls BEAM via a stub looking as follows: @@ -287,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) * b nbif_callemu * .long callee_exp * + * or if nbif_callemu is too far away: + * + * mov r0, #beamArity + * ldr r8, [pc,#0] // callee_exp + * ldr pc, [pc,#0] // nbif_callemu + * .long callee_exp + * .long nbif_callemu + * * I'm using r0 and r8 since they aren't used for - * parameter passing in native code. The branch to - * nbif_callemu may need to go via a trampoline. - * (Trampolines are allowed to modify r12, but they don't.) + * parameter passing in native code. */ - code = alloc_stub(4, &tramp_callemu); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32)); if (!code) return NULL; callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) { - callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) - abort(); + is_short_jmp = (callemu_offset >= -0x00800000 && + callemu_offset <= 0x007FFFFF); +#ifdef DEBUG + if (is_short_jmp && (callemu_offset % 3)==0) { + is_short_jmp = 0; } +#endif /* mov r0, #beamArity */ code[0] = 0xE3A00000 | (beamArity & 0xFF); /* ldr r8, [pc,#0] // callee_exp */ code[1] = 0xE59F8000; - /* b nbif_callemu */ - code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + if (is_short_jmp) { + /* b nbif_callemu */ + code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + } + else { + /* ldr pc, [pc,#0] // nbif_callemu */ + code[2] = 0xE59FF000; + /* .long nbif_callemu */ + code[4] = (unsigned int)&nbif_callemu; + } /* .long callee_exp */ code[3] = (unsigned int)callee_exp; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 5*sizeof(Uint32)); return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA) { Uint32 oldI = *address; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 58eee1693f..f9a657124a 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -417,7 +417,6 @@ BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) ASSERT(bitoffs == 0); ASSERT(bitsize == 0); trampolines = NIL; - // XXX: Trampolines are not tracked and freed address = hipe_alloc_code(nrbytes, BIF_ARG_2, &trampolines, BIF_P); if (!address) { Uint nrcallees; -- cgit v1.2.3 From 65fbc0464cce8266024129fcf10ef0906f907eb4 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 12 Oct 2016 19:00:53 +0200 Subject: erts: Fix old leak for ppc hipe code * Use erts_alloc(ERTS_ALC_T_HIPE_EXEC,_) * Each module has its own trampolines --- erts/emulator/hipe/hipe_ppc.c | 178 ++++++++++-------------------------------- 1 file changed, 42 insertions(+), 136 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index a1a6e2ad02..4413748936 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes) asm volatile("sync\n\tisync"); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -/* Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} - static int check_callees(Eterm callees) { Eterm *tuple; @@ -119,136 +90,71 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) + +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) { - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; + Uint32* trampoline = address; + int i; - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - Eterm mfa = tuple_val(callees)[trampnr]; - Eterm m = tuple_val(mfa)[1]; - Eterm f = tuple_val(mfa)[2]; - unsigned int a = unsigned_val(tuple_val(mfa)[3]); - unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a); - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { + for (i = 0; i < nrcallees; ++i) { #if defined(__powerpc64__) - if (nrfreewords < 7) - return NULL; - nrfreewords -= 7; - tramp_pos = trampoline = tramp_pos - 7; - trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ - trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ - trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ - trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[5] = 0x7D6903A6; /* mtctr r11 */ - trampoline[6] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 7*sizeof(int)); +# define TRAMPOLINE_WORDS 7 + trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ + trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ + trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ + trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[5] = 0x7D6903A6; /* mtctr r11 */ + trampoline[6] = 0x4E800420; /* bctr */ #else - if (nrfreewords < 4) - return NULL; - nrfreewords -= 4; - tramp_pos = trampoline = tramp_pos - 4; - trampoline[0] = 0x39600000; /* addi r11,r0,0 */ - trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ - trampoline[2] = 0x7D6903A6; /* mtctr r11 */ - trampoline[3] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 4*sizeof(int)); +# define TRAMPOLINE_WORDS 4 + trampoline[0] = 0x39600000; /* addi r11,r0,0 */ + trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ + trampoline[2] = 0x7D6903A6; /* mtctr r11 */ + trampoline[3] = 0x4E800420; /* bctr */ #endif - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); + + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); + + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } void hipe_free_code(void* code, unsigned int bytes) { - /*SVERK: Leaking code memory */ -} - -static unsigned int *alloc_stub(Uint nrwords) -{ - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } void hipe_free_native_stub(void* stub) { - /*SVERK: Leaking code stubs */ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } static void patch_imm16(Uint32 *address, unsigned int imm16) @@ -298,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(7); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32)); if (!code) return NULL; @@ -322,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[6] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 7*sizeof(int)); + hipe_flush_icache_range(code, 7*sizeof(Uint32)); return code; } @@ -370,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; /* * Native code calls BEAM via a stub looking as follows: @@ -393,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(4); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32)); if (!code) return NULL; @@ -406,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[3] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 4*sizeof(Uint32)); return code; } -- cgit v1.2.3 From 6cd1469d0c8f9af602332b20772134c1241f6429 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 6 Oct 2016 20:09:21 +0200 Subject: erts: Fix bug in stack walk on risc Symptom: Got ra==NULL at nsp==nsp_end when running basic_SUITE:basic_arith on arm why has this not been detected before? --- erts/emulator/hipe/hipe_risc_stack.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index 7528a3ab00..4001bedeb6 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) ra = (unsigned long)p->hipe.nra; prev_ra = 0; i = 0; - for (;;) { + while (nsp < nsp_end) { if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; if (ra != prev_ra) { @@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) break; prev_ra = ra; } - if (nsp >= nsp_end) - break; sdesc = hipe_find_sdesc(ra); nsp += arity + sdesc_fsize(sdesc); arity = sdesc_arity(sdesc); -- cgit v1.2.3 From df9e09fc1f938f9902052dcb896b42558ee9779d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 7 Oct 2016 15:54:33 +0200 Subject: erts: Remove dead alloc stats in hipe_amd64.c erlang:system_info({allocator, exec_alloc}) have stats if we need it. --- erts/emulator/hipe/hipe_amd64.c | 32 -------------------------------- 1 file changed, 32 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 96b8c22265..3173afad0a 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - /* * Memory allocator for executable code. * @@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void) */ static void *alloc_code(unsigned int alloc_bytes) { - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } @@ -132,9 +106,6 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * void hipe_free_code(void* code, unsigned int bytes) { - ALLOC_CODE_STATS(--nr_allocs); - ALLOC_CODE_STATS(total_alloc -= bytes); - erts_free(ERTS_ALC_T_HIPE_EXEC, code); } @@ -244,9 +215,6 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) void hipe_free_native_stub(void* stub) { - ALLOC_CODE_STATS(++nr_allocs); - /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ - erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } -- cgit v1.2.3 From 8bb80fe76f5b4debe981e42ba70260c5a12b250f Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Fri, 14 Oct 2016 14:43:22 +0200 Subject: erts: Cleanup hipe trampoline code --- erts/emulator/hipe/hipe_amd64.c | 4 +-- erts/emulator/hipe/hipe_bif0.c | 70 +++++++---------------------------------- erts/emulator/hipe/hipe_bif0.h | 8 ----- erts/emulator/hipe/hipe_sparc.c | 4 +-- erts/emulator/hipe/hipe_x86.c | 4 +-- 5 files changed, 18 insertions(+), 72 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 3173afad0a..e3cff4a4ba 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Sint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Sint)destAddress - (Sint)callAddress - 4; if ((Sint)(Sint32)rel32 != rel32) return -1; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index f9a657124a..0402a4de98 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -819,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3) struct primop { HashBucket bucket; /* bucket.hvalue == atom_val(name) */ const void *address; -#if defined(__arm__) - void *trampoline; -#endif }; static struct primop primops[] = { @@ -880,29 +877,6 @@ static struct primop *primop_table_get(Eterm name) return hash_get(&primop_table, &tmpl); } -#if defined(__arm__) -static struct primop *primop_table_put(Eterm name) -{ - struct primop tmpl; - - init_primop_table(); - tmpl.bucket.hvalue = atom_val(name); - return hash_put(&primop_table, &tmpl); -} - -void *hipe_primop_get_trampoline(Eterm name) -{ - struct primop *primop = primop_table_get(name); - return primop ? primop->trampoline : NULL; -} - -void hipe_primop_set_trampoline(Eterm name, void *trampoline) -{ - struct primop *primop = primop_table_put(name); - primop->trampoline = trampoline; -} -#endif - /* * hipe_bifs_primop_address(Atom) -> address or false */ @@ -1062,9 +1036,6 @@ struct hipe_mfa_info { void *new_address; struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ struct hipe_mfa_info* next_in_mod; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - void *trampoline; -#endif #ifdef DEBUG Export* dbg_export; #endif @@ -1094,7 +1065,9 @@ static struct { struct hipe_ref { struct hipe_ref_head head; /* list of refs to same calleee */ void *address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void *trampoline; +#endif unsigned int flags; struct hipe_ref* next_from_modi; /* list of refs from same module instance */ #if defined(DEBUG) @@ -1181,9 +1154,6 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->callers.next = &res->callers; res->callers.prev = &res->callers; res->next_in_mod = NULL; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - res->trampoline = NULL; -#endif #ifdef DEBUG res->dbg_export = NULL; #endif @@ -1294,30 +1264,6 @@ static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) hipe_mfa_info_table_rwunlock(); } -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity) -{ - struct hipe_mfa_info *p; - void *trampoline; - - hipe_mfa_info_table_rlock(); - p = hipe_mfa_info_table_get_locked(m, f, arity); - trampoline = p ? p->trampoline : NULL; - hipe_mfa_info_table_runlock(); - return trampoline; -} - -void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline) -{ - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(m, f, arity); - p->trampoline = trampoline; - hipe_mfa_info_table_rwunlock(); -} -#endif - BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { struct hipe_mfa mfa; @@ -1531,7 +1477,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; +#endif ref->flags = flags; /* @@ -1815,8 +1763,14 @@ void hipe_redirect_to_module(Module* modp) if (ref->flags & REF_FLAG_IS_LOAD_MFA) res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline); + else { +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void* trampoline = ref->trampoline; +#else + void* trampoline = NULL; +#endif + res = hipe_patch_call(ref->address, p->remote_address, trampoline); + } if (res) fprintf(stderr, "%s: patch failed", __FUNCTION__); } diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 2b7c01b975..02f0d6c823 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -32,14 +32,6 @@ extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a); -extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline); -#endif -#if defined(__arm__) -extern void *hipe_primop_get_trampoline(Eterm name); -extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); -#endif /* needed in beam_load.c */ int hipe_need_blocking(Module*); diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 75bfb4fef1..876b20bb15 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -87,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint32 relDest, newI; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress); newI = (1 << 30) | (relDest >> 2); *(Uint32*)callAddress = newI; diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index b091bee6fb..c7e24673ac 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -61,8 +61,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Uint)destAddress - (Uint)callAddress - 4; *(Uint32*)callAddress = rel32; hipe_flush_icache_word(callAddress); -- cgit v1.2.3 From 3582e6f420d84ddac64b55cb13100f1ae7f31b08 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 13 Oct 2016 20:59:58 +0200 Subject: erts: Replace unsafe Module.first_hipe_ref with hash table mod2mfa_tab --- erts/emulator/hipe/hipe_bif0.c | 196 +++++++++++++++++++++++++++++++---------- erts/emulator/hipe/hipe_bif0.h | 6 +- erts/emulator/hipe/hipe_load.c | 4 +- 3 files changed, 153 insertions(+), 53 deletions(-) (limited to 'erts/emulator/hipe') diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 0402a4de98..4c9757e8d3 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1024,6 +1024,7 @@ struct hipe_ref_head { * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { + HashBucket mod2mfa; struct { unsigned long hvalue; struct hipe_mfa_info *next; @@ -1057,6 +1058,61 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; +Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */ + +static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa) +{ + return mfa->mod2mfa.hvalue; +} + +static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa) +{ + return tmpl->hvalue != mfa->mod2mfa.hvalue; +} + +static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl) +{ + return tmpl; /* hash_put always use mfa itself at template */ +} + +static void mod2mfa_free(struct hipe_mfa_info* mfa) +{ +} + +static void mod2mfa_tab_init(void) +{ + HashFunctions f; + static int init_done = 0; + + if (init_done) + return; + init_done = 1; + + f.hash = (H_FUN) mod2mfa_hash; + f.cmp = (HCMP_FUN) mod2mfa_cmp; + f.alloc = (HALLOC_FUN) mod2mfa_alloc; + f.free = (HFREE_FUN) mod2mfa_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; + + hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f); +} + +static struct hipe_mfa_info* mod2mfa_get(Module* modp) +{ + HashBucket tmpl; + tmpl.hvalue = modp->module; + return hash_get(&mod2mfa_tab, &tmpl); +} + +static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) +{ + mfa->mod2mfa.hvalue = atom_val(mfa->m); + return hash_put(&mod2mfa_tab, mfa); +} + + /* * An external native call site M:F(...) @@ -1103,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } +static ERTS_INLINE +struct hipe_mfa_info* mod2mfa_get_safe(Module* modp) +{ + struct hipe_mfa_info* mfa; + hipe_mfa_info_table_rlock(); + mfa = mod2mfa_get(modp); + hipe_mfa_info_table_runlock(); + return mfa; +} + #define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) @@ -1173,6 +1239,8 @@ void hipe_mfa_info_table_init(void) hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size); hipe_mfa_info_table_init_lock(); + + mod2mfa_tab_init(); } static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity) @@ -1199,8 +1267,8 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned long h; unsigned int i; struct hipe_mfa_info *p; + struct hipe_mfa_info *first_in_mod; unsigned int size; - Module* modp; h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; @@ -1221,10 +1289,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); - modp = erts_put_active_module(m); - ASSERT(modp); - p->next_in_mod = modp->first_hipe_mfa; - modp->first_hipe_mfa = p; + first_in_mod = mod2mfa_put(p); + if (p != first_in_mod) { + p->next_in_mod = first_in_mod->next_in_mod; + first_in_mod->next_in_mod = p; + } + else { + p->next_in_mod = NULL; + } DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); @@ -1288,7 +1360,7 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) /* Ask if we need to block all threads - * while loading/deleting (beam) code for this module? + * while loading/deleting code for this module? */ int hipe_need_blocking(Module* modp) { @@ -1298,7 +1370,7 @@ int hipe_need_blocking(Module* modp) * or native code to make unaccessible. */ hipe_mfa_info_table_rlock(); - for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { ASSERT(!p->new_address); if (p->callers.next != &p->callers || !p->is_stub) { break; @@ -1519,19 +1591,25 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix()); - struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; struct hipe_mfa_info* p; - ASSERT(modp); - for (;;) { - p = *prevp; - ASSERT(p && p->m == unlink_me->m); - if (p == unlink_me) { - *prevp = p->next_in_mod; - break; - } - prevp = &p->next_in_mod; + p = hash_get(&mod2mfa_tab, unlink_me); + ASSERT(p); + if (p == unlink_me) { + hash_erase(&mod2mfa_tab, p); + if (p->next_in_mod) + mod2mfa_put(p->next_in_mod); + } + else { + struct hipe_mfa_info** prevp; + + do { + prevp = &p->next_in_mod; + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + } while (p != unlink_me); + + *prevp = p->next_in_mod; } } @@ -1561,18 +1639,11 @@ static void hipe_purge_all_refs(void) for (i = 0; i < nrbuckets; ++i) { ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { - Module* modp; struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - ASSERT(mfa->callers.next == &mfa->callers); - - modp = erts_get_module(mfa->m, erts_active_code_ix()); - if (modp) { - /* unsafe write to active module */ - modp->first_hipe_mfa = NULL; - } - erts_free(ERTS_ALC_T_HIPE, mfa); + hash_erase(&mod2mfa_tab, mfa); + erts_free(ERTS_ALC_T_HIPE, mfa); } } hipe_mfa_info_table.used = 0; @@ -1598,18 +1669,22 @@ int hipe_purge_need_blocking(Module* modp) modp->old.hipe_code->first_hipe_sdesc) return 1; } - return !modp->curr.code_hdr && modp->first_hipe_mfa; + if (!modp->curr.code_hdr) { + return mod2mfa_get_safe(modp) != NULL; + } + return 0; } -void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module, + int is_blocking) { struct hipe_ref* ref = first_ref; + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + while (ref) { struct hipe_ref* free_ref = ref; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, ref->callee->m, ref->callee->f, ref->callee->a); DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, @@ -1632,8 +1707,12 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) if (ref->head.next == ref->head.prev) { struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); if (p->is_stub) { + if (!is_blocking) + hipe_mfa_info_table_rwlock(); unlink_mfa_from_mod(p); purge_mfa(p); + if (!is_blocking) + hipe_mfa_info_table_rwunlock(); } } @@ -1642,14 +1721,18 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module) } } -void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module) +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module, + int is_blocking) { struct hipe_sdesc* sdesc = first_sdesc; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ + while (sdesc) { struct hipe_sdesc* free_sdesc = sdesc; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); ASSERT(make_atom(sdesc->m_aix) == module); @@ -1659,10 +1742,12 @@ void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module) } -void hipe_purge_module(Module* modp) +void hipe_purge_module(Module* modp, int is_blocking) { ASSERT(modp); + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); if (modp->old.hipe_code) { @@ -1670,8 +1755,10 @@ void hipe_purge_module(Module* modp) * Remove all hipe_ref's (external calls) from the old module instance */ if (modp->old.hipe_code->first_hipe_ref) { + ERTS_SMP_LC_ASSERT(is_blocking); + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, - make_atom(modp->module)); + make_atom(modp->module), is_blocking); modp->old.hipe_code->first_hipe_ref = NULL; } @@ -1679,8 +1766,10 @@ void hipe_purge_module(Module* modp) * Remove all hipe_sdesc's for the old module instance */ if (modp->old.hipe_code->first_hipe_sdesc) { + ERTS_SMP_LC_ASSERT(is_blocking); + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, - make_atom(modp->module)); + make_atom(modp->module), is_blocking); modp->old.hipe_code->first_hipe_sdesc = NULL; } @@ -1693,17 +1782,28 @@ void hipe_purge_module(Module* modp) * Remove unreferred hipe_mfa_info's * when all module instances are removed (like in init:restart) */ - if (modp->curr.code_hdr == NULL) { - struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; - struct hipe_mfa_info* p = *prevp; - ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking()); - for (; p; p = *prevp) { - if (p->callers.next == &p->callers) { - *prevp = p->next_in_mod; - purge_mfa(p); + if (is_blocking && modp->curr.code_hdr == NULL) { + struct hipe_mfa_info* was_first = mod2mfa_get(modp); + struct hipe_mfa_info* is_first = was_first; + struct hipe_mfa_info** prevp = &is_first; + struct hipe_mfa_info *p; + + if (was_first) { + for (p = was_first ; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + if (p != was_first) + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + if (was_first != is_first) { + hash_erase(&mod2mfa_tab, was_first); + purge_mfa(was_first); + if (is_first) + mod2mfa_put(is_first); } - else - prevp = &p->next_in_mod; } } } @@ -1719,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp) ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { if (p->new_address) { if (p->is_stub) { hipe_free_native_stub(p->remote_address); diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 02f0d6c823..4a59bacc6e 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -36,9 +36,9 @@ extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned in /* needed in beam_load.c */ int hipe_need_blocking(Module*); int hipe_purge_need_blocking(Module*); -void hipe_purge_refs(struct hipe_ref*, Eterm); -void hipe_purge_sdescs(struct hipe_sdesc*, Eterm); -void hipe_purge_module(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking); +void hipe_purge_module(Module*, int is_blocking); void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c index fcbcf1a6a4..2998ed87a2 100644 --- a/erts/emulator/hipe/hipe_load.c +++ b/erts/emulator/hipe/hipe_load.c @@ -50,11 +50,11 @@ void hipe_free_loader_state(HipeLoaderState *stp) stp->data_segment_size = 0; if (stp->new_hipe_refs) { - hipe_purge_refs(stp->new_hipe_refs, stp->module); + hipe_purge_refs(stp->new_hipe_refs, stp->module, 0); stp->new_hipe_refs = NULL; } if (stp->new_hipe_sdesc) { - hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module); + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0); stp->new_hipe_sdesc = NULL; } -- cgit v1.2.3