aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--erts/emulator/beam/beam_bif_load.c2
-rw-r--r--erts/emulator/beam/module.c7
-rw-r--r--erts/emulator/beam/module.h3
-rw-r--r--erts/emulator/hipe/hipe_bif0.c196
-rw-r--r--erts/emulator/hipe/hipe_bif0.h6
-rw-r--r--erts/emulator/hipe/hipe_load.c4
6 files changed, 155 insertions, 63 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 0a4d89d51b..c31abbb84f 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1711,7 +1711,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
modp->old.catches = BEAM_CATCHES_NIL;
erts_remove_from_ranges(code);
#ifdef HIPE
- hipe_purge_module(modp);
+ hipe_purge_module(modp, is_blocking);
#endif
ERTS_BIF_PREP_RET(ret, am_true);
}
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index e9b18ba525..3e622c0f3a 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -91,9 +91,6 @@ static Module* module_alloc(Module* tmpl)
erts_module_instance_init(&obj->curr);
erts_module_instance_init(&obj->old);
obj->on_load = 0;
-#ifdef HIPE
- obj->first_hipe_mfa = NULL;
-#endif
DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
return obj;
}
@@ -128,6 +125,7 @@ void init_module_table(void)
erts_smp_atomic_init_nob(&tot_module_bytes, 0);
}
+
Module*
erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
{
@@ -208,9 +206,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
dst_mod->curr = src_mod->curr;
dst_mod->old = src_mod->old;
dst_mod->on_load = src_mod->on_load;
-#ifdef HIPE
- dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa;
-#endif
}
void module_start_staging(void)
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 9de610ccf7..1d6d9dd463 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -47,9 +47,6 @@ typedef struct erl_module {
struct erl_module_instance curr;
struct erl_module_instance old; /* protected by "old_code" rwlock */
struct erl_module_instance* on_load;
-#ifdef HIPE
- struct hipe_mfa_info* first_hipe_mfa;
-#endif
} Module;
void erts_module_instance_init(struct erl_module_instance* modi);
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 0402a4de98..4c9757e8d3 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1024,6 +1024,7 @@ struct hipe_ref_head {
* - maps MFA to most recent trampoline [if powerpc or arm]
*/
struct hipe_mfa_info {
+ HashBucket mod2mfa;
struct {
unsigned long hvalue;
struct hipe_mfa_info *next;
@@ -1057,6 +1058,61 @@ static struct {
erts_smp_rwmtx_t lock;
} hipe_mfa_info_table;
+Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */
+
+static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa)
+{
+ return mfa->mod2mfa.hvalue;
+}
+
+static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa)
+{
+ return tmpl->hvalue != mfa->mod2mfa.hvalue;
+}
+
+static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl)
+{
+ return tmpl; /* hash_put always use mfa itself at template */
+}
+
+static void mod2mfa_free(struct hipe_mfa_info* mfa)
+{
+}
+
+static void mod2mfa_tab_init(void)
+{
+ HashFunctions f;
+ static int init_done = 0;
+
+ if (init_done)
+ return;
+ init_done = 1;
+
+ f.hash = (H_FUN) mod2mfa_hash;
+ f.cmp = (HCMP_FUN) mod2mfa_cmp;
+ f.alloc = (HALLOC_FUN) mod2mfa_alloc;
+ f.free = (HFREE_FUN) mod2mfa_free;
+ f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+ f.meta_free = (HMFREE_FUN) erts_free;
+ f.meta_print = (HMPRINT_FUN) erts_print;
+
+ hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f);
+}
+
+static struct hipe_mfa_info* mod2mfa_get(Module* modp)
+{
+ HashBucket tmpl;
+ tmpl.hvalue = modp->module;
+ return hash_get(&mod2mfa_tab, &tmpl);
+}
+
+static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa)
+{
+ mfa->mod2mfa.hvalue = atom_val(mfa->m);
+ return hash_put(&mod2mfa_tab, mfa);
+}
+
+
/*
* An external native call site M:F(...)
@@ -1103,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void)
erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock);
}
+static ERTS_INLINE
+struct hipe_mfa_info* mod2mfa_get_safe(Module* modp)
+{
+ struct hipe_mfa_info* mfa;
+ hipe_mfa_info_table_rlock();
+ mfa = mod2mfa_get(modp);
+ hipe_mfa_info_table_runlock();
+ return mfa;
+}
+
#define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A))
static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size)
@@ -1173,6 +1239,8 @@ void hipe_mfa_info_table_init(void)
hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size);
hipe_mfa_info_table_init_lock();
+
+ mod2mfa_tab_init();
}
static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity)
@@ -1199,8 +1267,8 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
unsigned long h;
unsigned int i;
struct hipe_mfa_info *p;
+ struct hipe_mfa_info *first_in_mod;
unsigned int size;
- Module* modp;
h = HIPE_MFA_HASH(m, f, arity);
i = h & hipe_mfa_info_table.mask;
@@ -1221,10 +1289,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */
hipe_mfa_info_table_grow();
- modp = erts_put_active_module(m);
- ASSERT(modp);
- p->next_in_mod = modp->first_hipe_mfa;
- modp->first_hipe_mfa = p;
+ first_in_mod = mod2mfa_put(p);
+ if (p != first_in_mod) {
+ p->next_in_mod = first_in_mod->next_in_mod;
+ first_in_mod->next_in_mod = p;
+ }
+ else {
+ p->next_in_mod = NULL;
+ }
DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p);
@@ -1288,7 +1360,7 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
/* Ask if we need to block all threads
- * while loading/deleting (beam) code for this module?
+ * while loading/deleting code for this module?
*/
int hipe_need_blocking(Module* modp)
{
@@ -1298,7 +1370,7 @@ int hipe_need_blocking(Module* modp)
* or native code to make unaccessible.
*/
hipe_mfa_info_table_rlock();
- for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+ for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
ASSERT(!p->new_address);
if (p->callers.next != &p->callers || !p->is_stub) {
break;
@@ -1519,19 +1591,25 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me)
{
- Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix());
- struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
struct hipe_mfa_info* p;
- ASSERT(modp);
- for (;;) {
- p = *prevp;
- ASSERT(p && p->m == unlink_me->m);
- if (p == unlink_me) {
- *prevp = p->next_in_mod;
- break;
- }
- prevp = &p->next_in_mod;
+ p = hash_get(&mod2mfa_tab, unlink_me);
+ ASSERT(p);
+ if (p == unlink_me) {
+ hash_erase(&mod2mfa_tab, p);
+ if (p->next_in_mod)
+ mod2mfa_put(p->next_in_mod);
+ }
+ else {
+ struct hipe_mfa_info** prevp;
+
+ do {
+ prevp = &p->next_in_mod;
+ p = *prevp;
+ ASSERT(p && p->m == unlink_me->m);
+ } while (p != unlink_me);
+
+ *prevp = p->next_in_mod;
}
}
@@ -1561,18 +1639,11 @@ static void hipe_purge_all_refs(void)
for (i = 0; i < nrbuckets; ++i) {
ASSERT(bucket[i] == NULL);
while (bucket[i] != NULL) {
- Module* modp;
struct hipe_mfa_info* mfa = bucket[i];
bucket[i] = mfa->bucket.next;
- ASSERT(mfa->callers.next == &mfa->callers);
-
- modp = erts_get_module(mfa->m, erts_active_code_ix());
- if (modp) {
- /* unsafe write to active module */
- modp->first_hipe_mfa = NULL;
- }
- erts_free(ERTS_ALC_T_HIPE, mfa);
+ hash_erase(&mod2mfa_tab, mfa);
+ erts_free(ERTS_ALC_T_HIPE, mfa);
}
}
hipe_mfa_info_table.used = 0;
@@ -1598,18 +1669,22 @@ int hipe_purge_need_blocking(Module* modp)
modp->old.hipe_code->first_hipe_sdesc)
return 1;
}
- return !modp->curr.code_hdr && modp->first_hipe_mfa;
+ if (!modp->curr.code_hdr) {
+ return mod2mfa_get_safe(modp) != NULL;
+ }
+ return 0;
}
-void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module)
+void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module,
+ int is_blocking)
{
struct hipe_ref* ref = first_ref;
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
while (ref) {
struct hipe_ref* free_ref = ref;
- ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
ref->callee->m, ref->callee->f, ref->callee->a);
DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
@@ -1632,8 +1707,12 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module)
if (ref->head.next == ref->head.prev) {
struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
if (p->is_stub) {
+ if (!is_blocking)
+ hipe_mfa_info_table_rwlock();
unlink_mfa_from_mod(p);
purge_mfa(p);
+ if (!is_blocking)
+ hipe_mfa_info_table_rwunlock();
}
}
@@ -1642,14 +1721,18 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module)
}
}
-void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module)
+void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module,
+ int is_blocking)
{
struct hipe_sdesc* sdesc = first_sdesc;
+
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+ ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */
+
while (sdesc) {
struct hipe_sdesc* free_sdesc = sdesc;
- ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
ASSERT(make_atom(sdesc->m_aix) == module);
@@ -1659,10 +1742,12 @@ void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module)
}
-void hipe_purge_module(Module* modp)
+void hipe_purge_module(Module* modp, int is_blocking)
{
ASSERT(modp);
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
if (modp->old.hipe_code) {
@@ -1670,8 +1755,10 @@ void hipe_purge_module(Module* modp)
* Remove all hipe_ref's (external calls) from the old module instance
*/
if (modp->old.hipe_code->first_hipe_ref) {
+ ERTS_SMP_LC_ASSERT(is_blocking);
+
hipe_purge_refs(modp->old.hipe_code->first_hipe_ref,
- make_atom(modp->module));
+ make_atom(modp->module), is_blocking);
modp->old.hipe_code->first_hipe_ref = NULL;
}
@@ -1679,8 +1766,10 @@ void hipe_purge_module(Module* modp)
* Remove all hipe_sdesc's for the old module instance
*/
if (modp->old.hipe_code->first_hipe_sdesc) {
+ ERTS_SMP_LC_ASSERT(is_blocking);
+
hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc,
- make_atom(modp->module));
+ make_atom(modp->module), is_blocking);
modp->old.hipe_code->first_hipe_sdesc = NULL;
}
@@ -1693,17 +1782,28 @@ void hipe_purge_module(Module* modp)
* Remove unreferred hipe_mfa_info's
* when all module instances are removed (like in init:restart)
*/
- if (modp->curr.code_hdr == NULL) {
- struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
- struct hipe_mfa_info* p = *prevp;
- ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking());
- for (; p; p = *prevp) {
- if (p->callers.next == &p->callers) {
- *prevp = p->next_in_mod;
- purge_mfa(p);
+ if (is_blocking && modp->curr.code_hdr == NULL) {
+ struct hipe_mfa_info* was_first = mod2mfa_get(modp);
+ struct hipe_mfa_info* is_first = was_first;
+ struct hipe_mfa_info** prevp = &is_first;
+ struct hipe_mfa_info *p;
+
+ if (was_first) {
+ for (p = was_first ; p; p = *prevp) {
+ if (p->callers.next == &p->callers) {
+ *prevp = p->next_in_mod;
+ if (p != was_first)
+ purge_mfa(p);
+ }
+ else
+ prevp = &p->next_in_mod;
+ }
+ if (was_first != is_first) {
+ hash_erase(&mod2mfa_tab, was_first);
+ purge_mfa(was_first);
+ if (is_first)
+ mod2mfa_put(is_first);
}
- else
- prevp = &p->next_in_mod;
}
}
}
@@ -1719,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp)
ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
- for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+ for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
if (p->new_address) {
if (p->is_stub) {
hipe_free_native_stub(p->remote_address);
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index 02f0d6c823..4a59bacc6e 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -36,9 +36,9 @@ extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned in
/* needed in beam_load.c */
int hipe_need_blocking(Module*);
int hipe_purge_need_blocking(Module*);
-void hipe_purge_refs(struct hipe_ref*, Eterm);
-void hipe_purge_sdescs(struct hipe_sdesc*, Eterm);
-void hipe_purge_module(Module*);
+void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking);
+void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking);
+void hipe_purge_module(Module*, int is_blocking);
void hipe_redirect_to_module(Module* modp);
/* these are also needed in hipe_amd64.c */
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index fcbcf1a6a4..2998ed87a2 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -50,11 +50,11 @@ void hipe_free_loader_state(HipeLoaderState *stp)
stp->data_segment_size = 0;
if (stp->new_hipe_refs) {
- hipe_purge_refs(stp->new_hipe_refs, stp->module);
+ hipe_purge_refs(stp->new_hipe_refs, stp->module, 0);
stp->new_hipe_refs = NULL;
}
if (stp->new_hipe_sdesc) {
- hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module);
+ hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0);
stp->new_hipe_sdesc = NULL;
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-16 21:48:53 +0000