aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_bif_load.c761
-rw-r--r--erts/emulator/beam/beam_bp.c36
-rw-r--r--erts/emulator/beam/beam_catches.c136
-rw-r--r--erts/emulator/beam/beam_catches.h11
-rw-r--r--erts/emulator/beam/beam_debug.c17
-rw-r--r--erts/emulator/beam/beam_emu.c42
-rw-r--r--erts/emulator/beam/beam_load.c443
-rw-r--r--erts/emulator/beam/beam_load.h48
-rw-r--r--erts/emulator/beam/beam_ranges.c349
-rw-r--r--erts/emulator/beam/bif.c30
-rw-r--r--erts/emulator/beam/bif.h18
-rw-r--r--erts/emulator/beam/bif.tab9
-rw-r--r--erts/emulator/beam/break.c59
-rw-r--r--erts/emulator/beam/code_ix.c152
-rw-r--r--erts/emulator/beam/code_ix.h141
-rw-r--r--erts/emulator/beam/dist.c16
-rw-r--r--erts/emulator/beam/erl_alloc.c8
-rw-r--r--erts/emulator/beam/erl_alloc.types2
-rw-r--r--erts/emulator/beam/erl_bif_binary.c69
-rw-r--r--erts/emulator/beam/erl_bif_chksum.c13
-rw-r--r--erts/emulator/beam/erl_bif_re.c11
-rw-r--r--erts/emulator/beam/erl_bif_trace.c103
-rw-r--r--erts/emulator/beam/erl_db.c50
-rw-r--r--erts/emulator/beam/erl_db_tree.c12
-rw-r--r--erts/emulator/beam/erl_fun.h2
-rw-r--r--erts/emulator/beam/erl_init.c8
-rw-r--r--erts/emulator/beam/erl_lock_check.c4
-rw-r--r--erts/emulator/beam/erl_nif.c42
-rw-r--r--erts/emulator/beam/erl_process.c58
-rw-r--r--erts/emulator/beam/erl_process.h10
-rw-r--r--erts/emulator/beam/erl_unicode.c79
-rw-r--r--erts/emulator/beam/export.c352
-rw-r--r--erts/emulator/beam/export.h37
-rw-r--r--erts/emulator/beam/external.c2
-rw-r--r--erts/emulator/beam/global.h27
-rw-r--r--erts/emulator/beam/index.c23
-rw-r--r--erts/emulator/beam/index.h14
-rw-r--r--erts/emulator/beam/module.c159
-rw-r--r--erts/emulator/beam/module.h64
39 files changed, 2319 insertions, 1098 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 78a9d76a20..768c38dae1 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -37,25 +37,83 @@
static void set_default_trace_pattern(Eterm module);
static Eterm check_process_code(Process* rp, Module* modp);
-static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp);
-static void delete_export_references(Eterm module);
-static int purge_module(int module);
+static void delete_code(Module* modp);
static void decrement_refc(BeamInstr* code);
static int is_native(BeamInstr* code);
static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
-static void remove_from_address_table(BeamInstr* code);
-Eterm
-load_module_2(BIF_ALIST_2)
+
+
+BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
{
- Eterm reason;
- Eterm* hp;
- int sz;
- byte* code;
+ Module* modp;
Eterm res;
+ ErtsCodeIndex code_ix;
+
+ if (is_not_atom(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P, BADARG);
+ }
+ code_ix = erts_active_code_ix();
+ if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) {
+ return am_undefined;
+ }
+ erts_rlock_old_code(code_ix);
+ res = ((modp->curr.code && is_native(modp->curr.code)) ||
+ (modp->old.code != 0 && is_native(modp->old.code))) ?
+ am_true : am_false;
+ erts_runlock_old_code(code_ix);
+ return res;
+}
+
+BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
+{
+ Module* modp;
+ Eterm res;
+
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3],
+ BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+ }
+
+ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_smp_thr_progress_block();
+
+ modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
+
+ if (modp && modp->curr.num_breakpoints > 0) {
+ ASSERT(modp->curr.code != NULL);
+ erts_clear_module_break(modp);
+ ASSERT(modp->curr.num_breakpoints == 0);
+ }
+
+ erts_start_staging_code_ix();
+
+ res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+
+ if (res == BIF_ARG_1) {
+ erts_end_staging_code_ix();
+ erts_commit_staging_code_ix();
+ }
+ else {
+ erts_abort_staging_code_ix();
+ }
+ erts_smp_thr_progress_unblock();
+ erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_release_code_write_permission();
+ return res;
+}
+
+BIF_RETTYPE
+prepare_loading_2(BIF_ALIST_2)
+{
byte* temp_alloc = NULL;
- struct LoaderState* stp;
+ byte* code;
+ Uint sz;
+ Binary* magic;
+ Eterm reason;
+ Eterm* hp;
+ Eterm res;
if (is_not_atom(BIF_ARG_1)) {
error:
@@ -65,108 +123,277 @@ load_module_2(BIF_ALIST_2)
if ((code = erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc)) == NULL) {
goto error;
}
- hp = HAlloc(BIF_P, 3);
- /*
- * Read the BEAM file and prepare the module for loading.
- */
- stp = erts_alloc_loader_state();
+ magic = erts_alloc_loader_state();
sz = binary_size(BIF_ARG_2);
- reason = erts_prepare_loading(stp, BIF_P, BIF_P->group_leader,
+ reason = erts_prepare_loading(magic, BIF_P, BIF_P->group_leader,
&BIF_ARG_1, code, sz);
erts_free_aligned_binary_bytes(temp_alloc);
if (reason != NIL) {
+ hp = HAlloc(BIF_P, 3);
res = TUPLE2(hp, am_error, reason);
BIF_RET(res);
}
+ hp = HAlloc(BIF_P, PROC_BIN_SIZE);
+ res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic);
+ erts_refc_dec(&magic->refc, 1);
+ BIF_RET(res);
+}
- /*
- * Stop all other processes and finish the loading of the module.
- */
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erts_smp_thr_progress_block();
+struct m {
+ Binary* code;
+ Eterm module;
+ Module* modp;
+ Uint exception;
+};
- reason = erts_finish_loading(stp, BIF_P, 0, &BIF_ARG_1);
- if (reason != NIL) {
- res = TUPLE2(hp, am_error, reason);
- } else {
- set_default_trace_pattern(BIF_ARG_1);
- res = TUPLE2(hp, am_module, BIF_ARG_1);
- }
+static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int);
- erts_smp_thr_progress_unblock();
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
- BIF_RET(res);
+static Eterm
+exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions)
+{
+ Eterm* hp = HAlloc(p, 3 + 2*exceptions);
+ Eterm res = NIL;
+
+ mp += exceptions - 1;
+ while (exceptions > 0) {
+ if (mp->exception) {
+ res = CONS(hp, mp->module, res);
+ hp += 2;
+ exceptions--;
+ }
+ mp--;
+ }
+ return TUPLE2(hp, tag, res);
}
-BIF_RETTYPE purge_module_1(BIF_ALIST_1)
+
+BIF_RETTYPE
+finish_loading_1(BIF_ALIST_1)
{
- int purge_res;
+ int i;
+ int n;
+ struct m* p = NULL;
+ Uint exceptions;
+ Eterm res;
+ int is_blocking = 0;
+ int do_commit = 0;
- if (is_not_atom(BIF_ARG_1)) {
- BIF_ERROR(BIF_P, BADARG);
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ ERTS_BIF_YIELD1(bif_export[BIF_finish_loading_1], BIF_P, BIF_ARG_1);
}
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erts_smp_thr_progress_block();
+ /*
+ * Validate the argument before we start loading; it must be a
+ * proper list where each element is a magic binary containing
+ * prepared (not previously loaded) code.
+ *
+ * First count the number of elements and allocate an array
+ * to keep the elements in.
+ */
- erts_export_consolidate();
- purge_res = purge_module(atom_val(BIF_ARG_1));
+ n = list_length(BIF_ARG_1);
+ if (n == -1) {
+ ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
+ goto done;
+ }
+ p = erts_alloc(ERTS_ALC_T_LOADER_TMP, n*sizeof(struct m));
- erts_smp_thr_progress_unblock();
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ /*
+ * We now know that the argument is a proper list. Validate
+ * and collect the binaries into the array.
+ */
- if (purge_res < 0) {
- BIF_ERROR(BIF_P, BADARG);
+ for (i = 0; i < n; i++) {
+ Eterm* cons = list_val(BIF_ARG_1);
+ Eterm term = CAR(cons);
+ ProcBin* pb;
+
+ if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
+ ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
+ goto done;
+ }
+ pb = (ProcBin*) binary_val(term);
+ p[i].code = pb->val;
+ p[i].module = erts_module_for_prepared_code(p[i].code);
+ if (p[i].module == NIL) {
+ ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
+ goto done;
+ }
+ BIF_ARG_1 = CDR(cons);
}
- BIF_RET(am_true);
-}
-BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
-{
- Module* modp;
+ /*
+ * Since we cannot handle atomic loading of a group of modules
+ * if one or more of them uses on_load, we will only allow one
+ * element in the list. This limitation is intended to be
+ * lifted in the future.
+ */
- if (is_not_atom(BIF_ARG_1)) {
- BIF_ERROR(BIF_P, BADARG);
+ if (n > 1) {
+ ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT);
+ goto done;
}
- if ((modp = erts_get_module(BIF_ARG_1)) == NULL) {
- return am_undefined;
+
+ /*
+ * All types are correct. There cannot be a BADARG from now on.
+ * Before we can start loading, we must check whether any of
+ * the modules already has old code. To avoid a race, we must
+ * not allow other process to initiate a code loading operation
+ * from now on.
+ */
+
+ res = am_ok;
+ erts_start_staging_code_ix();
+
+ for (i = 0; i < n; i++) {
+ p[i].modp = erts_put_module(p[i].module);
+ }
+ for (i = 0; i < n; i++) {
+ if (p[i].modp->curr.num_breakpoints > 0 ||
+ p[i].modp->curr.num_traced_exports > 0 ||
+ erts_is_default_trace_enabled()) {
+ /* tracing involved, fallback with thread blocking */
+ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_smp_thr_progress_block();
+ is_blocking = 1;
+ break;
+ }
}
- return ((modp->code && is_native(modp->code)) ||
- (modp->old_code != 0 && is_native(modp->old_code))) ?
- am_true : am_false;
-}
-BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
-{
- Eterm res;
+ if (is_blocking) {
+ for (i = 0; i < n; i++) {
+ if (p[i].modp->curr.num_breakpoints) {
+ erts_clear_module_break(p[i].modp);
+ ASSERT(p[i].modp->curr.num_breakpoints == 0);
+ }
+ }
+ }
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erts_smp_thr_progress_block();
+ exceptions = 0;
+ for (i = 0; i < n; i++) {
+ p[i].exception = 0;
+ if (p[i].modp->curr.code && p[i].modp->old.code) {
+ p[i].exception = 1;
+ exceptions++;
+ }
+ }
- erts_export_consolidate();
- res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+ if (exceptions) {
+ res = exception_list(BIF_P, am_not_purged, p, exceptions);
+ } else {
+ /*
+ * Now we can load all code. This can't fail.
+ */
- erts_smp_thr_progress_unblock();
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
- return res;
+ exceptions = 0;
+ for (i = 0; i < n; i++) {
+ Eterm mod;
+ Eterm retval;
+
+ erts_refc_inc(&p[i].code->refc, 1);
+ retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod);
+ ASSERT(retval == NIL || retval == am_on_load);
+ if (retval == am_on_load) {
+ p[i].exception = 1;
+ exceptions++;
+ }
+ }
+
+ /*
+ * Check whether any module has an on_load_handler.
+ */
+
+ if (exceptions) {
+ res = exception_list(BIF_P, am_on_load, p, exceptions);
+ }
+ do_commit = 1;
+ }
+
+done:
+ return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n);
+}
+
+static Eterm
+staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
+ struct m* loaded, int nloaded)
+{
+#ifdef ERTS_SMP
+ if (is_blocking || !commit)
+#endif
+ {
+ if (commit) {
+ erts_end_staging_code_ix();
+ erts_commit_staging_code_ix();
+ if (loaded) {
+ int i;
+ for (i=0; i < nloaded; i++) {
+ set_default_trace_pattern(loaded[i].module);
+ }
+ }
+ }
+ else {
+ erts_abort_staging_code_ix();
+ }
+ if (loaded) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+ }
+ if (is_blocking) {
+ erts_smp_thr_progress_unblock();
+ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+ }
+ erts_release_code_write_permission();
+ return res;
+ }
+#ifdef ERTS_SMP
+ else {
+ ErtsThrPrgrVal later;
+ ASSERT(is_value(res));
+
+ if (loaded) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+ }
+ erts_end_staging_code_ix();
+ /*
+ * Now we must wait for all schedulers to do a memory barrier before
+ * we can activate and let them access the new staged code. This allows
+ * schedulers to read active code_ix in a safe way while executing
+ * without any memory barriers at all.
+ */
+
+ later = erts_thr_progress_later();
+ erts_thr_progress_wakeup(c_p->scheduler_data, later);
+ erts_notify_code_ix_activation(c_p, later);
+ erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+ /*
+ * handle_code_ix_activation() will do the rest "later"
+ * and resume this process to return 'res'.
+ */
+ ERTS_BIF_YIELD_RETURN(c_p, res);
+ }
+#endif
}
BIF_RETTYPE
check_old_code_1(BIF_ALIST_1)
{
+ ErtsCodeIndex code_ix;
Module* modp;
+ Eterm res = am_false;
if (is_not_atom(BIF_ARG_1)) {
BIF_ERROR(BIF_P, BADARG);
}
- modp = erts_get_module(BIF_ARG_1);
- if (modp == NULL) { /* Doesn't exist. */
- BIF_RET(am_false);
- } else if (modp->old_code == NULL) { /* No old code. */
- BIF_RET(am_false);
+ code_ix = erts_active_code_ix();
+ modp = erts_get_module(BIF_ARG_1, code_ix);
+ if (modp != NULL) {
+ erts_rlock_old_code(code_ix);
+ if (modp->old.code != NULL) {
+ res = am_true;
+ }
+ erts_runlock_old_code(code_ix);
}
- BIF_RET(am_true);
+ BIF_RET(res);
}
Eterm
@@ -180,14 +407,20 @@ check_process_code_2(BIF_ALIST_2)
}
if (is_internal_pid(BIF_ARG_1)) {
Eterm res;
+ ErtsCodeIndex code_ix;
if (internal_pid_index(BIF_ARG_1) >= erts_max_processes)
goto error;
- modp = erts_get_module(BIF_ARG_2);
+ code_ix = erts_active_code_ix();
+ modp = erts_get_module(BIF_ARG_2, code_ix);
if (modp == NULL) { /* Doesn't exist. */
return am_false;
- } else if (modp->old_code == NULL) { /* No old code. */
+ }
+ erts_rlock_old_code(code_ix);
+ if (modp->old.code == NULL) { /* No old code. */
+ erts_runlock_old_code(code_ix);
return am_false;
}
+ erts_runlock_old_code(code_ix);
#ifdef ERTS_SMP
rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN,
@@ -202,7 +435,14 @@ check_process_code_2(BIF_ALIST_2)
ERTS_BIF_YIELD2(bif_export[BIF_check_process_code_2], BIF_P,
BIF_ARG_1, BIF_ARG_2);
}
- res = check_process_code(rp, modp);
+ erts_rlock_old_code(code_ix);
+ if (modp->old.code != NULL) { /* must check again */
+ res = check_process_code(rp, modp);
+ }
+ else {
+ res = am_false;
+ }
+ erts_runlock_old_code(code_ix);
#ifdef ERTS_SMP
if (BIF_P != rp) {
erts_resume(rp, ERTS_PROC_LOCK_MAIN);
@@ -223,56 +463,71 @@ check_process_code_2(BIF_ALIST_2)
BIF_RETTYPE delete_module_1(BIF_ALIST_1)
{
- int res;
+ ErtsCodeIndex code_ix;
+ Module* modp;
+ int is_blocking = 0;
+ int success = 0;
+ Eterm res = NIL;
- if (is_not_atom(BIF_ARG_1))
- goto badarg;
+ if (is_not_atom(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P, BADARG);
+ }
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erts_smp_thr_progress_block();
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ ERTS_BIF_YIELD1(bif_export[BIF_delete_module_1], BIF_P, BIF_ARG_1);
+ }
{
- Module *modp = erts_get_module(BIF_ARG_1);
+ erts_start_staging_code_ix();
+ code_ix = erts_staging_code_ix();
+ modp = erts_get_module(BIF_ARG_1, code_ix);
if (!modp) {
res = am_undefined;
}
- else if (modp->old_code != 0) {
+ else if (modp->old.code != 0) {
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
erts_dsprintf(dsbufp, "Module %T must be purged before loading\n",
BIF_ARG_1);
erts_send_error_to_logger(BIF_P->group_leader, dsbufp);
- res = am_badarg;
+ ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
}
else {
- delete_export_references(BIF_ARG_1);
- delete_code(BIF_P, 0, modp);
+ if (modp->curr.num_breakpoints > 0 ||
+ modp->curr.num_traced_exports > 0) {
+ /* we have tracing, retry single threaded */
+ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_smp_thr_progress_block();
+ is_blocking = 1;
+ if (modp->curr.num_breakpoints) {
+ erts_clear_module_break(modp);
+ ASSERT(modp->curr.num_breakpoints == 0);
+ }
+ }
+ delete_code(modp);
res = am_true;
+ success = 1;
}
}
-
- erts_smp_thr_progress_unblock();
- erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-
- if (res == am_badarg) {
- badarg:
- BIF_ERROR(BIF_P, BADARG);
- }
- BIF_RET(res);
+ return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0);
}
BIF_RETTYPE module_loaded_1(BIF_ALIST_1)
{
Module* modp;
+ ErtsCodeIndex code_ix;
+ Eterm res = am_false;
if (is_not_atom(BIF_ARG_1)) {
BIF_ERROR(BIF_P, BADARG);
}
- if ((modp = erts_get_module(BIF_ARG_1)) == NULL ||
- modp->code == NULL ||
- modp->code[MI_ON_LOAD_FUNCTION_PTR] != 0) {
- BIF_RET(am_false);
+ code_ix = erts_active_code_ix();
+ if ((modp = erts_get_module(BIF_ARG_1, code_ix)) != NULL) {
+ if (modp->curr.code != NULL
+ && modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] == 0) {
+ res = am_true;
+ }
}
- BIF_RET(am_true);
+ BIF_RET(res);
}
BIF_RETTYPE pre_loaded_0(BIF_ALIST_0)
@@ -282,27 +537,28 @@ BIF_RETTYPE pre_loaded_0(BIF_ALIST_0)
BIF_RETTYPE loaded_0(BIF_ALIST_0)
{
+ ErtsCodeIndex code_ix = erts_active_code_ix();
+ Module* modp;
Eterm previous = NIL;
Eterm* hp;
int i;
int j = 0;
-
- for (i = 0; i < module_code_size(); i++) {
- if (module_code(i) != NULL &&
- ((module_code(i)->code_length != 0) ||
- (module_code(i)->old_code_length != 0))) {
+
+ for (i = 0; i < module_code_size(code_ix); i++) {
+ if ((modp = module_code(i, code_ix)) != NULL &&
+ ((modp->curr.code_length != 0) ||
+ (modp->old.code_length != 0))) {
j++;
}
}
if (j > 0) {
hp = HAlloc(BIF_P, j*2);
- for (i = 0; i < module_code_size(); i++) {
- if (module_code(i) != NULL &&
- ((module_code(i)->code_length != 0) ||
- (module_code(i)->old_code_length != 0))) {
- previous = CONS(hp, make_atom(module_code(i)->module),
- previous);
+ for (i = 0; i < module_code_size(code_ix); i++) {
+ if ((modp=module_code(i,code_ix)) != NULL &&
+ ((modp->curr.code_length != 0) ||
+ (modp->old.code_length != 0))) {
+ previous = CONS(hp, make_atom(modp->module), previous);
hp += 2;
}
}
@@ -312,54 +568,65 @@ BIF_RETTYPE loaded_0(BIF_ALIST_0)
BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1)
{
- Module* modp = erts_get_module(BIF_ARG_1);
- Eterm on_load;
+ Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
- if (!modp || modp->code == 0) {
- error:
- BIF_ERROR(BIF_P, BADARG);
+ if (modp && modp->curr.code) {
+ BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]);
}
- if ((on_load = modp->code[MI_ON_LOAD_FUNCTION_PTR]) == 0) {
- goto error;
+ else {
+ BIF_ERROR(BIF_P, BADARG);
}
- BIF_TRAP_CODE_PTR_0(BIF_P, on_load);
}
BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
{
- Module* modp = erts_get_module(BIF_ARG_1);
+ ErtsCodeIndex code_ix;
+ Module* modp;
Eterm on_load;
- if (!modp || modp->code == 0) {
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2],
+ BIF_P, BIF_ARG_1, BIF_ARG_2);
+ }
+
+ /* ToDo: Use code_ix staging instead of thread blocking */
+
+ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_smp_thr_progress_block();
+
+ code_ix = erts_active_code_ix();
+ modp = erts_get_module(BIF_ARG_1, code_ix);
+
+ if (!modp || modp->curr.code == 0) {
error:
+ erts_smp_thr_progress_unblock();
+ erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_release_code_write_permission();
BIF_ERROR(BIF_P, BADARG);
}
- if ((on_load = modp->code[MI_ON_LOAD_FUNCTION_PTR]) == 0) {
+ if ((on_load = modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]) == 0) {
goto error;
}
if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) {
goto error;
}
- erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
- erts_smp_thr_progress_block();
-
if (BIF_ARG_2 == am_true) {
int i;
/*
* The on_load function succeded. Fix up export entries.
*/
- for (i = 0; i < export_list_size(); i++) {
- Export *ep = export_list(i);
+ for (i = 0; i < export_list_size(code_ix); i++) {
+ Export *ep = export_list(i,code_ix);
if (ep != NULL &&
ep->code[0] == BIF_ARG_1 &&
ep->code[4] != 0) {
- ep->address = (void *) ep->code[4];
+ ep->addressv[code_ix] = (void *) ep->code[4];
ep->code[4] = 0;
}
}
- modp->code[MI_ON_LOAD_FUNCTION_PTR] = 0;
+ modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] = 0;
set_default_trace_pattern(BIF_ARG_1);
} else if (BIF_ARG_2 == am_false) {
BeamInstr* code;
@@ -370,19 +637,21 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
* This is an combination of delete and purge. We purge
* the current code; the old code is not touched.
*/
- erts_total_code_size -= modp->code_length;
- code = modp->code;
- end = (BeamInstr *)((char *)code + modp->code_length);
+ erts_total_code_size -= modp->curr.code_length;
+ code = modp->curr.code;
+ end = (BeamInstr *)((char *)code + modp->curr.code_length);
erts_cleanup_funs_on_purge(code, end);
- beam_catches_delmod(modp->catches, code, modp->code_length);
+ beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length,
+ erts_active_code_ix());
erts_free(ERTS_ALC_T_CODE, (void *) code);
- modp->code = NULL;
- modp->code_length = 0;
- modp->catches = BEAM_CATCHES_NIL;
- remove_from_address_table(code);
+ modp->curr.code = NULL;
+ modp->curr.code_length = 0;
+ modp->curr.catches = BEAM_CATCHES_NIL;
+ erts_remove_from_ranges(code);
}
erts_smp_thr_progress_unblock();
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_release_code_write_permission();
BIF_RET(am_true);
}
@@ -429,10 +698,10 @@ check_process_code(Process* rp, Module* modp)
/*
* Pick up limits for the module.
*/
- start = modp->old_code;
- end = (BeamInstr *)((char *)start + modp->old_code_length);
+ start = modp->old.code;
+ end = (BeamInstr *)((char *)start + modp->old.code_length);
mod_start = (char *) start;
- mod_size = modp->old_code_length;
+ mod_size = modp->old.code_length;
/*
* Check if current instruction or continuation pointer points into module.
@@ -566,10 +835,10 @@ check_process_code(Process* rp, Module* modp)
done_gc = 1;
FLAGS(rp) |= F_NEED_FULLSWEEP;
(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
- literals = (Eterm *) modp->old_code[MI_LITERALS_START];
- lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals;
+ literals = (Eterm *) modp->old.code[MI_LITERALS_START];
+ lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals;
oh = (struct erl_off_heap_header *)
- modp->old_code[MI_LITERALS_OFF_HEAP];
+ modp->old.code[MI_LITERALS_OFF_HEAP];
erts_garbage_collect_literals(rp, literals, lit_size, oh);
}
}
@@ -628,53 +897,82 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
#undef in_area
-
-static int
-purge_module(int module)
+BIF_RETTYPE purge_module_1(BIF_ALIST_1)
{
+ ErtsCodeIndex code_ix;
BeamInstr* code;
BeamInstr* end;
Module* modp;
+ int is_blocking = 0;
+ Eterm ret;
- /*
- * Correct module?
- */
-
- if ((modp = erts_get_module(make_atom(module))) == NULL) {
- return -2;
+ if (is_not_atom(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P, BADARG);
}
- /*
- * Any code to purge?
- */
- if (modp->old_code == 0) {
- return -1;
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ ERTS_BIF_YIELD1(bif_export[BIF_purge_module_1], BIF_P, BIF_ARG_1);
}
+ code_ix = erts_active_code_ix();
+
/*
- * Unload any NIF library
+ * Correct module?
*/
- if (modp->old_nif != NULL) {
- erts_unload_nif(modp->old_nif);
- modp->old_nif = NULL;
+
+ if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) {
+ ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
}
+ else {
+ erts_rwlock_old_code(code_ix);
- /*
- * Remove the old code.
- */
- ASSERT(erts_total_code_size >= modp->old_code_length);
- erts_total_code_size -= modp->old_code_length;
- code = modp->old_code;
- end = (BeamInstr *)((char *)code + modp->old_code_length);
- erts_cleanup_funs_on_purge(code, end);
- beam_catches_delmod(modp->old_catches, code, modp->old_code_length);
- decrement_refc(code);
- erts_free(ERTS_ALC_T_CODE, (void *) code);
- modp->old_code = NULL;
- modp->old_code_length = 0;
- modp->old_catches = BEAM_CATCHES_NIL;
- remove_from_address_table(code);
- return 0;
+ /*
+ * Any code to purge?
+ */
+ if (modp->old.code == 0) {
+ ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+ }
+ else {
+ /*
+ * Unload any NIF library
+ */
+ if (modp->old.nif != NULL) {
+ /* ToDo: Do unload nif without blocking */
+ erts_rwunlock_old_code(code_ix);
+ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_smp_thr_progress_block();
+ is_blocking = 1;
+ erts_rwlock_old_code(code_ix);
+ erts_unload_nif(modp->old.nif);
+ modp->old.nif = NULL;
+ }
+
+ /*
+ * Remove the old code.
+ */
+ ASSERT(erts_total_code_size >= modp->old.code_length);
+ erts_total_code_size -= modp->old.code_length;
+ code = modp->old.code;
+ end = (BeamInstr *)((char *)code + modp->old.code_length);
+ erts_cleanup_funs_on_purge(code, end);
+ beam_catches_delmod(modp->old.catches, code, modp->old.code_length,
+ code_ix);
+ decrement_refc(code);
+ erts_free(ERTS_ALC_T_CODE, (void *) code);
+ modp->old.code = NULL;
+ modp->old.code_length = 0;
+ modp->old.catches = BEAM_CATCHES_NIL;
+ erts_remove_from_ranges(code);
+ ERTS_BIF_PREP_RET(ret, am_true);
+ }
+ erts_rwunlock_old_code(code_ix);
+ }
+ if (is_blocking) {
+ erts_smp_thr_progress_unblock();
+ erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ }
+ erts_release_code_write_permission();
+ return ret;
}
static void
@@ -694,92 +992,50 @@ decrement_refc(BeamInstr* code)
}
}
-static void
-remove_from_address_table(BeamInstr* code)
-{
- int i;
-
- for (i = 0; i < num_loaded_modules; i++) {
- if (modules[i].start == code) {
- num_loaded_modules--;
- while (i < num_loaded_modules) {
- modules[i] = modules[i+1];
- i++;
- }
- mid_module = &modules[num_loaded_modules/2];
- return;
- }
- }
- ASSERT(0); /* Not found? */
-}
-
-
/*
- * Move code from current to old.
+ * Move code from current to old and null all export entries for the module
*/
-static void
-delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp)
-{
-#ifdef ERTS_ENABLE_LOCK_CHECK
-#ifdef ERTS_SMP
- if (c_p && c_p_locks)
- erts_proc_lc_chk_only_proc_main(c_p);
- else
-#endif
- erts_lc_check_exact(NULL, 0);
-#endif
-
- /*
- * Clear breakpoints if any
- */
- if (modp->code != NULL && modp->code[MI_NUM_BREAKPOINTS] > 0) {
- if (c_p && c_p_locks)
- erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
- erts_smp_thr_progress_block();
- erts_clear_module_break(modp);
- modp->code[MI_NUM_BREAKPOINTS] = 0;
- erts_smp_thr_progress_unblock();
- if (c_p && c_p_locks)
- erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
- }
- modp->old_code = modp->code;
- modp->old_code_length = modp->code_length;
- modp->old_catches = modp->catches;
- modp->old_nif = modp->nif;
- modp->code = NULL;
- modp->code_length = 0;
- modp->catches = BEAM_CATCHES_NIL;
- modp->nif = NULL;
-}
-
-
-/* null all references on the export table for the module called with the
- atom index below */
-
static void
-delete_export_references(Eterm module)
+delete_code(Module* modp)
{
+ ErtsCodeIndex code_ix = erts_staging_code_ix();
+ Eterm module = make_atom(modp->module);
int i;
- ASSERT(is_atom(module));
-
- for (i = 0; i < export_list_size(); i++) {
- Export *ep = export_list(i);
+ for (i = 0; i < export_list_size(code_ix); i++) {
+ Export *ep = export_list(i, code_ix);
if (ep != NULL && (ep->code[0] == module)) {
- if (ep->address == ep->code+3 &&
- (ep->code[3] == (BeamInstr) em_apply_bif)) {
- continue;
+ if (ep->addressv[code_ix] == ep->code+3) {
+ if (ep->code[3] == (BeamInstr) em_apply_bif) {
+ continue;
+ }
+ else if (ep->code[3] == (BeamInstr) em_call_traced_function) {
+ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+ ASSERT(modp->curr.num_traced_exports > 0);
+ --modp->curr.num_traced_exports;
+ MatchSetUnref(ep->match_prog_set);
+ ep->match_prog_set = NULL;
+ }
+ else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler
+ || !erts_initialized);
}
- ep->address = ep->code+3;
+ ep->addressv[code_ix] = ep->code+3;
ep->code[3] = (BeamInstr) em_call_error_handler;
ep->code[4] = 0;
- MatchSetUnref(ep->match_prog_set);
- ep->match_prog_set = NULL;
+ ASSERT(ep->match_prog_set == NULL);
}
}
+
+ ASSERT(modp->curr.num_breakpoints == 0);
+ ASSERT(modp->curr.num_traced_exports == 0);
+ modp->old = modp->curr;
+ modp->curr.code = NULL;
+ modp->curr.code_length = 0;
+ modp->curr.catches = BEAM_CATCHES_NIL;
+ modp->curr.nif = NULL;
}
-
+
Eterm
beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
@@ -791,11 +1047,10 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
* if not, delete old code; error if old code already exists.
*/
- if (modp->code != NULL && modp->old_code != NULL) {
+ if (modp->curr.code != NULL && modp->old.code != NULL) {
return am_not_purged;
- } else if (modp->old_code == NULL) { /* Make the current version old. */
- delete_code(c_p, c_p_locks, modp);
- delete_export_references(module);
+ } else if (modp->old.code == NULL) { /* Make the current version old. */
+ delete_code(modp);
}
return NIL;
}
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 53f283ba39..ef48d7d05e 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -478,9 +478,9 @@ erts_find_local_func(Eterm mfa[3]) {
BeamInstr* code_ptr;
Uint i,n;
- if ((modp = erts_get_module(mfa[0])) == NULL)
+ if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL)
return NULL;
- if ((code_base = (BeamInstr **) modp->code) == NULL)
+ if ((code_base = (BeamInstr **) modp->curr.code) == NULL)
return NULL;
n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
for (i = 0; i < n; ++i) {
@@ -850,12 +850,13 @@ static int set_break(Eterm mfa[3], int specified,
{
Module *modp;
int num_processed = 0;
+ ErtsCodeIndex code_ix = erts_active_code_ix();
if (!specified) {
/* Find and process all modules in the system... */
int current;
- int last = module_code_size();
+ int last = module_code_size(code_ix);
for (current = 0; current < last; current++) {
- modp = module_code(current);
+ modp = module_code(current, code_ix);
ASSERT(modp != NULL);
num_processed +=
set_module_break(modp, mfa, specified,
@@ -864,7 +865,7 @@ static int set_break(Eterm mfa[3], int specified,
}
} else {
/* Process a single module */
- if ((modp = erts_get_module(mfa[0])) != NULL) {
+ if ((modp = erts_get_module(mfa[0], code_ix)) != NULL) {
num_processed +=
set_module_break(modp, mfa, specified,
match_spec, break_op, count_op,
@@ -884,7 +885,7 @@ static int set_module_break(Module *modp, Eterm mfa[3], int specified,
ASSERT(break_op);
ASSERT(modp);
- code_base = (BeamInstr **) modp->code;
+ code_base = (BeamInstr **) modp->curr.code;
if (code_base == NULL) {
return 0;
}
@@ -914,10 +915,10 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif,
Uint ix = 0;
if (bif == BREAK_IS_ERL) {
- code_base = (BeamInstr **)modp->code;
+ code_base = (BeamInstr **)modp->curr.code;
ASSERT(code_base);
ASSERT(code_base <= (BeamInstr **)pc);
- ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *)));
+ ASSERT((BeamInstr **)pc < code_base + (modp->curr.code_length/sizeof(BeamInstr *)));
} else {
ASSERT(*pc == (BeamInstr) em_apply_bif);
ASSERT(modp == NULL);
@@ -1098,29 +1099,30 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif,
}
if (bif == BREAK_IS_ERL) {
- ++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]);
+ ++modp->curr.num_breakpoints;
}
return 1;
}
static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op)
{
+ ErtsCodeIndex code_ix = erts_active_code_ix();
int num_processed = 0;
Module *modp;
if (!specified) {
/* Iterate over all modules */
int current;
- int last = module_code_size();
+ int last = module_code_size(code_ix);
for (current = 0; current < last; current++) {
- modp = module_code(current);
+ modp = module_code(current, code_ix);
ASSERT(modp != NULL);
num_processed += clear_module_break(modp, mfa, specified, break_op);
}
} else {
/* Process a single module */
- if ((modp = erts_get_module(mfa[0])) != NULL) {
+ if ((modp = erts_get_module(mfa[0], code_ix)) != NULL) {
num_processed +=
clear_module_break(modp, mfa, specified, break_op);
}
@@ -1137,7 +1139,7 @@ static int clear_module_break(Module *m, Eterm mfa[3], int specified,
BeamInstr n;
ASSERT(m);
- code_base = (BeamInstr **) m->code;
+ code_base = (BeamInstr **) m->curr.code;
if (code_base == NULL) {
return 0;
}
@@ -1161,10 +1163,10 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre
BeamInstr **code_base = NULL;
if (bif == BREAK_IS_ERL) {
- code_base = (BeamInstr **)m->code;
+ code_base = (BeamInstr **)m->curr.code;
ASSERT(code_base);
ASSERT(code_base <= (BeamInstr **)pc);
- ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *)));
+ ASSERT((BeamInstr **)pc < code_base + (m->curr.code_length/sizeof(BeamInstr *)));
} else {
ASSERT(*pc == (BeamInstr) em_apply_bif);
ASSERT(m == NULL);
@@ -1274,8 +1276,8 @@ static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr bre
}
Free(bd);
if (bif == BREAK_IS_ERL) {
- ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0);
- --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]);
+ ASSERT(m->curr.num_breakpoints > 0);
+ --m->curr.num_breakpoints;
}
if (*rs) {
for (ix = 1; ix < erts_no_schedulers; ++ix) {
diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c
index 406ef1db5f..92f7ffe5a2 100644
--- a/erts/emulator/beam/beam_catches.c
+++ b/erts/emulator/beam/beam_catches.c
@@ -31,78 +31,144 @@ typedef struct {
unsigned cdr;
} beam_catch_t;
-static int free_list;
-static unsigned high_mark;
-static unsigned tabsize;
-static beam_catch_t *beam_catches;
+#ifdef DEBUG
+# define IF_DEBUG(x) x
+#else
+# define IF_DEBUG(x)
+#endif
+
+struct bc_pool {
+ int free_list;
+ unsigned high_mark;
+ unsigned tabsize;
+ beam_catch_t *beam_catches;
+ /*
+ * Note that the 'beam_catches' area is shared by pools. Used slots
+ * are readonly as long as the module is not purgable. The free-list is
+ * protected by the code_ix lock.
+ */
+
+ IF_DEBUG(int is_staging;)
+};
+
+static struct bc_pool bccix[ERTS_NUM_CODE_IX];
void beam_catches_init(void)
{
- tabsize = DEFAULT_TABSIZE;
- free_list = -1;
- high_mark = 0;
+ int i;
+
+ bccix[0].tabsize = DEFAULT_TABSIZE;
+ bccix[0].free_list = -1;
+ bccix[0].high_mark = 0;
+ bccix[0].beam_catches = erts_alloc(ERTS_ALC_T_CODE,
+ sizeof(beam_catch_t)*DEFAULT_TABSIZE);
+ IF_DEBUG(bccix[0].is_staging = 0);
+ for (i=1; i<ERTS_NUM_CODE_IX; i++) {
+ bccix[i] = bccix[i-1];
+ }
+ /* For initial load: */
+ IF_DEBUG(bccix[erts_staging_code_ix()].is_staging = 1);
+}
- beam_catches = erts_alloc(ERTS_ALC_T_CODE, sizeof(beam_catch_t)*DEFAULT_TABSIZE);
+
+static void gc_old_vec(beam_catch_t* vec)
+{
+ int i;
+ for (i=0; i<ERTS_NUM_CODE_IX; i++) {
+ if (bccix[i].beam_catches == vec) {
+ return;
+ }
+ }
+ erts_free(ERTS_ALC_T_CODE, vec);
+}
+
+
+void beam_catches_start_staging(void)
+{
+ ErtsCodeIndex dst = erts_staging_code_ix();
+ ErtsCodeIndex src = erts_active_code_ix();
+ beam_catch_t* prev_vec = bccix[dst].beam_catches;
+
+ ASSERT(!bccix[src].is_staging && !bccix[dst].is_staging);
+
+ bccix[dst] = bccix[src];
+ gc_old_vec(prev_vec);
+ IF_DEBUG(bccix[dst].is_staging = 1);
+}
+
+void beam_catches_end_staging(int commit)
+{
+ IF_DEBUG(bccix[erts_staging_code_ix()].is_staging = 0);
}
unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr)
{
int i;
+ struct bc_pool* p = &bccix[erts_staging_code_ix()];
+ ASSERT(p->is_staging);
/*
* Allocate from free_list while it is non-empty.
* If free_list is empty, allocate at high_mark.
- *
- * This avoids the need to initialise the free list in
- * beam_catches_init(), which would cost O(TABSIZ) time.
*/
- if( free_list >= 0 ) {
- i = free_list;
- free_list = beam_catches[i].cdr;
- } else if( high_mark < tabsize ) {
- i = high_mark;
- high_mark++;
- } else {
- /* No free slots and table is full: realloc table */
- tabsize = 2*tabsize;
- beam_catches = erts_realloc(ERTS_ALC_T_CODE, beam_catches, sizeof(beam_catch_t)*tabsize);
- i = high_mark;
- high_mark++;
+ if (p->free_list >= 0) {
+ i = p->free_list;
+ p->free_list = p->beam_catches[i].cdr;
+ }
+ else {
+ if (p->high_mark >= p->tabsize) {
+ /* No free slots and table is full: realloc table */
+ beam_catch_t* prev_vec = p->beam_catches;
+ unsigned newsize = p->tabsize*2;
+
+ p->beam_catches = erts_alloc(ERTS_ALC_T_CODE,
+ newsize*sizeof(beam_catch_t));
+ sys_memcpy(p->beam_catches, prev_vec,
+ p->tabsize*sizeof(beam_catch_t));
+ gc_old_vec(prev_vec);
+ p->tabsize = newsize;
+ }
+ i = p->high_mark++;
}
- beam_catches[i].cp = cp;
- beam_catches[i].cdr = cdr;
+ p->beam_catches[i].cp = cp;
+ p->beam_catches[i].cdr = cdr;
return i;
}
BeamInstr *beam_catches_car(unsigned i)
{
- if( i >= tabsize ) {
+ struct bc_pool* p = &bccix[erts_active_code_ix()];
+
+ if (i >= p->tabsize ) {
erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
}
- return beam_catches[i].cp;
+ return p->beam_catches[i].cp;
}
-void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes)
+void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes,
+ ErtsCodeIndex code_ix)
{
+ struct bc_pool* p = &bccix[code_ix];
unsigned i, cdr;
+ ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging);
for(i = head; i != (unsigned)-1;) {
- if( i >= tabsize ) {
+ if (i >= p->tabsize) {
erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
}
- if( (char*)beam_catches[i].cp - (char*)code >= code_bytes ) {
+ if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) {
erl_exit(1,
"beam_catches_delmod: item %#x has cp %#lx which is not "
"in module's range [%#lx,%#lx[\r\n",
- i, (long)beam_catches[i].cp,
+ i, (long)p->beam_catches[i].cp,
(long)code, (long)((char*)code + code_bytes));
}
- beam_catches[i].cp = 0;
- cdr = beam_catches[i].cdr;
- beam_catches[i].cdr = free_list;
- free_list = i;
+ p->beam_catches[i].cp = 0;
+ cdr = p->beam_catches[i].cdr;
+ p->beam_catches[i].cdr = p->free_list;
+ p->free_list = i;
i = cdr;
}
}
diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h
index 6223427f0d..b2bd2351a5 100644
--- a/erts/emulator/beam/beam_catches.h
+++ b/erts/emulator/beam/beam_catches.h
@@ -20,12 +20,21 @@
#ifndef __BEAM_CATCHES_H
#define __BEAM_CATCHES_H
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+#include "sys.h"
+#include "code_ix.h"
+
#define BEAM_CATCHES_NIL (-1)
void beam_catches_init(void);
+void beam_catches_start_staging(void);
+void beam_catches_end_staging(int commit);
unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr);
BeamInstr *beam_catches_car(unsigned i);
-void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes);
+void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes,
+ ErtsCodeIndex);
#define catch_pc(x) beam_catches_car(catch_val((x)))
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 8041c92162..e69cbc3048 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -114,6 +114,10 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
mfa[2] = signed_val(mfa[2]);
}
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_breakpoint_2],
+ BIF_P, BIF_ARG_1, BIF_ARG_2);
+ }
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
@@ -125,7 +129,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2)
erts_smp_thr_progress_unblock();
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-
+ erts_release_code_write_permission();
return res;
error:
@@ -207,6 +211,7 @@ erts_debug_disassemble_1(BIF_ALIST_1)
BIF_RET(am_false);
}
} else if (is_tuple(addr)) {
+ ErtsCodeIndex code_ix;
Module* modp;
Eterm mod;
Eterm name;
@@ -225,14 +230,14 @@ erts_debug_disassemble_1(BIF_ALIST_1)
goto error;
}
arity = signed_val(tp[3]);
- modp = erts_get_module(mod);
+ code_ix = erts_active_code_ix();
+ modp = erts_get_module(mod, code_ix);
/*
* Try the export entry first to allow disassembly of special functions
* such as erts_debug:apply/4. Then search for it in the module.
*/
-
- if ((ep = erts_find_function(mod, name, arity)) != NULL) {
+ if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) {
/* XXX: add "&& ep->address != ep->code+3" condition?
* Consider a traced function.
* Its ep will have ep->address == ep->code+3.
@@ -241,9 +246,9 @@ erts_debug_disassemble_1(BIF_ALIST_1)
* But this code_ptr will point to the start of the Export,
* not the function's func_info instruction. BOOM !?
*/
- code_ptr = ((BeamInstr *) ep->address) - 5;
+ code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5;
funcinfo = code_ptr+2;
- } else if (modp == NULL || (code_base = modp->code) == NULL) {
+ } else if (modp == NULL || (code_base = modp->curr.code) == NULL) {
BIF_RET(am_undef);
} else {
n = code_base[MI_NUM_FUNCTIONS];
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c65b2be106..d6189c6c65 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -541,7 +541,7 @@ extern int count_instructions;
do { \
if (FCALLS > 0) { \
Eterm* dis_next; \
- SET_I(((Export *) Arg(0))->address); \
+ SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
dis_next = (Eterm *) *I; \
FCALLS--; \
CHECK_ARGS(I); \
@@ -550,7 +550,7 @@ extern int count_instructions;
&& FCALLS > neg_o_reds) { \
goto save_calls1; \
} else { \
- SET_I(((Export *) Arg(0))->address); \
+ SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
CHECK_ARGS(I); \
goto context_switch; \
} \
@@ -5065,7 +5065,7 @@ void process_main(void)
save_calls(c_p, (Export *) Arg(0));
- SET_I(((Export *) Arg(0))->address);
+ SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);
dis_next = (Eterm *) *I;
FCALLS--;
@@ -5742,7 +5742,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
/*
* Search for the error_handler module.
*/
- ep = erts_find_function(erts_proc_get_error_handler(p), func, 3);
+ ep = erts_find_function(erts_proc_get_error_handler(p), func, 3,
+ erts_active_code_ix());
if (ep == NULL) { /* No error handler */
p->current = fi;
p->freason = EXC_UNDEF;
@@ -5772,7 +5773,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
reg[0] = fi[0];
reg[1] = fi[1];
reg[2] = args;
- return ep->address;
+ return ep->addressv[erts_active_code_ix()];
}
@@ -5786,7 +5787,7 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity,
* there is no error handler module.
*/
- if ((ep = erts_find_export_entry(erts_proc_get_error_handler(p),
+ if ((ep = erts_active_export_entry(erts_proc_get_error_handler(p),
am_undefined_function, 3)) == NULL) {
return NULL;
} else {
@@ -5893,13 +5894,13 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
* Note: All BIFs have export entries; thus, no special case is needed.
*/
- if ((ep = erts_find_export_entry(module, function, arity)) == NULL) {
+ if ((ep = erts_active_export_entry(module, function, arity)) == NULL) {
if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL) goto error;
} else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
save_calls(p, ep);
}
- return ep->address;
+ return ep->addressv[erts_active_code_ix()];
}
static BeamInstr*
@@ -5941,14 +5942,14 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
* Note: All BIFs have export entries; thus, no special case is needed.
*/
- if ((ep = erts_find_export_entry(module, function, arity)) == NULL) {
+ if ((ep = erts_active_export_entry(module, function, arity)) == NULL) {
if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL)
goto error;
} else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
save_calls(p, ep);
}
- return ep->address;
+ return ep->addressv[erts_active_code_ix()];
}
int
@@ -6118,7 +6119,7 @@ call_fun(Process* p, /* Current process. */
Export* ep;
Module* modp;
Eterm module;
-
+ ErtsCodeIndex code_ix = erts_active_code_ix();
/*
* No arity. There is no module loaded that defines the fun,
@@ -6126,9 +6127,9 @@ call_fun(Process* p, /* Current process. */
* representation (the module has never been loaded),
* or the module defining the fun has been unloaded.
*/
-
module = fe->module;
- if ((modp = erts_get_module(module)) != NULL && modp->code != NULL) {
+ if ((modp = erts_get_module(module, code_ix)) != NULL
+ && modp->curr.code != NULL) {
/*
* There is a module loaded, but obviously the fun is not
* defined in it. We must not call the error_handler
@@ -6143,7 +6144,7 @@ call_fun(Process* p, /* Current process. */
*/
ep = erts_find_function(erts_proc_get_error_handler(p),
- am_undefined_lambda, 3);
+ am_undefined_lambda, 3, code_ix);
if (ep == NULL) { /* No error handler */
p->current = NULL;
p->freason = EXC_UNDEF;
@@ -6153,7 +6154,7 @@ call_fun(Process* p, /* Current process. */
reg[1] = fun;
reg[2] = args;
reg[3] = NIL;
- return ep->address;
+ return ep->addressv[erts_active_code_ix()];
}
}
} else if (is_export_header(hdr)) {
@@ -6164,7 +6165,7 @@ call_fun(Process* p, /* Current process. */
actual_arity = (int) ep->code[2];
if (arity == actual_arity) {
- return ep->address;
+ return ep->addressv[erts_active_code_ix()];
} else {
/*
* Wrong arity. First build a list of the arguments.
@@ -6215,8 +6216,8 @@ call_fun(Process* p, /* Current process. */
erts_send_warning_to_logger(p->group_leader, dsbufp);
}
- if ((ep = erts_find_export_entry(module, function, arity)) == NULL) {
- ep = erts_find_export_entry(erts_proc_get_error_handler(p),
+ if ((ep = erts_active_export_entry(module, function, arity)) == NULL) {
+ ep = erts_active_export_entry(erts_proc_get_error_handler(p),
am_undefined_function, 3);
if (ep == NULL) {
p->freason = EXC_UNDEF;
@@ -6239,7 +6240,7 @@ call_fun(Process* p, /* Current process. */
reg[1] = function;
reg[2] = args;
}
- return ep->address;
+ return ep->addressv[erts_active_code_ix()];
} else {
badfun:
p->current = NULL;
@@ -6344,7 +6345,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
if ((ep = export_get(&e)) == NULL) {
return 0;
}
- return ep->address == ep->code+3 && (ep->code[3] == (BeamInstr) em_apply_bif);
+ return ep->addressv[erts_active_code_ix()] == ep->code+3
+ && (ep->code[3] == (BeamInstr) em_apply_bif);
}
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d54fe603d8..d3f55a2ba4 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -352,27 +352,6 @@ typedef struct LoaderState {
int loc_size; /* Size of location info in bytes (2/4) */
} LoaderState;
-/*
- * Layout of the line table.
- */
-
-#define MI_LINE_FNAME_PTR 0
-#define MI_LINE_LOC_TAB 1
-#define MI_LINE_LOC_SIZE 2
-#define MI_LINE_FUNC_TAB 3
-
-#define LINE_INVALID_LOCATION (0)
-
-/*
- * Macros for manipulating locations.
- */
-
-#define IS_VALID_LOCATION(File, Line) \
- ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1))
-#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line))
-#define LOC_FILE(Loc) ((Loc) >> 24)
-#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
-
#define GetTagAndValue(Stp, Tag, Val) \
do { \
BeamInstr __w; \
@@ -496,7 +475,8 @@ typedef struct LoaderState {
} while (0)
-static void free_state(LoaderState* stp);
+static void free_loader_state(Binary* magic);
+static void loader_state_dtor(Binary* magic);
static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
Eterm group_leader, Eterm module,
BeamInstr* code, Uint size);
@@ -549,22 +529,9 @@ static Eterm native_addresses(Process* p, Eterm mod);
int patch_funentries(Eterm Patchlist);
int patch(Eterm Addresses, Uint fe);
static int safe_mul(UWord a, UWord b, UWord* resp);
-static void lookup_loc(FunctionInfo* fi, BeamInstr* pc,
- BeamInstr* modp, int idx);
-
static int must_swap_floats;
-/*
- * The following variables keep a sorted list of address ranges for
- * each module. It allows us to quickly find a function given an
- * instruction pointer.
- */
-Range* modules = NULL; /* Sorted lists of module addresses. */
-int num_loaded_modules; /* Number of loaded modules. */
-int allocated_modules; /* Number of slots allocated. */
-Range* mid_module = NULL; /* Cached search start point */
-
Uint erts_total_code_size;
/**********************************************************************/
@@ -580,11 +547,7 @@ void init_load(void)
f.fd = 1.0;
must_swap_floats = (f.fw[0] == 0);
- allocated_modules = 128;
- modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS,
- allocated_modules*sizeof(Range));
- mid_module = modules;
- num_loaded_modules = 0;
+ erts_init_ranges();
}
static void
@@ -596,7 +559,7 @@ define_file(LoaderState* stp, char* name, int idx)
}
Eterm
-erts_load_module(Process *c_p,
+erts_preload_module(Process *c_p,
ErtsProcLocks c_p_locks,
Eterm group_leader, /* Group leader or NIL if none. */
Eterm* modp, /*
@@ -606,15 +569,16 @@ erts_load_module(Process *c_p,
byte* code, /* Points to the code to load */
Uint size) /* Size of code to load. */
{
- LoaderState* stp = erts_alloc_loader_state();
+ Binary* magic = erts_alloc_loader_state();
Eterm retval;
- retval = erts_prepare_loading(stp, c_p, group_leader, modp,
+ ASSERT(!erts_initialized);
+ retval = erts_prepare_loading(magic, c_p, group_leader, modp,
code, size);
if (retval != NIL) {
return retval;
}
- return erts_finish_loading(stp, c_p, c_p_locks, modp);
+ return erts_finish_loading(magic, c_p, c_p_locks, modp);
}
/* #define LOAD_MEMORY_HARD_DEBUG 1*/
@@ -630,11 +594,13 @@ extern void check_allocated_block(Uint type, void *blk);
#endif
Eterm
-erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader,
+erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
Eterm* modp, byte* code, Uint unloaded_size)
{
Eterm retval = am_badfile;
+ LoaderState* stp;
+ stp = ERTS_MAGIC_BIN_DATA(magic);
stp->module = *modp;
stp->group_leader = group_leader;
@@ -680,8 +646,6 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader,
stp->code[MI_COMPILE_PTR] = 0;
stp->code[MI_COMPILE_SIZE] = 0;
stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
- stp->code[MI_NUM_BREAKPOINTS] = 0;
-
/*
* Read the atom table.
@@ -775,23 +739,24 @@ erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader,
load_error:
if (retval != NIL) {
- free_state(stp);
+ free_loader_state(magic);
}
return retval;
}
Eterm
-erts_finish_loading(LoaderState* stp, Process* c_p,
+erts_finish_loading(Binary* magic, Process* c_p,
ErtsProcLocks c_p_locks, Eterm* modp)
{
Eterm retval;
+ LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
/*
* No other process may run since we will update the export
* table which is not protected by any locks.
*/
- ERTS_SMP_LC_ASSERT(erts_initialized == 0 ||
+ ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_is_code_ix_locked() ||
erts_smp_thr_progress_is_blocking());
/*
@@ -812,7 +777,6 @@ erts_finish_loading(LoaderState* stp, Process* c_p,
* exported and imported functions. This can't fail.
*/
- erts_export_consolidate();
CHKBLK(ERTS_ALC_T_CODE,stp->code);
final_touch(stp);
@@ -838,16 +802,20 @@ erts_finish_loading(LoaderState* stp, Process* c_p,
}
load_error:
- free_state(stp);
+ free_loader_state(magic);
return retval;
}
-LoaderState*
+Binary*
erts_alloc_loader_state(void)
{
LoaderState* stp;
+ Binary* magic;
- stp = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LoaderState));
+ magic = erts_create_magic_binary(sizeof(LoaderState),
+ loader_state_dtor);
+ erts_refc_inc(&magic->refc, 1);
+ stp = ERTS_MAGIC_BIN_DATA(magic);
stp->bin = NULL;
stp->function = THE_NON_VALUE; /* Function not known yet */
stp->arity = 0;
@@ -876,76 +844,123 @@ erts_alloc_loader_state(void)
stp->line_instr = 0;
stp->func_line = 0;
stp->fname = 0;
- return stp;
+ return magic;
+}
+
+/*
+ * Return the module name (a tagged atom) for the prepared code
+ * in the magic binary, or NIL if the binary does not contain
+ * prepared code.
+ */
+Eterm
+erts_module_for_prepared_code(Binary* magic)
+{
+ LoaderState* stp;
+
+ if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) {
+ return NIL;
+ }
+ stp = ERTS_MAGIC_BIN_DATA(magic);
+ if (stp->code != 0) {
+ return stp->module;
+ } else {
+ return NIL;
+ }
}
static void
-free_state(LoaderState* stp)
+free_loader_state(Binary* magic)
{
+ loader_state_dtor(magic);
+ if (erts_refc_dectest(&magic->refc, 0) == 0) {
+ erts_bin_free(magic);
+ }
+}
+
+/*
+ * This destructor function can safely be called multiple times.
+ */
+static void
+loader_state_dtor(Binary* magic)
+{
+ LoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
+
if (stp->bin != 0) {
driver_free_binary(stp->bin);
+ stp->bin = 0;
}
if (stp->code != 0) {
erts_free(ERTS_ALC_T_CODE, stp->code);
+ stp->code = 0;
}
- if (stp->labels != NULL) {
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->labels);
+ if (stp->labels != 0) {
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels);
+ stp->labels = 0;
}
- if (stp->atom != NULL) {
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->atom);
+ if (stp->atom != 0) {
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->atom);
+ stp->atom = 0;
}
- if (stp->import != NULL) {
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->import);
+ if (stp->import != 0) {
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->import);
+ stp->import = 0;
}
- if (stp->export != NULL) {
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->export);
+ if (stp->export != 0) {
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->export);
+ stp->export = 0;
}
if (stp->lambdas != stp->def_lambdas) {
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->lambdas);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->lambdas);
+ stp->lambdas = stp->def_lambdas;
}
- if (stp->literals != NULL) {
+ if (stp->literals != 0) {
int i;
for (i = 0; i < stp->num_literals; i++) {
- if (stp->literals[i].heap != NULL) {
- erts_free(ERTS_ALC_T_LOADER_TMP,
+ if (stp->literals[i].heap != 0) {
+ erts_free(ERTS_ALC_T_PREPARED_CODE,
(void *) stp->literals[i].heap);
+ stp->literals[i].heap = 0;
}
}
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literals);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals);
+ stp->literals = 0;
}
- while (stp->literal_patches != NULL) {
+ while (stp->literal_patches != 0) {
LiteralPatch* next = stp->literal_patches->next;
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literal_patches);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literal_patches);
stp->literal_patches = next;
}
- while (stp->string_patches != NULL) {
+ while (stp->string_patches != 0) {
StringPatch* next = stp->string_patches->next;
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->string_patches);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->string_patches);
stp->string_patches = next;
}
- while (stp->genop_blocks) {
- GenOpBlock* next = stp->genop_blocks->next;
- erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks);
- stp->genop_blocks = next;
- }
if (stp->line_item != 0) {
- erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_item);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, stp->line_item);
+ stp->line_item = 0;
}
if (stp->line_instr != 0) {
- erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_instr);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, stp->line_instr);
+ stp->line_instr = 0;
}
if (stp->func_line != 0) {
- erts_free(ERTS_ALC_T_LOADER_TMP, stp->func_line);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, stp->func_line);
+ stp->func_line = 0;
}
if (stp->fname != 0) {
- erts_free(ERTS_ALC_T_LOADER_TMP, stp->fname);
+ erts_free(ERTS_ALC_T_PREPARED_CODE, stp->fname);
+ stp->fname = 0;
}
- erts_free(ERTS_ALC_T_LOADER_TMP, stp);
+ /*
+ * The following data items should have been freed earlier.
+ */
+
+ ASSERT(stp->genop_blocks == 0);
}
static Eterm
@@ -955,7 +970,6 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
{
Module* modp;
Eterm retval;
- int i;
if ((retval = beam_make_current_old(c_p, c_p_locks, module)) != NIL) {
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
@@ -972,30 +986,15 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
erts_total_code_size += size;
modp = erts_put_module(module);
- modp->code = code;
- modp->code_length = size;
- modp->catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
+ modp->curr.code = code;
+ modp->curr.code_length = size;
+ modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
/*
- * Update address table (used for finding a function from a PC value).
+ * Update ranges (used for finding a function from a PC value).
*/
- if (num_loaded_modules == allocated_modules) {
- allocated_modules *= 2;
- modules = (Range *) erts_realloc(ERTS_ALC_T_MODULE_REFS,
- (void *) modules,
- allocated_modules * sizeof(Range));
- }
- for (i = num_loaded_modules; i > 0; i--) {
- if (code > modules[i-1].start) {
- break;
- }
- modules[i] = modules[i-1];
- }
- modules[i].start = code;
- modules[i].end = (BeamInstr *) (((byte *)code) + size);
- num_loaded_modules++;
- mid_module = &modules[num_loaded_modules/2];
+ erts_update_ranges(code, size);
return NIL;
}
@@ -1218,7 +1217,7 @@ load_atom_table(LoaderState* stp)
GetInt(stp, 4, stp->num_atoms);
stp->num_atoms++;
- stp->atom = erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->atom = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_atoms*sizeof(Eterm));
/*
@@ -1263,7 +1262,7 @@ load_import_table(LoaderState* stp)
int i;
GetInt(stp, 4, stp->num_imports);
- stp->import = erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->import = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_imports * sizeof(ImportEntry));
for (i = 0; i < stp->num_imports; i++) {
int n;
@@ -1294,7 +1293,7 @@ load_import_table(LoaderState* stp)
* If the export entry refers to a BIF, get the pointer to
* the BIF function.
*/
- if ((e = erts_find_export_entry(mod, func, arity)) != NULL) {
+ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) {
if (e->code[3] == (BeamInstr) em_apply_bif) {
stp->import[i].bf = (BifFunction) e->code[4];
if (func == am_load_nif && mod == am_erlang && arity == 2) {
@@ -1322,7 +1321,7 @@ read_export_table(LoaderState* stp)
stp->num_exps, stp->num_functions);
}
stp->export
- = (ExportEntry *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ = (ExportEntry *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
(stp->num_exps * sizeof(ExportEntry)));
for (i = 0; i < stp->num_exps; i++) {
@@ -1386,7 +1385,7 @@ read_export_table(LoaderState* stp)
static int
is_bif(Eterm mod, Eterm func, unsigned arity)
{
- Export* e = erts_find_export_entry(mod, func, arity);
+ Export* e = erts_active_export_entry(mod, func, arity);
if (e == NULL) {
return 0;
}
@@ -1409,9 +1408,12 @@ read_lambda_table(LoaderState* stp)
int i;
GetInt(stp, 4, stp->num_lambdas);
- stp->lambdas_allocated = stp->num_lambdas;
- stp->lambdas = (Lambda *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
- stp->num_lambdas * sizeof(Lambda));
+ if (stp->num_lambdas > stp->lambdas_allocated) {
+ ASSERT(stp->lambdas == stp->def_lambdas);
+ stp->lambdas_allocated = stp->num_lambdas;
+ stp->lambdas = (Lambda *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
+ stp->num_lambdas * sizeof(Lambda));
+ }
for (i = 0; i < stp->num_lambdas; i++) {
Uint n;
Uint32 Index;
@@ -1461,7 +1463,7 @@ read_literal_table(LoaderState* stp)
stp->file_p = uncompressed;
stp->file_left = (unsigned) uncompressed_sz;
GetInt(stp, 4, stp->num_literals);
- stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_literals * sizeof(Literal));
stp->allocated_literals = stp->num_literals;
@@ -1481,7 +1483,7 @@ read_literal_table(LoaderState* stp)
if ((heap_size = erts_decode_ext_size(p, sz)) < 0) {
LoadError1(stp, "literal %d: bad external format", i);
}
- hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
heap_size*sizeof(Eterm));
stp->literals[i].off_heap.first = 0;
stp->literals[i].off_heap.overhead = 0;
@@ -1553,7 +1555,7 @@ read_line_table(LoaderState* stp)
*/
num_line_items++;
- lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
num_line_items * sizeof(BeamInstr));
stp->line_item = lp;
stp->num_line_items = num_line_items;
@@ -1609,7 +1611,7 @@ read_line_table(LoaderState* stp)
*/
if (stp->num_fnames != 0) {
- stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_fnames *
sizeof(Eterm));
for (i = 0; i < stp->num_fnames; i++) {
@@ -1625,11 +1627,11 @@ read_line_table(LoaderState* stp)
/*
* Allocate the arrays to be filled while code is being loaded.
*/
- stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_line_instrs *
sizeof(LineInstr));
stp->current_li = 0;
- stp->func_line = (int *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->func_line = (int *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_functions *
sizeof(int));
@@ -1692,7 +1694,7 @@ read_code_header(LoaderState* stp)
* Initialize label table.
*/
- stp->labels = (Label *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->labels = (Label *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_labels * sizeof(Label));
for (i = 0; i < stp->num_labels; i++) {
stp->labels[i].value = 0;
@@ -1746,6 +1748,7 @@ load_code(LoaderState* stp)
GenOp* last_op = NULL;
GenOp** last_op_next = NULL;
int arity;
+ int retval = 1;
/*
* The size of the loaded func_info instruction is needed
@@ -2475,7 +2478,8 @@ load_code(LoaderState* stp)
stp->function = THE_NON_VALUE;
stp->genop = NULL;
stp->specific_op = -1;
- return 1;
+ retval = 1;
+ goto cleanup;
}
/*
@@ -2489,9 +2493,20 @@ load_code(LoaderState* stp)
}
}
-
load_error:
- return 0;
+ retval = 0;
+
+ cleanup:
+ /*
+ * Clean up everything that is not needed any longer.
+ */
+
+ while (stp->genop_blocks) {
+ GenOpBlock* next = stp->genop_blocks->next;
+ erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks);
+ stp->genop_blocks = next;
+ }
+ return retval;
}
@@ -4283,7 +4298,7 @@ final_touch(LoaderState* stp)
index = next;
}
modp = erts_put_module(stp->module);
- modp->catches = catches;
+ modp->curr.catches = catches;
/*
* Export functions.
@@ -4300,13 +4315,13 @@ final_touch(LoaderState* stp)
ep = erts_export_put(stp->module, stp->export[i].function,
stp->export[i].arity);
if (!on_load) {
- ep->address = address;
+ ep->addressv[erts_staging_code_ix()] = address;
} else {
/*
* Don't make any of the exported functions
* callable yet.
*/
- ep->address = ep->code+3;
+ ep->addressv[erts_staging_code_ix()] = ep->code+3;
ep->code[4] = (BeamInstr) address;
}
}
@@ -4951,7 +4966,7 @@ new_label(LoaderState* stp)
int num = stp->num_labels;
stp->num_labels++;
- stp->labels = (Label *) erts_realloc(ERTS_ALC_T_LOADER_TMP,
+ stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE,
(void *) stp->labels,
stp->num_labels * sizeof(Label));
stp->labels[num].value = 0;
@@ -4962,7 +4977,8 @@ new_label(LoaderState* stp)
static void
new_literal_patch(LoaderState* stp, int pos)
{
- LiteralPatch* p = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LiteralPatch));
+ LiteralPatch* p = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
+ sizeof(LiteralPatch));
p->pos = pos;
p->next = stp->literal_patches;
stp->literal_patches = p;
@@ -4971,7 +4987,7 @@ new_literal_patch(LoaderState* stp, int pos)
static void
new_string_patch(LoaderState* stp, int pos)
{
- StringPatch* p = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(StringPatch));
+ StringPatch* p = erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(StringPatch));
p->pos = pos;
p->next = stp->string_patches;
stp->string_patches = p;
@@ -4989,14 +5005,14 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
ASSERT(stp->num_literals == 0);
stp->allocated_literals = 8;
need = stp->allocated_literals * sizeof(Literal);
- stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
need);
} else if (stp->allocated_literals <= stp->num_literals) {
Uint need;
stp->allocated_literals *= 2;
need = stp->allocated_literals * sizeof(Literal);
- stp->literals = (Literal *) erts_realloc(ERTS_ALC_T_LOADER_TMP,
+ stp->literals = (Literal *) erts_realloc(ERTS_ALC_T_PREPARED_CODE,
(void *) stp->literals,
need);
}
@@ -5005,7 +5021,7 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
lit = stp->literals + stp->num_literals;
lit->offset = 0;
lit->heap_size = heap_size;
- lit->heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm));
+ lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm));
lit->term = make_boxed(lit->heap);
lit->off_heap.first = 0;
lit->off_heap.overhead = 0;
@@ -5024,7 +5040,7 @@ erts_module_info_0(Process* p, Eterm module)
return THE_NON_VALUE;
}
- if (erts_get_module(module) == NULL) {
+ if (erts_get_module(module, erts_active_code_ix()) == NULL) {
return THE_NON_VALUE;
}
@@ -5088,11 +5104,11 @@ functions_in_module(Process* p, /* Process whose heap to use. */
return THE_NON_VALUE;
}
- modp = erts_get_module(mod);
+ modp = erts_get_module(mod, erts_active_code_ix());
if (modp == NULL) {
return THE_NON_VALUE;
}
- code = modp->code;
+ code = modp->curr.code;
num_functions = code[MI_NUM_FUNCTIONS];
need = 5*num_functions;
hp = HAlloc(p, need);
@@ -5142,12 +5158,12 @@ native_addresses(Process* p, Eterm mod)
return THE_NON_VALUE;
}
- modp = erts_get_module(mod);
+ modp = erts_get_module(mod, erts_active_code_ix());
if (modp == NULL) {
return THE_NON_VALUE;
}
- code = modp->code;
+ code = modp->curr.code;
num_functions = code[MI_NUM_FUNCTIONS];
need = (6+BIG_UINT_HEAP_SIZE)*num_functions;
hp = HAlloc(p, need);
@@ -5187,18 +5203,20 @@ exported_from_module(Process* p, /* Process whose heap to use. */
Eterm* hp = NULL;
Eterm* hend = NULL;
Eterm result = NIL;
+ ErtsCodeIndex code_ix;
if (is_not_atom(mod)) {
return THE_NON_VALUE;
}
- for (i = 0; i < export_list_size(); i++) {
- Export* ep = export_list(i);
+ code_ix = erts_active_code_ix();
+ for (i = 0; i < export_list_size(code_ix); i++) {
+ Export* ep = export_list(i,code_ix);
if (ep->code[0] == mod) {
Eterm tuple;
- if (ep->address == ep->code+3 &&
+ if (ep->addressv[code_ix] == ep->code+3 &&
ep->code[3] == (BeamInstr) em_call_error_handler) {
/* There is a call to the function, but it does not exist. */
continue;
@@ -5242,11 +5260,11 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
return THE_NON_VALUE;
}
- modp = erts_get_module(mod);
+ modp = erts_get_module(mod, erts_active_code_ix());
if (modp == NULL) {
return THE_NON_VALUE;
}
- code = modp->code;
+ code = modp->curr.code;
ext = (byte *) code[MI_ATTR_PTR];
if (ext != NULL) {
hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]);
@@ -5282,11 +5300,11 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
return THE_NON_VALUE;
}
- modp = erts_get_module(mod);
+ modp = erts_get_module(mod, erts_active_code_ix());
if (modp == NULL) {
return THE_NON_VALUE;
}
- code = modp->code;
+ code = modp->curr.code;
ext = (byte *) code[MI_COMPILE_PTR];
if (ext != NULL) {
hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]);
@@ -5301,113 +5319,6 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
}
/*
- * Find a function from the given pc and fill information in
- * the FunctionInfo struct. If the full_info is non-zero, fill
- * in all available information (including location in the
- * source code). If no function is found, the 'current' field
- * will be set to NULL.
- */
-
-void
-erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
-{
- Range* low = modules;
- Range* high = low + num_loaded_modules;
- Range* mid = mid_module;
-
- fi->current = NULL;
- fi->needed = 5;
- fi->loc = LINE_INVALID_LOCATION;
- while (low < high) {
- if (pc < mid->start) {
- high = mid;
- } else if (pc > mid->end) {
- low = mid + 1;
- } else {
- BeamInstr** low1 = (BeamInstr **) (mid->start + MI_FUNCTIONS);
- BeamInstr** high1 = low1 + mid->start[MI_NUM_FUNCTIONS];
- BeamInstr** mid1;
-
- while (low1 < high1) {
- mid1 = low1 + (high1-low1) / 2;
- if (pc < mid1[0]) {
- high1 = mid1;
- } else if (pc < mid1[1]) {
- mid_module = mid;
- fi->current = mid1[0]+2;
- if (full_info) {
- BeamInstr** fp = (BeamInstr **) (mid->start +
- MI_FUNCTIONS);
- int idx = mid1 - fp;
- lookup_loc(fi, pc, mid->start, idx);
- }
- return;
- } else {
- low1 = mid1 + 1;
- }
- }
- return;
- }
- mid = low + (high-low) / 2;
- }
-}
-
-static void
-lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx)
-{
- Eterm* line = (Eterm *) modp[MI_LINE_TABLE];
- Eterm* low;
- Eterm* high;
- Eterm* mid;
- Eterm pc;
-
- if (line == 0) {
- return;
- }
-
- pc = (Eterm) (BeamInstr) orig_pc;
- fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR];
- low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx];
- high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1];
- while (high > low) {
- mid = low + (high-low) / 2;
- if (pc < mid[0]) {
- high = mid;
- } else if (pc < mid[1]) {
- int file;
- int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB];
-
- if (line[MI_LINE_LOC_SIZE] == 2) {
- Uint16* loc_table =
- (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB];
- fi->loc = loc_table[index];
- } else {
- Uint32* loc_table =
- (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB];
- ASSERT(line[MI_LINE_LOC_SIZE] == 4);
- fi->loc = loc_table[index];
- }
- if (fi->loc == LINE_INVALID_LOCATION) {
- return;
- }
- fi->needed += 3+2+3+2;
- file = LOC_FILE(fi->loc);
- if (file == 0) {
- /* Special case: Module name with ".erl" appended */
- Atom* mod_atom = atom_tab(atom_val(fi->current[0]));
- fi->needed += 2*(mod_atom->len+4);
- } else {
- Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
- fi->needed += 2*ap->len;
- }
- return;
- } else {
- low = mid + 1;
- }
- }
-}
-
-/*
* Build a single {M,F,A,Loction} item to be part of
* a stack trace.
*/
@@ -5487,6 +5398,7 @@ code_get_chunk_2(BIF_ALIST_2)
Process* p = BIF_P;
Eterm Bin = BIF_ARG_1;
Eterm Chunk = BIF_ARG_2;
+ Binary* magic = 0;
LoaderState* stp;
Uint chunk = 0;
ErlSubBin* sb;
@@ -5499,12 +5411,13 @@ code_get_chunk_2(BIF_ALIST_2)
Eterm real_bin;
byte* temp_alloc = NULL;
- stp = erts_alloc_loader_state();
+ magic = erts_alloc_loader_state();
+ stp = ERTS_MAGIC_BIN_DATA(magic);
if ((start = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) {
error:
erts_free_aligned_binary_bytes(temp_alloc);
- if (stp) {
- free_state(stp);
+ if (magic) {
+ free_loader_state(magic);
}
BIF_ERROR(p, BADARG);
}
@@ -5549,7 +5462,7 @@ code_get_chunk_2(BIF_ALIST_2)
done:
erts_free_aligned_binary_bytes(temp_alloc);
- free_state(stp);
+ free_loader_state(magic);
return res;
}
@@ -5562,14 +5475,16 @@ code_module_md5_1(BIF_ALIST_1)
{
Process* p = BIF_P;
Eterm Bin = BIF_ARG_1;
+ Binary* magic;
LoaderState* stp;
byte* bytes;
byte* temp_alloc = NULL;
Eterm res;
- stp = erts_alloc_loader_state();
+ magic = erts_alloc_loader_state();
+ stp = ERTS_MAGIC_BIN_DATA(magic);
if ((bytes = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) {
- free_state(stp);
+ free_loader_state(magic);
BIF_ERROR(p, BADARG);
}
stp->module = THE_NON_VALUE; /* Suppress diagnostiscs */
@@ -5583,7 +5498,7 @@ code_module_md5_1(BIF_ALIST_1)
done:
erts_free_aligned_binary_bytes(temp_alloc);
- free_state(stp);
+ free_loader_state(magic);
return res;
}
@@ -5639,7 +5554,7 @@ stub_read_export_table(LoaderState* stp)
stp->num_exps, stp->num_functions);
}
stp->export
- = (ExportEntry *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+ = (ExportEntry *) erts_alloc(ERTS_ALC_T_PREPARED_CODE,
stp->num_exps * sizeof(ExportEntry));
for (i = 0; i < stp->num_exps; i++) {
@@ -5687,7 +5602,7 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
for (i = 0; i < n; i++) {
if (stp->export[i].function == function && stp->export[i].arity == arity) {
Export* ep = erts_export_put(mod, function, arity);
- ep->address = fp+5;
+ ep->addressv[erts_staging_code_ix()] = fp+5;
return;
}
}
@@ -5866,6 +5781,7 @@ patch_funentries(Eterm Patchlist)
Eterm
erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
{
+ Binary* magic;
LoaderState* stp;
BeamInstr Funcs;
BeamInstr Patchlist;
@@ -5887,7 +5803,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
* Must initialize stp->lambdas here because the error handling code
* at label 'error' uses it.
*/
- stp = erts_alloc_loader_state();
+ magic = erts_alloc_loader_state();
+ stp = ERTS_MAGIC_BIN_DATA(magic);
if (is_not_atom(Mod)) {
goto error;
@@ -5967,7 +5884,6 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
code[MI_COMPILE_PTR] = 0;
code[MI_COMPILE_SIZE] = 0;
code[MI_COMPILE_SIZE_ON_HEAP] = 0;
- code[MI_NUM_BREAKPOINTS] = 0;
code[MI_LITERALS_START] = 0;
code[MI_LITERALS_END] = 0;
code[MI_LITERALS_OFF_HEAP] = 0;
@@ -6075,13 +5991,13 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
if (patch_funentries(Patchlist)) {
erts_free_aligned_binary_bytes(temp_alloc);
- free_state(stp);
+ free_loader_state(magic);
return Mod;
}
error:
erts_free_aligned_binary_bytes(temp_alloc);
- free_state(stp);
+ free_loader_state(magic);
BIF_ERROR(p, BADARG);
}
@@ -6098,3 +6014,4 @@ static int safe_mul(UWord a, UWord b, UWord* resp)
return (res / b) == a;
}
}
+
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 997ba197db..f1506a8684 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -50,10 +50,6 @@ extern BeamInstr beam_debug_apply[];
extern BeamInstr* em_call_error_handler;
extern BeamInstr* em_apply_bif;
extern BeamInstr* em_call_traced_function;
-typedef struct {
- BeamInstr* start; /* Pointer to start of module. */
- BeamInstr* end; /* Points one word beyond last function in module. */
-} Range;
/*
* The following variables keep a sorted list of address ranges for
@@ -61,11 +57,6 @@ typedef struct {
* instruction pointer.
*/
-extern Range* modules;
-extern int num_loaded_modules;
-extern int allocated_modules;
-extern Range* mid_module;
-
/* Total code size in bytes */
extern Uint erts_total_code_size;
/*
@@ -94,27 +85,22 @@ extern Uint erts_total_code_size;
#define MI_COMPILE_SIZE_ON_HEAP 6
/*
- * Number of breakpoints in module is stored in this word
- */
-#define MI_NUM_BREAKPOINTS 7
-
-/*
* Literal area (constant pool).
*/
-#define MI_LITERALS_START 8
-#define MI_LITERALS_END 9
-#define MI_LITERALS_OFF_HEAP 10
+#define MI_LITERALS_START 7
+#define MI_LITERALS_END 8
+#define MI_LITERALS_OFF_HEAP 9
/*
* Pointer to the on_load function (or NULL if none).
*/
-#define MI_ON_LOAD_FUNCTION_PTR 11
+#define MI_ON_LOAD_FUNCTION_PTR 10
/*
* Pointer to the line table (or NULL if none).
*/
-#define MI_LINE_TABLE 12
+#define MI_LINE_TABLE 11
/*
* Start of function pointer table. This table contains pointers to
@@ -125,5 +111,27 @@ extern Uint erts_total_code_size;
* this table.
*/
-#define MI_FUNCTIONS 13
+#define MI_FUNCTIONS 12
+
+/*
+ * Layout of the line table.
+ */
+
+#define MI_LINE_FNAME_PTR 0
+#define MI_LINE_LOC_TAB 1
+#define MI_LINE_LOC_SIZE 2
+#define MI_LINE_FUNC_TAB 3
+
+#define LINE_INVALID_LOCATION (0)
+
+/*
+ * Macros for manipulating locations.
+ */
+
+#define IS_VALID_LOCATION(File, Line) \
+ ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1))
+#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line))
+#define LOC_FILE(Loc) ((Loc) >> 24)
+#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
+
#endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c
new file mode 100644
index 0000000000..0f2d5d0c2a
--- /dev/null
+++ b/erts/emulator/beam/beam_ranges.c
@@ -0,0 +1,349 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "beam_load.h"
+
+typedef struct {
+ BeamInstr* start; /* Pointer to start of module. */
+ erts_smp_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */
+} Range;
+
+/* Range 'end' needs to be atomic as we purge module
+ by setting end=start in active code_ix */
+#define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end))
+
+static Range* find_range(BeamInstr* pc);
+static void lookup_loc(FunctionInfo* fi, BeamInstr* pc,
+ BeamInstr* modp, int idx);
+
+/*
+ * The following variables keep a sorted list of address ranges for
+ * each module. It allows us to quickly find a function given an
+ * instruction pointer.
+ */
+struct ranges {
+ Range* modules; /* Sorted lists of module addresses. */
+ Sint n; /* Number of range entries. */
+ Sint allocated; /* Number of allocated entries. */
+ erts_smp_atomic_t mid; /* Cached search start point */
+};
+static struct ranges r[ERTS_NUM_CODE_IX];
+static erts_smp_atomic_t mem_used;
+
+#ifdef HARD_DEBUG
+static void check_consistency(struct ranges* p)
+{
+ int i;
+
+ ASSERT(p->n <= p->allocated);
+ ASSERT((Uint)(p->mid - p->modules) < p->n ||
+ (p->mid == p->modules && p->n == 0));
+ for (i = 0; i < p->n; i++) {
+ ASSERT(p->modules[i].start <= RANGE_END(&p->modules[i]));
+ ASSERT(!i || RANGE_END(&p->modules[i-1]) < p->modules[i].start);
+ }
+}
+# define CHECK(r) check_consistency(r)
+#else
+# define CHECK(r)
+#endif /* HARD_DEBUG */
+
+
+void
+erts_init_ranges(void)
+{
+ Sint i;
+
+ erts_smp_atomic_init_nob(&mem_used, 0);
+ for (i = 0; i < ERTS_NUM_CODE_IX; i++) {
+ r[i].modules = 0;
+ r[i].n = 0;
+ r[i].allocated = 0;
+ erts_smp_atomic_init_nob(&r[i].mid, 0);
+ }
+}
+
+void
+erts_start_staging_ranges(void)
+{
+ ErtsCodeIndex dst = erts_staging_code_ix();
+
+ if (r[dst].modules) {
+ erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated);
+ erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules);
+ r[dst].modules = NULL;
+ }
+}
+
+void
+erts_end_staging_ranges(int commit)
+{
+ ErtsCodeIndex dst = erts_staging_code_ix();
+
+ if (commit && r[dst].modules == NULL) {
+ Sint i;
+ Sint n;
+
+ /* No modules added, just clone src and remove purged code. */
+ ErtsCodeIndex src = erts_active_code_ix();
+
+ erts_smp_atomic_add_nob(&mem_used, r[src].n);
+ r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS,
+ r[src].n * sizeof(Range));
+ r[dst].allocated = r[src].n;
+ n = 0;
+ for (i = 0; i < r[src].n; i++) {
+ Range* rp = r[src].modules+i;
+ if (rp->start < RANGE_END(rp)) {
+ /* Only insert a module that has not been purged. */
+ r[dst].modules[n] = *rp;
+ n++;
+ }
+ }
+ r[dst].n = n;
+ erts_smp_atomic_set_nob(&r[dst].mid,
+ (erts_aint_t) (r[dst].modules + n / 2));
+ }
+}
+
+void
+erts_update_ranges(BeamInstr* code, Uint size)
+{
+ ErtsCodeIndex dst = erts_staging_code_ix();
+ ErtsCodeIndex src = erts_active_code_ix();
+ Sint i;
+ Sint n;
+ Sint need;
+
+ if (src == dst) {
+ ASSERT(!erts_initialized);
+
+ /*
+ * During start-up of system, the indices are the same.
+ * Handle this by faking a source area.
+ */
+ src = (src+1) % ERTS_NUM_CODE_IX;
+ if (r[src].modules) {
+ erts_smp_atomic_add_nob(&mem_used, -r[src].allocated);
+ erts_free(ERTS_ALC_T_MODULE_REFS, r[src].modules);
+ }
+ r[src] = r[dst];
+ r[dst].modules = 0;
+ }
+
+ CHECK(&r[src]);
+
+ ASSERT(r[dst].modules == NULL);
+ need = r[dst].allocated = r[src].n + 1;
+ erts_smp_atomic_add_nob(&mem_used, need);
+ r[dst].modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS,
+ need * sizeof(Range));
+ n = 0;
+ for (i = 0; i < r[src].n; i++) {
+ Range* rp = r[src].modules+i;
+ if (code < rp->start) {
+ r[dst].modules[n].start = code;
+ erts_smp_atomic_init_nob(&r[dst].modules[n].end,
+ (erts_aint_t)(((byte *)code) + size));
+ ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code);
+ n++;
+ break;
+ }
+ if (rp->start < RANGE_END(rp)) {
+ /* Only insert a module that has not been purged. */
+ r[dst].modules[n].start = rp->start;
+ erts_smp_atomic_init_nob(&r[dst].modules[n].end,
+ (erts_aint_t)(RANGE_END(rp)));
+ ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start);
+ n++;
+ }
+ }
+
+ while (i < r[src].n) {
+ Range* rp = r[src].modules+i;
+ if (rp->start < RANGE_END(rp)) {
+ /* Only insert a module that has not been purged. */
+ r[dst].modules[n].start = rp->start;
+ erts_smp_atomic_init_nob(&r[dst].modules[n].end,
+ (erts_aint_t)(RANGE_END(rp)));
+ ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start);
+ n++;
+ }
+ i++;
+ }
+
+ if (n == 0 || code > r[dst].modules[n-1].start) {
+ r[dst].modules[n].start = code;
+ erts_smp_atomic_init_nob(&r[dst].modules[n].end,
+ (erts_aint_t)(((byte *)code) + size));
+ ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code);
+ n++;
+ }
+
+ ASSERT(n <= r[src].n+1);
+ r[dst].n = n;
+ erts_smp_atomic_set_nob(&r[dst].mid,
+ (erts_aint_t) (r[dst].modules + n / 2));
+
+ CHECK(&r[dst]);
+ CHECK(&r[src]);
+}
+
+void
+erts_remove_from_ranges(BeamInstr* code)
+{
+ Range* rp = find_range(code);
+ erts_smp_atomic_set_nob(&rp->end, (erts_aint_t)rp->start);
+}
+
+UWord
+erts_ranges_sz(void)
+{
+ return erts_smp_atomic_read_nob(&mem_used) * sizeof(Range);
+}
+
+/*
+ * Find a function from the given pc and fill information in
+ * the FunctionInfo struct. If the full_info is non-zero, fill
+ * in all available information (including location in the
+ * source code). If no function is found, the 'current' field
+ * will be set to NULL.
+ */
+
+void
+erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
+{
+ BeamInstr** low;
+ BeamInstr** high;
+ BeamInstr** mid;
+ Range* rp;
+
+ fi->current = NULL;
+ fi->needed = 5;
+ fi->loc = LINE_INVALID_LOCATION;
+ rp = find_range(pc);
+ if (rp == 0) {
+ return;
+ }
+
+ low = (BeamInstr **) (rp->start + MI_FUNCTIONS);
+ high = low + rp->start[MI_NUM_FUNCTIONS];
+ while (low < high) {
+ mid = low + (high-low) / 2;
+ if (pc < mid[0]) {
+ high = mid;
+ } else if (pc < mid[1]) {
+ fi->current = mid[0]+2;
+ if (full_info) {
+ BeamInstr** fp = (BeamInstr **) (rp->start +
+ MI_FUNCTIONS);
+ int idx = mid - fp;
+ lookup_loc(fi, pc, rp->start, idx);
+ }
+ return;
+ } else {
+ low = mid + 1;
+ }
+ }
+}
+
+static Range*
+find_range(BeamInstr* pc)
+{
+ ErtsCodeIndex active = erts_active_code_ix();
+ Range* low = r[active].modules;
+ Range* high = low + r[active].n;
+ Range* mid = (Range *) erts_smp_atomic_read_nob(&r[active].mid);
+
+ CHECK(&r[active]);
+ while (low < high) {
+ if (pc < mid->start) {
+ high = mid;
+ } else if (pc > RANGE_END(mid)) {
+ low = mid + 1;
+ } else {
+ erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid);
+ return mid;
+ }
+ mid = low + (high-low) / 2;
+ }
+ return 0;
+}
+
+static void
+lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx)
+{
+ Eterm* line = (Eterm *) modp[MI_LINE_TABLE];
+ Eterm* low;
+ Eterm* high;
+ Eterm* mid;
+ Eterm pc;
+
+ if (line == 0) {
+ return;
+ }
+
+ pc = (Eterm) (BeamInstr) orig_pc;
+ fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR];
+ low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx];
+ high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1];
+ while (high > low) {
+ mid = low + (high-low) / 2;
+ if (pc < mid[0]) {
+ high = mid;
+ } else if (pc < mid[1]) {
+ int file;
+ int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB];
+
+ if (line[MI_LINE_LOC_SIZE] == 2) {
+ Uint16* loc_table =
+ (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB];
+ fi->loc = loc_table[index];
+ } else {
+ Uint32* loc_table =
+ (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB];
+ ASSERT(line[MI_LINE_LOC_SIZE] == 4);
+ fi->loc = loc_table[index];
+ }
+ if (fi->loc == LINE_INVALID_LOCATION) {
+ return;
+ }
+ fi->needed += 3+2+3+2;
+ file = LOC_FILE(fi->loc);
+ if (file == 0) {
+ /* Special case: Module name with ".erl" appended */
+ Atom* mod_atom = atom_tab(atom_val(fi->current[0]));
+ fi->needed += 2*(mod_atom->len+4);
+ } else {
+ Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
+ fi->needed += 2*ap->len;
+ }
+ return;
+ } else {
+ low = mid + 1;
+ }
+ }
+}
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index f8305944a4..9b2394d92f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3709,7 +3709,8 @@ BIF_RETTYPE function_exported_3(BIF_ALIST_3)
is_not_small(BIF_ARG_3)) {
BIF_ERROR(BIF_P, BADARG);
}
- if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3)) == NULL) {
+ if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3),
+ erts_active_code_ix()) == NULL) {
BIF_RET(am_false);
}
BIF_RET(am_true);
@@ -4434,6 +4435,21 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
Export bif_return_trap_export;
+void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
+ Eterm (*bif)(BIF_ALIST_0))
+{
+ int i;
+ sys_memset((void *) ep, 0, sizeof(Export));
+ for (i=0; i<ERTS_NUM_CODE_IX; i++) {
+ ep->addressv[i] = &ep->code[3];
+ }
+ ep->code[0] = m;
+ ep->code[1] = f;
+ ep->code[2] = a;
+ ep->code[3] = (BeamInstr) em_apply_bif;
+ ep->code[4] = (BeamInstr) bif;
+}
+
void erts_init_bif(void)
{
reference0 = 0;
@@ -4449,17 +4465,13 @@ void erts_init_bif(void)
* yield the calling process traps to. The only thing it does:
* return the value passed as argument.
*/
- sys_memset((void *) &bif_return_trap_export, 0, sizeof(Export));
- bif_return_trap_export.address = &bif_return_trap_export.code[3];
- bif_return_trap_export.code[0] = am_erlang;
- bif_return_trap_export.code[1] = am_bif_return_trap;
+ erts_init_trap_export(&bif_return_trap_export, am_erlang, am_bif_return_trap,
#ifdef DEBUG
- bif_return_trap_export.code[2] = 2;
+ 2
#else
- bif_return_trap_export.code[2] = 1;
+ 1
#endif
- bif_return_trap_export.code[3] = (BeamInstr) em_apply_bif;
- bif_return_trap_export.code[4] = (BeamInstr) &bif_return_trap;
+ , &bif_return_trap);
flush_monitor_message_trap = erts_export_put(am_erlang,
am_flush_monitor_message,
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index d20089a9fb..7cb2c78815 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -125,7 +125,7 @@ do { \
#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \
do { \
(Proc)->arity = 0; \
- (Proc)->i = (BeamInstr*) ((Trap)->address); \
+ (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
(Proc)->freason = TRAP; \
(Ret) = THE_NON_VALUE; \
} while (0)
@@ -135,7 +135,7 @@ do { \
Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \
(Proc)->arity = 1; \
reg[0] = (Eterm) (A0); \
- (Proc)->i = (BeamInstr*) ((Trap)->address); \
+ (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
(Proc)->freason = TRAP; \
(Ret) = THE_NON_VALUE; \
} while (0)
@@ -146,7 +146,7 @@ do { \
(Proc)->arity = 2; \
reg[0] = (Eterm) (A0); \
reg[1] = (Eterm) (A1); \
- (Proc)->i = (BeamInstr*) ((Trap)->address); \
+ (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
(Proc)->freason = TRAP; \
(Ret) = THE_NON_VALUE; \
} while (0)
@@ -158,7 +158,7 @@ do { \
reg[0] = (Eterm) (A0); \
reg[1] = (Eterm) (A1); \
reg[2] = (Eterm) (A2); \
- (Proc)->i = (BeamInstr*) ((Trap)->address); \
+ (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
(Proc)->freason = TRAP; \
(Ret) = THE_NON_VALUE; \
} while (0)
@@ -170,13 +170,13 @@ do { \
reg[0] = (Eterm) (A0); \
reg[1] = (Eterm) (A1); \
reg[2] = (Eterm) (A2); \
- (Proc)->i = (BeamInstr*) ((Trap)->address); \
+ (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
(Proc)->freason = TRAP; \
} while (0)
#define BIF_TRAP0(p, Trap_) do { \
(p)->arity = 0; \
- (p)->i = (BeamInstr*) ((Trap_)->address); \
+ (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
(p)->freason = TRAP; \
return THE_NON_VALUE; \
} while(0)
@@ -185,7 +185,7 @@ do { \
Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \
(p)->arity = 1; \
reg[0] = (A0); \
- (p)->i = (BeamInstr*) ((Trap_)->address); \
+ (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
(p)->freason = TRAP; \
return THE_NON_VALUE; \
} while(0)
@@ -195,7 +195,7 @@ do { \
(p)->arity = 2; \
reg[0] = (A0); \
reg[1] = (A1); \
- (p)->i = (BeamInstr*) ((Trap_)->address); \
+ (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
(p)->freason = TRAP; \
return THE_NON_VALUE; \
} while(0)
@@ -206,7 +206,7 @@ do { \
reg[0] = (A0); \
reg[1] = (A1); \
reg[2] = (A2); \
- (p)->i = (BeamInstr*) ((Trap_)->address); \
+ (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
(p)->freason = TRAP; \
return THE_NON_VALUE; \
} while(0)
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 8cc568b16c..7f874a2d9c 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -142,8 +142,6 @@ bif erlang:list_to_pid/1
bif 'erl.lang.proc':string_to_pid/1 ebif_string_to_pid_1 list_to_pid_1
bif erlang:list_to_tuple/1
bif 'erl.lang.tuple':from_list/1 ebif_list_to_tuple_1
-bif erlang:load_module/2
-bif 'erl.system.code':load/2 ebif_load_module_2
bif erlang:loaded/0
bif 'erl.system.code':loaded/0 ebif_loaded_0
bif erlang:localtime/0
@@ -812,6 +810,13 @@ bif erlang:check_old_code/1
#
bif erlang:universaltime_to_posixtime/1
bif erlang:posixtime_to_universaltime/1
+
+#
+# New in R16B.
+#
+bif erlang:prepare_loading/2
+bif erlang:finish_loading/1
+
#
# Obsolete
#
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 6f5020dc14..39f91be7fc 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -377,17 +377,22 @@ loaded(int to, void *to_arg)
int old = 0;
int cur = 0;
BeamInstr* code;
+ Module* modp;
+ ErtsCodeIndex code_ix;
+
+ code_ix = erts_active_code_ix();
+ erts_rlock_old_code(code_ix);
/*
* Calculate and print totals.
*/
- for (i = 0; i < module_code_size(); i++) {
- if (module_code(i) != NULL &&
- ((module_code(i)->code_length != 0) ||
- (module_code(i)->old_code_length != 0))) {
- cur += module_code(i)->code_length;
- if (module_code(i)->old_code_length != 0) {
- old += module_code(i)->old_code_length;
+ for (i = 0; i < module_code_size(code_ix); i++) {
+ if ((modp = module_code(i, code_ix)) != NULL &&
+ ((modp->curr.code_length != 0) ||
+ (modp->old.code_length != 0))) {
+ cur += modp->curr.code_length;
+ if (modp->old.code_length != 0) {
+ old += modp->old.code_length;
}
}
}
@@ -398,21 +403,22 @@ loaded(int to, void *to_arg)
* Print one line per module.
*/
- for (i = 0; i < module_code_size(); i++) {
+ for (i = 0; i < module_code_size(code_ix); i++) {
+ modp = module_code(i, code_ix);
if (!ERTS_IS_CRASH_DUMPING) {
/*
* Interactive dump; keep it brief.
*/
- if (module_code(i) != NULL &&
- ((module_code(i)->code_length != 0) ||
- (module_code(i)->old_code_length != 0))) {
- erts_print(to, to_arg, "%T", make_atom(module_code(i)->module));
- cur += module_code(i)->code_length;
- erts_print(to, to_arg, " %d", module_code(i)->code_length );
- if (module_code(i)->old_code_length != 0) {
+ if (modp != NULL &&
+ ((modp->curr.code_length != 0) ||
+ (modp->old.code_length != 0))) {
+ erts_print(to, to_arg, "%T", make_atom(modp->module));
+ cur += modp->curr.code_length;
+ erts_print(to, to_arg, " %d", modp->curr.code_length );
+ if (modp->old.code_length != 0) {
erts_print(to, to_arg, " (%d old)",
- module_code(i)->old_code_length );
- old += module_code(i)->old_code_length;
+ modp->old.code_length );
+ old += modp->old.code_length;
}
erts_print(to, to_arg, "\n");
}
@@ -420,15 +426,15 @@ loaded(int to, void *to_arg)
/*
* To crash dump; make it parseable.
*/
- if (module_code(i) != NULL &&
- ((module_code(i)->code_length != 0) ||
- (module_code(i)->old_code_length != 0))) {
+ if (modp != NULL &&
+ ((modp->curr.code_length != 0) ||
+ (modp->old.code_length != 0))) {
erts_print(to, to_arg, "=mod:");
- erts_print(to, to_arg, "%T", make_atom(module_code(i)->module));
+ erts_print(to, to_arg, "%T", make_atom(modp->module));
erts_print(to, to_arg, "\n");
erts_print(to, to_arg, "Current size: %d\n",
- module_code(i)->code_length);
- code = module_code(i)->code;
+ modp->curr.code_length);
+ code = modp->curr.code;
if (code != NULL && code[MI_ATTR_PTR]) {
erts_print(to, to_arg, "Current attributes: ");
dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR],
@@ -440,9 +446,9 @@ loaded(int to, void *to_arg)
code[MI_COMPILE_SIZE]);
}
- if (module_code(i)->old_code_length != 0) {
- erts_print(to, to_arg, "Old size: %d\n", module_code(i)->old_code_length);
- code = module_code(i)->old_code;
+ if (modp->old.code_length != 0) {
+ erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length);
+ code = modp->old.code;
if (code[MI_ATTR_PTR]) {
erts_print(to, to_arg, "Old attributes: ");
dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR],
@@ -457,6 +463,7 @@ loaded(int to, void *to_arg)
}
}
}
+ erts_runlock_old_code(code_ix);
}
diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
new file mode 100644
index 0000000000..ae4cca1e58
--- /dev/null
+++ b/erts/emulator/beam/code_ix.c
@@ -0,0 +1,152 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include "code_ix.h"
+#include "global.h"
+#include "beam_catches.h"
+
+
+
+#if 0
+# define CIX_TRACE(text) erts_fprintf(stderr, "CIX_TRACE: " text " act=%u load=%u\r\n", erts_active_code_ix(), erts_staging_code_ix())
+#else
+# define CIX_TRACE(text)
+#endif
+
+erts_smp_atomic32_t the_active_code_index;
+erts_smp_atomic32_t the_staging_code_index;
+
+static int the_code_ix_lock = 0;
+struct code_ix_queue_item {
+ Process *p;
+ struct code_ix_queue_item* next;
+};
+static struct code_ix_queue_item* the_code_ix_queue = NULL;
+static erts_smp_mtx_t the_code_ix_queue_lock;
+
+void erts_code_ix_init(void)
+{
+ /* We start emulator by initializing preloaded modules
+ * single threaded with active and staging set both to zero.
+ * Preloading is finished by a commit that will set things straight.
+ */
+ erts_smp_atomic32_init_nob(&the_active_code_index, 0);
+ erts_smp_atomic32_init_nob(&the_staging_code_index, 0);
+ erts_smp_mtx_init(&the_code_ix_queue_lock, "code_ix_queue");
+ CIX_TRACE("init");
+}
+
+void erts_start_staging_code_ix(void)
+{
+ beam_catches_start_staging();
+ export_start_staging();
+ module_start_staging();
+ erts_start_staging_ranges();
+ CIX_TRACE("start");
+}
+
+
+void erts_end_staging_code_ix(void)
+{
+ beam_catches_end_staging(1);
+ export_end_staging(1);
+ module_end_staging(1);
+ erts_end_staging_ranges(1);
+ CIX_TRACE("end");
+}
+
+void erts_commit_staging_code_ix(void)
+{
+ ErtsCodeIndex ix;
+ /* We need to this lock as we are now making the staging export table active */
+ export_staging_lock();
+ ix = erts_staging_code_ix();
+ erts_smp_atomic32_set_nob(&the_active_code_index, ix);
+ ix = (ix + 1) % ERTS_NUM_CODE_IX;
+ erts_smp_atomic32_set_nob(&the_staging_code_index, ix);
+ export_staging_unlock();
+ CIX_TRACE("activate");
+}
+
+void erts_abort_staging_code_ix(void)
+{
+ beam_catches_end_staging(0);
+ export_end_staging(0);
+ module_end_staging(0);
+ erts_end_staging_ranges(0);
+ CIX_TRACE("abort");
+}
+
+
+/*
+ * Calller _must_ yield if we return 0
+ */
+int erts_try_seize_code_write_permission(Process* c_p)
+{
+ int success;
+#ifdef ERTS_SMP
+ ASSERT(!erts_smp_thr_progress_is_blocking()); /* to avoid deadlock */
+#endif
+
+ erts_smp_mtx_lock(&the_code_ix_queue_lock);
+ success = !the_code_ix_lock;
+ if (success) {
+ the_code_ix_lock = 1;
+ }
+ else { /* Already locked */
+ struct code_ix_queue_item* qitem;
+ qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem));
+ qitem->p = c_p;
+ erts_smp_proc_inc_refc(c_p);
+ qitem->next = the_code_ix_queue;
+ the_code_ix_queue = qitem;
+ erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+ }
+ erts_smp_mtx_unlock(&the_code_ix_queue_lock);
+ return success;
+}
+
+void erts_release_code_write_permission(void)
+{
+ erts_smp_mtx_lock(&the_code_ix_queue_lock);
+ while (the_code_ix_queue != NULL) { /* unleash the entire herd */
+ struct code_ix_queue_item* qitem = the_code_ix_queue;
+ erts_smp_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS);
+ if (!ERTS_PROC_IS_EXITING(qitem->p)) {
+ erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS);
+ }
+ erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS);
+ the_code_ix_queue = qitem->next;
+ erts_smp_proc_dec_refc(qitem->p);
+ erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem);
+ }
+ the_code_ix_lock = 0;
+ erts_smp_mtx_unlock(&the_code_ix_queue_lock);
+}
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+int erts_is_code_ix_locked(void)
+{
+ return the_code_ix_lock;
+}
+#endif
diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h
new file mode 100644
index 0000000000..6b2680044e
--- /dev/null
+++ b/erts/emulator/beam/code_ix.h
@@ -0,0 +1,141 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/* Description:
+ * This is the interface that facilitate changing the beam code
+ * (load,upgrade,delete) while allowing executing Erlang processes to
+ * access the code without any locks or other expensive memory barriers.
+ *
+ * The basic idea is to maintain several "logical copies" of the code. These
+ * copies are identified by a global 'code index', an integer of 0, 1 or 2.
+ * The code index is used as argument to code access structures like
+ * export, module, beam_catches, beam_ranges.
+ *
+ * The current 'active' code index is used to access the current running
+ * code. The 'staging' code index is used by the process that performs
+ * a code change operation. When a code change operation completes
+ * succesfully, the staging code index becomes the new active code index.
+ *
+ * The third code index is not explicitly used. It can be thought of as
+ * the "previous active" or the "next staging" index. It is needed to make
+ * sure that we do not reuse a code index for staging until we are sure
+ * that no executing BIFs are still referring it.
+ * We could get by with only two (0 and 1), but that would require that we
+ * must wait for all schedulers to re-schedule before each code change
+ * operation can start staging.
+ *
+ * Note that the 'code index' is very loosely coupled to the concept of
+ * 'current' and 'old' module versions. You can almost say that they are
+ * orthogonal to each other. Code index is an emulator global concept while
+ * 'current' and 'old' is specific for each module.
+ */
+
+#ifndef __CODE_IX_H__
+#define __CODE_IX_H__
+
+#ifndef __SYS_H__
+# ifdef HAVE_CONFIG_H
+# include "config.h"
+# endif
+# include "sys.h"
+#endif
+struct process;
+
+
+#define ERTS_NUM_CODE_IX 3
+typedef unsigned ErtsCodeIndex;
+
+
+/* Called once at emulator initialization.
+ */
+void erts_code_ix_init(void);
+
+/* Return active code index.
+ * Is guaranteed to be valid until the calling BIF returns.
+ * To get a consistent view of the code, only one call to erts_active_code_ix()
+ * should be made and the returned ix reused within the same BIF call.
+ */
+ERTS_GLB_INLINE
+ErtsCodeIndex erts_active_code_ix(void);
+
+/* Return staging code ix.
+ * Only used by a process performing code loading/upgrading/deleting/purging.
+ * code_ix must be locked.
+ */
+ERTS_GLB_INLINE
+ErtsCodeIndex erts_staging_code_ix(void);
+
+/* Try seize exclusive code write permission. Needed for code staging.
+ * Main process lock (only) must be held.
+ * System thread progress must not be blocked.
+ * Caller is suspended and *must* yield if 0 is returned.
+ */
+int erts_try_seize_code_write_permission(struct process*);
+
+/* Release code write permission.
+ * Will resume any suspended waiters.
+ */
+void erts_release_code_write_permission(void);
+
+/* Prepare the "staging area" to be a complete copy of the active code.
+ * Code write permission must have been seized.
+ * Must be followed by calls to either "end" and "commit" or "abort" before
+ * code write permission can be released.
+ */
+void erts_start_staging_code_ix(void);
+
+/* End the staging.
+ * Preceded by "start" and must be followed by "commit".
+ */
+void erts_end_staging_code_ix(void);
+
+/* Set staging code index as new active code index.
+ * Preceded by "end".
+ */
+void erts_commit_staging_code_ix(void);
+
+/* Abort the staging.
+ * Preceded by "start".
+ */
+void erts_abort_staging_code_ix(void);
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+int erts_is_code_ix_locked(void);
+#endif
+
+
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+extern erts_smp_atomic32_t the_active_code_index;
+extern erts_smp_atomic32_t the_staging_code_index;
+
+ERTS_GLB_INLINE ErtsCodeIndex erts_active_code_ix(void)
+{
+ return erts_smp_atomic32_read_nob(&the_active_code_index);
+}
+ERTS_GLB_INLINE ErtsCodeIndex erts_staging_code_ix(void)
+{
+ return erts_smp_atomic32_read_nob(&the_staging_code_index);
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* !__CODE_IX_H__ */
+
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index bee61e7273..00d57ceab5 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -2295,15 +2295,15 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
goto error;
/* Check that all trap functions are defined !! */
- if (dsend2_trap->address == NULL ||
- dsend3_trap->address == NULL ||
+ if (dsend2_trap->addressv[0] == NULL ||
+ dsend3_trap->addressv[0] == NULL ||
/* dsend_nosuspend_trap->address == NULL ||*/
- dlink_trap->address == NULL ||
- dunlink_trap->address == NULL ||
- dmonitor_node_trap->address == NULL ||
- dgroup_leader_trap->address == NULL ||
- dmonitor_p_trap->address == NULL ||
- dexit_trap->address == NULL) {
+ dlink_trap->addressv[0] == NULL ||
+ dunlink_trap->addressv[0] == NULL ||
+ dmonitor_node_trap->addressv[0] == NULL ||
+ dgroup_leader_trap->addressv[0] == NULL ||
+ dmonitor_p_trap->addressv[0] == NULL ||
+ dexit_trap->addressv[0] == NULL) {
goto error;
}
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index df27186680..ac92822aca 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -2185,9 +2185,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
if (want.code) {
size.code = module_table_sz();
size.code += export_table_sz();
- size.code += export_list_size() * sizeof(Export);
+ size.code += export_entries_sz();
size.code += erts_fun_table_sz();
- size.code += allocated_modules*sizeof(Range);
+ size.code += erts_ranges_sz();
size.code += erts_total_code_size;
}
@@ -2335,7 +2335,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
values[i].arity = 2;
values[i].name = "export_list";
- values[i].ui[0] = export_list_size() * sizeof(Export);
+ values[i].ui[0] = export_entries_sz();
i++;
values[i].arity = 2;
@@ -2350,7 +2350,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
values[i].arity = 2;
values[i].name = "module_refs";
- values[i].ui[0] = allocated_modules*sizeof(Range);
+ values[i].ui[0] = erts_ranges_sz();
i++;
values[i].arity = 2;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 90a6c0cbee..de7d5599bb 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -164,6 +164,7 @@ type MSG_REF FIXED_SIZE PROCESSES msg_ref
type MSG_ROOTS TEMPORARY PROCESSES msg_roots
type ROOTSET TEMPORARY PROCESSES root_set
type LOADER_TMP TEMPORARY CODE loader_tmp
+type PREPARED_CODE SHORT_LIVED CODE prepared_code
type BIF_TIMER_TABLE LONG_LIVED SYSTEM bif_timer_table
type SL_BIF_TIMER SHORT_LIVED PROCESSES bif_timer_sl
type LL_BIF_TIMER STANDARD PROCESSES bif_timer_ll
@@ -263,6 +264,7 @@ type ZLIB STANDARD SYSTEM zlib
type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map
type AUX_WORK_TMO LONG_LIVED SYSTEM aux_work_timeouts
type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q
+type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q
+if threads_no_smp
# Need thread safe allocs, but std_alloc and fix_alloc are not;
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index cc4f2be8eb..474151d454 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -72,52 +72,29 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
void erts_init_bif_binary(void)
{
- sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export));
- binary_match_trap_export.address = &binary_match_trap_export.code[3];
- binary_match_trap_export.code[0] = am_erlang;
- binary_match_trap_export.code[1] = am_binary_match_trap;
- binary_match_trap_export.code[2] = 3;
- binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif;
- binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap;
-
- sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export));
- binary_matches_trap_export.address = &binary_matches_trap_export.code[3];
- binary_matches_trap_export.code[0] = am_erlang;
- binary_matches_trap_export.code[1] = am_binary_matches_trap;
- binary_matches_trap_export.code[2] = 3;
- binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif;
- binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap;
-
- sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export));
- binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3];
- binary_longest_prefix_trap_export.code[0] = am_erlang;
- binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap;
- binary_longest_prefix_trap_export.code[2] = 3;
- binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif;
- binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap;
-
- sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export));
- binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3];
- binary_longest_suffix_trap_export.code[0] = am_erlang;
- binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap;
- binary_longest_suffix_trap_export.code[2] = 3;
- binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif;
- binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap;
-
- sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export));
- binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3];
- binary_bin_to_list_trap_export.code[0] = am_erlang;
- binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap;
- binary_bin_to_list_trap_export.code[2] = 3;
- binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif;
- binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap;
- sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export));
- binary_copy_trap_export.address = &binary_copy_trap_export.code[3];
- binary_copy_trap_export.code[0] = am_erlang;
- binary_copy_trap_export.code[1] = am_binary_copy_trap;
- binary_copy_trap_export.code[2] = 2;
- binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif;
- binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap;
+ erts_init_trap_export(&binary_match_trap_export,
+ am_erlang, am_binary_match_trap, 3,
+ &binary_match_trap);
+
+ erts_init_trap_export(&binary_matches_trap_export,
+ am_erlang, am_binary_matches_trap, 3,
+ &binary_matches_trap);
+
+ erts_init_trap_export(&binary_longest_prefix_trap_export,
+ am_erlang, am_binary_longest_prefix_trap, 3,
+ &binary_longest_prefix_trap);
+
+ erts_init_trap_export(&binary_longest_suffix_trap_export,
+ am_erlang, am_binary_longest_suffix_trap, 3,
+ &binary_longest_suffix_trap);
+
+ erts_init_trap_export(&binary_bin_to_list_trap_export,
+ am_erlang, am_binary_bin_to_list_trap, 3,
+ &binary_bin_to_list_trap);
+
+ erts_init_trap_export(&binary_copy_trap_export,
+ am_erlang, am_binary_copy_trap, 2,
+ &binary_copy_trap);
max_loop_limit = 0;
return;
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index 06b7ffdf32..ff5ce3cc7a 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -42,16 +42,9 @@ static Export chksum_md5_2_exp;
void erts_init_bif_chksum(void)
{
/* Non visual BIF to trap to. */
- memset(&chksum_md5_2_exp, 0, sizeof(Export));
- chksum_md5_2_exp.address =
- &chksum_md5_2_exp.code[3];
- chksum_md5_2_exp.code[0] = am_erlang;
- chksum_md5_2_exp.code[1] = am_atom_put("md5_trap",8);
- chksum_md5_2_exp.code[2] = 2;
- chksum_md5_2_exp.code[3] =
- (BeamInstr) em_apply_bif;
- chksum_md5_2_exp.code[4] =
- (BeamInstr) &md5_2;
+ erts_init_trap_export(&chksum_md5_2_exp,
+ am_erlang, am_atom_put("md5_trap",8), 2,
+ &md5_2);
}
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index 6b843d2e08..b036c5ef5c 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -71,14 +71,9 @@ void erts_init_bif_re(void)
erts_pcre_stack_free = &erts_erts_pcre_stack_free;
default_table = NULL; /* ISO8859-1 default, forced into pcre */
max_loop_limit = CONTEXT_REDS * LOOP_FACTOR;
-
- sys_memset((void *) &re_exec_trap_export, 0, sizeof(Export));
- re_exec_trap_export.address = &re_exec_trap_export.code[3];
- re_exec_trap_export.code[0] = am_erlang;
- re_exec_trap_export.code[1] = am_re_run_trap;
- re_exec_trap_export.code[2] = 3;
- re_exec_trap_export.code[3] = (BeamInstr) em_apply_bif;
- re_exec_trap_export.code[4] = (BeamInstr) &re_exec_trap;
+
+ erts_init_trap_export(&re_exec_trap_export, am_erlang, am_re_run_trap, 3,
+ &re_exec_trap);
grun_trap_exportp = erts_export_put(am_re,am_grun,3);
urun_trap_exportp = erts_export_put(am_re,am_urun,3);
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index b0a58c80ea..1c99fcdaa6 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -60,8 +60,8 @@ static Eterm trace_info_pid(Process* p, Eterm pid_spec, Eterm key);
static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key);
static Eterm trace_info_on_load(Process* p, Eterm key);
-static int setup_func_trace(Export* ep, void* match_prog);
-static int reset_func_trace(Export* ep);
+static int setup_func_trace(Export* ep, void* match_prog, ErtsCodeIndex);
+static int reset_func_trace(Export* ep, ErtsCodeIndex);
static void reset_bif_trace(int bif_index);
static void setup_bif_trace(int bif_index);
static void set_trace_bif(int bif_index, void* match_prog);
@@ -98,7 +98,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
{
DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
int i;
- int matches = 0;
+ int matches = -1;
int specified = 0;
enum erts_break_op on;
Binary* match_prog_set;
@@ -108,6 +108,9 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
Process *meta_tracer_proc = p;
Eterm meta_tracer_pid = p->id;
+ if (!erts_try_seize_code_write_permission(p)) {
+ ERTS_BIF_YIELD3(bif_export[BIF_trace_pattern_3], p, MFA, Pattern, flaglist);
+ }
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
@@ -241,7 +244,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
erts_default_meta_match_spec = NULL;
erts_default_meta_tracer_pid = NIL;
}
- MatchSetUnref(match_prog_set);
if (erts_default_trace_pattern_flags.breakpoint &&
flags.breakpoint) {
/* Breakpoint trace -> breakpoint trace */
@@ -297,8 +299,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
erts_default_trace_pattern_is_on = !!flags.breakpoint;
}
}
-
- goto done;
+ matches = 0;
} else if (is_tuple(MFA)) {
Eterm *tp = tuple_val(MFA);
if (tp[0] != make_arityval(3)) {
@@ -322,35 +323,29 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
if (is_small(mfa[2])) {
mfa[2] = signed_val(mfa[2]);
}
- } else {
- goto error;
- }
- if (meta_tracer_proc) {
- meta_tracer_proc->trace_flags |= F_TRACER;
- }
-
-
- matches = erts_set_trace_pattern(mfa, specified,
- match_prog_set, match_prog_set,
- on, flags, meta_tracer_pid);
- MatchSetUnref(match_prog_set);
-
- done:
- UnUseTmpHeap(3,p);
- erts_smp_thr_progress_unblock();
- erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+ if (meta_tracer_proc) {
+ meta_tracer_proc->trace_flags |= F_TRACER;
+ }
- return make_small(matches);
+ matches = erts_set_trace_pattern(mfa, specified,
+ match_prog_set, match_prog_set,
+ on, flags, meta_tracer_pid);
+ }
error:
-
MatchSetUnref(match_prog_set);
-
UnUseTmpHeap(3,p);
erts_smp_thr_progress_unblock();
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
- BIF_ERROR(p, BADARG);
+ erts_release_code_write_permission();
+
+ if (matches >= 0) {
+ return make_small(matches);
+ }
+ else {
+ BIF_ERROR(p, BADARG);
+ }
}
void
@@ -372,7 +367,10 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
*meta_tracer_pid = erts_default_meta_tracer_pid;
}
-
+int erts_is_default_trace_enabled(void)
+{
+ return erts_default_trace_pattern_is_on;
+}
Uint
erts_trace_flag2bit(Eterm flag)
@@ -466,6 +464,10 @@ Eterm trace_3(BIF_ALIST_3)
BIF_ERROR(p, BADARG);
}
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ ERTS_BIF_YIELD3(bif_export[BIF_trace_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+ }
+
if (is_nil(tracer) || is_internal_pid(tracer)) {
Process *tracer_proc = erts_pid2proc(p,
ERTS_PROC_LOCK_MAIN,
@@ -730,6 +732,7 @@ Eterm trace_3(BIF_ALIST_3)
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
}
#endif
+ erts_release_code_write_permission();
BIF_RET(make_small(matches));
@@ -745,6 +748,7 @@ Eterm trace_3(BIF_ALIST_3)
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
}
#endif
+ erts_release_code_write_permission();
BIF_ERROR(p, BADARG);
}
@@ -990,7 +994,7 @@ static int function_is_traced(Process *p,
e.code[1] = mfa[1];
e.code[2] = mfa[2];
if ((ep = export_get(&e)) != NULL) {
- if (ep->address == ep->code+3 &&
+ if (ep->addressv[erts_active_code_ix()] == ep->code+3 &&
ep->code[3] != (BeamInstr) em_call_error_handler) {
if (ep->code[3] == (BeamInstr) em_call_traced_function) {
*ms = ep->match_prog_set;
@@ -1333,6 +1337,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
int on, struct trace_pattern_flags flags,
Eterm meta_tracer_pid)
{
+ const ErtsCodeIndex code_ix = erts_active_code_ix();
int matches = 0;
int i;
@@ -1340,8 +1345,8 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
* First work on normal functions (not real BIFs).
*/
- for (i = 0; i < export_list_size(); i++) {
- Export* ep = export_list(i);
+ for (i = 0; i < export_list_size(code_ix); i++) {
+ Export* ep = export_list(i, code_ix);
int j;
if (ExportIsBuiltIn(ep)) {
@@ -1355,11 +1360,11 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
if (j == specified) {
if (on) {
if (! flags.breakpoint)
- matches += setup_func_trace(ep, match_prog_set);
+ matches += setup_func_trace(ep, match_prog_set, code_ix);
else
- reset_func_trace(ep);
+ reset_func_trace(ep, code_ix);
} else if (! flags.breakpoint) {
- matches += reset_func_trace(ep);
+ matches += reset_func_trace(ep, code_ix);
}
}
}
@@ -1522,9 +1527,11 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
*/
static int
-setup_func_trace(Export* ep, void* match_prog)
+setup_func_trace(Export* ep, void* match_prog, ErtsCodeIndex code_ix)
{
- if (ep->address == ep->code+3) {
+ Module* modp;
+
+ if (ep->addressv[code_ix] == ep->code+3) {
if (ep->code[3] == (BeamInstr) em_call_error_handler) {
return 0;
} else if (ep->code[3] == (BeamInstr) em_call_traced_function) {
@@ -1543,15 +1550,19 @@ setup_func_trace(Export* ep, void* match_prog)
/*
* Currently no trace support for native code.
*/
- if (erts_is_native_break(ep->address)) {
+ if (erts_is_native_break(ep->addressv[code_ix])) {
return 0;
}
-
+
ep->code[3] = (BeamInstr) em_call_traced_function;
- ep->code[4] = (BeamInstr) ep->address;
- ep->address = ep->code+3;
+ ep->code[4] = (BeamInstr) ep->addressv[code_ix];
+ ep->addressv[code_ix] = ep->code+3;
ep->match_prog_set = match_prog;
MatchSetRef(ep->match_prog_set);
+
+ modp = erts_get_module(ep->code[0], code_ix);
+ ASSERT(modp);
+ modp->curr.num_traced_exports++;
return 1;
}
@@ -1584,13 +1595,17 @@ static void set_trace_bif(int bif_index, void* match_prog) {
*/
static int
-reset_func_trace(Export* ep)
+reset_func_trace(Export* ep, ErtsCodeIndex code_ix)
{
- if (ep->address == ep->code+3) {
+ if (ep->addressv[code_ix] == ep->code+3) {
if (ep->code[3] == (BeamInstr) em_call_error_handler) {
return 0;
} else if (ep->code[3] == (BeamInstr) em_call_traced_function) {
- ep->address = (Uint *) ep->code[4];
+ Module* modp = erts_get_module(ep->code[0], code_ix);
+ ASSERT(modp);
+ modp->curr.num_traced_exports--;
+
+ ep->addressv[code_ix] = (Uint *) ep->code[4];
MatchSetUnref(ep->match_prog_set);
ep->match_prog_set = NULL;
return 1;
@@ -1605,7 +1620,7 @@ reset_func_trace(Export* ep)
/*
* Currently no trace support for native code.
*/
- if (erts_is_native_break(ep->address)) {
+ if (erts_is_native_break(ep->addressv[code_ix])) {
return 0;
}
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 51bdf53823..5cbfa65378 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -2816,7 +2816,6 @@ void init_db(void)
{
DbTable init_tb;
int i;
- extern BeamInstr* em_apply_bif;
Eterm *hp;
unsigned bits;
size_t size;
@@ -2949,49 +2948,24 @@ void init_db(void)
}
/* Non visual BIF to trap to. */
- memset(&ets_select_delete_continue_exp, 0, sizeof(Export));
- ets_select_delete_continue_exp.address =
- &ets_select_delete_continue_exp.code[3];
- ets_select_delete_continue_exp.code[0] = am_ets;
- ets_select_delete_continue_exp.code[1] = am_atom_put("delete_trap",11);
- ets_select_delete_continue_exp.code[2] = 1;
- ets_select_delete_continue_exp.code[3] =
- (BeamInstr) em_apply_bif;
- ets_select_delete_continue_exp.code[4] =
- (BeamInstr) &ets_select_delete_1;
+ erts_init_trap_export(&ets_select_delete_continue_exp,
+ am_ets, am_atom_put("delete_trap",11), 1,
+ &ets_select_delete_1);
/* Non visual BIF to trap to. */
- memset(&ets_select_count_continue_exp, 0, sizeof(Export));
- ets_select_count_continue_exp.address =
- &ets_select_count_continue_exp.code[3];
- ets_select_count_continue_exp.code[0] = am_ets;
- ets_select_count_continue_exp.code[1] = am_atom_put("count_trap",11);
- ets_select_count_continue_exp.code[2] = 1;
- ets_select_count_continue_exp.code[3] =
- (BeamInstr) em_apply_bif;
- ets_select_count_continue_exp.code[4] =
- (BeamInstr) &ets_select_count_1;
+ erts_init_trap_export(&ets_select_count_continue_exp,
+ am_ets, am_atom_put("count_trap",11), 1,
+ &ets_select_count_1);
/* Non visual BIF to trap to. */
- memset(&ets_select_continue_exp, 0, sizeof(Export));
- ets_select_continue_exp.address =
- &ets_select_continue_exp.code[3];
- ets_select_continue_exp.code[0] = am_ets;
- ets_select_continue_exp.code[1] = am_atom_put("select_trap",11);
- ets_select_continue_exp.code[2] = 1;
- ets_select_continue_exp.code[3] =
- (BeamInstr) em_apply_bif;
- ets_select_continue_exp.code[4] =
- (BeamInstr) &ets_select_trap_1;
+ erts_init_trap_export(&ets_select_continue_exp,
+ am_ets, am_atom_put("select_trap",11), 1,
+ &ets_select_trap_1);
/* Non visual BIF to trap to. */
- memset(&ets_delete_continue_exp, 0, sizeof(Export));
- ets_delete_continue_exp.address = &ets_delete_continue_exp.code[3];
- ets_delete_continue_exp.code[0] = am_ets;
- ets_delete_continue_exp.code[1] = am_atom_put("delete_trap",11);
- ets_delete_continue_exp.code[2] = 1;
- ets_delete_continue_exp.code[3] = (BeamInstr) em_apply_bif;
- ets_delete_continue_exp.code[4] = (BeamInstr) &ets_delete_trap;
+ erts_init_trap_export(&ets_delete_continue_exp,
+ am_ets, am_atom_put("delete_trap",11), 1,
+ &ets_delete_trap);
hp = ms_delete_all_buff;
ms_delete_all = CONS(hp, am_true, NIL);
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 312050b931..faa7f31d99 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -452,16 +452,8 @@ DbTableMethod db_tree =
void db_initialize_tree(void)
{
- memset(&ets_select_reverse_exp, 0, sizeof(Export));
- ets_select_reverse_exp.address =
- &ets_select_reverse_exp.code[3];
- ets_select_reverse_exp.code[0] = am_ets;
- ets_select_reverse_exp.code[1] = am_reverse;
- ets_select_reverse_exp.code[2] = 3;
- ets_select_reverse_exp.code[3] =
- (BeamInstr) em_apply_bif;
- ets_select_reverse_exp.code[4] =
- (BeamInstr) &ets_select_reverse;
+ erts_init_trap_export(&ets_select_reverse_exp, am_ets, am_reverse, 3,
+ &ets_select_reverse);
return;
};
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 2f165afa06..6023fa0448 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -79,8 +79,6 @@ ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index);
ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
byte* uniq, int index, int arity);
-ErlFunEntry* erts_get_fun_entry2(Eterm mod, int old_uniq, int old_index,
- byte* uniq, int index, int arity);
void erts_erase_fun_entry(ErlFunEntry* fe);
#ifndef HYBRID /* FIND ME! */
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 717315d8bd..4db23a0a18 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -264,6 +264,7 @@ erl_init(int ncpu)
erts_init_trace();
erts_init_binary();
erts_init_bits();
+ erts_code_ix_init();
erts_init_fun_table();
init_atom_table();
init_export_table();
@@ -352,7 +353,8 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
Eterm env;
start_mod = am_atom_put(modname, sys_strlen(modname));
- if (erts_find_function(start_mod, am_start, 2) == NULL) {
+ if (erts_find_function(start_mod, am_start, 2,
+ erts_active_code_ix()) == NULL) {
erl_exit(5, "No function %s:start/2\n", modname);
}
@@ -451,7 +453,7 @@ load_preloaded(void)
if ((code = sys_preload_begin(&preload_p[i])) == 0)
erl_exit(1, "Failed to find preloaded code for module %s\n",
name);
- res = erts_load_module(NULL, 0, NIL, &module_name, code, length);
+ res = erts_preload_module(NULL, 0, NIL, &module_name, code, length);
sys_preload_end(&preload_p[i]);
if (res != NIL)
erl_exit(1,"Failed loading preloaded module %s (%T)\n",
@@ -1471,6 +1473,8 @@ erl_start(int argc, char **argv)
init_shared_memory(boot_argc, boot_argv);
load_preloaded();
+ erts_end_staging_code_ix();
+ erts_commit_staging_code_ix();
erts_initialized = 1;
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 09e85893c3..8c7e1f36ee 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -84,6 +84,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "reg_tab", NULL },
{ "migration_info_update", NULL },
{ "proc_main", "pid" },
+ { "old_code", "address" },
#ifdef HIPE
{ "hipe_mfait_lock", NULL },
#endif
@@ -93,6 +94,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "proc_msgq", "pid" },
{ "dist_entry", "address" },
{ "dist_entry_links", "address" },
+ { "code_ix_queue", NULL },
{ "proc_status", "pid" },
{ "proc_tab", NULL },
{ "ports_snapshot", NULL },
@@ -443,7 +445,7 @@ print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix)
"%s'%s:%p%s'%s%s",
prefix,
lname,
- boxed_val(extra),
+ _unchecked_boxed_val(extra),
lock_type(flags),
rw_op_str(flags),
suffix);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 58a09986d2..bdbeb3bcf6 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1513,6 +1513,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
if (len < 0) {
BIF_ERROR(BIF_P, BADARG);
}
+
lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1);
if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) {
@@ -1521,6 +1522,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
}
lib_name[len] = '\0';
+ if (!erts_try_seize_code_write_permission(BIF_P)) {
+ erts_free(ERTS_ALC_T_TMP, lib_name);
+ ERTS_BIF_YIELD2(bif_export[BIF_load_nif_2],
+ BIF_P, BIF_ARG_1, BIF_ARG_2);
+ }
+
/* Block system (is this the right place to do it?) */
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
@@ -1534,11 +1541,11 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
ASSERT(caller != NULL);
mod_atom = caller[0];
ASSERT(is_atom(mod_atom));
- mod=erts_get_module(mod_atom);
+ mod=erts_get_module(mod_atom, erts_active_code_ix());
ASSERT(mod != NULL);
- if (!in_area(caller, mod->code, mod->code_length)) {
- ASSERT(in_area(caller, mod->old_code, mod->old_code_length));
+ if (!in_area(caller, mod->curr.code, mod->curr.code_length)) {
+ ASSERT(in_area(caller, mod->old.code, mod->old.code_length));
ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old "
"module '%T' not allowed", mod_atom);
@@ -1584,7 +1591,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
BeamInstr** code_pp;
ErlNifFunc* f = &entry->funcs[i];
if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom)
- || (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) {
+ || (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) {
ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u",
mod_atom, f->name, f->arity);
}
@@ -1613,18 +1620,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
erts_refc_init(&lib->rt_dtor_cnt, 0);
lib->mod = mod;
env.mod_nif = lib;
- if (mod->nif != NULL) { /* Reload */
+ if (mod->curr.nif != NULL) { /* Reload */
int k;
- lib->priv_data = mod->nif->priv_data;
+ lib->priv_data = mod->curr.nif->priv_data;
- ASSERT(mod->nif->entry != NULL);
+ ASSERT(mod->curr.nif->entry != NULL);
if (entry->reload == NULL) {
ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library.");
goto error;
}
/* Check that no NIF is removed */
- for (k=0; k < mod->nif->entry->num_of_funcs; k++) {
- ErlNifFunc* old_func = &mod->nif->entry->funcs[k];
+ for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) {
+ ErlNifFunc* old_func = &mod->curr.nif->entry->funcs[k];
for (i=0; i < entry->num_of_funcs; i++) {
if (old_func->arity == entry->funcs[i].arity
&& sys_strcmp(old_func->name, entry->funcs[i].name) == 0) {
@@ -1645,24 +1652,24 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful.");
}
else {
- mod->nif->entry = NULL; /* to prevent 'unload' callback */
- erts_unload_nif(mod->nif);
+ mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */
+ erts_unload_nif(mod->curr.nif);
reload_warning = 1;
}
}
else {
lib->priv_data = NULL;
- if (mod->old_nif != NULL) { /* Upgrade */
- void* prev_old_data = mod->old_nif->priv_data;
+ if (mod->old.nif != NULL) { /* Upgrade */
+ void* prev_old_data = mod->old.nif->priv_data;
if (entry->upgrade == NULL) {
ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library.");
goto error;
}
erts_pre_nif(&env, BIF_P, lib);
- veto = entry->upgrade(&env, &lib->priv_data, &mod->old_nif->priv_data, BIF_ARG_2);
+ veto = entry->upgrade(&env, &lib->priv_data, &mod->old.nif->priv_data, BIF_ARG_2);
erts_post_nif(&env);
if (veto) {
- mod->old_nif->priv_data = prev_old_data;
+ mod->old.nif->priv_data = prev_old_data;
ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful.");
}
/*else if (mod->old_nif->priv_data != prev_old_data) {
@@ -1682,12 +1689,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
/*
** Everything ok, patch the beam code with op_call_nif
*/
- mod->nif = lib;
+ mod->curr.nif = lib;
for (i=0; i < entry->num_of_funcs; i++)
{
BeamInstr* code_ptr;
erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom);
- code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity);
+ code_ptr = *get_func_pp(mod->curr.code, f_atom, entry->funcs[i].arity);
if (code_ptr[1] == 0) {
code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif);
@@ -1715,6 +1722,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
erts_smp_thr_progress_unblock();
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+ erts_release_code_write_permission();
erts_free(ERTS_ALC_T_TMP, lib_name);
if (reload_warning) {
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 138acfeb2c..6c136571d5 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -366,6 +366,9 @@ dbg_chk_aux_work_val(erts_aint32_t value)
#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
#endif
+#ifdef ERTS_SMP
+ valid |= ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION;
+#endif
if (~valid & value)
erl_exit(ERTS_ABORT_EXIT,
@@ -861,7 +864,7 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi,
}
}
-#if 0 /* Currently not used */
+#ifdef ERTS_SMP
static ERTS_INLINE void
set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
@@ -882,7 +885,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
}
}
-#endif
+#endif /* ERTS_SMP */
static ERTS_INLINE erts_aint32_t
set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
@@ -1145,7 +1148,45 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
}
}
+#endif /* ERTS_USE_ASYNC_READY_Q */
+
+#ifdef ERTS_SMP
+void
+erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later)
+{
+ ErtsAuxWorkData* awdp = &p->scheduler_data->aux_work_data;
+ ASSERT(awdp->code_ix_activation.code_stager == NULL);
+ awdp->code_ix_activation.code_stager = p;
+ awdp->code_ix_activation.thr_prgr = later;
+ erts_smp_proc_inc_refc(p);
+ set_aux_work_flags_wakeup_relb(p->scheduler_data->ssi,
+ ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION);
+}
+
+static erts_aint32_t
+handle_code_ix_activation(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+ Process* p;
+ if (!erts_thr_progress_has_reached(awdp->code_ix_activation.thr_prgr)) {
+ return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION;
+ }
+ unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION);
+ p = awdp->code_ix_activation.code_stager;
+ ASSERT(p);
+#ifdef DEBUG
+ awdp->code_ix_activation.code_stager = NULL;
#endif
+ erts_commit_staging_code_ix();
+ erts_release_code_write_permission();
+ erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
+ if (!ERTS_PROC_IS_EXITING(p)) {
+ erts_resume(p, ERTS_PROC_LOCK_STATUS);
+ }
+ erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+ erts_smp_proc_dec_refc(p);
+ return aux_work & ~ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION;
+}
+#endif /* ERTS_SMP */
static ERTS_INLINE erts_aint32_t
handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
@@ -1451,6 +1492,10 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work)
handle_mseg_cache_check);
#endif
+#ifdef ERTS_SMP
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION,
+ handle_code_ix_activation);
+#endif
ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
return aux_work;
@@ -9599,13 +9644,8 @@ init_processes_bif(void)
+ 1);
/* processes_trap/2 is a hidden BIF that the processes/0 BIF traps to. */
- sys_memset((void *) &processes_trap_export, 0, sizeof(Export));
- processes_trap_export.address = &processes_trap_export.code[3];
- processes_trap_export.code[0] = am_erlang;
- processes_trap_export.code[1] = am_processes_trap;
- processes_trap_export.code[2] = 2;
- processes_trap_export.code[3] = (BeamInstr) em_apply_bif;
- processes_trap_export.code[4] = (BeamInstr) &processes_trap;
+ erts_init_trap_export(&processes_trap_export, am_erlang, am_processes_trap, 2,
+ &processes_trap);
#if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
erts_get_emu_time(&debug_tv_start);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c23810f15a..3f19d92fcd 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -264,6 +264,7 @@ typedef enum {
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8)
#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10)
+#define ERTS_SSI_AUX_WORK_CODE_IX_ACTIVATION (((erts_aint32_t) 1) << 11)
typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
@@ -429,6 +430,12 @@ typedef struct {
void *queue;
} async_ready;
#endif
+#ifdef ERTS_SMP
+ struct {
+ Process* code_stager;
+ ErtsThrPrgrVal thr_prgr;
+ } code_ix_activation;
+#endif
} ErtsAuxWorkData;
struct ErtsSchedulerData_ {
@@ -1110,6 +1117,9 @@ void erts_smp_notify_check_children_needed(void);
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
+#ifdef ERTS_SMP
+void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
+#endif
void erts_schedule_misc_aux_work(int sched_id,
void (*func)(void *),
void *arg);
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 6d5eae73b0..049226ab7d 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -77,66 +77,25 @@ void erts_init_unicode(void)
{
max_loop_limit = CONTEXT_REDS * LOOP_FACTOR;
/* Non visual BIFs to trap to. */
- memset(&characters_to_utf8_trap_exp, 0, sizeof(Export));
- characters_to_utf8_trap_exp.address =
- &characters_to_utf8_trap_exp.code[3];
- characters_to_utf8_trap_exp.code[0] = am_erlang;
- characters_to_utf8_trap_exp.code[1] =
- am_atom_put("characters_to_utf8_trap",23);
- characters_to_utf8_trap_exp.code[2] = 3;
- characters_to_utf8_trap_exp.code[3] =
- (BeamInstr) em_apply_bif;
- characters_to_utf8_trap_exp.code[4] =
- (BeamInstr) &characters_to_utf8_trap;
-
- memset(&characters_to_list_trap_1_exp, 0, sizeof(Export));
- characters_to_list_trap_1_exp.address =
- &characters_to_list_trap_1_exp.code[3];
- characters_to_list_trap_1_exp.code[0] = am_erlang;
- characters_to_list_trap_1_exp.code[1] =
- am_atom_put("characters_to_list_trap_1",25);
- characters_to_list_trap_1_exp.code[2] = 3;
- characters_to_list_trap_1_exp.code[3] =
- (BeamInstr) em_apply_bif;
- characters_to_list_trap_1_exp.code[4] =
- (BeamInstr) &characters_to_list_trap_1;
-
- memset(&characters_to_list_trap_2_exp, 0, sizeof(Export));
- characters_to_list_trap_2_exp.address =
- &characters_to_list_trap_2_exp.code[3];
- characters_to_list_trap_2_exp.code[0] = am_erlang;
- characters_to_list_trap_2_exp.code[1] =
- am_atom_put("characters_to_list_trap_2",25);
- characters_to_list_trap_2_exp.code[2] = 3;
- characters_to_list_trap_2_exp.code[3] =
- (BeamInstr) em_apply_bif;
- characters_to_list_trap_2_exp.code[4] =
- (BeamInstr) &characters_to_list_trap_2;
-
-
- memset(&characters_to_list_trap_3_exp, 0, sizeof(Export));
- characters_to_list_trap_3_exp.address =
- &characters_to_list_trap_3_exp.code[3];
- characters_to_list_trap_3_exp.code[0] = am_erlang;
- characters_to_list_trap_3_exp.code[1] =
- am_atom_put("characters_to_list_trap_3",25);
- characters_to_list_trap_3_exp.code[2] = 3;
- characters_to_list_trap_3_exp.code[3] =
- (BeamInstr) em_apply_bif;
- characters_to_list_trap_3_exp.code[4] =
- (BeamInstr) &characters_to_list_trap_3;
-
- memset(&characters_to_list_trap_4_exp, 0, sizeof(Export));
- characters_to_list_trap_4_exp.address =
- &characters_to_list_trap_4_exp.code[3];
- characters_to_list_trap_4_exp.code[0] = am_erlang;
- characters_to_list_trap_4_exp.code[1] =
- am_atom_put("characters_to_list_trap_4",25);
- characters_to_list_trap_4_exp.code[2] = 1;
- characters_to_list_trap_4_exp.code[3] =
- (BeamInstr) em_apply_bif;
- characters_to_list_trap_4_exp.code[4] =
- (BeamInstr) &characters_to_list_trap_4;
+ erts_init_trap_export(&characters_to_utf8_trap_exp,
+ am_erlang, am_atom_put("characters_to_utf8_trap",23), 3,
+ &characters_to_utf8_trap);
+
+ erts_init_trap_export(&characters_to_list_trap_1_exp,
+ am_erlang, am_atom_put("characters_to_list_trap_1",25), 3,
+ &characters_to_list_trap_1);
+
+ erts_init_trap_export(&characters_to_list_trap_2_exp,
+ am_erlang, am_atom_put("characters_to_list_trap_2",25), 3,
+ &characters_to_list_trap_2);
+
+ erts_init_trap_export(&characters_to_list_trap_3_exp,
+ am_erlang, am_atom_put("characters_to_list_trap_3",25), 3,
+ &characters_to_list_trap_3);
+
+ erts_init_trap_export(&characters_to_list_trap_4_exp,
+ am_erlang, am_atom_put("characters_to_list_trap_4",25), 1,
+ &characters_to_list_trap_4);
c_to_b_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_binary_int,2);
c_to_l_int_trap_exportp = erts_export_put(am_unicode,am_characters_to_list_int,2);
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index fb0ee99119..229641cb32 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -32,98 +32,163 @@
#define EXPORT_HASH(m,f,a) ((m)*(f)+(a))
-static IndexTable export_table; /* Not locked. */
-static Hash secondary_export_table; /* Locked. */
+#ifdef DEBUG
+# define IF_DEBUG(x) x
+#else
+# define IF_DEBUG(x)
+#endif
-#include "erl_smp.h"
+static IndexTable export_tables[ERTS_NUM_CODE_IX]; /* Active not locked */
-static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. */
+static erts_smp_atomic_t total_entries_bytes;
-#define export_read_lock() erts_smp_rwmtx_rlock(&export_table_lock)
-#define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock)
-#define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock)
-#define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock)
+#include "erl_smp.h"
+
+/* This lock protects the staging export table from concurrent access
+ * AND it protects the staging table from becoming active.
+ */
+erts_smp_mtx_t export_staging_lock;
extern BeamInstr* em_call_error_handler;
extern BeamInstr* em_call_traced_function;
+struct export_entry
+{
+ IndexSlot slot; /* MUST BE LOCATED AT TOP OF STRUCT!!! */
+ Export* ep;
+};
+
+/* Helper struct that brings things together in one allocation
+*/
+struct export_blob
+{
+ Export exp;
+ struct export_entry entryv[ERTS_NUM_CODE_IX];
+ /* Note that entryv is not indexed by "code_ix".
+ */
+};
+
+/* Helper struct only used as template
+*/
+struct export_templ
+{
+ struct export_entry entry;
+ Export exp;
+};
+
+static struct export_blob* entry_to_blob(struct export_entry* ee)
+{
+ return (struct export_blob*)
+ ((char*)ee->ep - offsetof(struct export_blob,exp));
+}
+
void
export_info(int to, void *to_arg)
{
#ifdef ERTS_SMP
int lock = !ERTS_IS_CRASH_DUMPING;
if (lock)
- export_read_lock();
+ export_staging_lock();
#endif
- index_info(to, to_arg, &export_table);
- hash_info(to, to_arg, &secondary_export_table);
+ index_info(to, to_arg, &export_tables[erts_active_code_ix()]);
+ hash_info(to, to_arg, &export_tables[erts_staging_code_ix()].htable);
#ifdef ERTS_SMP
if (lock)
- export_read_unlock();
+ export_staging_unlock();
#endif
}
static HashValue
-export_hash(Export* x)
+export_hash(struct export_entry* ee)
{
+ Export* x = ee->ep;
return EXPORT_HASH(x->code[0], x->code[1], x->code[2]);
}
static int
-export_cmp(Export* tmpl, Export* obj)
+export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e)
{
+ Export* tmpl = tmpl_e->ep;
+ Export* obj = obj_e->ep;
return !(tmpl->code[0] == obj->code[0] &&
tmpl->code[1] == obj->code[1] &&
tmpl->code[2] == obj->code[2]);
}
-static Export*
-export_alloc(Export* tmpl)
+static struct export_entry*
+export_alloc(struct export_entry* tmpl_e)
{
- Export* obj = (Export*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(Export));
-
- obj->fake_op_func_info_for_hipe[0] = 0;
- obj->fake_op_func_info_for_hipe[1] = 0;
- obj->code[0] = tmpl->code[0];
- obj->code[1] = tmpl->code[1];
- obj->code[2] = tmpl->code[2];
- obj->slot.index = -1;
- obj->address = obj->code+3;
- obj->code[3] = (BeamInstr) em_call_error_handler;
- obj->code[4] = 0;
- obj->match_prog_set = NULL;
- return obj;
+ struct export_blob* blob;
+ unsigned ix;
+
+ if (tmpl_e->slot.index == -1) { /* Template, allocate blob */
+ Export* tmpl = tmpl_e->ep;
+ Export* obj;
+
+ blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob));
+ erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob));
+ obj = &blob->exp;
+ obj->fake_op_func_info_for_hipe[0] = 0;
+ obj->fake_op_func_info_for_hipe[1] = 0;
+ obj->code[0] = tmpl->code[0];
+ obj->code[1] = tmpl->code[1];
+ obj->code[2] = tmpl->code[2];
+ obj->code[3] = (BeamInstr) em_call_error_handler;
+ obj->code[4] = 0;
+ obj->match_prog_set = NULL;
+
+ for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) {
+ obj->addressv[ix] = obj->code+3;
+
+ blob->entryv[ix].slot.index = -1;
+ blob->entryv[ix].ep = &blob->exp;
+ }
+ ix = 0;
+ }
+ else { /* Existing entry in another table, use free entry in blob */
+ blob = entry_to_blob(tmpl_e);
+ for (ix = 0; blob->entryv[ix].slot.index >= 0; ix++) {
+ ASSERT(ix < ERTS_NUM_CODE_IX);
+ }
+ }
+ return &blob->entryv[ix];
}
-
-static void
-export_free(Export* obj)
+static void
+export_free(struct export_entry* obj)
{
- erts_free(ERTS_ALC_T_EXPORT, (void*) obj);
+ struct export_blob* blob = entry_to_blob(obj);
+ int i;
+ obj->slot.index = -1;
+ for (i=0; i < ERTS_NUM_CODE_IX; i++) {
+ if (blob->entryv[i].slot.index >= 0) {
+ return;
+ }
+ }
+ erts_free(ERTS_ALC_T_EXPORT, blob);
+ erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob));
}
-
void
init_export_table(void)
{
HashFunctions f;
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
- rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
- rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+ int i;
- erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab");
+ erts_smp_mtx_init(&export_staging_lock, "export_tab");
+ erts_smp_atomic_init_nob(&total_entries_bytes, 0);
f.hash = (H_FUN) export_hash;
f.cmp = (HCMP_FUN) export_cmp;
f.alloc = (HALLOC_FUN) export_alloc;
f.free = (HFREE_FUN) export_free;
- erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_table, "export_list",
- EXPORT_INITIAL_SIZE, EXPORT_LIMIT, f);
- hash_init(ERTS_ALC_T_EXPORT_TABLE, &secondary_export_table,
- "secondary_export_table", 50, f);
+ for (i=0; i<ERTS_NUM_CODE_IX; i++) {
+ erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_tables[i], "export_list",
+ EXPORT_INITIAL_SIZE, EXPORT_LIMIT, f);
+ }
}
/*
@@ -138,29 +203,42 @@ init_export_table(void)
* called through such an export entry.
* 3) This function is suitable for the implementation of erlang:apply/3.
*/
+extern Export* /* inline-helper */
+erts_find_export_entry(Eterm m, Eterm f, unsigned int a,ErtsCodeIndex code_ix);
Export*
-erts_find_export_entry(Eterm m, Eterm f, unsigned int a)
+erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix)
{
HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a);
int ix;
HashBucket* b;
- ix = hval % export_table.htable.size;
- b = export_table.htable.bucket[ix];
+ ix = hval % export_tables[code_ix].htable.size;
+ b = export_tables[code_ix].htable.bucket[ix];
/*
* Note: We have inlined the code from hash.c for speed.
*/
while (b != (HashBucket*) 0) {
- Export* ep = (Export *) b;
+ Export* ep = ((struct export_entry*) b)->ep;
if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) {
- break;
+ return ep;
}
b = b->next;
}
- return (Export*)b;
+ return NULL;
+}
+
+static struct export_entry* init_template(struct export_templ* templ,
+ Eterm m, Eterm f, unsigned a)
+{
+ templ->entry.ep = &templ->exp;
+ templ->entry.slot.index = -1;
+ templ->exp.code[0] = m;
+ templ->exp.code[1] = f;
+ templ->exp.code[2] = a;
+ return &templ->entry;
}
@@ -176,47 +254,41 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a)
*/
Export*
-erts_find_function(Eterm m, Eterm f, unsigned int a)
+erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix)
{
- Export e;
- Export* ep;
-
- e.code[0] = m;
- e.code[1] = f;
- e.code[2] = a;
+ struct export_templ templ;
+ struct export_entry* ee;
- ep = hash_get(&export_table.htable, (void*) &e);
- if (ep != NULL && ep->address == ep->code+3 &&
- ep->code[3] != (BeamInstr) em_call_traced_function) {
- ep = NULL;
+ ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a));
+ if (ee == NULL || (ee->ep->addressv[code_ix] == ee->ep->code+3 &&
+ ee->ep->code[3] != (BeamInstr) em_call_traced_function)) {
+ return NULL;
}
- return ep;
+ return ee->ep;
}
/*
* Returns a pointer to an existing export entry for a MFA,
* or creates a new one and returns the pointer.
*
- * This function provides unlocked write access to the main export
- * table. It should only be used during start up or when
- * all other threads are blocked.
+ * This function acts on the staging export table. It should only be used
+ * to load new code.
*/
Export*
erts_export_put(Eterm mod, Eterm func, unsigned int arity)
{
- Export e;
- int ix;
+ ErtsCodeIndex code_ix = erts_staging_code_ix();
+ struct export_templ templ;
+ struct export_entry* ee;
- ERTS_SMP_LC_ASSERT(erts_initialized == 0
- || erts_smp_thr_progress_is_blocking());
ASSERT(is_atom(mod));
ASSERT(is_atom(func));
- e.code[0] = mod;
- e.code[1] = func;
- e.code[2] = arity;
- ix = index_put(&export_table, (void*) &e);
- return (Export*) erts_index_lookup(&export_table, ix);
+ export_staging_lock();
+ ee = (struct export_entry*) index_put_entry(&export_tables[code_ix],
+ init_template(&templ, mod, func, arity));
+ export_staging_unlock();
+ return ee->ep;
}
/*
@@ -224,77 +296,117 @@ erts_export_put(Eterm mod, Eterm func, unsigned int arity)
* export entry (making a call through it will cause the error_handler to
* be called).
*
- * Stub export entries will be placed in the secondary export table.
- * erts_export_consolidate() will move all stub export entries into the
- * main export table (will be done the next time code is loaded).
+ * Stub export entries will be placed in the loader export table.
*/
Export*
erts_export_get_or_make_stub(Eterm mod, Eterm func, unsigned int arity)
{
- Export e;
+ ErtsCodeIndex code_ix;
Export* ep;
+ IF_DEBUG(int retrying = 0;)
ASSERT(is_atom(mod));
ASSERT(is_atom(func));
-
- e.code[0] = mod;
- e.code[1] = func;
- e.code[2] = arity;
- ep = erts_find_export_entry(mod, func, arity);
- if (ep == 0) {
- /*
- * The code is not loaded (yet). Put the export in the secondary
- * export table, to avoid having to lock the main export table.
- */
- export_write_lock();
- ep = (Export *) hash_put(&secondary_export_table, (void*) &e);
- export_write_unlock();
- }
+
+ do {
+ code_ix = erts_active_code_ix();
+ ep = erts_find_export_entry(mod, func, arity, code_ix);
+ if (ep == 0) {
+ /*
+ * The code is not loaded (yet). Put the export in the staging
+ * export table, to avoid having to lock the active export table.
+ */
+ export_staging_lock();
+ if (erts_active_code_ix() == code_ix) {
+ struct export_templ templ;
+ struct export_entry* entry;
+
+ IndexTable* tab = &export_tables[erts_staging_code_ix()];
+ init_template(&templ, mod, func, arity);
+ entry = (struct export_entry *) index_put_entry(tab, &templ.entry);
+ ep = entry->ep;
+ ASSERT(ep);
+ }
+ else { /* race */
+ ASSERT(!retrying);
+ IF_DEBUG(retrying = 1);
+ }
+ export_staging_unlock();
+ }
+ } while (!ep);
return ep;
}
-/*
- * To be called before loading code (with other threads blocked).
- * This function will move all export entries from the secondary
- * export table into the primary.
- */
-void
-erts_export_consolidate(void)
+Export *export_list(int i, ErtsCodeIndex code_ix)
{
-#ifdef DEBUG
- HashInfo hi;
-#endif
-
- ERTS_SMP_LC_ASSERT(erts_initialized == 0
- || erts_smp_thr_progress_is_blocking());
-
- export_write_lock();
- erts_index_merge(&secondary_export_table, &export_table);
- erts_hash_merge(&secondary_export_table, &export_table.htable);
- export_write_unlock();
-#ifdef DEBUG
- hash_get_info(&hi, &export_table.htable);
- ASSERT(export_table.entries == hi.objs);
-#endif
+ return ((struct export_entry*) erts_index_lookup(&export_tables[code_ix], i))->ep;
}
-Export *export_list(int i)
+int export_list_size(ErtsCodeIndex code_ix)
{
- return (Export*) erts_index_lookup(&export_table, i);
+ return export_tables[code_ix].entries;
}
-int export_list_size(void)
+int export_table_sz(void)
{
- return export_table.entries;
+ int i, bytes = 0;
+
+ export_staging_lock();
+ for (i=0; i<ERTS_NUM_CODE_IX; i++) {
+ bytes += index_table_sz(&export_tables[i]);
+ }
+ export_staging_unlock();
+ return bytes;
+}
+int export_entries_sz(void)
+{
+ return erts_smp_atomic_read_nob(&total_entries_bytes);
+}
+Export *export_get(Export *e)
+{
+ struct export_entry ee;
+ struct export_entry* entry;
+
+ ee.ep = e;
+ entry = (struct export_entry*)hash_get(&export_tables[erts_active_code_ix()].htable, &ee);
+ return entry ? entry->ep : NULL;
}
-int export_table_sz(void)
+IF_DEBUG(static ErtsCodeIndex debug_start_load_ix = 0;)
+
+
+void export_start_staging(void)
{
- return index_table_sz(&export_table);
+ ErtsCodeIndex dst_ix = erts_staging_code_ix();
+ ErtsCodeIndex src_ix = erts_active_code_ix();
+ IndexTable* dst = &export_tables[dst_ix];
+ IndexTable* src = &export_tables[src_ix];
+ struct export_entry* src_entry;
+ struct export_entry* dst_entry;
+ int i;
+
+ ASSERT(dst_ix != src_ix);
+ ASSERT(debug_start_load_ix == -1);
+
+ export_staging_lock();
+ /*
+ * Insert all entries in src into dst table
+ */
+ for (i = 0; i < src->entries; i++) {
+ src_entry = (struct export_entry*) erts_index_lookup(src, i);
+ src_entry->ep->addressv[dst_ix] = src_entry->ep->addressv[src_ix];
+ dst_entry = (struct export_entry*) index_put_entry(dst, src_entry);
+ ASSERT(entry_to_blob(src_entry) == entry_to_blob(dst_entry));
+ }
+ export_staging_unlock();
+
+ IF_DEBUG(debug_start_load_ix = dst_ix);
}
-Export *export_get(Export *e)
+void export_end_staging(int commit)
{
- return hash_get(&export_table.htable, e);
+ ASSERT(debug_start_load_ix == erts_staging_code_ix());
+ IF_DEBUG(debug_start_load_ix = -1);
}
+
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index c604fdf7c3..ec9fcb26f2 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -28,13 +28,15 @@
#include "index.h"
#endif
+#include "code_ix.h"
+
/*
** Export entry
*/
+
typedef struct export
{
- IndexSlot slot; /* MUST BE LOCATED AT TOP OF STRUCT!!! */
- void* address; /* Pointer to code for function. */
+ void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */
struct binary* match_prog_set; /* Match program for tracing. */
BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
@@ -59,21 +61,38 @@ typedef struct export
void init_export_table(void);
void export_info(int, void *);
-Export* erts_find_export_entry(Eterm m, Eterm f, unsigned int a);
+ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a);
Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity);
-
Export* erts_export_get_or_make_stub(Eterm, Eterm, unsigned);
-void erts_export_consolidate(void);
-Export *export_list(int);
-int export_list_size(void);
+Export *export_list(int,ErtsCodeIndex);
+int export_list_size(ErtsCodeIndex);
int export_table_sz(void);
+int export_entries_sz(void);
Export *export_get(Export*);
+void export_start_staging(void);
+void export_end_staging(int commit);
+
+extern erts_smp_mtx_t export_staging_lock;
+#define export_staging_lock() erts_smp_mtx_lock(&export_staging_lock)
+#define export_staging_unlock() erts_smp_mtx_unlock(&export_staging_lock)
#include "beam_load.h" /* For em_* extern declarations */
#define ExportIsBuiltIn(EntryPtr) \
-(((EntryPtr)->address == (EntryPtr)->code + 3) && \
+(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \
((EntryPtr)->code[3] == (BeamInstr) em_apply_bif))
-#endif
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Export*
+erts_active_export_entry(Eterm m, Eterm f, unsigned int a)
+{
+ extern Export* erts_find_export_entry(Eterm m, Eterm f, unsigned a, ErtsCodeIndex);
+ return erts_find_export_entry(m, f, a, erts_active_code_ix());
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* __EXPORT_H__ */
+
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9d52ed4e98..25f593640c 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2562,7 +2562,7 @@ dec_term_atom_common:
goto error;
}
if (edep && (edep->flags & ERTS_DIST_EXT_BTT_SAFE)) {
- if (!erts_find_export_entry(mod, name, arity))
+ if (!erts_active_export_entry(mod, name, arity))
goto error;
}
*objp = make_export(hp);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index f1335f600d..6eac0db363 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -28,6 +28,7 @@
#include "hash.h"
#include "index.h"
#include "atom.h"
+#include "code_ix.h"
#include "export.h"
#include "module.h"
#include "register.h"
@@ -847,6 +848,8 @@ void erts_queue_monitor_message(Process *,
Eterm,
Eterm,
Eterm);
+void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
+ Eterm (*bif)(Process*,Eterm*));
void erts_init_bif(void);
Eterm erl_send(Process *p, Eterm to, Eterm msg);
@@ -869,24 +872,33 @@ typedef struct {
Eterm* fname_ptr; /* Pointer to fname table */
} FunctionInfo;
-struct LoaderState* erts_alloc_loader_state(void);
-Eterm erts_prepare_loading(struct LoaderState*, Process *c_p,
+Binary* erts_alloc_loader_state(void);
+Eterm erts_module_for_prepared_code(Binary* magic);
+Eterm erts_prepare_loading(Binary* loader_state, Process *c_p,
Eterm group_leader, Eterm* modp,
byte* code, Uint size);
-Eterm erts_finish_loading(struct LoaderState* stp, Process* c_p,
+Eterm erts_finish_loading(Binary* loader_state, Process* c_p,
ErtsProcLocks c_p_locks, Eterm* modp);
-Eterm erts_load_module(Process *c_p, ErtsProcLocks c_p_locks,
- Eterm group_leader, Eterm* mod, byte* code, Uint size);
+Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks,
+ Eterm group_leader, Eterm* mod, byte* code, Uint size);
void init_load(void);
BeamInstr* find_function_from_pc(BeamInstr* pc);
Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp,
Eterm args, Eterm* mfa_p);
-void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info);
void erts_set_current_function(FunctionInfo* fi, BeamInstr* current);
Eterm erts_module_info_0(Process* p, Eterm module);
Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
+/* beam_ranges.c */
+void erts_init_ranges(void);
+void erts_start_staging_ranges(void);
+void erts_end_staging_ranges(int commit);
+void erts_update_ranges(BeamInstr* code, Uint size);
+void erts_remove_from_ranges(BeamInstr* code);
+UWord erts_ranges_sz(void);
+void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info);
+
/* break.c */
void init_break_handler(void);
void erts_set_ignore_break(void);
@@ -1483,7 +1495,7 @@ void erts_cleanup_offheap(ErlOffHeap *offheap);
Uint erts_fit_in_bits(Uint);
int list_length(Eterm);
-Export* erts_find_function(Eterm, Eterm, unsigned int);
+Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex);
int erts_is_builtin(Eterm, Eterm, int);
Uint32 make_broken_hash(Eterm);
Uint32 block_hash(byte *, unsigned, Uint32);
@@ -1761,6 +1773,7 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
Binary **meta_match_spec,
struct trace_pattern_flags *trace_pattern_flags,
Eterm *meta_tracer_pid);
+int erts_is_default_trace_enabled(void);
void erts_bif_trace_init(void);
/*
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index a4a3007f93..25d5cce0f3 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -68,14 +68,14 @@ erts_index_init(ErtsAlcType_t type, IndexTable* t, char* name,
return t;
}
-int
-index_put(IndexTable* t, void* tmpl)
+IndexSlot*
+index_put_entry(IndexTable* t, void* tmpl)
{
int ix;
IndexSlot* p = (IndexSlot*) hash_put(&t->htable, tmpl);
if (p->index >= 0) {
- return p->index;
+ return p;
}
ix = t->entries;
@@ -92,7 +92,7 @@ index_put(IndexTable* t, void* tmpl)
t->entries++;
p->index = ix;
t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p;
- return ix;
+ return p;
}
int index_get(IndexTable* t, void* tmpl)
@@ -135,3 +135,18 @@ void erts_index_merge(Hash* src, IndexTable* dst)
}
}
}
+
+void index_erase_latest_from(IndexTable* t, Uint from_ix)
+{
+ if(from_ix < (Uint)t->entries) {
+ int ix;
+ for (ix = from_ix; ix < t->entries; ix++) {
+ IndexSlot* obj = t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK];
+ hash_erase(&t->htable, obj);
+ }
+ t->entries = from_ix;
+ }
+ else {
+ ASSERT(from_ix == t->entries);
+ }
+}
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 4eb9b1f992..3afe48d007 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -55,12 +55,24 @@ void index_info(int, void *, IndexTable*);
int index_table_sz(IndexTable *);
int index_get(IndexTable*, void*);
-int index_put(IndexTable*, void*);
+
+IndexSlot* index_put_entry(IndexTable*, void*);
void erts_index_merge(Hash*, IndexTable*);
+/* Erase all entries with index 'ix' and higher
+*/
+void index_erase_latest_from(IndexTable*, Uint ix);
+
+ERTS_GLB_INLINE int index_put(IndexTable*, void*);
ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int index_put(IndexTable* t, void* tmpl)
+{
+ return index_put_entry(t, tmpl)->index;
+}
+
ERTS_GLB_INLINE IndexSlot*
erts_index_lookup(IndexTable* t, Uint ix)
{
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index b93b1ad09a..1ef71cda79 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -26,21 +26,32 @@
#include "global.h"
#include "module.h"
+#ifdef DEBUG
+# define IF_DEBUG(x) x
+#else
+# define IF_DEBUG(x)
+#endif
+
#define MODULE_SIZE 50
#define MODULE_LIMIT (64*1024)
-static IndexTable module_table;
+static IndexTable module_tables[ERTS_NUM_CODE_IX];
-/*
- * SMP note: We don't need to look accesses to the module table because
- * there is one only scheduler thread when we update it.
+erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX];
+
+static erts_smp_atomic_t tot_module_bytes;
+
+/* SMP note: Active module table lookup and current module instance can be
+ * read without any locks. Old module instances are protected by
+ * "the_old_code_rwlocks" as purging is done on active module table.
+ * Staging table is protected by the "code_ix lock".
*/
#include "erl_smp.h"
void module_info(int to, void *to_arg)
{
- index_info(to, to_arg, &module_table);
+ index_info(to, to_arg, &module_tables[erts_active_code_ix()]);
}
@@ -59,45 +70,67 @@ static int module_cmp(Module* tmpl, Module* obj)
static Module* module_alloc(Module* tmpl)
{
Module* obj = (Module*) erts_alloc(ERTS_ALC_T_MODULE, sizeof(Module));
+ erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module));
obj->module = tmpl->module;
- obj->code = 0;
- obj->old_code = 0;
- obj->code_length = 0;
- obj->old_code_length = 0;
+ obj->curr.code = 0;
+ obj->old.code = 0;
+ obj->curr.code_length = 0;
+ obj->old.code_length = 0;
obj->slot.index = -1;
- obj->nif = NULL;
- obj->old_nif = NULL;
+ obj->curr.nif = NULL;
+ obj->old.nif = NULL;
+ obj->curr.num_breakpoints = 0;
+ obj->old.num_breakpoints = 0;
+ obj->curr.num_traced_exports = 0;
+ obj->old.num_traced_exports = 0;
return obj;
}
+static void module_free(Module* mod)
+{
+ erts_free(ERTS_ALC_T_MODULE, mod);
+ erts_smp_atomic_add_nob(&tot_module_bytes, -sizeof(Module));
+}
void init_module_table(void)
{
HashFunctions f;
+ int i;
f.hash = (H_FUN) module_hash;
f.cmp = (HCMP_FUN) module_cmp;
f.alloc = (HALLOC_FUN) module_alloc;
- f.free = 0;
+ f.free = (HFREE_FUN) module_free;
+
+ for (i = 0; i < ERTS_NUM_CODE_IX; i++) {
+ erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code",
+ MODULE_SIZE, MODULE_LIMIT, f);
+ }
- erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_table, "module_code",
- MODULE_SIZE, MODULE_LIMIT, f);
+ for (i=0; i<ERTS_NUM_CODE_IX; i++) {
+ erts_smp_rwmtx_init_x(&the_old_code_rwlocks[i], "old_code", make_small(i));
+ }
+ erts_smp_atomic_init_nob(&tot_module_bytes, 0);
}
Module*
-erts_get_module(Eterm mod)
+erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
{
Module e;
int index;
+ IndexTable* mod_tab;
ASSERT(is_atom(mod));
+
+ mod_tab = &module_tables[code_ix];
+
e.module = atom_val(mod);
- index = index_get(&module_table, (void*) &e);
+ index = index_get(mod_tab, (void*) &e);
if (index == -1) {
return NULL;
} else {
- return (Module*) erts_index_lookup(&module_table, index);
+ return (Module*) erts_index_lookup(mod_tab, index);
}
}
@@ -105,27 +138,101 @@ Module*
erts_put_module(Eterm mod)
{
Module e;
- int index;
+ IndexTable* mod_tab;
+ int oldsz, newsz;
+ Module* res;
ASSERT(is_atom(mod));
ERTS_SMP_LC_ASSERT(erts_initialized == 0
- || erts_smp_thr_progress_is_blocking());
+ || erts_is_code_ix_locked());
+
+ mod_tab = &module_tables[erts_staging_code_ix()];
e.module = atom_val(mod);
- index = index_put(&module_table, (void*) &e);
- return (Module*) erts_index_lookup(&module_table, index);
+ oldsz = index_table_sz(mod_tab);
+ res = (Module*) index_put_entry(mod_tab, (void*) &e);
+ newsz = index_table_sz(mod_tab);
+ erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
+ return res;
}
-Module *module_code(int i)
+Module *module_code(int i, ErtsCodeIndex code_ix)
{
- return (Module*) erts_index_lookup(&module_table, i);
+ return (Module*) erts_index_lookup(&module_tables[code_ix], i);
}
-int module_code_size(void)
+int module_code_size(ErtsCodeIndex code_ix)
{
- return module_table.entries;
+ return module_tables[code_ix].entries;
}
int module_table_sz(void)
{
- return index_table_sz(&module_table);
+ return erts_smp_atomic_read_nob(&tot_module_bytes);
+}
+
+#ifdef DEBUG
+static ErtsCodeIndex dbg_load_code_ix = 0;
+#endif
+
+static int entries_at_start_staging = 0;
+
+void module_start_staging(void)
+{
+ IndexTable* src = &module_tables[erts_active_code_ix()];
+ IndexTable* dst = &module_tables[erts_staging_code_ix()];
+ Module* src_mod;
+ Module* dst_mod;
+ int i, oldsz, newsz;
+
+ ASSERT(dbg_load_code_ix == -1);
+ ASSERT(dst->entries <= src->entries);
+
+ /*
+ * Make sure our existing modules are up-to-date
+ */
+ for (i = 0; i < dst->entries; i++) {
+ src_mod = (Module*) erts_index_lookup(src, i);
+ dst_mod = (Module*) erts_index_lookup(dst, i);
+ ASSERT(src_mod->module == dst_mod->module);
+
+ dst_mod->curr = src_mod->curr;
+ dst_mod->old = src_mod->old;
+ }
+
+ /*
+ * Copy all new modules from active table
+ */
+ oldsz = index_table_sz(dst);
+ for (i = dst->entries; i < src->entries; i++) {
+ src_mod = (Module*) erts_index_lookup(src, i);
+ dst_mod = (Module*) index_put_entry(dst, src_mod);
+ ASSERT(dst_mod != src_mod);
+
+ dst_mod->curr = src_mod->curr;
+ dst_mod->old = src_mod->old;
+ }
+ newsz = index_table_sz(dst);
+ erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
+
+ entries_at_start_staging = dst->entries;
+ IF_DEBUG(dbg_load_code_ix = erts_staging_code_ix());
+}
+
+void module_end_staging(int commit)
+{
+ ASSERT(dbg_load_code_ix == erts_staging_code_ix());
+
+ if (!commit) { /* abort */
+ IndexTable* tab = &module_tables[erts_staging_code_ix()];
+ int oldsz, newsz;
+
+ ASSERT(entries_at_start_staging <= tab->entries);
+ oldsz = index_table_sz(tab);
+ index_erase_latest_from(tab, entries_at_start_staging);
+ newsz = index_table_sz(tab);
+ erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
+ }
+
+ IF_DEBUG(dbg_load_code_ix = -1);
}
+
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 694e4ab72f..6e123a0ffe 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -24,28 +24,72 @@
#include "index.h"
#endif
+struct erl_module_instance {
+ BeamInstr* code;
+ int code_length; /* Length of loaded code in bytes. */
+ unsigned catches;
+ struct erl_module_nif* nif;
+ int num_breakpoints;
+ int num_traced_exports;
+};
typedef struct erl_module {
IndexSlot slot; /* Must be located at top of struct! */
int module; /* Atom index for module (not tagged). */
- BeamInstr* code;
- BeamInstr* old_code;
- int code_length; /* Length of loaded code in bytes. */
- int old_code_length; /* Length of old loaded code in bytes */
- unsigned catches, old_catches;
- struct erl_module_nif* nif;
- struct erl_module_nif* old_nif;
+ struct erl_module_instance curr;
+ struct erl_module_instance old; /* protected by "old_code" rwlock */
} Module;
-Module* erts_get_module(Eterm mod);
+Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix);
Module* erts_put_module(Eterm mod);
void init_module_table(void);
+void module_start_staging(void);
+void module_end_staging(int commit);
void module_info(int, void *);
-Module *module_code(int);
-int module_code_size(void);
+Module *module_code(int, ErtsCodeIndex);
+int module_code_size(ErtsCodeIndex);
int module_table_sz(void);
+ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex);
+ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex);
+ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex);
+ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex);
+#ifdef ERTS_ENABLE_LOCK_CHECK
+int erts_is_old_code_rlocked(ErtsCodeIndex);
+#endif
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+extern erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX];
+
+ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex code_ix)
+{
+ erts_smp_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]);
+}
+ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex code_ix)
+{
+ erts_smp_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]);
+}
+ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex code_ix)
+{
+ erts_smp_rwmtx_rlock(&the_old_code_rwlocks[code_ix]);
+}
+ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex code_ix)
+{
+ erts_smp_rwmtx_runlock(&the_old_code_rwlocks[code_ix]);
+}
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ERTS_GLB_INLINE int erts_is_old_code_rlocked(ErtsCodeIndex code_ix)
+{
+ return erts_smp_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]);
+}
#endif
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+
+#endif /* !__MODULE_H__ */