aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/bif.tab6
-rw-r--r--erts/emulator/beam/big.c7
-rw-r--r--erts/emulator/beam/erl_bif_atomics.c2
-rw-r--r--erts/emulator/beam/erl_init.c101
-rw-r--r--erts/emulator/beam/erl_process.c158
-rw-r--r--erts/emulator/beam/erl_process.h2
6 files changed, 195 insertions, 81 deletions
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index d4ba90a61a..0e218811d8 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -725,3 +725,9 @@ bif erts_internal:counters_get/2
bif erts_internal:counters_add/3
bif erts_internal:counters_put/3
bif erts_internal:counters_info/1
+
+#
+# New in 21.2.3
+#
+
+bif erts_internal:spawn_system_process/3
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 84338769e0..b373d3a577 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1159,8 +1159,11 @@ static dsize_t I_band(ErtsDigit* x, dsize_t xl, short xsgn,
*r++ = ~c1 & ~c2;
x++; y++;
}
- while(xl--)
- *r++ = ~*x++;
+ while(xl--) {
+ DSUBb(*x,0,b1,c1);
+ *r++ = ~c1;
+ x++;
+ }
}
}
return I_btrail(r0, r, sign);
diff --git a/erts/emulator/beam/erl_bif_atomics.c b/erts/emulator/beam/erl_bif_atomics.c
index 092dbb3bd3..029831bd95 100644
--- a/erts/emulator/beam/erl_bif_atomics.c
+++ b/erts/emulator/beam/erl_bif_atomics.c
@@ -133,7 +133,7 @@ static ERTS_INLINE Eterm bld_atomic(Process* proc, AtomicsRef* p,
if ((Uint64)val <= MAX_SMALL)
return make_small((Sint) val);
else {
- Uint hsz = ERTS_UINT64_HEAP_SIZE(val);
+ Uint hsz = ERTS_UINT64_HEAP_SIZE((Uint64)val);
Eterm* hp = HAlloc(proc, hsz);
return erts_uint64_to_big(val, &hp);
}
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 99e788c718..41bec17dfa 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -376,6 +376,27 @@ erl_init(int ncpu,
}
static Eterm
+erl_spawn_system_process(Process* parent, Eterm mod, Eterm func, Eterm args,
+ ErlSpawnOpts *so)
+{
+ Eterm res;
+ int arity;
+
+ ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(parent));
+ arity = erts_list_length(args);
+
+ if (erts_find_function(mod, func, arity, erts_active_code_ix()) == NULL) {
+ erts_exit(ERTS_ERROR_EXIT, "No function %T:%T/%i\n", mod, func, arity);
+ }
+
+ so->flags |= SPO_SYSTEM_PROC;
+
+ res = erl_create_process(parent, mod, func, args, so);
+
+ return res;
+}
+
+static Eterm
erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv)
{
int i;
@@ -386,12 +407,6 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
Process parent;
ErlSpawnOpts so;
Eterm env;
-
- start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1);
- if (erts_find_function(start_mod, am_start, 2,
- erts_active_code_ix()) == NULL) {
- erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname);
- }
/*
* We need a dummy parent process to be able to call erl_create_process().
@@ -399,6 +414,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
erts_init_empty_process(&parent);
erts_proc_lock(&parent, ERTS_PROC_LOCK_MAIN);
+
hp = HAlloc(&parent, argc*2 + 4);
args = NIL;
for (i = argc-1; i >= 0; i--) {
@@ -411,44 +427,73 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char**
hp += 2;
args = CONS(hp, env, args);
- so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC;
- res = erl_create_process(&parent, start_mod, am_start, args, &so);
+ start_mod = erts_atom_put((byte *) modname, sys_strlen(modname),
+ ERTS_ATOM_ENC_LATIN1, 1);
+
+ so.flags = erts_default_spo_flags;
+ res = erl_spawn_system_process(&parent, start_mod, am_start, args, &so);
+ ASSERT(is_internal_pid(res));
+
erts_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN);
erts_cleanup_empty_process(&parent);
+
return res;
}
static Eterm
erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq, int prio)
-{
- Eterm start_mod;
- Process* parent;
+{
+ Process *parent;
ErlSpawnOpts so;
- Eterm res;
-
- start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1);
- if (erts_find_function(start_mod, am_start, 0,
- erts_active_code_ix()) == NULL) {
- erts_exit(ERTS_ERROR_EXIT, "No function %s:start/0\n", modname);
- }
+ Eterm mod, res;
parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN);
+ mod = erts_atom_put((byte *) modname, sys_strlen(modname),
+ ERTS_ATOM_ENC_LATIN1, 1);
+
+ so.flags = erts_default_spo_flags|SPO_USE_ARGS;
- so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC|SPO_USE_ARGS;
- if (off_heap_msgq)
+ if (off_heap_msgq) {
so.flags |= SPO_OFF_HEAP_MSGQ;
- so.min_heap_size = H_MIN_SIZE;
- so.min_vheap_size = BIN_VH_MIN_SIZE;
- so.max_heap_size = H_MAX_SIZE;
- so.max_heap_flags = H_MAX_FLAGS;
- so.priority = prio;
- so.max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs);
- so.scheduler = 0;
- res = erl_create_process(parent, start_mod, am_start, NIL, &so);
+ }
+
+ so.min_heap_size = H_MIN_SIZE;
+ so.min_vheap_size = BIN_VH_MIN_SIZE;
+ so.max_heap_size = H_MAX_SIZE;
+ so.max_heap_flags = H_MAX_FLAGS;
+ so.priority = prio;
+ so.max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs);
+ so.scheduler = 0;
+
+ res = erl_spawn_system_process(parent, mod, am_start, NIL, &so);
+ ASSERT(is_internal_pid(res));
+
erts_proc_unlock(parent, ERTS_PROC_LOCK_MAIN);
+
return res;
}
+Eterm erts_internal_spawn_system_process_3(BIF_ALIST_3) {
+ Eterm mod, func, args, res;
+ ErlSpawnOpts so;
+
+ mod = BIF_ARG_1;
+ func = BIF_ARG_2;
+ args = BIF_ARG_3;
+
+ ASSERT(is_atom(mod));
+ ASSERT(is_atom(func));
+ ASSERT(erts_list_length(args) >= 0);
+
+ so.flags = erts_default_spo_flags;
+ res = erl_spawn_system_process(BIF_P, mod, func, args, &so);
+
+ if (is_non_value(res)) {
+ BIF_ERROR(BIF_P, so.error_code);
+ }
+
+ BIF_RET(res);
+}
Eterm
erts_preloaded(Process* p)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 73c99f7fb0..dca502c939 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -92,10 +92,6 @@
#undef HARDDEBUG
#endif
-#ifdef HARDDEBUG
-#define HARDDEBUG_RUNQS
-#endif
-
#ifdef HIPE
#include "hipe_mode_switch.h" /* for hipe_init_process() */
#include "hipe_signal.h" /* for hipe_thread_signal_init() */
@@ -3280,6 +3276,46 @@ prepare_for_sys_schedule(void)
#define prepare_for_sys_schedule() 0
#endif
+#ifdef HARDDEBUG
+#define ERTS_HDBG_CHK_SLEEP_LIST(SL, L, F, FN) \
+ check_sleepers_list((SL), (L), (F), (FN))
+static void check_sleepers_list(ErtsSchedulerSleepList *sl,
+ int lock,
+ ErtsSchedulerSleepInfo *find,
+ ErtsSchedulerSleepInfo *find_not)
+{
+ ErtsSchedulerSleepInfo *last_out;
+ int found = 0;
+
+ if (lock)
+ erts_spin_lock(&sl->lock);
+
+ ERTS_ASSERT(!find_not || (!find_not->next && !find_not->prev));
+
+ last_out = sl->list;
+ if (last_out) {
+ ErtsSchedulerSleepInfo *tmp = last_out;
+ do {
+ ERTS_ASSERT(tmp->next);
+ ERTS_ASSERT(tmp->prev);
+ ERTS_ASSERT(tmp->next->prev == tmp);
+ ERTS_ASSERT(tmp->prev->next == tmp);
+ ERTS_ASSERT(tmp != find_not);
+ if (tmp == find)
+ found = !0;
+ tmp = tmp->next;
+
+ } while (tmp != last_out);
+ }
+ ERTS_ASSERT(!find || found);
+
+ if (lock)
+ erts_spin_unlock(&sl->lock);
+}
+#else
+#define ERTS_HDBG_CHK_SLEEP_LIST(SL, L, F, FN) ((void) 0)
+#endif
+
static void
scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
{
@@ -3293,23 +3329,29 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
- if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
- erts_spin_lock(&rq->sleepers.lock);
flgs = sched_prep_spin_wait(ssi);
if (flgs & ERTS_SSI_FLG_SUSPENDED) {
/* Go suspend instead... */
- if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
- erts_spin_unlock(&rq->sleepers.lock);
return;
}
if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
- ssi->prev = NULL;
- ssi->next = rq->sleepers.list;
- if (rq->sleepers.list)
- rq->sleepers.list->prev = ssi;
- rq->sleepers.list = ssi;
- erts_spin_unlock(&rq->sleepers.lock);
+ erts_spin_lock(&rq->sleepers.lock);
+ ERTS_HDBG_CHK_SLEEP_LIST(&rq->sleepers, 0, NULL, ssi);
+ ASSERT(!ssi->next); /* Not in sleepers list */
+ ASSERT(!ssi->prev);
+ if (!rq->sleepers.list) {
+ ssi->next = ssi->prev = ssi;
+ rq->sleepers.list = ssi;
+ }
+ else {
+ ssi->prev = rq->sleepers.list;
+ ssi->next = rq->sleepers.list->next;
+ ssi->prev->next = ssi;
+ ssi->next->prev = ssi;
+ }
+ ERTS_HDBG_CHK_SLEEP_LIST(&rq->sleepers, 0, ssi, NULL);
+ erts_spin_unlock(&rq->sleepers.lock);
dirty_active(esdp, -1);
}
@@ -3451,8 +3493,28 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
(ERTS_SSI_FLG_SUSPENDED
| ERTS_SSI_FLG_MSB_EXEC));
- if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+ if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
dirty_sched_wall_time_change(esdp, working = 1);
+ erts_spin_lock(&rq->sleepers.lock);
+ ERTS_HDBG_CHK_SLEEP_LIST(&rq->sleepers, 0, ssi->next ? ssi : NULL, NULL);
+ if (ssi->next) { /* Still in list... */
+ if (ssi->next == ssi) {
+ ASSERT(rq->sleepers.list == ssi);
+ ASSERT(ssi->prev == ssi);
+ rq->sleepers.list = NULL;
+ }
+ else {
+ ASSERT(ssi->prev != ssi);
+ if (rq->sleepers.list == ssi)
+ rq->sleepers.list = ssi->next;
+ ssi->prev->next = ssi->next;
+ ssi->next->prev = ssi->prev;
+ }
+ ssi->next = ssi->prev = NULL;
+ }
+ ERTS_HDBG_CHK_SLEEP_LIST(&rq->sleepers, 0, NULL, ssi);
+ erts_spin_unlock(&rq->sleepers.lock);
+ }
else if (!thr_prgr_active) {
erts_thr_progress_active(erts_thr_prgr_data(esdp), thr_prgr_active = 1);
sched_wall_time_change(esdp, 1);
@@ -3540,56 +3602,44 @@ wake_scheduler(ErtsRunQueue *rq)
}
static void
-wake_dirty_schedulers(ErtsRunQueue *rq, int one)
+wake_dirty_scheduler(ErtsRunQueue *rq)
{
- ErtsSchedulerSleepInfo *ssi;
+ ErtsSchedulerSleepInfo *lo_ssi, *fo_ssi;
ErtsSchedulerSleepList *sl;
ASSERT(ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
sl = &rq->sleepers;
erts_spin_lock(&sl->lock);
- ssi = sl->list;
- if (!ssi) {
+ ERTS_HDBG_CHK_SLEEP_LIST(&rq->sleepers, 0, NULL, NULL);
+ lo_ssi = sl->list;
+ if (!lo_ssi) {
erts_spin_unlock(&sl->lock);
- if (one)
- wake_scheduler(rq);
- } else if (one) {
+ wake_scheduler(rq);
+ }
+ else {
erts_aint32_t flgs;
- if (ssi->prev)
- ssi->prev->next = ssi->next;
- else {
- ASSERT(sl->list == ssi);
- sl->list = ssi->next;
+ fo_ssi = lo_ssi->next;
+ ASSERT(fo_ssi->prev == lo_ssi);
+ if (fo_ssi == lo_ssi) {
+ ASSERT(lo_ssi->prev == lo_ssi);
+ sl->list = NULL;
+ }
+ else {
+ ASSERT(lo_ssi->prev != lo_ssi);
+ lo_ssi->next = fo_ssi->next;
+ fo_ssi->next->prev = fo_ssi->prev;
}
- if (ssi->next)
- ssi->next->prev = ssi->prev;
-
- erts_spin_unlock(&sl->lock);
-
- ERTS_THR_MEMORY_BARRIER;
- flgs = ssi_flags_set_wake(ssi);
- erts_sched_finish_poke(ssi, flgs);
- } else {
- sl->list = NULL;
+ fo_ssi->next = fo_ssi->prev = NULL;
+ ERTS_HDBG_CHK_SLEEP_LIST(&rq->sleepers, 0, NULL, fo_ssi);
erts_spin_unlock(&sl->lock);
ERTS_THR_MEMORY_BARRIER;
- do {
- ErtsSchedulerSleepInfo *wake_ssi = ssi;
- ssi = ssi->next;
- erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi));
- } while (ssi);
+ flgs = ssi_flags_set_wake(fo_ssi);
+ erts_sched_finish_poke(fo_ssi, flgs);
}
}
-static void
-wake_dirty_scheduler(ErtsRunQueue *rq)
-{
- wake_dirty_schedulers(rq, 1);
-}
-
-
#define ERTS_NO_USED_RUNQS_SHIFT 16
#define ERTS_NO_RUNQS_MASK 0xffffU
@@ -5946,6 +5996,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) {
ErtsSchedulerSleepInfo *ssi = &aligned_dirty_cpu_sched_sleep_info[ix].ssi;
erts_atomic32_init_nob(&ssi->flags, 0);
+ ssi->next = NULL;
+ ssi->prev = NULL;
ssi->event = NULL; /* initialized in sched_dirty_cpu_thread_func */
erts_atomic32_init_nob(&ssi->aux_work, 0);
}
@@ -5956,6 +6008,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
ErtsSchedulerSleepInfo *ssi = &aligned_dirty_io_sched_sleep_info[ix].ssi;
erts_atomic32_init_nob(&ssi->flags, 0);
+ ssi->next = NULL;
+ ssi->prev = NULL;
ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */
erts_atomic32_init_nob(&ssi->aux_work, 0);
}
@@ -7475,7 +7529,13 @@ suspend_scheduler(ErtsSchedulerData *esdp)
return;
}
+#ifdef HARDDEBUG
+ if (sched_type != ERTS_SCHED_NORMAL)
+ ERTS_HDBG_CHK_SLEEP_LIST(&esdp->run_queue->sleepers, !0, NULL, ssi);
+#endif
+
if (erts_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) {
+
ASSERT(no == 1);
if (!msb_scheduler_type_switch(sched_type, esdp, no))
return;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 9a22858f2a..43937f216c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -357,7 +357,7 @@ typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
typedef struct {
erts_spinlock_t lock;
- ErtsSchedulerSleepInfo *list;
+ ErtsSchedulerSleepInfo *list; /* circular lifo list; points to last out */
} ErtsSchedulerSleepList;
struct ErtsSchedulerSleepInfo_ {
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-03 23:59:51 +0000