aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--erts/emulator/beam/beam_emu.c10
-rw-r--r--erts/emulator/beam/bif.c4
-rw-r--r--erts/emulator/beam/dist.c155
-rw-r--r--erts/emulator/beam/dist.h1
-rw-r--r--erts/emulator/beam/erl_alloc.c14
-rw-r--r--erts/emulator/beam/erl_alloc.types3
-rw-r--r--erts/emulator/beam/erl_alloc_util.c28
-rw-r--r--erts/emulator/beam/erl_arith.c107
-rw-r--r--erts/emulator/beam/erl_bif_info.c14
-rw-r--r--erts/emulator/beam/erl_bif_lists.c5
-rw-r--r--erts/emulator/beam/erl_bif_port.c4
-rw-r--r--erts/emulator/beam/erl_bif_timer.c4
-rw-r--r--erts/emulator/beam/erl_bif_trace.c15
-rw-r--r--erts/emulator/beam/erl_bits.h2
-rw-r--r--erts/emulator/beam/erl_db.c86
-rw-r--r--erts/emulator/beam/erl_db_util.c5
-rw-r--r--erts/emulator/beam/erl_debug.c24
-rw-r--r--erts/emulator/beam/erl_debug.h8
-rw-r--r--erts/emulator/beam/erl_lock_check.c2
-rw-r--r--erts/emulator/beam/erl_node_tables.c13
-rw-r--r--erts/emulator/beam/erl_process.c14
-rw-r--r--erts/emulator/beam/erl_process.h7
-rw-r--r--erts/emulator/beam/erl_term.c14
-rw-r--r--erts/emulator/beam/erl_term.h59
-rw-r--r--erts/emulator/beam/erl_trace.c207
-rw-r--r--erts/emulator/beam/erl_vm.h9
-rw-r--r--erts/emulator/beam/external.c11
-rw-r--r--erts/emulator/beam/global.h57
-rw-r--r--erts/emulator/beam/io.c14
-rw-r--r--erts/emulator/beam/utils.c11
-rw-r--r--erts/emulator/sys/common/erl_mseg.c56
31 files changed, 773 insertions, 190 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2f7f48193d..66c6eb7819 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1090,7 +1090,11 @@ void process_main(void)
*/
register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
- Eterm tmp_big[2]; /* Temporary buffer for small bignums. */
+#if HEAP_ON_C_STACK
+ Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */
+#else
+ Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
+#endif
#ifndef ERTS_SMP
static Eterm save_reg[ERTS_X_REGS_ALLOCATED];
@@ -1141,7 +1145,6 @@ void process_main(void)
* Note: c_p->arity must be set to reflect the number of useful terms in
* c_p->arg_reg before calling the scheduler.
*/
-
if (!init_done) {
init_done = 1;
goto init_emulator;
@@ -1168,6 +1171,9 @@ void process_main(void)
reg = c_p->scheduler_data->save_reg;
freg = c_p->scheduler_data->freg;
#endif
+#if !HEAP_ON_C_STACK
+ tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
+#endif
ERL_BITS_RELOAD_STATEP(c_p);
{
int reds;
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 1b670585a7..0b21cf3afd 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -616,13 +616,15 @@ local_name_monitor(Process *p, Eterm target_name)
rp = erts_whereis_process(p, p_locks, target_name, ERTS_PROC_LOCK_LINK,
ERTS_P2P_FLG_ALLOW_OTHER_X);
if (!rp) {
- Eterm lhp[3];
+ DeclareTmpHeap(lhp,3,p);
Eterm item;
+ UseTmpHeap(3,p);
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
p_locks &= ~ERTS_PROC_LOCK_LINK;
item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname);
erts_queue_monitor_message(p, &p_locks,
mon_ref, am_process, item, am_noproc);
+ UnUseTmpHeap(3,p);
}
else if (rp != p) {
erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, rp->id,
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index e3094404e2..64ab88fc45 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -228,6 +228,7 @@ int is_node_name_atom(Eterm a)
typedef struct {
DistEntry *dep;
+ Eterm *lhp;
} NetExitsContext;
/*
@@ -253,8 +254,9 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
erts_destroy_monitor(rmon);
}
} else {
- Eterm lhp[3];
+ DeclareTmpHeapNoproc(lhp,3);
Eterm watched;
+ UseTmpHeapNoproc(3);
ASSERT(mon->type == MON_TARGET);
rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
/* ASSERT(rmon != NULL); can happen during process exit */
@@ -271,6 +273,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
watched, am_noconnection);
erts_destroy_monitor(rmon);
}
+ UnUseTmpHeapNoproc(3);
}
erts_smp_proc_unlock(rp, rp_locks);
done:
@@ -632,19 +635,27 @@ static void clear_dist_entry(DistEntry *dep)
int
erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote)
{
- Eterm ctl_heap[4];
+ DeclareTmpHeapNoproc(ctl_heap,4);
Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_LINK), local, remote);
+ int res;
+ UseTmpHeapNoproc(4);
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ UnUseTmpHeapNoproc(4);
+ return res;
}
int
erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote)
{
- Eterm ctl_heap[4];
+ DeclareTmpHeapNoproc(ctl_heap,4);
Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_UNLINK), local, remote);
+ int res;
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ UseTmpHeapNoproc(4);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ UnUseTmpHeapNoproc(4);
+ return res;
}
@@ -656,7 +667,10 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
Eterm ref, Eterm reason)
{
Eterm ctl;
- Eterm ctl_heap[6];
+ DeclareTmpHeapNoproc(ctl_heap,6);
+ int res;
+
+ UseTmpHeapNoproc(6);
ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT),
watched, watcher, ref, reason);
@@ -667,7 +681,9 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
erts_smp_de_links_unlock(dsdp->dep);
#endif
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+ UnUseTmpHeapNoproc(6);
+ return res;
}
/* We want to monitor a process (named or unnamed) on another node, we send:
@@ -678,13 +694,17 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
Eterm ref)
{
Eterm ctl;
- Eterm ctl_heap[5];
+ DeclareTmpHeapNoproc(ctl_heap,5);
+ int res;
+ UseTmpHeapNoproc(5);
ctl = TUPLE4(&ctl_heap[0],
make_small(DOP_MONITOR_P),
watcher, watched, ref);
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ UnUseTmpHeapNoproc(5);
+ return res;
}
/* A local process monitoring a remote one wants to stop monitoring, either
@@ -696,23 +716,29 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
Eterm watched, Eterm ref, int force)
{
Eterm ctl;
- Eterm ctl_heap[5];
+ DeclareTmpHeapNoproc(ctl_heap,5);
+ int res;
+ UseTmpHeapNoproc(5);
ctl = TUPLE4(&ctl_heap[0],
make_small(DOP_DEMONITOR_P),
watcher, watched, ref);
- return dsig_send(dsdp, ctl, THE_NON_VALUE, force);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, force);
+ UnUseTmpHeapNoproc(5);
+ return res;
}
int
erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
{
Eterm ctl;
- Eterm ctl_heap[5];
+ DeclareTmpHeapNoproc(ctl_heap,5);
Eterm token = NIL;
Process *sender = dsdp->proc;
+ int res;
+ UseTmpHeapNoproc(5);
if (SEQ_TRACE_TOKEN(sender) != NIL) {
seq_trace_update_send(sender);
token = SEQ_TRACE_TOKEN(sender);
@@ -724,17 +750,21 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
make_small(DOP_SEND_TT), am_Cookie, remote, token);
else
ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote);
- return dsig_send(dsdp, ctl, message, 0);
+ res = dsig_send(dsdp, ctl, message, 0);
+ UnUseTmpHeapNoproc(5);
+ return res;
}
int
erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
{
Eterm ctl;
- Eterm ctl_heap[6];
+ DeclareTmpHeapNoproc(ctl_heap,6);
Eterm token = NIL;
Process *sender = dsdp->proc;
+ int res;
+ UseTmpHeapNoproc(6);
if (SEQ_TRACE_TOKEN(sender) != NIL) {
seq_trace_update_send(sender);
token = SEQ_TRACE_TOKEN(sender);
@@ -747,7 +777,9 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
else
ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND),
sender->id, am_Cookie, remote_name);
- return dsig_send(dsdp, ctl, message, 0);
+ res = dsig_send(dsdp, ctl, message, 0);
+ UnUseTmpHeapNoproc(6);
+ return res;
}
/* local has died, deliver the exit signal to remote */
@@ -756,8 +788,10 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
Eterm reason, Eterm token)
{
Eterm ctl;
- Eterm ctl_heap[6];
+ DeclareTmpHeapNoproc(ctl_heap,6);
+ int res;
+ UseTmpHeapNoproc(6);
if (token != NIL) {
seq_trace_update_send(dsdp->proc);
seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local);
@@ -767,38 +801,58 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason);
}
/* forced, i.e ignore busy */
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+ UnUseTmpHeapNoproc(6);
+ return res;
}
int
erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason)
{
- Eterm ctl_heap[5];
- Eterm ctl = TUPLE4(&ctl_heap[0],
- make_small(DOP_EXIT), local, remote, reason);
+ DeclareTmpHeapNoproc(ctl_heap,5);
+ int res;
+ Eterm ctl;
+
+ UseTmpHeapNoproc(5);
+ ctl = TUPLE4(&ctl_heap[0],
+ make_small(DOP_EXIT), local, remote, reason);
/* forced, i.e ignore busy */
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+ UnUseTmpHeapNoproc(5);
+ return res;
}
int
erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason)
{
- Eterm ctl_heap[5];
- Eterm ctl = TUPLE4(&ctl_heap[0],
- make_small(DOP_EXIT2), local, remote, reason);
+ DeclareTmpHeapNoproc(ctl_heap,5);
+ int res;
+ Eterm ctl;
+
+ UseTmpHeapNoproc(5);
+ ctl = TUPLE4(&ctl_heap[0],
+ make_small(DOP_EXIT2), local, remote, reason);
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ UnUseTmpHeapNoproc(5);
+ return res;
}
int
erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
{
- Eterm ctl_heap[4];
- Eterm ctl = TUPLE3(&ctl_heap[0],
- make_small(DOP_GROUP_LEADER), leader, remote);
+ DeclareTmpHeapNoproc(ctl_heap,4);
+ int res;
+ Eterm ctl;
+
+ UseTmpHeapNoproc(4);
+ ctl = TUPLE3(&ctl_heap[0],
+ make_small(DOP_GROUP_LEADER), leader, remote);
- return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+ UnUseTmpHeapNoproc(4);
+ return res;
}
#if defined(PURIFY)
@@ -832,6 +886,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
**
** assert hlen == 0 !!!
*/
+
int erts_net_message(Port *prt,
DistEntry *dep,
byte *hbuf,
@@ -839,6 +894,7 @@ int erts_net_message(Port *prt,
byte *buf,
int len)
{
+#define DIST_CTL_DEFAULT_SIZE 64
ErtsDistExternal ede;
byte *t;
Sint ctl_len;
@@ -850,7 +906,7 @@ int erts_net_message(Port *prt,
Eterm *tuple;
Eterm reason;
Process* rp;
- Eterm ctl_default[64];
+ DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE);
Eterm* ctl = ctl_default;
ErlOffHeap off_heap;
Eterm* hp;
@@ -864,6 +920,7 @@ int erts_net_message(Port *prt,
int orig_len = len;
#endif
+ UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
/* Thanks to Luke Gorrie */
off_heap.mso = NULL;
#ifndef HYBRID /* FIND ME! */
@@ -876,12 +933,16 @@ int erts_net_message(Port *prt,
ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
- if (!erts_is_alive)
+ if (!erts_is_alive) {
+ UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
return 0;
+ }
if (hlen > 0)
goto data_error;
- if (len == 0) /* HANDLE TICK !!! */
+ if (len == 0) { /* HANDLE TICK !!! */
+ UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
return 0;
+ }
#ifdef ERTS_RAW_DIST_MSG_DBG
erts_fprintf(stderr, "<< ");
@@ -922,7 +983,8 @@ int erts_net_message(Port *prt,
goto data_error;
}
orig_ctl_len = ctl_len;
- if (ctl_len > sizeof(ctl_default)/sizeof(ctl_default[0])) {
+
+ if (ctl_len > DIST_CTL_DEFAULT_SIZE) {
ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm));
}
hp = ctl;
@@ -1202,7 +1264,7 @@ int erts_net_message(Port *prt,
{DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */
- Eterm lhp[3];
+ DeclareTmpHeapNoproc(lhp,3);
Eterm sysname;
ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK;
@@ -1237,6 +1299,7 @@ int erts_net_message(Port *prt,
erts_smp_proc_unlock(rp, rp_locks);
break;
}
+ UseTmpHeapNoproc(3);
watched = (is_not_nil(mon->name)
? TUPLE2(&lhp[0], mon->name, sysname)
@@ -1246,6 +1309,7 @@ int erts_net_message(Port *prt,
ref, am_process, watched, reason);
erts_smp_proc_unlock(rp, rp_locks);
erts_destroy_monitor(mon);
+ UnUseTmpHeapNoproc(3);
break;
}
@@ -1384,6 +1448,7 @@ int erts_net_message(Port *prt,
erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
}
#endif
+ UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
ERTS_SMP_CHK_NO_PROC_LOCKS;
return 0;
@@ -1402,6 +1467,7 @@ int erts_net_message(Port *prt,
erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
}
#endif
+ UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
erts_do_exit_port(prt, dep->cid, am_killed);
ERTS_SMP_CHK_NO_PROC_LOCKS;
return -1;
@@ -2547,12 +2613,15 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
int visible = 0;
int hidden = 0;
int this = 0;
- Uint buf[2]; /* For one cons-cell */
+ DeclareTmpHeap(buf,2,BIF_P); /* For one cons-cell */
DistEntry *dep;
Eterm arg_list = BIF_ARG_1;
#ifdef DEBUG
Eterm* endp;
#endif
+
+ UseTmpHeap(2,BIF_P);
+
if (is_atom(BIF_ARG_1))
arg_list = CONS(buf, BIF_ARG_1, NIL);
@@ -2563,13 +2632,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
case am_known: visible = hidden = not_connected = this = 1; break;
case am_this: this = 1; break;
case am_connected: visible = hidden = 1; break;
- default: BIF_ERROR(BIF_P, BADARG); break;
+ default: goto error; break;
}
arg_list = CDR(list_val(arg_list));
}
- if (is_not_nil(arg_list))
- BIF_ERROR(BIF_P, BADARG);
+ if (is_not_nil(arg_list)) {
+ goto error;
+ }
length = 0;
@@ -2591,7 +2661,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
if (length == 0) {
erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
- BIF_RET(result);
+ goto done;
}
hp = HAlloc(BIF_P, 2*length);
@@ -2620,7 +2690,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
}
ASSERT(endp == hp);
erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+
+done:
+ UnUseTmpHeap(2,BIF_P);
BIF_RET(result);
+
+error:
+ UnUseTmpHeap(2,BIF_P);
+ BIF_ERROR(BIF_P,BADARG);
}
/**********************************************************************/
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index ea1abcaeed..cea575798f 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -287,4 +287,5 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32);
extern Uint erts_dist_cache_size(void);
+
#endif
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index b853ec0f01..4403a19f4a 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1605,12 +1605,13 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
}
else {
- Eterm tmp_heap[2];
+ DeclareTmpHeapNoproc(tmp_heap,2);
Eterm wanted_list;
if (is_nil(earg))
return NIL;
+ UseTmpHeapNoproc(2);
if (is_not_atom(earg))
wanted_list = earg;
else {
@@ -1690,15 +1691,18 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
atoms[length] = am_maximum;
uintps[length++] = &size.maximum;
}
- }
- else
+ } else {
+ UnUseTmpHeapNoproc(2);
return am_badarg;
+ }
break;
default:
+ UnUseTmpHeapNoproc(2);
return am_badarg;
}
wanted_list = CDR(list_val(wanted_list));
}
+ UnUseTmpHeapNoproc(2);
if (is_not_nil(wanted_list))
return am_badarg;
}
@@ -2285,8 +2289,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz)
SysAllocStat sas;
Eterm opts_am;
Eterm opts;
- Eterm as[4];
- Eterm ts[4];
+ Eterm as[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
+ Eterm ts[4]; /* Ok even if !HEAP_ON_C_STACK, not really heap data on stack */
int l;
if (only_sz)
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index f701f71c7d..f808d3b66e 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -322,6 +322,9 @@ type SSB SHORT_LIVED PROCESSES ssb
# Types used by system specific code
#
+type TEMP_TERM TEMPORARY SYSTEM temp_term
+type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term
+
type DRV_TAB LONG_LIVED SYSTEM drv_tab
type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state
type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 9b7bc24c1c..8b71b51912 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -630,6 +630,11 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0);
+#if HALFWORD_HEAP
+ if (!blk) {
+ blk = create_carrier(allctr, *blk_szp, CFLG_MBC|CFLG_FORCE_MSEG);
+ }
+#else
if (!blk) {
blk = create_carrier(allctr, *blk_szp, CFLG_MBC);
if (!blk) {
@@ -640,6 +645,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC);
}
}
+#endif
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
if (IS_MBC_BLK(blk)) {
@@ -2522,7 +2528,12 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
INC_CC(allctr->calls.this_alloc);
if (size >= allctr->sbc_threshold) {
+#if HALFWORD_HEAP
+ Block_t *blk = create_carrier(allctr, size,
+ CFLG_SBC | CFLG_FORCE_MSEG);
+#else
Block_t *blk = create_carrier(allctr, size, CFLG_SBC);
+#endif
res = blk ? BLK2UMEM(blk) : NULL;
}
else
@@ -2780,13 +2791,21 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
Block_t *new_blk;
if(IS_SBC_BLK(blk)) {
do_carrier_resize:
+#if HALFWORD_HEAP
+ new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG);
+#else
new_blk = resize_carrier(allctr, blk, size, CFLG_SBC);
+#endif
res = new_blk ? BLK2UMEM(new_blk) : NULL;
}
else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
else {
+#if HALFWORD_HEAP
+ new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG);
+#else
new_blk = create_carrier(allctr, size, CFLG_SBC);
+#endif
if (new_blk) {
res = BLK2UMEM(new_blk);
sys_memcpy((void *) res,
@@ -3208,12 +3227,21 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (allctr->main_carrier_size) {
Block_t *blk;
+#if HALFWORD_HEAP
+ blk = create_carrier(allctr,
+ allctr->main_carrier_size,
+ CFLG_MBC
+ | CFLG_FORCE_SIZE
+ | CFLG_FORCE_MSEG
+ | CFLG_MAIN_CARRIER);
+#else
blk = create_carrier(allctr,
allctr->main_carrier_size,
CFLG_MBC
| CFLG_FORCE_SIZE
| CFLG_FORCE_SYS_ALLOC
| CFLG_MAIN_CARRIER);
+#endif
if (!blk)
goto error;
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index 126ec7cc73..64fad9fe0e 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -41,6 +41,16 @@
# define MAX(x, y) (((x) > (y)) ? (x) : (y))
#endif
+#if !HEAP_ON_C_STACK
+# define DECLARE_TMP(VariableName,N,P) \
+ Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N))
+#else
+# define DECLARE_TMP(VariableName,N,P) \
+ Eterm VariableName[2]
+#endif
+# define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp)))
+
+
static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right);
static ERTS_INLINE void maybe_shrink(Process* p, Eterm* hp, Eterm res, Uint alloc)
@@ -169,7 +179,7 @@ shift(Process* p, Eterm arg1, Eterm arg2, int right)
{
Sint i;
Sint ires;
- Eterm tmp_big1[2];
+ DECLARE_TMP(tmp_big1,0,p);
Eterm* bigp;
Uint need;
@@ -312,8 +322,8 @@ BIF_RETTYPE bnot_1(BIF_ALIST_1)
Eterm
erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm res;
Eterm hdr;
FloatDef f1, f2;
@@ -458,8 +468,8 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2)
Eterm
erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm hdr;
Eterm res;
FloatDef f1, f2;
@@ -602,8 +612,8 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2)
Eterm
erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm hdr;
Eterm res;
FloatDef f1, f2;
@@ -627,8 +637,8 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
} else if (arg2 == SMALL_ONE) {
return(arg1);
} else {
- Eterm big_res[3];
-
+ DeclareTmpHeap(big_res,3,p);
+ UseTmpHeap(3,p);
/*
* The following code is optimized for the case that
* result is small (which should be the most common case
@@ -636,6 +646,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
*/
res = small_times(signed_val(arg1), signed_val(arg2), big_res);
if (is_small(res)) {
+ UnUseTmpHeap(3,p);
return res;
} else {
/*
@@ -657,6 +668,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
if (arity > 1) {
*hp = big_res[2];
}
+ UnUseTmpHeap(3,p);
return res;
}
}
@@ -915,8 +927,8 @@ erts_mixed_div(Process* p, Eterm arg1, Eterm arg2)
Eterm
erts_int_div(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
int ires;
switch (NUMBER_CODE(arg1, arg2)) {
@@ -967,8 +979,8 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2)
Eterm
erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
int ires;
switch (NUMBER_CODE(arg1, arg2)) {
@@ -979,7 +991,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
if (arg1 != make_small(MIN_SMALL)) {
return arg1;
} else {
- Eterm tmp = small_to_big(signed_val(arg1), tmp_big1);
+ Eterm tmp;
+ tmp = small_to_big(signed_val(arg1), tmp_big1);
if ((ires = big_ucomp(tmp, arg2)) == 0) {
return SMALL_ZERO;
} else {
@@ -1013,8 +1026,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
Eterm erts_band(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm* hp;
int need;
@@ -1041,8 +1054,8 @@ Eterm erts_band(Process* p, Eterm arg1, Eterm arg2)
Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm* hp;
int need;
@@ -1069,8 +1082,8 @@ Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2)
Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2)
{
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm* hp;
int need;
@@ -1148,8 +1161,8 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live)
{
Eterm arg1;
Eterm arg2;
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm res;
Eterm hdr;
FloatDef f1, f2;
@@ -1237,10 +1250,10 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live)
need_heap = BIG_NEED_SIZE(sz);
if (ERTS_NEED_GC(p, need_heap)) {
erts_garbage_collect(p, need_heap, reg, live+2);
- if (arg1 != make_big(tmp_big1)) {
+ if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
arg1 = reg[live];
}
- if (arg2 != make_big(tmp_big2)) {
+ if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
arg2 = reg[live+1];
}
}
@@ -1316,8 +1329,8 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live)
{
Eterm arg1;
Eterm arg2;
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm hdr;
Eterm res;
FloatDef f1, f2;
@@ -1394,10 +1407,10 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live)
need_heap = BIG_NEED_SIZE(sz);
if (ERTS_NEED_GC(p, need_heap)) {
erts_garbage_collect(p, need_heap, reg, live+2);
- if (arg1 != make_big(tmp_big1)) {
+ if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
arg1 = reg[live];
}
- if (arg2 != make_big(tmp_big2)) {
+ if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
arg2 = reg[live+1];
}
}
@@ -1482,8 +1495,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
{
Eterm arg1;
Eterm arg2;
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
Eterm hdr;
Eterm res;
FloatDef f1, f2;
@@ -1509,7 +1522,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
} else if (arg2 == SMALL_ONE) {
return(arg1);
} else {
- Eterm big_res[3];
+ DeclareTmpHeap(big_res,3,p);
+ UseTmpHeap(3,p);
/*
* The following code is optimized for the case that
@@ -1519,6 +1533,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
res = small_times(signed_val(arg1), signed_val(arg2),
big_res);
if (is_small(res)) {
+ UnUseTmpHeap(3,p);
return res;
} else {
/*
@@ -1546,6 +1561,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
if (arity > 1) {
*hp = big_res[2];
}
+ UnUseTmpHeap(3,p);
return res;
}
}
@@ -1609,17 +1625,17 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
need_heap = BIG_NEED_SIZE(sz);
if (ERTS_NEED_GC(p, need_heap)) {
erts_garbage_collect(p, need_heap, reg, live+2);
- if (arg1 != make_big(tmp_big1)) {
+ if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
arg1 = reg[live];
}
- if (arg2 != make_big(tmp_big2)) {
+ if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
arg2 = reg[live+1];
}
}
hp = p->htop;
p->htop += need_heap;
res = big_times(arg1, arg2, hp);
- trim_heap(p, hp, res);
+ trim_heap(p, hp, res);
/*
* Note that the result must be big in this case, since
@@ -1828,8 +1844,8 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live)
{
Eterm arg1;
Eterm arg2;
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
int ires;
arg1 = reg[live];
@@ -1866,10 +1882,10 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live)
need = BIG_NEED_SIZE(i-ires+1) + BIG_NEED_SIZE(i);
if (ERTS_NEED_GC(p, need)) {
erts_garbage_collect(p, need, reg, live+2);
- if (arg1 != make_big(tmp_big1)) {
+ if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
arg1 = reg[live];
}
- if (arg2 != make_big(tmp_big2)) {
+ if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
arg2 = reg[live+1];
}
}
@@ -1894,8 +1910,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
{
Eterm arg1;
Eterm arg2;
- Eterm tmp_big1[2];
- Eterm tmp_big2[2];
+ DECLARE_TMP(tmp_big1,0,p);
+ DECLARE_TMP(tmp_big2,1,p);
int ires;
arg1 = reg[live];
@@ -1908,7 +1924,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
if (arg1 != make_small(MIN_SMALL)) {
return arg1;
} else {
- Eterm tmp = small_to_big(signed_val(arg1), tmp_big1);
+ Eterm tmp;
+ tmp = small_to_big(signed_val(arg1), tmp_big1);
if ((ires = big_ucomp(tmp, arg2)) == 0) {
return SMALL_ZERO;
} else {
@@ -1928,10 +1945,10 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
if (ERTS_NEED_GC(p, need)) {
erts_garbage_collect(p, need, reg, live+2);
- if (arg1 != make_big(tmp_big1)) {
+ if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
arg1 = reg[live];
}
- if (arg2 != make_big(tmp_big2)) {
+ if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
arg2 = reg[live+1];
}
}
@@ -1956,8 +1973,8 @@ Eterm erts_gc_##func(Process* p, Eterm* reg, Uint live) \
{ \
Eterm arg1; \
Eterm arg2; \
- Eterm tmp_big1[2]; \
- Eterm tmp_big2[2]; \
+ DECLARE_TMP(tmp_big1,0,p); \
+ DECLARE_TMP(tmp_big2,1,p); \
Eterm* hp; \
int need; \
\
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a34d400ed8..8d3618393a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -59,6 +59,12 @@
/* Keep erts_system_version as a global variable for easy access from a core */
static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
" (erts-" ERLANG_VERSION ")"
+#if HALFWORD_HEAP
+ " [halfword]"
+#endif
+#if !HEAP_ON_C_STACK
+ " [no-c-stack-objects]"
+#endif
#ifndef OTP_RELEASE
" [source]"
#endif
@@ -624,12 +630,18 @@ static Eterm pi_1_keys[] = {
#define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm))
static Eterm pi_1_keys_list;
-static Uint pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS];
+#if HEAP_ON_C_STACK
+static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS];
+#endif
static void
process_info_init(void)
{
+#if HEAP_ON_C_STACK
Eterm *hp = &pi_1_keys_list_heap[0];
+#else
+ Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS);
+#endif
int i;
pi_1_keys_list = NIL;
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index a9e8dd86f7..5df4783fcb 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -89,13 +89,14 @@ BIF_RETTYPE append_2(BIF_ALIST_2)
BIF_RET(copy);
}
+#define SMALL_VEC_SIZE 10
BIF_RETTYPE subtract_2(BIF_ALIST_2)
{
Eterm list;
Eterm* hp;
Uint need;
Eterm res;
- Eterm small_vec[10]; /* Preallocated memory for small lists */
+ Eterm small_vec[SMALL_VEC_SIZE]; /* Preallocated memory for small lists */
Eterm* vec_p;
Eterm* vp;
int i;
@@ -115,7 +116,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2)
BIF_RET(BIF_ARG_1);
/* allocate element vector */
- if (n <= sizeof(small_vec)/sizeof(small_vec[0]))
+ if (n <= SMALL_VEC_SIZE)
vec_p = small_vec;
else
vec_p = (Eterm*) erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index f454f2e12d..f9d97ee1ab 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -716,15 +716,17 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
}
} else if (option == am_cd) {
Eterm iolist;
- Eterm heap[4];
+ DeclareTmpHeap(heap,4,p);
int r;
+ UseTmpHeap(4,p);
heap[0] = *tp;
heap[1] = make_list(heap+2);
heap[2] = make_small(0);
heap[3] = NIL;
iolist = make_list(heap);
r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
+ UnUseTmpHeap(4,p);
if (r < 0) {
goto badarg;
}
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 172bb37952..1740569b69 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -178,8 +178,8 @@ eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2)
#else
#define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS)
#endif
- Uint r1_hp[MAX_REF_HEAP_SZ];
- Uint r2_hp[MAX_REF_HEAP_SZ];
+ DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2));
+ Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ;
return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2));
#undef MAX_REF_HEAP_SZ
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 7dff5e0eeb..6623eea4f3 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -86,7 +86,7 @@ trace_pattern_2(Process* p, Eterm MFA, Eterm Pattern)
Eterm
trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
{
- Eterm mfa[3];
+ DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
int i;
int matches = 0;
int specified = 0;
@@ -101,6 +101,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
erts_smp_block_system(0);
+ UseTmpHeap(3,p);
/*
* Check and compile the match specification.
*/
@@ -312,7 +313,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
MatchSetUnref(match_prog_set);
done:
-
+ UnUseTmpHeap(3,p);
erts_smp_release_system();
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
@@ -322,6 +323,7 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
MatchSetUnref(match_prog_set);
+ UnUseTmpHeap(3,p);
erts_smp_release_system();
erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
BIF_ERROR(p, BADARG);
@@ -1011,7 +1013,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
{
Eterm* tp;
Eterm* hp;
- Eterm mfa[3];
+ DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
Binary *ms = NULL, *ms_meta = NULL;
Sint count = 0;
Eterm traced = am_false;
@@ -1020,6 +1022,9 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
Eterm meta = am_false;
int r;
+
+ UseTmpHeap(3,p);
+
if (!is_tuple(func_spec)) {
goto error;
}
@@ -1037,9 +1042,11 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
r = function_is_traced(mfa, &ms, &ms_meta, &meta, &count);
switch (r) {
case FUNC_TRACE_NOEXIST:
+ UnUseTmpHeap(3,p);
hp = HAlloc(p, 3);
return TUPLE2(hp, key, am_undefined);
case FUNC_TRACE_UNTRACED:
+ UnUseTmpHeap(3,p);
hp = HAlloc(p, 3);
return TUPLE2(hp, key, am_false);
case FUNC_TRACE_GLOBAL_TRACE:
@@ -1120,10 +1127,12 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
default:
goto error;
}
+ UnUseTmpHeap(3,p);
hp = HAlloc(p, 3);
return TUPLE2(hp, key, retval);
error:
+ UnUseTmpHeap(3,p);
BIF_ERROR(p, BADARG);
}
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index e3f8e0b679..d482d325a1 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -63,7 +63,7 @@ typedef struct erl_bin_match_struct{
#define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1)
#define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms))
-#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*)(_Ms - TAG_PRIMARY_BOXED))->mb)
+#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb)
#if defined(ERTS_SMP)
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 15b1c6bb56..08c448882f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -704,12 +704,13 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
int cret = DB_ERROR_BADITEM;
Eterm list;
Eterm iter;
- Eterm cell[2];
+ DeclareTmpHeap(cell,2,BIF_P);
DbUpdateHandle handle;
if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
BIF_ERROR(BIF_P, BADARG);
}
+ UseTmpHeap(2,BIF_P);
if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
goto bail_out;
}
@@ -762,6 +763,7 @@ finalize:
tb->common.meth->db_finalize_dbterm(&handle);
bail_out:
+ UnUseTmpHeap(2,BIF_P);
db_unlock(tb, LCK_WRITE_REC);
switch (cret) {
@@ -794,8 +796,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
Eterm* ret_list_currp = NULL;
Eterm* ret_list_prevp = NULL;
Eterm iter;
- Eterm cell[2];
- Eterm tuple[3];
+ DeclareTmpHeap(cell,5,BIF_P);
+ Eterm *tuple = cell+2;
DbUpdateHandle handle;
Uint halloc_size = 0; /* overestimated heap usage */
Eterm* htop; /* actual heap usage */
@@ -805,6 +807,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
BIF_ERROR(BIF_P, BADARG);
}
+
+ UseTmpHeap(5,BIF_P);
+
if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
goto bail_out;
}
@@ -951,6 +956,7 @@ finalize:
tb->common.meth->db_finalize_dbterm(&handle);
bail_out:
+ UnUseTmpHeap(5,BIF_P);
db_unlock(tb, LCK_WRITE_REC);
switch (cret) {
@@ -1185,7 +1191,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
Sint keypos;
int is_named, is_fine_locked;
int cret;
- Eterm meta_tuple[3];
+ DeclareTmpHeap(meta_tuple,3,BIF_P);
DbTableMethod* meth;
if (is_not_atom(BIF_ARG_1)) {
@@ -1375,6 +1381,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
#endif
+ UseTmpHeap(3,BIF_P);
+
db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
if (db_put_hash(meta_pid_to_tab,
TUPLE2(meta_tuple, BIF_P->id, make_small(slot)),
@@ -1383,6 +1391,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
}
db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
+ UnUseTmpHeap(3,BIF_P);
+
BIF_RET(ret);
}
@@ -1519,7 +1529,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
}
if (tb->common.owner != BIF_P->id) {
- Eterm meta_tuple[3];
+ DeclareTmpHeap(meta_tuple,3,BIF_P);
/*
* The table is being deleted by a process other than its owner.
@@ -1527,6 +1537,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
* current process will be killed (e.g. by an EXIT signal), we will
* now transfer the ownership to the current process.
*/
+ UseTmpHeap(3,BIF_P);
db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
make_small(tb->common.slot));
@@ -1538,6 +1549,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
TUPLE2(meta_tuple,BIF_P->id,make_small(tb->common.slot)),
0);
db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
+ UnUseTmpHeap(3,BIF_P);
}
/* disable inheritance */
free_heir_data(tb);
@@ -1571,7 +1583,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
{
Process* to_proc = NULL;
ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
- Eterm buf[5];
+ DeclareTmpHeap(buf,5,BIF_P);
Eterm to_pid = BIF_ARG_2;
Eterm from_pid;
DbTable* tb = NULL;
@@ -1593,6 +1605,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
goto badarg; /* or should we be idempotent? return false maybe */
}
+ UseTmpHeap(5,BIF_P);
db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
make_small(tb->common.slot));
@@ -1610,6 +1623,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, from_pid, BIF_ARG_3),
0);
erts_smp_proc_unlock(to_proc, to_locks);
+ UnUseTmpHeap(5,BIF_P);
BIF_RET(am_true);
badarg:
@@ -1626,9 +1640,10 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
Eterm heir = THE_NON_VALUE;
Eterm heir_data = THE_NON_VALUE;
Uint32 protection = 0;
- Eterm fakelist[2];
+ DeclareTmpHeap(fakelist,2,BIF_P);
Eterm tail;
+ UseTmpHeap(2,BIF_P);
for (tail = is_tuple(BIF_ARG_2) ? CONS(fakelist, BIF_ARG_2, NIL) : BIF_ARG_2;
is_list(tail);
tail = CDR(list_val(tail))) {
@@ -1681,9 +1696,11 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
}
db_unlock (tb,LCK_WRITE);
+ UnUseTmpHeap(2,BIF_P);
BIF_RET(am_true);
badarg:
+ UnUseTmpHeap(2,BIF_P);
if (tb != NULL) {
db_unlock(tb,LCK_WRITE);
}
@@ -1949,29 +1966,37 @@ BIF_RETTYPE ets_match_1(BIF_ALIST_1)
BIF_RETTYPE ets_match_2(BIF_ALIST_2)
{
Eterm ms;
- Eterm buff[8];
+ DeclareTmpHeap(buff,8,BIF_P);
Eterm *hp = buff;
- /*hp = HAlloc(BIF_P, 8);*/
+ Eterm res;
+
+ UseTmpHeap(8,BIF_P);
ms = CONS(hp, am_DollarDollar, NIL);
hp += 2;
ms = TUPLE3(hp, BIF_ARG_2, NIL, ms);
hp += 4;
ms = CONS(hp, ms, NIL);
- return ets_select_2(BIF_P, BIF_ARG_1, ms);
+ res = ets_select_2(BIF_P, BIF_ARG_1, ms);
+ UnUseTmpHeap(8,BIF_P);
+ return res;
}
BIF_RETTYPE ets_match_3(BIF_ALIST_3)
{
Eterm ms;
- Eterm buff[8];
+ DeclareTmpHeap(buff,8,BIF_P);
Eterm *hp = buff;
- /*hp = HAlloc(BIF_P, 8);*/
+ Eterm res;
+
+ UseTmpHeap(8,BIF_P);
ms = CONS(hp, am_DollarDollar, NIL);
hp += 2;
ms = TUPLE3(hp, BIF_ARG_2, NIL, ms);
hp += 4;
ms = CONS(hp, ms, NIL);
- return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+ res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+ UnUseTmpHeap(8,BIF_P);
+ return res;
}
@@ -2385,29 +2410,37 @@ BIF_RETTYPE ets_match_object_1(BIF_ALIST_1)
BIF_RETTYPE ets_match_object_2(BIF_ALIST_2)
{
Eterm ms;
- Eterm buff[8];
+ DeclareTmpHeap(buff,8,BIF_P);
Eterm *hp = buff;
- /*hp = HAlloc(BIF_P, 8);*/
+ Eterm res;
+
+ UseTmpHeap(8,BIF_P);
ms = CONS(hp, am_DollarUnderscore, NIL);
hp += 2;
ms = TUPLE3(hp, BIF_ARG_2, NIL, ms);
hp += 4;
ms = CONS(hp, ms, NIL);
- return ets_select_2(BIF_P, BIF_ARG_1, ms);
+ res = ets_select_2(BIF_P, BIF_ARG_1, ms);
+ UnUseTmpHeap(8,BIF_P);
+ return res;
}
BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
{
Eterm ms;
- Eterm buff[8];
+ DeclareTmpHeap(buff,8,BIF_P);
Eterm *hp = buff;
- /*hp = HAlloc(BIF_P, 8);*/
+ Eterm res;
+
+ UseTmpHeap(8,BIF_P);
ms = CONS(hp, am_DollarUnderscore, NIL);
hp += 2;
ms = TUPLE3(hp, BIF_ARG_2, NIL, ms);
hp += 4;
ms = CONS(hp, ms, NIL);
- return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+ res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+ UnUseTmpHeap(8,BIF_P);
+ return res;
}
/*
@@ -2843,7 +2876,7 @@ static int give_away_to_heir(Process* p, DbTable* tb)
{
Process* to_proc;
ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
- Eterm buf[5];
+ DeclareTmpHeap(buf,5,p);
Eterm to_pid;
Eterm heir_data;
@@ -2888,6 +2921,7 @@ retry:
erts_smp_proc_unlock(to_proc, to_locks);
return 0; /* heir dead and pid reused, table still mine */
}
+ UseTmpHeap(5,p);
db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
make_small(tb->common.slot));
@@ -2899,7 +2933,7 @@ retry:
TUPLE2(buf,to_pid,make_small(tb->common.slot)),
0);
db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-
+ UnUseTmpHeap(5,p);
db_unlock(tb,LCK_WRITE);
heir_data = tb->common.heir_data;
if (!is_immed(heir_data)) {
@@ -3145,7 +3179,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
static void fix_table_locked(Process* p, DbTable* tb)
{
DbFixation *fix;
- Eterm meta_tuple[3];
+ DeclareTmpHeap(meta_tuple,3,p);
#ifdef ERTS_SMP
erts_smp_mtx_lock(&tb->common.fixlock);
@@ -3179,12 +3213,15 @@ static void fix_table_locked(Process* p, DbTable* tb)
erts_smp_mtx_unlock(&tb->common.fixlock);
#endif
p->flags |= F_USING_DB;
+ UseTmpHeap(3,p);
db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
if (db_put_hash(meta_pid_to_fixed_tab,
TUPLE2(meta_tuple, p->id, make_small(tb->common.slot)),
0) != DB_ERROR_NONE) {
+ UnUseTmpHeap(3,p);
erl_exit(1,"Could not insert ets metadata in safe_fixtable.");
}
+ UnUseTmpHeap(3,p);
db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
}
@@ -3285,10 +3322,13 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data)
}
if (!is_immed(heir_data)) {
- Eterm tmp[2];
+ DeclareTmpHeap(tmp,2,me);
+
+ UseTmpHeap(2,me);
/* Make a dummy 1-tuple around data to use db_get_term() */
heir_data = (Eterm) db_get_term(&tb->common, NULL, 0,
TUPLE1(tmp,heir_data));
+ UnUseTmpHeap(2,me);
ASSERT(!is_immed(heir_data));
}
tb->common.heir_data = heir_data;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 8c373451fd..de12b9bdbf 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2300,7 +2300,7 @@ void db_free_dmc_err_info(DMCErrInfo *ei){
*/
Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
{
- Eterm big_tmp[2];
+ DeclareTmpHeapNoproc(big_tmp,2);
Eterm res;
Sint ires;
Eterm arg1;
@@ -2318,6 +2318,7 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
}
}
else {
+ UseTmpHeapNoproc(2);
switch(NUMBER_CODE(counter, incr)) {
case SMALL_BIG:
arg1 = small_to_big(signed_val(counter), big_tmp);
@@ -2332,12 +2333,14 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
arg2 = counter;
break;
default:
+ UnUseTmpHeapNoproc(2);
return THE_NON_VALUE;
}
res = big_plus(arg1, arg2, *hpp);
if (is_big(res)) {
*hpp += BIG_NEED_SIZE(big_size(res));
}
+ UnUseTmpHeapNoproc(2);
return res;
}
}
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index e5c3c76fdd..99b05fcf26 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -895,5 +895,29 @@ void print_memory_info(Process *p)
#endif
erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes);
}
+#if !HEAP_ON_C_STACK && defined(DEBUG)
+Eterm *erts_debug_allocate_tmp_heap(int size, Process *p)
+{
+ ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+ int offset = sd->num_tmp_heap_used;
+
+ ASSERT(offset+size <= TMP_HEAP_SIZE);
+ return (sd->tmp_heap)+offset;
+}
+void erts_debug_use_tmp_heap(int size, Process *p)
+{
+ ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+
+ sd->num_tmp_heap_used += size;
+ ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE);
+}
+void erts_debug_unuse_tmp_heap(int size, Process *p)
+{
+ ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+
+ sd->num_tmp_heap_used -= size;
+ ASSERT(sd->num_tmp_heap_used >= 0);
+}
+#endif
#endif
diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h
index 74f4a00b63..b69f2c52cd 100644
--- a/erts/emulator/beam/erl_debug.h
+++ b/erts/emulator/beam/erl_debug.h
@@ -18,9 +18,10 @@
*/
#ifndef _ERL_DEBUG_H_
#define _ERL_DEBUG_H_
-
#ifdef DEBUG
+#include "erl_term.h"
+
#ifdef HIPE
#include "hipe_debug.h"
#endif
@@ -92,6 +93,11 @@ extern void print_tagged_memory(Eterm *start, Eterm *end);
extern void print_untagged_memory(Eterm *start, Eterm *end);
extern void print_memory(Process *p);
extern void print_memory_info(Process *p);
+#if defined(DEBUG) && !HEAP_ON_C_STACK
+extern Eterm *erts_debug_allocate_tmp_heap(int, Process *);
+extern void erts_debug_use_tmp_heap(int, Process *);
+extern void erts_debug_unuse_tmp_heap(int, Process *);
+#endif
#ifdef HYBRID
extern void print_ma_info(void);
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 074b08ea57..4858be1b75 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1231,6 +1231,8 @@ void
erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags)
{
lck->id = erts_lc_get_lock_order_id(name);
+
+ /* XXX:PaN What to do with the extra information? */
lck->extra = make_boxed(&lck->extra);
lck->flags = flags;
lck->inited = ERTS_LC_INITITALIZED;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 42b28d987c..f3f10201b5 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1109,9 +1109,10 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
break;
}
if (insert_bin) {
- Uint id_heap[BIG_UINT_HEAP_SIZE];
+ DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE);
Uint *hp = &id_heap[0];
InsertedBin *nib;
+ UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
a.id = erts_bld_uint(&hp, NULL, (Uint) pb->val);
erts_match_prog_foreach_offheap(pb->val,
insert_offheap2,
@@ -1120,6 +1121,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id)
nib->bin_val = pb->val;
nib->next = inserted_bins;
inserted_bins = nib;
+ UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
}
}
}
@@ -1190,12 +1192,15 @@ static void
insert_bif_timer(Eterm receiver, Eterm msg, ErlHeapFragment *bp, void *arg)
{
if (bp) {
- Eterm heap[3];
+ DeclareTmpHeapNoproc(heap,3);
+
+ UseTmpHeapNoproc(3);
insert_offheap(&bp->off_heap,
TIMER_REF,
(is_internal_pid(receiver)
? receiver
: TUPLE2(&heap[0], AM_process, receiver)));
+ UnUseTmpHeapNoproc(3);
}
}
@@ -1230,7 +1235,7 @@ setup_reference_table(void)
DistEntry *dep;
HashInfo hi;
int i;
- Eterm heap[3];
+ DeclareTmpHeapNoproc(heap,3);
inserted_bins = NULL;
@@ -1251,6 +1256,7 @@ setup_reference_table(void)
/* Go through the hole system, and build a table of all references
to ErlNode and DistEntry structures */
+ UseTmpHeapNoproc(3);
insert_node(erts_this_node,
SYSTEM_REF,
TUPLE2(&heap[0], AM_system, am_undefined));
@@ -1261,6 +1267,7 @@ setup_reference_table(void)
HEAP_REF,
TUPLE2(&heap[0], AM_processes, am_undefined));
#endif
+ UnUseTmpHeapNoproc(3);
/* Insert all processes */
for (i = 0; i < erts_max_processes; i++)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 996806fc75..f5b1308ca4 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2187,6 +2187,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
esdp->match_pseudo_process = NULL;
esdp->free_process = NULL;
#endif
+#if !HEAP_ON_C_STACK
+ esdp->num_tmp_heap_used = 0;
+#endif
esdp->no = (Uint) ix+1;
esdp->current_process = NULL;
esdp->current_port = NULL;
@@ -7778,9 +7781,10 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
erts_port_release(prt);
} else if (is_internal_pid(mon->pid)) {/* local by name or pid */
Eterm watched;
- Eterm lhp[3];
+ DeclareTmpHeapNoproc(lhp,3);
ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
| ERTS_PROC_LOCKS_MSG_SEND);
+ UseTmpHeapNoproc(3);
rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
if (rp == NULL) {
goto done;
@@ -7795,6 +7799,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process,
watched, pcontext->reason);
}
+ UnUseTmpHeapNoproc(3);
/* else: demonitor while we exited, i.e. do nothing... */
erts_smp_proc_unlock(rp, rp_locks);
} else { /* external by pid or name */
@@ -8228,11 +8233,12 @@ continue_exit_process(Process *p
* Pre-build the EXIT tuple if there are any links.
*/
if (lnk) {
- Eterm tmp_heap[4];
+ DeclareTmpHeap(tmp_heap,4,p);
Eterm exit_tuple;
Uint exit_tuple_sz;
Eterm* hp;
+ UseTmpHeap(4,p);
hp = &tmp_heap[0];
exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason);
@@ -8243,11 +8249,13 @@ continue_exit_process(Process *p
ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz};
erts_sweep_links(lnk, &doit_exit_link, &context);
}
+ UnUseTmpHeap(4,p);
}
{
ExitMonitorContext context = {reason, p};
- erts_sweep_monitors(mon,&doit_exit_monitor,&context);
+ erts_sweep_monitors(mon,&doit_exit_monitor,&context); /* Allocates TmpHeap, but we
+ have none here */
}
if (scb)
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index f58b6932b3..4466442fbc 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -351,6 +351,13 @@ struct ErtsSchedulerData_ {
void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
Process *free_process;
#endif
+#if !HEAP_ON_C_STACK
+ Eterm tmp_heap[TMP_HEAP_SIZE];
+ int num_tmp_heap_used;
+ Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
+ Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE];
+ Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
+#endif
Process *current_process;
Uint no; /* Scheduler number */
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 2924abbd51..c0ed3cb0a6 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -68,7 +68,9 @@ unsigned tag_val_def(Eterm x)
static char msg[32];
switch (x & _TAG_PRIMARY_MASK) {
- case TAG_PRIMARY_LIST: return LIST_DEF;
+ case TAG_PRIMARY_LIST:
+ ET_ASSERT(_list_precond(x),file,line);
+ return LIST_DEF;
case TAG_PRIMARY_BOXED: {
Eterm hdr = *boxed_val(x);
ET_ASSERT(is_header(hdr),file,line);
@@ -121,12 +123,12 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \
return _unchecked_##FUN(x); \
}
-ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_aligned);
+ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer);
ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,is_boxed);
-ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_aligned);
+ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,_boxed_precond);
+ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer);
ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,is_list);
+ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,_list_precond);
ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small);
ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small);
ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom);
@@ -163,7 +165,7 @@ ET_DEFINE_CHECKED(Uint32*,external_ref_data,Eterm,is_external_ref);
ET_DEFINE_CHECKED(struct erl_node_*,external_ref_node,Eterm,is_external_ref);
ET_DEFINE_CHECKED(Eterm*,export_val,Eterm,is_export);
-ET_DEFINE_CHECKED(Eterm,make_cp,Uint*,_is_aligned);
+ET_DEFINE_CHECKED(Eterm,make_cp,Uint*,_is_taggable_pointer);
ET_DEFINE_CHECKED(Uint*,cp_val,Eterm,is_CP);
ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch);
ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index a6596558fa..016d6e66a7 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -20,6 +20,35 @@
#ifndef __ERL_TERM_H
#define __ERL_TERM_H
+#ifdef ARCH_64
+#define HALFWORD_HEAP 1
+#define HALFWORD_ASSERT 1
+#endif
+
+#if HALFWORD_HEAP
+# define HEAP_ON_C_STACK 0
+# if HALFWORD_ASSERT
+# ifdef ET_DEBUG
+# undef ET_DEBUG
+# endif
+# define ET_DEBUG 1
+# endif
+# if 1
+# define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL
+# define COMPRESS_POINTER(AnUint) (AnUint)
+# define EXPAND_POINTER(APointer) (APointer)
+# else
+# define CHECK_POINTER_MASK 0x0UL
+# define COMPRESS_POINTER(AnUint) (AnUint)
+# define EXPAND_POINTER(APointer) (APointer)
+# endif
+#else
+# define HEAP_ON_C_STACK 1
+# define CHECK_POINTER_MASK 0x0UL
+# define COMPRESS_POINTER(AnUint) (AnUint)
+# define EXPAND_POINTER(APointer) (APointer)
+#endif
+
struct erl_node_; /* Declared in erl_node_tables.h */
/*
@@ -158,9 +187,16 @@ struct erl_node_; /* Declared in erl_node_tables.h */
/* boxed object access methods */
+#if HALFWORD_HEAP
+#define _is_taggable_pointer(x) (((Uint)(x) & (CHECK_POINTER_MASK | 0x3)) == 0)
+#define _boxed_precond(x) (is_boxed(x))
+#else
+#define _is_taggable_pointer(x) (((Uint)(x) & 0x3) == 0)
+#define _boxed_precond(x) (is_boxed(x))
+#endif
#define _is_aligned(x) (((Uint)(x) & 0x3) == 0)
-#define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED)
-_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*)
+#define _unchecked_make_boxed(x) COMPRESS_POINTER((Uint)(x) + TAG_PRIMARY_BOXED)
+_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*);
#define make_boxed(x) _ET_APPLY(make_boxed,(x))
#if 1
#define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED))
@@ -170,13 +206,13 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm)
#else
#define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
#endif
-#define _unchecked_boxed_val(x) ((Eterm*)((x) - TAG_PRIMARY_BOXED))
-_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm)
+#define _unchecked_boxed_val(x) EXPAND_POINTER((Eterm*)((x) - TAG_PRIMARY_BOXED))
+_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm);
#define boxed_val(x) _ET_APPLY(boxed_val,(x))
/* cons cell ("list") access methods */
-#define _unchecked_make_list(x) ((Uint)(x) + TAG_PRIMARY_LIST)
-_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*)
+#define _unchecked_make_list(x) COMPRESS_POINTER((Uint)(x) + TAG_PRIMARY_LIST)
+_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*);
#define make_list(x) _ET_APPLY(make_list,(x))
#if 1
#define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST))
@@ -187,8 +223,13 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
#define is_list(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST)
#define is_not_list(x) (!is_list((x)))
#endif
-#define _unchecked_list_val(x) ((Eterm*)((x) - TAG_PRIMARY_LIST))
-_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm)
+#if HALFWORD_HEAP
+#define _list_precond(x) (is_list(x))
+#else
+#define _list_precond(x) (is_list(x))
+#endif
+#define _unchecked_list_val(x) EXPAND_POINTER((Eterm*)((x) - TAG_PRIMARY_LIST))
+_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm);
#define list_val(x) _ET_APPLY(list_val,(x))
#define CONS(hp, car, cdr) \
@@ -198,7 +239,7 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Eterm)
#define CDR(x) ((x)[1])
/* generic tagged pointer (boxed or list) access methods */
-#define _unchecked_ptr_val(x) ((Eterm*)((x) & ~((Uint) 0x3)))
+#define _unchecked_ptr_val(x) EXPAND_POINTER((Eterm*)((x) & ~((Uint) 0x3)))
#define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/
#define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm)))
#define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 2842c2361a..f98d5ee522 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -397,11 +397,13 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
*/
static void
do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
- Eterm local_heap[4+5+5];
+#define LOCAL_HEAP_SIZE (4+5+5)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
Eterm message;
Eterm *hp;
Eterm mfarity;
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
ASSERT(is_pid(pid));
ASSERT(is_tuple(timestamp));
ASSERT(*tuple_val(timestamp) == make_arityval(3));
@@ -426,6 +428,8 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
pid,
SYS_MSG_TYPE_UNDEFINED,
message);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
}
#endif
@@ -442,7 +446,9 @@ send_to_port(Process *c_p, Eterm message,
Eterm *tracer_pid, Uint *tracee_flags) {
Port* trace_port;
#ifndef ERTS_SMP
- Eterm ts, local_heap[4], *hp;
+#define LOCAL_HEAP_SIZE (4)
+ Eterm ts, *hp;
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
#endif
ASSERT(is_internal_port(*tracer_pid));
@@ -486,6 +492,8 @@ send_to_port(Process *c_p, Eterm message,
* (e.g. getting_linked) need not be the current process. That other
* process might not have timestamps enabled.
*/
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
if (*tracee_flags & F_TIMESTAMP) {
ASSERT(is_tuple(message));
hp = tuple_val(message);
@@ -522,6 +530,8 @@ send_to_port(Process *c_p, Eterm message,
*/
do_send_schedfix_to_port(trace_port, c_p->id, ts);
}
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
#endif
}
@@ -589,7 +599,10 @@ seq_trace_send_to_port(Process *c_p,
{
Port* trace_port;
#ifndef ERTS_SMP
- Eterm ts, local_heap[4], *hp;
+ Eterm ts, *hp;
+#define LOCAL_HEAP_SIZE (4)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
#endif
ASSERT(is_internal_port(seq_tracer));
@@ -607,6 +620,9 @@ seq_trace_send_to_port(Process *c_p,
if (INVALID_TRACER_PORT(trace_port, seq_tracer)) {
invalid_tracer_port:
system_seq_tracer = am_false;
+#ifndef ERTS_SMP
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#endif
return;
}
@@ -620,6 +636,7 @@ seq_trace_send_to_port(Process *c_p,
message);
#ifndef ERTS_SMP
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
return;
}
/* Make a fake schedule only if the current process is traced
@@ -660,6 +677,8 @@ seq_trace_send_to_port(Process *c_p,
*/
do_send_schedfix_to_port(trace_port, c_p->id, ts);
}
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
#endif
}
@@ -719,7 +738,8 @@ send_to_tracer(Process *tracee,
static void
trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
{
- Eterm local_heap[5+4+1+TS_HEAP_WORDS];
+#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS)
+ DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
Eterm tmp, mess, *hp;
ErlHeapFragment *bp = NULL;
ErlOffHeap *off_heap;
@@ -768,8 +788,10 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
curr_func = p->current != NULL;
}
+ UseTmpHeap(LOCAL_HEAP_SIZE,p);
+
if (to_port)
- hp = &local_heap[0];
+ hp = local_heap;
else {
Uint size = 5;
if (curr_func)
@@ -802,6 +824,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
}
send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched);
+ UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
+#undef LOCAL_HEAP_SIZE
}
/* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
@@ -848,7 +872,10 @@ trace_send(Process *p, Eterm to, Eterm msg)
}
if (is_internal_port(p->tracer_proc)) {
- Eterm local_heap[11];
+#define LOCAL_HEAP_SIZE (11)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
mess = TUPLE5(hp, am_trace, p->id, operation, msg, to);
hp += 6;
@@ -857,6 +884,8 @@ trace_send(Process *p, Eterm to, Eterm msg)
hp = patch_ts(mess, hp);
}
send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
Uint need;
@@ -908,7 +937,10 @@ trace_receive(Process *rp, Eterm msg)
Eterm* hp;
if (is_internal_port(rp->tracer_proc)) {
- Eterm local_heap[10];
+#define LOCAL_HEAP_SIZE (10)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
mess = TUPLE4(hp, am_trace, rp->id, am_receive, msg);
hp += 5;
@@ -917,6 +949,8 @@ trace_receive(Process *rp, Eterm msg)
hp = patch_ts(mess, hp);
}
send_to_port(rp, mess, &rp->tracer_proc, &rp->trace_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
Uint hsz;
@@ -1018,7 +1052,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
}
if (is_internal_port(seq_tracer)) {
- Eterm local_heap[64];
+#define LOCAL_HEAP_SIZE (64)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
label = SEQ_TRACE_T_LABEL(token);
lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token),
@@ -1043,6 +1080,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
mess = TUPLE4(hp, am_seq_trace, label, mess, ts);
seq_trace_send_to_port(process, seq_tracer, mess, ts);
}
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
#ifndef ERTS_SMP
@@ -1145,13 +1184,17 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
void
erts_trace_return_to(Process *p, Uint *pc)
{
+#define LOCAL_HEAP_SIZE (4+5+5)
Eterm* hp;
Eterm mfa;
Eterm mess;
- Eterm local_heap[4+5+5];
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
Eterm *code_ptr = find_function_from_pc(pc);
+
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
if (!code_ptr) {
@@ -1196,6 +1239,8 @@ erts_trace_return_to(Process *p, Uint *pc)
mess = copy_struct(mess, size, &hp, off_heap);
ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
}
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
}
@@ -1262,7 +1307,9 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
arity = fi[2];
if (is_internal_port(*tracer_pid)) {
- Eterm local_heap[4+6+5];
+#define LOCAL_HEAP_SIZE (4+6+5)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
hp = local_heap;
mfa = TUPLE3(hp, mod, name, make_small(arity));
hp += 4;
@@ -1273,6 +1320,8 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
hp = patch_ts(mess, hp);
}
send_to_port(p, mess, tracer_pid, tracee_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
ErlHeapFragment *bp;
@@ -1385,7 +1434,10 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
}
if (is_internal_port(*tracer_pid)) {
- Eterm local_heap[4+3+6+5];
+#define LOCAL_HEAP_SIZE (4+3+6+5)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2]));
hp += 4;
@@ -1393,13 +1445,15 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
hp += 3;
mess = TUPLE5(hp, am_trace, p->id, am_exception_from, mfa_tuple, cv);
hp += 6;
- ASSERT((hp - local_heap)*sizeof(*hp) <= sizeof(local_heap));
+ ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE);
erts_smp_mtx_lock(&smq_mtx);
if (*tracee_flags & F_TIMESTAMP) {
hp = patch_ts(mess, hp); /* hp += 5 */
- ASSERT((hp - local_heap)*sizeof(*hp) == sizeof(local_heap));
+ ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE);
}
send_to_port(p, mess, tracer_pid, tracee_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
ErlHeapFragment *bp;
@@ -1483,7 +1537,8 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
Eterm tracee;
#endif
Eterm transformed_args[MAX_ARG];
- ErlSubBin sub_bin_heap;
+ DeclareTmpHeap(sub_bin_heap_et,ERL_SUB_BIN_SIZE,p);
+ ErlSubBin *sub_bin_heap = (ErlSubBin *) sub_bin_heap_et;
ASSERT(tracer_pid);
if (*tracer_pid == am_true) {
@@ -1535,18 +1590,19 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
* temporarily convert any match contexts to sub binaries.
*/
arity = mfa[2];
+ UseTmpHeap(ERL_SUB_BIN_SIZE,p);
#ifdef DEBUG
- sub_bin_heap.thing_word = 0;
+ sub_bin_heap->thing_word = 0;
#endif
for (i = 0; i < arity; i++) {
Eterm arg = args[i];
if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) {
ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg);
ErlBinMatchBuffer* mb = &ms->mb;
- ErlSubBin* sb = &sub_bin_heap;
+ ErlSubBin* sb = sub_bin_heap;
Uint bit_size;
- ASSERT(sub_bin_heap.thing_word == 0); /* At most one of match context */
+ ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */
bit_size = mb->size - mb->offset;
sb->thing_word = HEADER_SUB_BIN;
@@ -1564,7 +1620,12 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
args = transformed_args;
if (is_internal_port(*tracer_pid)) {
+#if HEAP_ON_C_STACK
Eterm local_heap[64+MAX_ARG];
+#else
+ Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+ sizeof(Eterm)*(64+MAX_ARG));
+#endif
hp = local_heap;
if (!erts_is_valid_tracer_port(*tracer_pid)) {
@@ -1579,6 +1640,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
if (is_not_nil(tracee))
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
#endif
+#if !HEAP_ON_C_STACK
+ erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return 0;
}
@@ -1605,6 +1670,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
&return_flags);
if (is_non_value(pam_result)) {
erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+ erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return 0;
}
}
@@ -1612,16 +1681,28 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
/* Meta trace */
if (pam_result == am_false) {
erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+ erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return return_flags;
}
} else {
/* Non-meta trace */
if (*tracee_flags & F_TRACE_SILENT) {
erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+ erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return 0;
}
if (pam_result == am_false) {
erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+ erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return return_flags;
}
if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
@@ -1664,6 +1745,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
send_to_port(p, mess, tracer_pid, tracee_flags);
erts_smp_mtx_unlock(&smq_mtx);
erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+ erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return *tracer_pid == NIL ? 0 : return_flags;
} else {
@@ -1706,6 +1791,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
if (is_not_nil(tracee))
erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
#endif
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return 0;
}
@@ -1731,6 +1817,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
&return_flags);
if (is_non_value(pam_result)) {
erts_match_set_release_result(p);
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return 0;
}
}
@@ -1738,16 +1825,19 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
/* Meta trace */
if (pam_result == am_false) {
erts_match_set_release_result(p);
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return return_flags;
}
} else {
/* Non-meta trace */
if (*tracee_flags & F_TRACE_SILENT) {
erts_match_set_release_result(p);
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return 0;
}
if (pam_result == am_false) {
erts_match_set_release_result(p);
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return return_flags;
}
if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
@@ -1831,6 +1921,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
ASSERT(hp == limit);
ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
erts_smp_mtx_unlock(&smq_mtx);
+ UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
return return_flags;
}
}
@@ -1851,7 +1942,10 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
int need;
if (is_internal_port(t_p->tracer_proc)) {
- Eterm local_heap[5+5];
+#define LOCAL_HEAP_SIZE (5+5)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
mess = TUPLE4(hp, am_trace, t_p->id, what, data);
hp += 5;
@@ -1868,6 +1962,8 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
c_p,
#endif
mess, &t_p->tracer_proc, &t_p->trace_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
Eterm tmp;
@@ -1919,7 +2015,10 @@ trace_proc_spawn(Process *p, Eterm pid,
Eterm* hp;
if (is_internal_port(p->tracer_proc)) {
- Eterm local_heap[4+6+5];
+#define LOCAL_HEAP_SIZE (4+6+5)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
mfa = TUPLE3(hp, mod, func, args);
hp += 4;
@@ -1930,6 +2029,8 @@ trace_proc_spawn(Process *p, Eterm pid,
hp = patch_ts(mess, hp);
}
send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
Eterm tmp;
@@ -2078,7 +2179,8 @@ erts_bif_trace(int bif_index, Process* p,
if (reason != TRAP) {
Eterm class;
Eterm value = p->fvalue;
- Eterm nocatch[3];
+ DeclareTmpHeapNoproc(nocatch,3);
+ UseTmpHeapNoproc(3);
/* Expand error value like in handle_error() */
if (reason & EXF_ARGLIST) {
Eterm *tp;
@@ -2126,6 +2228,7 @@ erts_bif_trace(int bif_index, Process* p,
}
}
}
+ UnUseTmpHeapNoproc(3);
if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) {
erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
p->trace_flags |= F_EXCEPTION_TRACE;
@@ -2213,15 +2316,19 @@ trace_gc(Process *p, Eterm what)
BIN_OLD_VHEAP(p),
BIN_OLD_VHEAP_SZ(p)
};
- Eterm local_heap[(sizeof(values)/sizeof(Uint))
- *(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)
- + 5/*4-tuple */ + TS_HEAP_WORDS];
+#define LOCAL_HEAP_SIZE \
+ (sizeof(values)/sizeof(Eterm)) * \
+ (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \
+ 5/*4-tuple */ + TS_HEAP_WORDS
+ DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
#ifdef DEBUG
Eterm* limit;
#endif
ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm));
+ UseTmpHeap(LOCAL_HEAP_SIZE,p);
+
if (is_internal_port(p->tracer_proc)) {
hp = local_heap;
#ifdef DEBUG
@@ -2252,7 +2359,7 @@ trace_gc(Process *p, Eterm what)
#ifdef DEBUG
limit = hp + size;
- ASSERT(size <= sizeof(local_heap)/sizeof(Eterm));
+ ASSERT(size <= LOCAL_HEAP_SIZE);
#endif
msg = erts_bld_atom_uint_2tup_list(&hp,
@@ -2275,6 +2382,8 @@ trace_gc(Process *p, Eterm what)
else
ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, msg, bp);
erts_smp_mtx_unlock(&smq_mtx);
+ UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
+#undef LOCAL_HEAP_SIZE
}
@@ -2465,7 +2574,9 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
Uint Ms, s, us;
#ifndef ERTS_SMP
- Eterm local_heap[4 + 7];
+#define LOCAL_HEAP_SIZE (4 + 7)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
hp = local_heap;
#else
ErlHeapFragment *bp;
@@ -2498,6 +2609,8 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
#ifndef ERTS_SMP
profile_send(msg);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
#else
enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
#endif
@@ -2510,7 +2623,10 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
Eterm *hp, msg, timestamp;
#ifndef ERTS_SMP
- Eterm local_heap[4 + 7];
+#define LOCAL_HEAP_SIZE (4 + 7)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
#else
ErlHeapFragment *bp;
@@ -2528,6 +2644,8 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7;
#ifndef ERTS_SMP
profile_send(msg);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
#else
enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
#endif
@@ -2558,7 +2676,10 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
Eterm* hp;
if (is_internal_port(p->tracer_proc)) {
- Eterm local_heap[5+6];
+#define LOCAL_HEAP_SIZE (5+6)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name);
@@ -2569,6 +2690,8 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
}
/* No fake schedule */
send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
ErlHeapFragment *bp;
@@ -2613,7 +2736,10 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
Eterm* hp;
if (is_internal_port(t_p->tracer_proc)) {
- Eterm local_heap[5+5];
+#define LOCAL_HEAP_SIZE (5+5)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
mess = TUPLE4(hp, am_trace, t_p->id, what, data);
hp += 5;
@@ -2623,6 +2749,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
}
/* No fake schedule */
send_to_port(NULL, mess, &t_p->tracer_proc, &t_p->trace_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
ErlHeapFragment *bp;
@@ -2674,7 +2802,10 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
Eterm sched_id = am_undefined;
if (is_internal_port(p->tracer_proc)) {
- Eterm local_heap[5+6];
+#define LOCAL_HEAP_SIZE (5+6)
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
@@ -2700,6 +2831,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
/* No fake scheduling */
send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
erts_smp_mtx_unlock(&smq_mtx);
} else {
ErlHeapFragment *bp;
@@ -2750,7 +2883,11 @@ profile_runnable_port(Port *p, Eterm status) {
Eterm count = make_small(0);
#ifndef ERTS_SMP
- Eterm local_heap[4 + 6];
+#define LOCAL_HEAP_SIZE (4 + 6)
+
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
#else
@@ -2771,6 +2908,8 @@ profile_runnable_port(Port *p, Eterm status) {
#ifndef ERTS_SMP
profile_send(msg);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
#else
enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
#endif
@@ -2785,7 +2924,11 @@ profile_runnable_proc(Process *p, Eterm status){
Eterm where = am_undefined;
#ifndef ERTS_SMP
- Eterm local_heap[4 + 6 + 4];
+#define LOCAL_HEAP_SIZE (4 + 6 + 4)
+
+ DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+ UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
hp = local_heap;
#else
ErlHeapFragment *bp;
@@ -2818,6 +2961,8 @@ profile_runnable_proc(Process *p, Eterm status){
msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6;
#ifndef ERTS_SMP
profile_send(msg);
+ UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
#else
enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
#endif
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 50b3e5b61c..1b61bca440 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -50,6 +50,15 @@
#define MAX_ARG 256 /* Max number of arguments allowed */
#define MAX_REG 1024 /* Max number of x(N) registers used */
+/* Scheduler stores data for temporary heaps if
+ !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary
+ heap data on the C stack or if we use the buffers in the scheduler data. */
+#define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers
+ small heap for transient heap data */
+#define CMP_TMP_HEAP_SIZE 2 /* cmp wants its own tmp-heap... */
+#define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */
+#define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */
+
/*
* The new arithmetic operations need some extra X registers in the register array.
*/
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 099eddd195..f7b498e414 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1563,8 +1563,10 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
put_int32(val, ep);
ep += 4;
} else {
- Eterm tmp_big[2];
- Eterm big = small_to_big(val, tmp_big);
+ DeclareTmpHeapNoproc(tmp_big,2);
+ Eterm big;
+ UseTmpHeapNoproc(2);
+ big = small_to_big(val, tmp_big);
*ep++ = SMALL_BIG_EXT;
n = big_bytes(big);
ASSERT(n < 256);
@@ -1572,6 +1574,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
ep += 1;
*ep++ = big_sign(big);
ep = big_to_bytes(big, ep);
+ UnUseTmpHeapNoproc(2);
}
}
break;
@@ -2650,9 +2653,11 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
else if (sizeof(Sint) == 4 || IS_SSMALL28(val))
result += 1 + 4; /* INTEGER_EXT */
else {
- Eterm tmp_big[2];
+ DeclareTmpHeapNoproc(tmp_big,2);
+ UseTmpHeapNoproc(2);
i = big_bytes(small_to_big(val, tmp_big));
result += 1 + 1 + 1 + i; /* SMALL_BIG_EXT */
+ UnUseTmpHeapNoproc(2);
}
}
break;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index cab249a53f..03e97a0ce4 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1839,4 +1839,61 @@ erts_alloc_message_heap(Uint size,
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+#if !HEAP_ON_C_STACK
+# if defined(DEBUG)
+# define DeclareTmpHeap(VariableName,Size,Process) \
+ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process)
+# define DeclareTmpHeapNoproc(VariableName,Size) \
+ Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL)
+# define UseTmpHeap(Size,Proc) \
+ do { \
+ erts_debug_use_tmp_heap((Size),(Proc)); \
+ } while (0)
+# define UnUseTmpHeap(Size,Proc) \
+ do { \
+ erts_debug_unuse_tmp_heap((Size),(Proc)); \
+ } while (0)
+# define UseTmpHeapNoproc(Size) \
+ do { \
+ erts_debug_use_tmp_heap(Size,NULL); \
+ } while (0)
+# define UnUseTmpHeapNoproc(Size) \
+ do { \
+ erts_debug_unuse_tmp_heap(Size,NULL); \
+ } while (0)
+# else
+# define DeclareTmpHeap(VariableName,Size,Process) \
+ Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
+# define DeclareTmpHeapNoproc(VariableName,Size) \
+ Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used)
+# define UseTmpHeap(Size,Proc) \
+ do { \
+ ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \
+ } while (0)
+# define UnUseTmpHeap(Size,Proc) \
+ do { \
+ ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \
+ } while (0)
+# define UseTmpHeapNoproc(Size) \
+ do { \
+ erts_get_scheduler_data()->num_tmp_heap_used += (Size); \
+ } while (0)
+# define UnUseTmpHeapNoproc(Size) \
+ do { \
+ erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \
+ } while (0)
+
+
+# endif
+
+#else
+# define DeclareTmpHeap(VariableName,Size,Process) \
+ Eterm VariableName[Size]
+# define DeclareTmpHeapNoproc(VariableName,Size) \
+ Eterm VariableName[Size]
+# define UseTmpHeap(Size,Proc) /* Nothing */
+# define UnUseTmpHeap(Size,Proc) /* Nothing */
+# define UseTmpHeapNoproc(Size) /* Nothing */
+# define UnUseTmpHeapNoproc(Size) /* Nothing */
+#endif
#endif
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3309b77086..ac25eb9d5c 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -4079,7 +4079,8 @@ int driver_monitor_process(ErlDrvPort port,
Port *prt = erts_drvport2port(port);
Process *rp;
Eterm ref;
- Eterm buf[REF_THING_SIZE];
+ DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
+
if (prt->drv_ptr->process_exit == NULL) {
return -1;
}
@@ -4089,12 +4090,14 @@ int driver_monitor_process(ErlDrvPort port,
if (!rp) {
return 1;
}
+ UseTmpHeapNoproc(REF_THING_SIZE);
ref = erts_make_ref_in_buffer(buf);
erts_add_monitor(&(prt->monitors), MON_ORIGIN, ref, rp->id, NIL);
erts_add_monitor(&(rp->monitors), MON_TARGET, ref, prt->id, NIL);
erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
ref_to_driver_monitor(ref,monitor);
+ UnUseTmpHeapNoproc(REF_THING_SIZE);
return 0;
}
@@ -4104,10 +4107,12 @@ int driver_demonitor_process(ErlDrvPort port,
Port *prt = erts_drvport2port(port);
Process *rp;
Eterm ref;
- Eterm buf[REF_THING_SIZE];
+ DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
ErtsMonitor *mon;
Eterm to;
+
+ UseTmpHeapNoproc(REF_THING_SIZE);
memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE);
ref = make_internal_ref(buf);
mon = erts_lookup_monitor(prt->monitors, ref);
@@ -4134,6 +4139,7 @@ int driver_demonitor_process(ErlDrvPort port,
erts_destroy_monitor(rmon);
}
}
+ UnUseTmpHeapNoproc(REF_THING_SIZE);
return 0;
}
@@ -4142,10 +4148,11 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
{
Port *prt = erts_drvport2port(port);
Eterm ref;
- Eterm buf[REF_THING_SIZE];
+ DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
ErtsMonitor *mon;
Eterm to;
+ UseTmpHeapNoproc(REF_THING_SIZE);
memcpy(buf,monitor,sizeof(Eterm)*REF_THING_SIZE);
ref = make_internal_ref(buf);
mon = erts_lookup_monitor(prt->monitors, ref);
@@ -4155,6 +4162,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
ASSERT(mon->type == MON_ORIGIN);
to = mon->pid;
ASSERT(is_internal_pid(to));
+ UnUseTmpHeapNoproc(REF_THING_SIZE);
return (ErlDrvTermData) to;
}
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 31efddc0f2..f59fbbee33 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1002,7 +1002,7 @@ Uint32
make_hash2(Eterm term)
{
Uint32 hash;
- Eterm tmp_big[2];
+ DeclareTmpHeapNoproc(tmp_big,2);
/* (HCONST * {2, ..., 14}) mod 2^32 */
#define HCONST_2 0x3c6ef372UL
@@ -1041,7 +1041,6 @@ make_hash2(Eterm term)
} while(0)
#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
-
/* Optimization. Simple cases before declaration of estack. */
if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
switch (term & _TAG_IMMED1_MASK) {
@@ -1070,6 +1069,7 @@ make_hash2(Eterm term)
Eterm tmp;
DECLARE_ESTACK(s);
+ UseTmpHeapNoproc(2);
hash = 0;
for (;;) {
switch (primary_tag(term)) {
@@ -1314,6 +1314,7 @@ make_hash2(Eterm term)
hash2_common:
if (ESTACK_ISEMPTY(s)) {
DESTROY_ESTACK(s);
+ UnUseTmpHeapNoproc(2);
return hash;
}
term = ESTACK_POP(s);
@@ -2581,7 +2582,11 @@ tailrecur_ne:
{
FloatDef f1, f2;
Eterm big;
- Eterm big_buf[2];
+#if HEAP_ON_C_STACK
+ Eterm big_buf[2]; /* If HEAP_ON_C_STACK */
+#else
+ Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap;
+#endif
switch(_NUMBER_CODE(a_tag, b_tag)) {
case SMALL_BIG:
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index f4e21bc05f..c859b192f7 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -87,11 +87,22 @@ static int is_cache_check_requested;
/* Mmap ... */
#define MMAP_PROT (PROT_READ|PROT_WRITE)
+
+#if HALFWORD_HEAP
+# ifdef MAP_32BIT
+# define RANGE_FLAG (MAP_32BIT)
+# else
+# error "Cannot have halfword heap if unable to restrict mmap areas
+# endif
+#else
+# define RANGE_FLAG (0)
+#endif
+
#ifdef MAP_ANON
-# define MMAP_FLAGS (MAP_ANON|MAP_PRIVATE)
+# define MMAP_FLAGS (MAP_ANON|MAP_PRIVATE|RANGE_FLAG)
# define MMAP_FD (-1)
#else
-# define MMAP_FLAGS (MAP_PRIVATE)
+# define MMAP_FLAGS (MAP_PRIVATE|RANGE_FLAG)
# define MMAP_FD mmap_fd
static int mmap_fd;
#endif
@@ -310,6 +321,12 @@ mseg_create(Uint size)
MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
if (seg == (void *) MAP_FAILED)
seg = NULL;
+#if HALFWORD_HEAP
+ if ((unsigned long) seg & CHECK_POINTER_MASK) {
+ erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg);
+ return NULL;
+ }
+#endif
#else
#error "Missing mseg_create() implementation"
#endif
@@ -1300,6 +1317,37 @@ erts_mseg_unit_size(void)
{
return page_size;
}
+#if HAVE_MMAP && HALFWORD_HEAP
+#ifdef MAP_NORESERVE
+#define RESERVE_FLAGS (MMAP_FLAGS | MAP_NORESERVE)
+#else
+#define RESERVE_FLAGS (MMAP_FLAGS)
+#endif
+static void halfword_reserve(void)
+{
+#if 0
+ void *ptr, *understack;
+ unsigned long x = 0x80000000;
+ long i = 0;
+
+ understack = mmap(NULL,GET_PAGE_SIZE, PROT_NONE, RESERVE_FLAGS,
+ MMAP_FD, 0);
+ while (x < (unsigned long) understack) {
+ ptr = mmap((void *) x, GET_PAGE_SIZE, PROT_NONE, RESERVE_FLAGS,
+ MMAP_FD, 0);
+ if ((unsigned long) ptr < 0x80000000) {
+ munmap(ptr, GET_PAGE_SIZE);
+ } else {
+ ++i;
+ }
+ x += GET_PAGE_SIZE;
+ }
+ erts_fprintf(stderr,"Reserved %ld pages [%d MB]...\n",i, (i*GET_PAGE_SIZE)/1024/1024);
+#else
+ return;
+#endif
+}
+#endif
void
erts_mseg_init(ErtsMsegInit_t *init)
@@ -1328,6 +1376,10 @@ erts_mseg_init(ErtsMsegInit_t *init)
erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n");
#endif
+#if HAVE_MMAP && HALFWORD_HEAP
+ halfword_reserve();
+#endif
+
page_size = GET_PAGE_SIZE;
page_shift = 1;