Fit all heap data into the 32-bit address range

This is the first step in the implementation of the half-word emulator,
a 64-bit emulator where all pointers to heap data will be stored
in 32-bit words. Code specific for this emulator variant is
conditionally compiled when the HALFWORD_HEAP define has
a non-zero value.

First force all pointers to heap data to fall into a single 32-bit range,
but still store them in 64-bit words.

Temporary term data stored on C stack is moved into scheduler specific
storage (allocated as heaps) and macros are added to make this
happen only in emulators where this is needed. For a vanilla VM the
temporary terms are still stored on the C stack.

1 files changed, 63 insertions, 23 deletions

diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 15b1c6bb56..08c448882f 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -704,12 +704,13 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
     int cret = DB_ERROR_BADITEM;
     Eterm list;
     Eterm iter;
-    Eterm cell[2];
+    DeclareTmpHeap(cell,2,BIF_P);
     DbUpdateHandle handle;
 
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
 	BIF_ERROR(BIF_P, BADARG);
     }
+    UseTmpHeap(2,BIF_P);
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
@@ -762,6 +763,7 @@ finalize:
     tb->common.meth->db_finalize_dbterm(&handle);
 
 bail_out:
+    UnUseTmpHeap(2,BIF_P);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
@@ -794,8 +796,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     Eterm* ret_list_currp = NULL;
     Eterm* ret_list_prevp = NULL;
     Eterm iter;
-    Eterm cell[2];
-    Eterm tuple[3];
+    DeclareTmpHeap(cell,5,BIF_P);
+    Eterm *tuple = cell+2;
     DbUpdateHandle handle;
     Uint halloc_size = 0; /* overestimated heap usage */
     Eterm* htop;          /* actual heap usage */
@@ -805,6 +807,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
 	BIF_ERROR(BIF_P, BADARG);
     }
+
+    UseTmpHeap(5,BIF_P);
+
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
@@ -951,6 +956,7 @@ finalize:
     tb->common.meth->db_finalize_dbterm(&handle);
 
 bail_out:
+    UnUseTmpHeap(5,BIF_P);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
@@ -1185,7 +1191,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     Sint keypos;
     int is_named, is_fine_locked;
     int cret;
-    Eterm meta_tuple[3];
+    DeclareTmpHeap(meta_tuple,3,BIF_P);
     DbTableMethod* meth;
 
     if (is_not_atom(BIF_ARG_1)) {
@@ -1375,6 +1381,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 		     erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
 #endif
 
+    UseTmpHeap(3,BIF_P);
+
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     if (db_put_hash(meta_pid_to_tab,
 		    TUPLE2(meta_tuple, BIF_P->id, make_small(slot)),
@@ -1383,6 +1391,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     }
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 
+    UnUseTmpHeap(3,BIF_P);
+
     BIF_RET(ret);
 }
 
@@ -1519,7 +1529,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     }
     
     if (tb->common.owner != BIF_P->id) {
-	Eterm meta_tuple[3];
+	DeclareTmpHeap(meta_tuple,3,BIF_P);
 
 	/*
 	 * The table is being deleted by a process other than its owner.
@@ -1527,6 +1537,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	 * current process will be killed (e.g. by an EXIT signal), we will
 	 * now transfer the ownership to the current process.
 	 */
+	UseTmpHeap(3,BIF_P);
 	db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			    make_small(tb->common.slot));
@@ -1538,6 +1549,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 		    TUPLE2(meta_tuple,BIF_P->id,make_small(tb->common.slot)),
 		    0);
 	db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
+	UnUseTmpHeap(3,BIF_P);
     }    
     /* disable inheritance */
     free_heir_data(tb);
@@ -1571,7 +1583,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 {
     Process* to_proc = NULL;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    Eterm buf[5];
+    DeclareTmpHeap(buf,5,BIF_P);
     Eterm to_pid = BIF_ARG_2;
     Eterm from_pid;
     DbTable* tb = NULL;
@@ -1593,6 +1605,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 	goto badarg;  /* or should we be idempotent? return false maybe */
     }
 
+    UseTmpHeap(5,BIF_P);
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			make_small(tb->common.slot));
@@ -1610,6 +1623,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 		      TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, from_pid, BIF_ARG_3), 
 		      0);
     erts_smp_proc_unlock(to_proc, to_locks);
+    UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
 
 badarg:
@@ -1626,9 +1640,10 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
     Eterm heir = THE_NON_VALUE;
     Eterm heir_data = THE_NON_VALUE;
     Uint32 protection = 0;
-    Eterm fakelist[2];
+    DeclareTmpHeap(fakelist,2,BIF_P);
     Eterm tail;
 
+    UseTmpHeap(2,BIF_P);
     for (tail = is_tuple(BIF_ARG_2) ? CONS(fakelist, BIF_ARG_2, NIL) : BIF_ARG_2;	
 	  is_list(tail);
 	  tail = CDR(list_val(tail))) {
@@ -1681,9 +1696,11 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
     }
 
     db_unlock (tb,LCK_WRITE);
+    UnUseTmpHeap(2,BIF_P);
     BIF_RET(am_true);
 
 badarg:
+    UnUseTmpHeap(2,BIF_P);
     if (tb != NULL) {
 	db_unlock(tb,LCK_WRITE);
     }
@@ -1949,29 +1966,37 @@ BIF_RETTYPE ets_match_1(BIF_ALIST_1)
 BIF_RETTYPE ets_match_2(BIF_ALIST_2)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select_2(BIF_P, BIF_ARG_1, ms);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 BIF_RETTYPE ets_match_3(BIF_ALIST_3)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 
@@ -2385,29 +2410,37 @@ BIF_RETTYPE ets_match_object_1(BIF_ALIST_1)
 BIF_RETTYPE ets_match_object_2(BIF_ALIST_2)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select_2(BIF_P, BIF_ARG_1, ms);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 /* 
@@ -2843,7 +2876,7 @@ static int give_away_to_heir(Process* p, DbTable* tb)
 {
     Process* to_proc;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    Eterm buf[5];
+    DeclareTmpHeap(buf,5,p);
     Eterm to_pid;
     Eterm heir_data;
 
@@ -2888,6 +2921,7 @@ retry:
 	erts_smp_proc_unlock(to_proc, to_locks);
 	return 0; /* heir dead and pid reused, table still mine */
     }
+    UseTmpHeap(5,p);
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			make_small(tb->common.slot));
@@ -2899,7 +2933,7 @@ retry:
 		TUPLE2(buf,to_pid,make_small(tb->common.slot)),
 		0);
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-    
+    UnUseTmpHeap(5,p);
     db_unlock(tb,LCK_WRITE);
     heir_data = tb->common.heir_data;
     if (!is_immed(heir_data)) {
@@ -3145,7 +3179,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 static void fix_table_locked(Process* p, DbTable* tb)
 {
     DbFixation *fix;
-    Eterm meta_tuple[3];
+    DeclareTmpHeap(meta_tuple,3,p);
 
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
@@ -3179,12 +3213,15 @@ static void fix_table_locked(Process* p, DbTable* tb)
     erts_smp_mtx_unlock(&tb->common.fixlock);
 #endif
     p->flags |= F_USING_DB;        
+    UseTmpHeap(3,p);
     db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
     if (db_put_hash(meta_pid_to_fixed_tab,
 		    TUPLE2(meta_tuple, p->id, make_small(tb->common.slot)),
 		    0) != DB_ERROR_NONE) {
+	UnUseTmpHeap(3,p);
 	erl_exit(1,"Could not insert ets metadata in safe_fixtable.");
     }	
+    UnUseTmpHeap(3,p);
     db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 }
 
@@ -3285,10 +3322,13 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data)
     }
 
     if (!is_immed(heir_data)) {
-	Eterm tmp[2];
+	DeclareTmpHeap(tmp,2,me);
+
+	UseTmpHeap(2,me);
 	/* Make a dummy 1-tuple around data to use db_get_term() */
 	heir_data = (Eterm) db_get_term(&tb->common, NULL, 0,
 					TUPLE1(tmp,heir_data));
+	UnUseTmpHeap(2,me);
 	ASSERT(!is_immed(heir_data));
     }
     tb->common.heir_data = heir_data;