Store pointers to heap data in 32-bit words

Store Erlang terms in 32-bit entities on the heap, expanding the
pointers to 64-bit when needed. This works because all terms are stored
on addresses in the 32-bit address range (the 32 most significant bits
of pointers to term data are always 0).

Introduce a new datatype called UWord (along with its companion SWord),
which is an integer having the exact same size as the machine word
(a void *), but might be larger than Eterm/Uint.

Store code as machine words, as the instructions are pointers to
executable code which might reside outside the 32-bit address range.

Continuation pointers are stored on the 32-bit stack and hence must
point to addresses in the low range, which means that loaded beam code
much be placed in the low 32-bit address range (but, as said earlier,
the instructions themselves are full words).

No Erlang term data can be stored on C stacks (enforced by an
earlier commit).

This version gives a prompt, but test cases still fail (and dump core).

The loader (and emulator loop) has instruction packing disabled.

The main issues has been in rewriting loader and actual virtual
machine. Subsystems (like distribution) does not work yet.

1 files changed, 224 insertions, 147 deletions

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 99fab28dce..b928b04210 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -51,9 +51,9 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 #define EXPORTED  2
 
 #ifdef NO_JUMP_TABLE
-#  define BeamOpCode(Op) ((Uint)(Op))
+#  define BeamOpCode(Op) ((UWord)(Op))
 #else
-#  define BeamOpCode(Op) ((Eterm)beam_ops[Op])
+#  define BeamOpCode(Op) ((UWord)beam_ops[Op])
 #endif
 
 #if defined(WORDS_BIGENDIAN)
@@ -94,7 +94,7 @@ typedef struct {
 
 typedef struct {
     unsigned type;		/* Type of operand. */
-    Uint val;			/* Value of operand. */
+    UWord val;			/* Value of operand. */
     Uint bigarity;		/* Arity for bignumbers (only). */
 } GenOpArg;
 
@@ -142,7 +142,7 @@ typedef struct {
 typedef struct {
     Eterm function;		/* Tagged atom for function. */
     int arity;			/* Arity. */
-    Eterm* address;		/* Address to function in code. */
+    UWord* address;		/* Address to function in code. */
 } ExportEntry;
 
 #define MakeIffId(a, b, c, d) \
@@ -274,13 +274,13 @@ typedef struct {
     int num_functions;		/* Number of functions in module. */
     int num_labels;		/* Number of labels. */
     int code_buffer_size;	/* Size of code buffer in words.  */
-    Eterm* code;		/* Loaded code. */
+    UWord* code;		/* Loaded code. */
     int ci;			/* Current index into loaded code. */
     Label* labels;
-    Uint put_strings;		/* Linked list of put_string instructions. */
-    Uint new_bs_put_strings;	/* Linked list of i_new_bs_put_string instructions. */
+    UWord put_strings;		/* Linked list of put_string instructions. */
+    UWord new_bs_put_strings;	/* Linked list of i_new_bs_put_string instructions. */
     StringPatch* string_patches; /* Linked list of position into string table to patch. */
-    Uint catches;		/* Linked list of catch_yf instructions. */
+    UWord catches;		/* Linked list of catch_yf instructions. */
     unsigned loaded_size;	/* Final size of code when loaded. */
     byte mod_md5[16];		/* MD5 for module code. */
     int may_load_nif;           /* true if NIFs may later be loaded for this module */  
@@ -341,7 +341,7 @@ typedef struct {
 
 #define GetTagAndValue(Stp, Tag, Val) \
    do { \
-      Uint __w; \
+      UWord __w; \
       GetByte(Stp, __w); \
       Tag = __w & 0x07; \
       if ((__w & 0x08) == 0) { \
@@ -388,7 +388,7 @@ typedef struct {
        goto load_error; \
     } else { \
        int __n = (N); \
-       Uint __result = 0; \
+       UWord __result = 0; \
        Stp->file_left -= (unsigned) __n; \
        while (__n-- > 0) { \
           __result = __result << 8 | *Stp->file_p++; \
@@ -465,7 +465,7 @@ static int bin_load(Process *c_p, ErtsProcLocks c_p_locks,
 static void init_state(LoaderState* stp);
 static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 			   Eterm group_leader, Eterm module,
-			   Eterm* code, Uint size, Uint catches);
+			   UWord* code, Uint size, UWord catches);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
 static int load_atom_table(LoaderState* stp);
@@ -499,8 +499,8 @@ static void load_printf(int line, LoaderState* context, char *fmt, ...);
 static int transform_engine(LoaderState* st);
 static void id_to_string(Uint id, char* s);
 static void new_genop(LoaderState* stp);
-static int get_int_val(LoaderState* stp, Uint len_code, Uint* result);
-static int get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result);
+static int get_int_val(LoaderState* stp, Uint len_code, UWord* result);
+static int get_erlang_integer(LoaderState* stp, Uint len_code, UWord* result);
 static int new_label(LoaderState* stp);
 static void new_literal_patch(LoaderState* stp, int pos);
 static void new_string_patch(LoaderState* stp, int pos);
@@ -513,7 +513,7 @@ static Eterm compilation_info_for_module(Process* p, Eterm mod);
 static Eterm native_addresses(Process* p, Eterm mod);
 int patch_funentries(Eterm Patchlist);
 int patch(Eterm Addresses, Uint fe);
-static int safe_mul(Uint a, Uint b, Uint* resp);
+static int safe_mul(UWord a, UWord b, UWord* resp);
 
 
 static int must_swap_floats;
@@ -592,6 +592,15 @@ erts_load_module(Process *c_p,
     return result;
 }
 
+#ifdef DEBUG
+extern void check_allocators(void);
+extern void check_allocated_block(Uint type, void *blk);
+#define CHKALLOC() check_allocators()
+#define CHKBLK(TYPE,BLK) if ((BLK) != NULL) check_allocated_block((TYPE),(BLK))
+#else
+#define CHKALLOC() /* nothing */
+#define CHKBLK(TYPE,BLK) /* nothing */
+#endif
 
 static int
 bin_load(Process *c_p, ErtsProcLocks c_p_locks,
@@ -608,6 +617,12 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Scan the IFF file.
      */
 
+#ifdef DEBUG
+    fprintf(stderr,"Loading a module\r\n");
+#endif
+
+    CHKALLOC();
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     state.file_name = "IFF header for Beam file";
     state.file_p = bytes;
     state.file_left = unloaded_size;
@@ -619,6 +634,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the header for the code chunk.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "code chunk header", CODE_CHUNK);
     if (!read_code_header(&state)) {
 	goto load_error;
@@ -628,6 +644,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the atom table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "atom table", ATOM_CHUNK);
     if (!load_atom_table(&state)) {
 	goto load_error;
@@ -637,6 +654,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the import table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "import table", IMP_CHUNK);
     if (!load_import_table(&state)) {
 	goto load_error;
@@ -646,6 +664,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the lambda (fun) table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     if (state.chunks[LAMBDA_CHUNK].size > 0) {
 	define_file(&state, "lambda (fun) table", LAMBDA_CHUNK);
 	if (!read_lambda_table(&state)) {
@@ -657,6 +676,7 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the literal table.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     if (state.chunks[LITERAL_CHUNK].size > 0) {
 	define_file(&state, "literals table (constant pool)", LITERAL_CHUNK);
 	if (!read_literal_table(&state)) {
@@ -668,18 +688,25 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Load the code chunk.
      */
 
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     state.file_name = "code chunk";
     state.file_p = state.code_start;
     state.file_left = state.code_size;
-    if (!load_code(&state) || !freeze_code(&state)) {
+    if (!load_code(&state)) {
 	goto load_error;
     }
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
+    if (!freeze_code(&state)) {
+	goto load_error;
+    }
+
 
     /*
      * Read and validate the export table.  (This must be done after
      * loading the code, because it contains labels.)
      */
     
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     define_file(&state, "export table", EXP_CHUNK);
     if (!read_export_table(&state)) {
 	goto load_error;
@@ -690,16 +717,19 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * exported and imported functions.  This can't fail.
      */
     
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     rval = insert_new_code(c_p, c_p_locks, state.group_leader, state.module,
 			   state.code, state.loaded_size, state.catches);
     if (rval < 0) {
 	goto load_error;
     }
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     final_touch(&state);
 
     /*
      * Loading succeded.
      */
+    CHKBLK(ERTS_ALC_T_CODE,state.code);
     rval = 0;
     state.code = NULL;		/* Prevent code from being freed. */
     *modp = state.module;
@@ -755,6 +785,9 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
 	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.genop_blocks);
 	state.genop_blocks = next;
     }
+#ifdef DEBUG
+    erts_fprintf(stderr,"Loaded %T\r\n",*modp);
+#endif
 
     return rval;
 }
@@ -791,7 +824,7 @@ init_state(LoaderState* stp)
 
 static int
 insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
-		Eterm group_leader, Eterm module, Eterm* code, Uint size, Uint catches)
+		Eterm group_leader, Eterm module, UWord* code, Uint size, UWord catches)
 {
     Module* modp;
     int rval;
@@ -833,7 +866,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 	modules[i] = modules[i-1];
     }
     modules[i].start = code;
-    modules[i].end = (Eterm *) (((byte *)code) + size);
+    modules[i].end = (UWord *) (((byte *)code) + size);
     num_loaded_modules++;
     mid_module = &modules[num_loaded_modules/2];
     return 0;
@@ -1083,7 +1116,7 @@ load_import_table(LoaderState* stp)
 	 * the BIF function.
 	 */
 	if ((e = erts_find_export_entry(mod, func, arity)) != NULL) {
-	    if (e->code[3] == (Uint) em_apply_bif) {
+	    if (e->code[3] == (UWord) em_apply_bif) {
 		stp->import[i].bf = (BifFunction) e->code[4];
 		if (func == am_load_nif && mod == am_erlang && arity == 2) {
 		    stp->may_load_nif = 1;
@@ -1151,7 +1184,7 @@ read_export_table(LoaderState* stp)
 	 * redefine).
 	 */
 	if ((e = erts_find_export_entry(stp->module, func, arity)) != NULL) {
-	    if (e->code[3] == (Uint) em_apply_bif) {
+	    if (e->code[3] == (UWord) em_apply_bif) {
 		int j;
 
 		for (j = 0; j < sizeof(allow_redef)/sizeof(allow_redef[0]); j++) {
@@ -1220,7 +1253,7 @@ static int
 read_literal_table(LoaderState* stp)
 {
     int i;
-    Uint uncompressed_sz;
+    UWord uncompressed_sz;
     byte* uncompressed = 0;
 
     GetInt(stp, 4, uncompressed_sz);
@@ -1338,8 +1371,8 @@ read_code_header(LoaderState* stp)
      * Initialize code area.
      */
     stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0);
-    stp->code = (Eterm*) erts_alloc(ERTS_ALC_T_CODE,
-				    sizeof(Eterm) * stp->code_buffer_size);
+    stp->code = (UWord *) erts_alloc(ERTS_ALC_T_CODE,
+				    sizeof(UWord) * stp->code_buffer_size);
 
     stp->code[MI_NUM_FUNCTIONS] = stp->num_functions;
     stp->ci = MI_FUNCTIONS + stp->num_functions + 1;
@@ -1365,16 +1398,18 @@ read_code_header(LoaderState* stp)
        LoadError2(Stp, "bad tag %d; expected %d", Actual, Expected); \
     } else {}
 
-#define Need(w) \
-    ASSERT(ci <= code_buffer_size); \
-    if (code_buffer_size < ci+(w)) { \
-        code_buffer_size = erts_next_heap_size(ci+(w), 0); \
-	stp->code = code \
-	    = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, \
-				     (void *) code, \
-				     code_buffer_size * sizeof(Eterm)); \
-    }
+#define CodeNeed(w) do {						\
+    ASSERT(ci <= code_buffer_size);					\
+    if (code_buffer_size < ci+(w)) {					\
+        code_buffer_size = erts_next_heap_size(ci+(w), 0);		\
+	stp->code = code						\
+	    = (UWord *) erts_realloc(ERTS_ALC_T_CODE,			\
+				     (void *) code,			\
+				     code_buffer_size * sizeof(UWord));	\
+    } 									\
+} while (0)
     
+#define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm))))
 
 
 static int
@@ -1387,7 +1422,7 @@ load_code(LoaderState* stp)
     char* sign;
     int arg;			/* Number of current argument. */
     int num_specific;		/* Number of specific ops for current. */
-    Eterm* code;
+    UWord* code;
     int code_buffer_size;
     int specific;
     Uint last_label = 0;	/* Number of last label. */
@@ -1446,7 +1481,7 @@ load_code(LoaderState* stp)
       if (((First) & 0x08) == 0) { \
 	 Val = (First) >> 4; \
       } else if (((First) & 0x10) == 0) { \
-         Uint __w; \
+         UWord __w; \
 	 GetByte(Stp, __w); \
 	 Val = (((First) >> 5) << 8) | __w; \
       } else { \
@@ -1455,7 +1490,7 @@ load_code(LoaderState* stp)
    } while (0)
 
 	for (arg = 0; arg < arity; arg++) {
-	    Uint first;
+	    UWord first;
 
 	    GetByte(stp, first);
 	    last_op->a[arg].type = first & 0x07;
@@ -1464,7 +1499,7 @@ load_code(LoaderState* stp)
 		if ((first & 0x08) == 0) {
 		    last_op->a[arg].val = first >> 4;
 		} else if ((first & 0x10) == 0) {
-		    Uint w;
+		    UWord w;
 		    GetByte(stp, w);
 		    ASSERT(first < 0x800);
 		    last_op->a[arg].val = ((first >> 5) << 8) | w;
@@ -1523,7 +1558,7 @@ load_code(LoaderState* stp)
 		break;
 	    case TAG_z:
 		{
-		    Uint ext_tag;
+		    UWord ext_tag;
 		    unsigned tag;
 
 		    GetValue(stp, first, ext_tag);
@@ -1531,14 +1566,14 @@ load_code(LoaderState* stp)
 		    case 0:	/* Floating point number */
 			{
 			    Eterm* hp;
-# ifndef ARCH_64
+#if !defined(ARCH_64) || HALFWORD_HEAP /* XXX:PaN - Should use ARCH_64 variant instead */
 			    Uint high, low;
 # endif
 			    last_op->a[arg].val = new_literal(stp, &hp,
 							      FLOAT_SIZE_OBJECT);
 			    hp[0] = HEADER_FLONUM;
 			    last_op->a[arg].type = TAG_q;
-# ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 			    GetInt(stp, 8, hp[1]);
 # else
 			    GetInt(stp, 4, high);
@@ -1575,10 +1610,10 @@ load_code(LoaderState* stp)
 			break;
 		    case 3: 	/* Allocation list. */
 			{
-			    Uint n;
-			    Uint type;
-			    Uint val;
-			    Uint words = 0;
+			    UWord n;
+			    UWord type;
+			    UWord val;
+			    UWord words = 0;
 			    
 			    stp->new_float_instructions = 1;
 			    GetTagAndValue(stp, tag, n);
@@ -1607,7 +1642,7 @@ load_code(LoaderState* stp)
 			}
 		    case 4:	/* Literal. */
 			{
-			    Uint val;
+			    UWord val;
 
 			    GetTagAndValue(stp, tag, val);
 			    VerifyTag(stp, tag, TAG_u);
@@ -1734,7 +1769,7 @@ load_code(LoaderState* stp)
 	    }
 
 	    stp->specific_op = specific;
-	    Need(opc[stp->specific_op].sz+2); /* Extra margin for packing */
+	    CodeNeed(opc[stp->specific_op].sz+2); /* Extra margin for packing */
 	    code[ci++] = BeamOpCode(stp->specific_op);
 	}
 	
@@ -1772,7 +1807,7 @@ load_code(LoaderState* stp)
 	    case 'c':		/* Tagged constant */
 		switch (tag) {
 		case TAG_i:
-		    code[ci++] = make_small(tmp_op->a[arg].val);
+		    code[ci++] = (UWord) make_small((Uint) tmp_op->a[arg].val);
 		    break;
 		case TAG_a:
 		    code[ci++] = tmp_op->a[arg].val;
@@ -1802,7 +1837,7 @@ load_code(LoaderState* stp)
 		    code[ci++] = make_yreg(tmp_op->a[arg].val);
 		    break;
 		case TAG_i:
-		    code[ci++] = make_small(tmp_op->a[arg].val);
+		    code[ci++] = (UWord) make_small((Uint)tmp_op->a[arg].val);
 		    break;
 		case TAG_a:
 		    code[ci++] = tmp_op->a[arg].val;
@@ -1896,12 +1931,12 @@ load_code(LoaderState* stp)
 		if (stp->import[i].bf == NULL) {
 		    LoadError1(stp, "not a BIF: import table index %d", i);
 		}
-		code[ci++] = (Eterm) stp->import[i].bf;
+		code[ci++] = (UWord) stp->import[i].bf;
 		break;
-	    case 'P':		/* Byte offset into tuple */
+	    case 'P':		/* Byte offset into tuple */ /* XXX:PaN - * sizeof(Eterm or Eterm *) ? */
 		VerifyTag(stp, tag, TAG_u);
 		tmp = tmp_op->a[arg].val;
-		code[ci++] = (Eterm) ((tmp_op->a[arg].val+1) * sizeof(Eterm *));
+		code[ci++] = (UWord) ((tmp_op->a[arg].val+1) * sizeof(Eterm));
 		break;
 	    case 'l':		/* Floating point register. */
 		VerifyTag(stp, tag_to_letter[tag], *sign);
@@ -1925,17 +1960,17 @@ load_code(LoaderState* stp)
 	for ( ; arg < tmp_op->arity; arg++) {
 	    switch (tmp_op->a[arg].type) {
 	    case TAG_i:
-		Need(1);
+		CodeNeed(1);
 		code[ci++] = make_small(tmp_op->a[arg].val);
 		break;
 	    case TAG_u:
 	    case TAG_a:
 	    case TAG_v:
-		Need(1);
+		CodeNeed(1);
 		code[ci++] = tmp_op->a[arg].val;
 		break;
 	    case TAG_f:
-		Need(1);
+		CodeNeed(1);
 		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
 		stp->labels[tmp_op->a[arg].val].patches = ci;
 		ci++;
@@ -1947,24 +1982,41 @@ load_code(LoaderState* stp)
 		    lit = stp->literals[tmp_op->a[arg].val].term;
 		    if (is_big(lit)) {
 			Eterm* bigp;
+			Eterm *tmp;
 			Uint size;
+			Uint term_size;
 
 			bigp = big_val(lit);
-			size = bignum_header_arity(*bigp);
-			Need(size+1);
-			code[ci++] = *bigp++;
-			while (size-- > 0) {
-			    code[ci++] = *bigp++;
+			term_size = bignum_header_arity(*bigp);
+			size = TermWords(term_size + 1);
+			CodeNeed(size);
+			tmp = (Eterm *) (code + ci);
+			*tmp++ = *bigp++;
+			while (term_size-- > 0) {
+			    *tmp++ = *bigp++;
 			}
+			ci +=size;
 		    } else if (is_float(lit)) {
-#ifdef ARCH_64
-			Need(1);
+#if defined(ARCH_64) && !HALFWORD_HEAP
+			CodeNeed(1);
 			code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1];
+#elif HALFWORD_HEAP
+			Eterm* fptr;
+			Uint size;
+			Eterm *tmp;
+
+			fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1;
+			size = TermWords(2);
+			CodeNeed(size);
+			tmp = (Eterm *) (code + ci);
+			*tmp++ = *fptr++;
+			*tmp = *fptr;
+			ci += size;
 #else
 			Eterm* fptr;
 
 			fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1;
-			Need(2);
+			CodeNeed(2);
 			code[ci++] = *fptr++;
 			code[ci++] = *fptr;
 #endif
@@ -1978,7 +2030,7 @@ load_code(LoaderState* stp)
 			   tag_to_letter[tmp_op->a[arg].type]);
 	    }
 	}
-
+#if !HALFWORD_HEAP /* XXX:PaN - just disabled during development */
 	/*
 	 * The packing engine.
 	 */
@@ -1987,7 +2039,7 @@ load_code(LoaderState* stp)
 	    Uint stack[8];	/* Stack. */
 	    Uint* sp = stack;	/* Points to next free position. */
 	    Uint packed = 0;	/* Accumulator for packed operations. */
-	    
+	    ASSERT(0); /* XXX:PaN - just to check that this does not happen when packing is disabled... */
 	    for (prog = opc[stp->specific_op].pack; *prog; prog++) {
 		switch (*prog) {
 		case 'g':	/* Get instruction; push on stack. */
@@ -2015,7 +2067,7 @@ load_code(LoaderState* stp)
 	    }
 	    ASSERT(sp == stack); /* Incorrect program? */
 	}
-
+#endif /* !HALFWORD_HEAP */
 	/*
 	 * Handle a few special cases.
 	 */
@@ -2037,9 +2089,9 @@ load_code(LoaderState* stp)
 			/* Must make room for call_nif op */
 			int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start);
 			ASSERT(pad > 0 && pad < MIN_FUNC_SZ);
-			Need(pad);
-			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(Eterm));
-			sys_memset(&code[finfo_ix], 0, pad*sizeof(Eterm));
+			CodeNeed(pad);
+			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(UWord));
+			sys_memset(&code[finfo_ix], 0, pad*sizeof(UWord));
 			ci += pad;
 			stp->labels[last_label].value += pad;
 		    }
@@ -2050,6 +2102,7 @@ load_code(LoaderState* stp)
 		 */
 		stp->function = code[ci-2];
 		stp->arity = code[ci-1];
+
 		ASSERT(stp->labels[last_label].value == ci - FINFO_SZ);
 		offset = MI_FUNCTIONS + function_number;
 		code[offset] = stp->labels[last_label].patches;
@@ -2161,7 +2214,6 @@ load_code(LoaderState* stp)
 	}
     }
     
-#undef Need
 
  load_error:
     return 0;
@@ -2373,7 +2425,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 		 GenOpArg Flags, GenOpArg Dst)
 {
     GenOp* op;
-    Uint bits;
+    UWord bits;
 
     NEW_GENOP(stp, op);
 
@@ -2838,14 +2890,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -2856,11 +2908,13 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	if (big_arity(big) > 1 || big_sign(big)) {
 	    goto error;
 	} else {
-	    (void) term_to_Uint(big, &op->a[1].val);
+	    Uint u;
+	    (void) term_to_Uint(big, &u);
+	    op->a[1].val = (UWord) u;
 	}
 #endif
     } else {
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     error:
 #endif
 	op->op = genop_i_wait_error_0;
@@ -2883,14 +2937,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#ifndef ARCH_64
+#if !defined(ARCH_64) || HALFWORD_HEAP
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -2901,11 +2955,13 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	if (big_arity(big) > 1 || big_sign(big)) {
 	    goto error;
 	} else {
-	    (void) term_to_Uint(big, &op->a[1].val);
+	    Uint u;
+	    (void) term_to_Uint(big, &u);
+	    op->a[1].val = (UWord) u;
 	}
 #endif
     } else {
-#ifndef ARCH_64
+#if !defined(ARCH_64) || HALFWORD_HEAP
     error:
 #endif
 	op->op = genop_i_wait_error_locked_0;
@@ -3321,7 +3377,7 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx)
     op->op = genop_i_make_fun_2;
     op->arity = 2;
     op->a[0].type = TAG_u;
-    op->a[0].val = (Uint) fe;
+    op->a[0].val = (UWord) fe;
     op->a[1].type = TAG_u;
     op->a[1].val = stp->lambdas[idx.val].num_free;
     op->next = NULL;
@@ -3376,7 +3432,8 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 static int
 freeze_code(LoaderState* stp)
 {
-    Eterm* code = stp->code;
+    UWord* code = stp->code;
+    Uint *literal_end = NULL;
     Uint index;
     int i;
     byte* str_table;
@@ -3401,46 +3458,49 @@ freeze_code(LoaderState* stp)
      * Calculate the final size of the code.
      */
 
-    size = (stp->ci + stp->total_literal_size) * sizeof(Eterm) +
+    size = (stp->ci * sizeof(UWord)) + (stp->total_literal_size * sizeof(Eterm)) +
 	strtab_size + attr_size + compile_size;
 
     /*
      * Move the code to its final location.
      */
 
-    code = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size);
-
+    code = (UWord *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size);
+    CHKBLK(ERTS_ALC_T_CODE,code);
     /*
      * Place a pointer to the op_int_code_end instruction in the
      * function table in the beginning of the file.
      */
 
-    code[MI_FUNCTIONS+stp->num_functions] = (Eterm) (code + stp->ci - 1);
+    code[MI_FUNCTIONS+stp->num_functions] = (UWord) (code + stp->ci - 1);
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Store the pointer to the on_load function.
      */
 
     if (stp->on_load) {
-	code[MI_ON_LOAD_FUNCTION_PTR] = (Eterm) (code + stp->on_load);
+	code[MI_ON_LOAD_FUNCTION_PTR] = (UWord) (code + stp->on_load);
     } else {
 	code[MI_ON_LOAD_FUNCTION_PTR] = 0;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
+    literal_end = (Uint *) (code+stp->ci);
     /*
      * Place the literal heap directly after the code and fix up all
      * put_literal instructions that refer to it.
      */
     {
-	Eterm* ptr;
-	Eterm* low;
-	Eterm* high;
+	Uint* ptr;
+	Uint* low;
+	Uint* high;
 	LiteralPatch* lp;
 
-	low = code+stp->ci;
+	low = (Uint *) (code+stp->ci);
 	high = low + stp->total_literal_size;
-	code[MI_LITERALS_START] = (Eterm) low;
-	code[MI_LITERALS_END] = (Eterm) high;
+	code[MI_LITERALS_START] = (UWord) low;
+	code[MI_LITERALS_END] = (UWord) high;
 	ptr = low;
 	for (i = 0; i < stp->num_literals; i++) {
 	    Uint offset;
@@ -3472,7 +3532,7 @@ freeze_code(LoaderState* stp)
 	}
 	lp = stp->literal_patches;
 	while (lp != 0) {
-	    Uint* op_ptr;
+	    UWord* op_ptr;
 	    Uint literal;
 	    Literal* lit;
 
@@ -3485,38 +3545,48 @@ freeze_code(LoaderState* stp)
 	    op_ptr[0] = literal;
 	    lp = lp->next;
 	}
-	stp->ci += stp->total_literal_size;
+	literal_end += stp->total_literal_size;
     }
     
     /*
      * Place the string table and, optionally, attributes, after the literal heap.
      */
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
-    sys_memcpy(code+stp->ci, stp->chunks[STR_CHUNK].start, strtab_size);
-    str_table = (byte *) (code+stp->ci);
+    sys_memcpy(literal_end, stp->chunks[STR_CHUNK].start, strtab_size);
+    CHKBLK(ERTS_ALC_T_CODE,code);
+    str_table = (byte *) literal_end;
     if (attr_size) {
 	byte* attr = str_table + strtab_size;
 	sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size);
-	code[MI_ATTR_PTR] = (Eterm) attr;
-	code[MI_ATTR_SIZE] = (Eterm) stp->chunks[ATTR_CHUNK].size;
+	code[MI_ATTR_PTR] = (UWord) attr;
+	code[MI_ATTR_SIZE] = (UWord) stp->chunks[ATTR_CHUNK].size;
 	decoded_size = erts_decode_ext_size(attr, attr_size, 0);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of module attributes");
  	}
 	code[MI_ATTR_SIZE_ON_HEAP] = decoded_size;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
     if (compile_size) {
 	byte* compile_info = str_table + strtab_size + attr_size;
+    CHKBLK(ERTS_ALC_T_CODE,code);
 	sys_memcpy(compile_info, stp->chunks[COMPILE_CHUNK].start,
 	       stp->chunks[COMPILE_CHUNK].size);
-	code[MI_COMPILE_PTR] = (Eterm) compile_info;
-	code[MI_COMPILE_SIZE] = (Eterm) stp->chunks[COMPILE_CHUNK].size;
+    CHKBLK(ERTS_ALC_T_CODE,code);
+	code[MI_COMPILE_PTR] = (UWord) compile_info;
+    CHKBLK(ERTS_ALC_T_CODE,code);
+	code[MI_COMPILE_SIZE] = (UWord) stp->chunks[COMPILE_CHUNK].size;
+    CHKBLK(ERTS_ALC_T_CODE,code);
 	decoded_size = erts_decode_ext_size(compile_info, compile_size, 0);
+    CHKBLK(ERTS_ALC_T_CODE,code);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of compilation information");
  	}
+    CHKBLK(ERTS_ALC_T_CODE,code);
 	code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
 
     /*
@@ -3529,9 +3599,10 @@ freeze_code(LoaderState* stp)
     while (index != 0) {
 	Uint next = code[index];
 	code[index] = BeamOpCode(op_put_string_IId);
-	code[index+2] = (Uint) (str_table + code[index+2] + code[index+1] - 1);
+	code[index+2] = (UWord) (str_table + code[index+2] + code[index+1] - 1);
 	index = next;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Go through all i_new_bs_put_strings instructions, restore the pointer to
@@ -3543,23 +3614,25 @@ freeze_code(LoaderState* stp)
     while (index != 0) {
 	Uint next = code[index];
 	code[index] = BeamOpCode(op_bs_put_string_II);
-	code[index+2] = (Uint) (str_table + code[index+2]);
+	code[index+2] = (UWord) (str_table + code[index+2]);
 	index = next;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     {
 	StringPatch* sp = stp->string_patches;
 
 	while (sp != 0) {
-	    Uint* op_ptr;
+	    UWord* op_ptr;
 	    byte* strp;
 
 	    op_ptr = code + sp->pos;
 	    strp = str_table + op_ptr[0];
-	    op_ptr[0] = (Eterm) strp;
+	    op_ptr[0] = (UWord) strp;
 	    sp = sp->next;
 	}
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Resolve all labels.
@@ -3579,10 +3652,11 @@ freeze_code(LoaderState* stp)
 	    ASSERT(this_patch < stp->ci);
 	    next_patch = code[this_patch];
 	    ASSERT(next_patch < stp->ci);
-	    code[this_patch] = (Uint) (code + value);
+	    code[this_patch] = (UWord) (code + value);
 	    this_patch = next_patch;
 	}
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Fix all catch_yf instructions.
@@ -3590,13 +3664,14 @@ freeze_code(LoaderState* stp)
     index = stp->catches;
     catches = BEAM_CATCHES_NIL;
     while (index != 0) {
-	Uint next = code[index];
+	UWord next = code[index];
 	code[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((Uint*)code[index+2], catches);
+	catches = beam_catches_cons((UWord *)code[index+2], catches);
 	code[index+2] = make_catch(catches);
 	index = next;
     }
     stp->catches = catches;
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Save the updated code pointer and code size.
@@ -3605,6 +3680,7 @@ freeze_code(LoaderState* stp)
     stp->code = code;
     stp->loaded_size = size;
 
+    CHKBLK(ERTS_ALC_T_CODE,code);
     return 1;
 
  load_error:
@@ -3638,7 +3714,7 @@ final_touch(LoaderState* stp)
 	     * callable yet.
 	     */
 	    ep->address = ep->code+3;
-	    ep->code[4] = (Eterm) stp->export[i].address;
+	    ep->code[4] = (UWord) stp->export[i].address;
 	}
     }
 
@@ -3650,14 +3726,14 @@ final_touch(LoaderState* stp)
 	Eterm mod;
 	Eterm func;
 	Uint arity;
-	Uint import;
+	UWord import;
 	Uint current;
 	Uint next;
 
 	mod = stp->import[i].module;
 	func = stp->import[i].function;
 	arity = stp->import[i].arity;
-	import = (Uint) erts_export_put(mod, func, arity);
+	import = (UWord) erts_export_put(mod, func, arity);
 	current = stp->import[i].patches;
 	while (current != 0) {
 	    ASSERT(current < stp->ci);
@@ -3675,7 +3751,7 @@ final_touch(LoaderState* stp)
 	for (i = 0; i < stp->num_lambdas; i++) {
 	    unsigned entry_label = stp->lambdas[i].label;
 	    ErlFunEntry* fe = stp->lambdas[i].fe;
-	    Eterm* code_ptr = (Eterm *) (stp->code + stp->labels[entry_label].value);
+	    UWord* code_ptr = (UWord *) (stp->code + stp->labels[entry_label].value);
 
 	    if (fe->address[0] != 0) {
 		/*
@@ -3807,7 +3883,7 @@ transform_engine(LoaderState* st)
 		if (i >= st->num_imports || st->import[i].bf == NULL)
 		    goto restart;
 		if (bif_number != -1 &&
-		    bif_export[bif_number]->code[4] != (Uint) st->import[i].bf) {
+		    bif_export[bif_number]->code[4] != (UWord) st->import[i].bf) {
 		    goto restart;
 		}
 	    }
@@ -4071,7 +4147,7 @@ load_printf(int line, LoaderState* context, char *fmt,...)
 
 
 static int
-get_int_val(LoaderState* stp, Uint len_code, Uint* result)
+get_int_val(LoaderState* stp, Uint len_code, UWord* result)
 {
     Uint count;
     Uint val;
@@ -4103,7 +4179,7 @@ get_int_val(LoaderState* stp, Uint len_code, Uint* result)
 
 
 static int
-get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result)
+get_erlang_integer(LoaderState* stp, Uint len_code, UWord* result)
 {
     Uint count;
     Sint val;
@@ -4124,11 +4200,12 @@ get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result)
 	count = len_code + 2;
     } else {
 	Uint tag;
+	UWord len_word;
 
 	ASSERT(len_code == 7);
-	GetTagAndValue(stp, tag, len_code);
+	GetTagAndValue(stp, tag, len_word);
 	VerifyTag(stp, TAG_u, tag);
-	count = len_code + 9;
+	count = len_word + 9;
     }
 
     /*
@@ -4376,7 +4453,7 @@ functions_in_module(Process* p, /* Process whose heap to use. */
 		     Eterm mod) /* Tagged atom for module. */
 {
     Module* modp;
-    Eterm* code;
+    UWord* code;
     int i;
     Uint num_functions;
     Eterm* hp;
@@ -4394,9 +4471,9 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     num_functions = code[MI_NUM_FUNCTIONS];
     hp = HAlloc(p, 5*num_functions);
     for (i = num_functions-1; i >= 0 ; i--) {
-	Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i];
-	Eterm name = func_info[3];
-	int arity = func_info[4];
+	UWord* func_info = (UWord *) code[MI_FUNCTIONS+i];
+	Eterm name = (Eterm) func_info[3];
+	int arity = (int) func_info[4];
 	Eterm tuple;
 
 	ASSERT(is_atom(name));
@@ -4419,7 +4496,7 @@ static Eterm
 native_addresses(Process* p, Eterm mod)
 {
     Module* modp;
-    Eterm* code;
+    UWord* code;
     int i;
     Eterm* hp;
     Uint num_functions;
@@ -4442,9 +4519,9 @@ native_addresses(Process* p, Eterm mod)
     hp = HAlloc(p, need);
     hp_end = hp + need;
     for (i = num_functions-1; i >= 0 ; i--) {
-	Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i];
-	Eterm name = func_info[3];
-	int arity = func_info[4];
+	UWord* func_info = (UWord *) code[MI_FUNCTIONS+i];
+	Eterm name = (Eterm) func_info[3];
+	int arity = (int) func_info[4];
 	Eterm tuple;
 
 	ASSERT(is_atom(name));
@@ -4486,7 +4563,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 	    Eterm tuple;
 	    
 	    if (ep->address == ep->code+3 &&
-		ep->code[3] == (Eterm) em_call_error_handler) {
+		ep->code[3] == (UWord) em_call_error_handler) {
 		/* There is a call to the function, but it does not exist. */ 
 		continue;
 	    }
@@ -4519,7 +4596,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
 
 {
     Module* modp;
-    Eterm* code;
+    UWord* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
@@ -4559,7 +4636,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 			    Eterm mod) /* Tagged atom for module. */
 {
     Module* modp;
-    Eterm* code;
+    UWord* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
@@ -4591,8 +4668,8 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 /*
  * Returns a pointer to {module, function, arity}, or NULL if not found.
  */
-Eterm*
-find_function_from_pc(Eterm* pc)
+UWord *
+find_function_from_pc(UWord* pc)
 {
     Range* low = modules;
     Range* high = low + num_loaded_modules;
@@ -4604,9 +4681,9 @@ find_function_from_pc(Eterm* pc)
 	} else if (pc > mid->end) {
 	    low = mid + 1;
 	} else {
-	    Eterm** low1 = (Eterm **) (mid->start + MI_FUNCTIONS);
-	    Eterm** high1 = low1 + mid->start[MI_NUM_FUNCTIONS];
-	    Eterm** mid1;
+	    UWord** low1 = (UWord **) (mid->start + MI_FUNCTIONS);
+	    UWord** high1 = low1 + mid->start[MI_NUM_FUNCTIONS];
+	    UWord** mid1;
 
 	    while (low1 < high1) {
 		mid1 = low1 + (high1-low1) / 2;
@@ -4719,10 +4796,10 @@ code_module_md5_1(Process* p, Eterm Bin)
 
 #define WORDS_PER_FUNCTION 6
 
-static Eterm*
-make_stub(Eterm* fp, Eterm mod, Eterm func, Uint arity, Uint native, Eterm OpCode)
+static UWord*
+make_stub(UWord* fp, Eterm mod, Eterm func, Uint arity, Uint native, UWord OpCode)
 {
-    fp[0] = (Eterm) BeamOp(op_i_func_info_IaaI);
+    fp[0] = (UWord) BeamOp(op_i_func_info_IaaI);
     fp[1] = native;
     fp[2] = mod;
     fp[3] = func;
@@ -4741,14 +4818,14 @@ static byte*
 stub_copy_info(LoaderState* stp,
 	       int chunk,	/* Chunk: ATTR_CHUNK or COMPILE_CHUNK */
 	       byte* info,	/* Where to store info. */
-	       Eterm* ptr_word,	/* Where to store pointer into info. */
-	       Eterm* size_word) /* Where to store size of info. */
+	       UWord* ptr_word,	/* Where to store pointer into info. */
+	       UWord* size_word) /* Where to store size of info. */
 {
     Sint decoded_size;
     Uint size = stp->chunks[chunk].size;
     if (size != 0) {
 	memcpy(info, stp->chunks[chunk].start, size);
-	*ptr_word = (Eterm) info;
+	*ptr_word = (UWord) info;
 	decoded_size = erts_decode_ext_size(info, size, 0);
 	if (decoded_size < 0) {
  	    return 0;
@@ -4791,7 +4868,7 @@ stub_read_export_table(LoaderState* stp)
 }
 
 static void
-stub_final_touch(LoaderState* stp, Eterm* fp)
+stub_final_touch(LoaderState* stp, UWord* fp)
 {
     int i;
     int n = stp->num_exps;
@@ -4978,12 +5055,12 @@ Eterm
 erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 {
     LoaderState state;
-    Eterm Funcs;
-    Eterm Patchlist;
+    UWord Funcs;
+    UWord Patchlist;
     Eterm* tp;
-    Eterm* code = NULL;
-    Eterm* ptrs;
-    Eterm* fp;
+    UWord* code = NULL;
+    UWord* ptrs;
+    UWord* fp;
     byte* info;
     Uint ci;
     int n;
@@ -5072,7 +5149,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Allocate memory for the stub module.
      */
 
-    code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(Eterm);
+    code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(UWord);
     code_size += state.chunks[ATTR_CHUNK].size;
     code_size += state.chunks[COMPILE_CHUNK].size;
     code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size);
@@ -5141,7 +5218,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	 * Set the pointer and make the stub. Put a return instruction
 	 * as the body until we know what kind of trap we should put there.
 	 */
-	ptrs[i] = (Eterm) fp;
+	ptrs[i] = (UWord) fp;
 #ifdef HIPE
 	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
 #else
@@ -5154,8 +5231,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Insert the last pointer and the int_code_end instruction.
      */
 
-    ptrs[i] = (Eterm) fp;
-    *fp++ = (Eterm) BeamOp(op_int_code_end);
+    ptrs[i] = (UWord) fp;
+    *fp++ = (UWord) BeamOp(op_int_code_end);
 
     /*
      * Copy attributes and compilation information.
@@ -5222,9 +5299,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 
 #undef WORDS_PER_FUNCTION
 
-static int safe_mul(Uint a, Uint b, Uint* resp)
+static int safe_mul(UWord a, UWord b, UWord* resp)
 {
-    Uint res = a * b;
+    UWord res = a * b;
     *resp = res;
 
     if (b == 0) {