Merge PR-1621 from margnus1/hipe-literal-tag

HiPE: Support for literal tag, tests and bugfixes

24 files changed, 366 insertions, 151 deletions

diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index 5ec6b6b44b..7d42d1b396 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -55,9 +55,6 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 #if defined(ARCH_64)
 #  define TAG_PTR_MASK__	0x7
 #  if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-#    ifdef HIPE
-#      error Hipe on 64-bit needs a real mmap as it does not support the literal tag
-#    endif
 #    define TAG_LITERAL_PTR	0x4
 #  else
 #    undef TAG_LITERAL_PTR
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index 7e3851f20e..625d8486fd 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -222,7 +222,7 @@ standard_bif_interface_2(nbif_rethrow, hipe_rethrow)
 standard_bif_interface_3(nbif_find_na_or_make_stub, hipe_find_na_or_make_stub)
 standard_bif_interface_2(nbif_nonclosure_address, hipe_nonclosure_address)
 nocons_nofail_primop_interface_0(nbif_fclearerror_error, hipe_fclearerror_error)
-standard_bif_interface_2(nbif_is_divisible, hipe_is_divisible)
+noproc_primop_interface_2(nbif_is_divisible, hipe_is_divisible)
 noproc_primop_interface_1(nbif_is_unicode, hipe_is_unicode)
 
 /*
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index 6ea120c65c..de00994d64 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -462,6 +462,15 @@ static const struct rts_param rts_params[] = {
       0
 #endif
     },
+    /* This flag is always defined, but its value is configuration-dependent. */
+    { 17, "ERTS_USE_LITERAL_TAG",
+      1,
+#if defined(TAG_LITERAL_PTR)
+      1
+#else
+      0
+#endif
+    },
     /* This parameter is always defined, but its value depends on ERTS_SMP. */
     { 19, "MSG_MESSAGE",
       1, offsetof(struct erl_mesg, m[0])
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index f5471285c2..99c34532b9 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -497,16 +497,12 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg)
     return 1;
 }
 
-/* Called via standard_bif_interface_2 */
-BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2)
+Uint hipe_is_divisible(Uint dividend, Uint divisor)
 {
-    /* Arguments are Eterm-sized unsigned integers */
-    Uint dividend = BIF_ARG_1;
-    Uint divisor = BIF_ARG_2;
     if (dividend % divisor) {
-	BIF_ERROR(BIF_P, BADARG);
+        return 0;
     } else {
-	return NIL;
+        return 1;
     }
 }
 
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index a4919f4886..d5081b8438 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -68,7 +68,7 @@ AEXTERN(Eterm,nbif_bs_put_utf16le,(Process*,Eterm,byte*,unsigned int));
 AEXTERN(Eterm,nbif_bs_get_utf16,(void));
 AEXTERN(Uint,nbif_is_unicode,(Eterm));
 AEXTERN(Eterm,nbif_bs_validate_unicode_retract,(void));
-AEXTERN(void,nbif_is_divisible,(Process*,Uint,Uint));
+AEXTERN(Uint,nbif_is_divisible,(Uint,Uint));
 
 AEXTERN(void,nbif_select_msg,(Process*));
 AEXTERN(Eterm,nbif_cmp_2,(void));
@@ -94,7 +94,7 @@ BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3);
 Uint hipe_is_unicode(Eterm);
 struct erl_bin_match_buffer;
 int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm);
-BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2);
+Uint hipe_is_divisible(Uint, Uint);
 
 #ifdef NO_FPE_SIGNALS
 AEXTERN(void,nbif_emulate_fpe,(Process*));
diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl
index 167f5c67bb..cb4c1cfbd3 100644
--- a/lib/hipe/icode/hipe_beam_to_icode.erl
+++ b/lib/hipe/icode/hipe_beam_to_icode.erl
@@ -794,7 +794,7 @@ trans_fun([{bs_append,{f,Lbl},Size,W,R,U,Binary,{field_flags,F},Dst}|
   SizeArg = trans_arg(Size),
   BinArg = trans_arg(Binary),
   IcodeDst = mk_var(Dst),
-  Offset = mk_var(reg),
+  Offset = mk_var(reg_gcsafe),
   Base = mk_var(reg),
   trans_bin_call({hipe_bs_primop,{bs_append,W,R,U,F}},Lbl,[SizeArg,BinArg],
 		[IcodeDst,Base,Offset],
@@ -805,7 +805,7 @@ trans_fun([{bs_private_append,{f,Lbl},Size,U,Binary,{field_flags,F},Dst}|
   SizeArg = trans_arg(Size),
   BinArg = trans_arg(Binary),
   IcodeDst = mk_var(Dst),
-  Offset = mk_var(reg),
+  Offset = mk_var(reg_gcsafe),
   Base = mk_var(reg),
   trans_bin_call({hipe_bs_primop,{bs_private_append,U,F}},
 		 Lbl,[SizeArg,BinArg],
@@ -844,7 +844,7 @@ trans_fun([{bs_init2,{f,Lbl},Size,_Words,_LiveRegs,{field_flags,Flags0},X}|
 	   Instructions], Env) ->
   Dst = mk_var(X),
   Flags = resolve_native_endianess(Flags0),
-  Offset = mk_var(reg),
+  Offset = mk_var(reg_gcsafe),
   Base = mk_var(reg),
   {Name, Args} =
     case Size of
@@ -860,7 +860,7 @@ trans_fun([{bs_init_bits,{f,Lbl},Size,_Words,_LiveRegs,{field_flags,Flags0},X}|
 	   Instructions], Env) ->
   Dst = mk_var(X),
   Flags = resolve_native_endianess(Flags0),
-  Offset = mk_var(reg),
+  Offset = mk_var(reg_gcsafe),
   Base = mk_var(reg),
   {Name, Args} =
     case Size of
@@ -1505,7 +1505,10 @@ clone_dst(Dest) ->
   New = 
     case hipe_icode:is_reg(Dest) of
       true ->
-	mk_var(reg);
+	case hipe_icode:reg_is_gcsafe(Dest) of
+	  true -> mk_var(reg_gcsafe);
+	  false -> mk_var(reg)
+	end;
       false ->
 	true = hipe_icode:is_var(Dest),	      
 	mk_var(new)
@@ -2126,7 +2129,12 @@ mk_var(reg) ->
   T = hipe_gensym:new_var(icode),
   V = (5*T)+4,
   hipe_gensym:update_vrange(icode,V),
-  hipe_icode:mk_reg(V).
+  hipe_icode:mk_reg(V);
+mk_var(reg_gcsafe) ->
+  T = hipe_gensym:new_var(icode),
+  V = (5*T)+4, % same namespace as 'reg'
+  hipe_gensym:update_vrange(icode,V),
+  hipe_icode:mk_reg_gcsafe(V).
 
 %%-----------------------------------------------------------------------
 %% Make an icode label of proper type
diff --git a/lib/hipe/icode/hipe_icode.erl b/lib/hipe/icode/hipe_icode.erl
index 24b7ac4783..bc3403b0c5 100644
--- a/lib/hipe/icode/hipe_icode.erl
+++ b/lib/hipe/icode/hipe_icode.erl
@@ -515,10 +515,12 @@
 	 annotate_variable/2,  %% annotate_var_or_reg(VarOrReg, Type)
 	 unannotate_variable/1,%% unannotate_var_or_reg(VarOrReg)
 	 mk_reg/1,               %% mk_reg(Id)
+	 mk_reg_gcsafe/1,        %% mk_reg_gcsafe(Id)
 	 mk_fvar/1,              %% mk_fvar(Id)
 	 mk_new_var/0,           %% mk_new_var()
 	 mk_new_fvar/0,          %% mk_new_fvar()
 	 mk_new_reg/0,           %% mk_new_reg()
+	 mk_new_reg_gcsafe/0,    %% mk_new_reg_gcsafe()
 	 mk_phi/1,               %% mk_phi(Id)
 	 mk_phi/2                %% mk_phi(Id, ArgList)
 	]).
@@ -1260,14 +1262,22 @@ is_var(_) -> false.
 -spec mk_reg(non_neg_integer()) -> #icode_variable{kind::'reg'}.
 mk_reg(V) -> #icode_variable{name=V, kind=reg}.
 
--spec reg_name(#icode_variable{kind::'reg'}) -> non_neg_integer().
-reg_name(#icode_variable{name=Name, kind=reg}) -> Name.
+-spec mk_reg_gcsafe(non_neg_integer()) -> #icode_variable{kind::'reg_gcsafe'}.
+mk_reg_gcsafe(V) -> #icode_variable{name=V, kind=reg_gcsafe}.
 
--spec reg_is_gcsafe(#icode_variable{kind::'reg'}) -> 'false'.
-reg_is_gcsafe(#icode_variable{kind=reg}) -> false. % for now
+-spec reg_name(#icode_variable{kind::'reg'|'reg_gcsafe'})
+	      -> non_neg_integer().
+reg_name(#icode_variable{name=Name, kind=reg})        -> Name;
+reg_name(#icode_variable{name=Name, kind=reg_gcsafe}) -> Name.
+
+-spec reg_is_gcsafe(#icode_variable{kind::'reg'}) -> 'false';
+		   (#icode_variable{kind::'reg_gcsafe'}) -> 'true'.
+reg_is_gcsafe(#icode_variable{kind=reg})        -> false;
+reg_is_gcsafe(#icode_variable{kind=reg_gcsafe}) -> true.
 
 -spec is_reg(icode_argument()) -> boolean().
-is_reg(#icode_variable{kind=reg}) -> true;
+is_reg(#icode_variable{kind=reg})        -> true;
+is_reg(#icode_variable{kind=reg_gcsafe}) -> true;
 is_reg(_) -> false.
 
 -spec mk_fvar(non_neg_integer()) -> #icode_variable{kind::'fvar'}.
@@ -1676,6 +1686,16 @@ mk_new_reg() ->
   mk_reg(hipe_gensym:get_next_var(icode)).
 
 %%
+%% @doc Makes a new gcsafe register; that is, a register that is allowed to be
+%% live over calls and other operations that might cause GCs and thus move heap
+%% data around.
+%%
+
+-spec mk_new_reg_gcsafe() -> icode_reg().
+mk_new_reg_gcsafe() ->
+  mk_reg_gcsafe(hipe_gensym:get_next_var(icode)).
+
+%%
 %% @doc Makes a new label.
 %%
 
diff --git a/lib/hipe/icode/hipe_icode.hrl b/lib/hipe/icode/hipe_icode.hrl
index 380ddd8371..7ed80a9ed4 100644
--- a/lib/hipe/icode/hipe_icode.hrl
+++ b/lib/hipe/icode/hipe_icode.hrl
@@ -41,9 +41,9 @@
 
 -type variable_annotation() :: {atom(), any(), fun((any()) -> string())}.
 
--record(icode_variable, {name :: non_neg_integer(), 
-			 kind :: 'var' | 'reg' | 'fvar',
-			 annotation = [] :: [] | variable_annotation()}). 
+-record(icode_variable, {name :: non_neg_integer(),
+			 kind :: 'var' | 'reg' | 'reg_gcsafe' | 'fvar',
+			 annotation = [] :: [] | variable_annotation()}).
 
 %%---------------------------------------------------------------------
 %% Type declarations for Icode instructions
@@ -66,7 +66,7 @@
 -type icode_funcall()   :: mfa() | icode_primop().
 
 -type icode_var()	:: #icode_variable{kind::'var'}.
--type icode_reg()	:: #icode_variable{kind::'reg'}.
+-type icode_reg()	:: #icode_variable{kind::'reg'|'reg_gcsafe'}.
 -type icode_fvar()	:: #icode_variable{kind::'fvar'}.
 -type icode_argument()	:: #icode_const{} | #icode_variable{}.
 -type icode_term_arg()	:: icode_var() | #icode_const{}.
diff --git a/lib/hipe/icode/hipe_icode_liveness.erl b/lib/hipe/icode/hipe_icode_liveness.erl
index 51e2855108..e61529a1bb 100644
--- a/lib/hipe/icode/hipe_icode_liveness.erl
+++ b/lib/hipe/icode/hipe_icode_liveness.erl
@@ -77,6 +77,7 @@ print_var(#icode_variable{name=V, kind=Kind, annotation=T}) ->
   case Kind of
     var -> io:format("v~p", [V]);
     reg -> io:format("r~p", [V]);
+    reg_gcsafe -> io:format("rs~p", [V]);
     fvar -> io:format("fv~p", [V])
   end,
   case T of
diff --git a/lib/hipe/icode/hipe_icode_pp.erl b/lib/hipe/icode/hipe_icode_pp.erl
index 5b017dca32..33d1e62884 100644
--- a/lib/hipe/icode/hipe_icode_pp.erl
+++ b/lib/hipe/icode/hipe_icode_pp.erl
@@ -230,7 +230,10 @@ pp_arg(Dev, Arg) ->
 	  case hipe_icode:is_reg(Arg) of
 	    true ->
 	      N = hipe_icode:reg_name(Arg),
-	      io:format(Dev, "r~p", [N]);
+	      case hipe_icode:reg_is_gcsafe(Arg) of
+		true  -> io:format(Dev, "rs~p", [N]);
+		false -> io:format(Dev, "r~p", [N])
+	      end;
 	    false ->
 	      N = hipe_icode:fvar_name(Arg),
 	      io:format(Dev, "fv~p", [N])
diff --git a/lib/hipe/main/hipe.app.src b/lib/hipe/main/hipe.app.src
index fb750dd418..66008a4178 100644
--- a/lib/hipe/main/hipe.app.src
+++ b/lib/hipe/main/hipe.app.src
@@ -179,6 +179,7 @@
 	     hipe_rtl_to_sparc,
 	     hipe_rtl_to_x86,
 	     hipe_rtl_varmap,
+	     hipe_rtl_verify_gcsafe,
 	     hipe_segment_trees,
 	     hipe_sdi,
 	     hipe_sparc,
diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl
index f5f5bf5830..acb9b7b062 100644
--- a/lib/hipe/main/hipe.erl
+++ b/lib/hipe/main/hipe.erl
@@ -1414,6 +1414,7 @@ opt_keys() ->
      use_clusters,
      use_jumptable,
      verbose,
+     verify_gcsafe,
      %% verbose_spills,
      x87].
 
@@ -1510,7 +1511,8 @@ opt_negations() ->
    {no_use_callgraph, use_callgraph},
    {no_use_clusters, use_clusters},
    {no_use_inline_atom_search, use_inline_atom_search},
-   {no_use_indexing, use_indexing}].
+   {no_use_indexing, use_indexing},
+   {no_verify_gcsafe, verify_gcsafe}].
 
 %% Don't use negative forms in right-hand sides of aliases and expansions!
 %% We only expand negations once, before the other expansions are done.
diff --git a/lib/hipe/main/hipe_main.erl b/lib/hipe/main/hipe_main.erl
index dca6fddec3..4b5eb4c63e 100644
--- a/lib/hipe/main/hipe_main.erl
+++ b/lib/hipe/main/hipe_main.erl
@@ -410,6 +410,11 @@ icode_to_rtl(MFA, Icode, Options, Servers) ->
         hipe_llvm_liveness:analyze(RtlCfg4)
     end,
   pp(RtlCfg5, MFA, rtl, pp_rtl, Options, Servers),
+  case proplists:get_bool(verify_gcsafe, Options) of
+    false -> ok;
+    true ->
+      ok = hipe_rtl_verify_gcsafe:check(RtlCfg5)
+  end,
   LinearRTL1 = hipe_rtl_cfg:linearize(RtlCfg5),
   LinearRTL2 = hipe_rtl_cleanup_const:cleanup(LinearRTL1),
   %% hipe_rtl:pp(standard_io, LinearRTL2),
diff --git a/lib/hipe/rtl/Makefile b/lib/hipe/rtl/Makefile
index 5abc9ec049..becdd0b7d8 100644
--- a/lib/hipe/rtl/Makefile
+++ b/lib/hipe/rtl/Makefile
@@ -50,7 +50,7 @@ HIPE_MODULES = hipe_rtl hipe_rtl_cfg \
 	  hipe_rtl_ssa hipe_rtl_ssa_const_prop \
 	  hipe_rtl_cleanup_const hipe_rtl_symbolic hipe_rtl_lcm \
 	  hipe_rtl_ssapre hipe_rtl_binary hipe_rtl_ssa_avail_expr \
-	  hipe_rtl_arch hipe_tagscheme
+	  hipe_rtl_arch hipe_tagscheme hipe_rtl_verify_gcsafe
 else
 HIPE_MODULES =
 endif
diff --git a/lib/hipe/rtl/hipe_rtl.erl b/lib/hipe/rtl/hipe_rtl.erl
index 04c9728d5c..33027f3259 100644
--- a/lib/hipe/rtl/hipe_rtl.erl
+++ b/lib/hipe/rtl/hipe_rtl.erl
@@ -1740,7 +1740,10 @@ pp_reg(Dev, Arg) ->
     true ->
       pp_hard_reg(Dev, reg_index(Arg));
     false ->
-      io:format(Dev, "r~w", [reg_index(Arg)])
+      case reg_is_gcsafe(Arg) of
+        true -> io:format(Dev, "rs~w", [reg_index(Arg)]);
+        false -> io:format(Dev, "r~w", [reg_index(Arg)])
+      end
   end.
 
 pp_var(Dev, Arg) ->
diff --git a/lib/hipe/rtl/hipe_rtl_binary_construct.erl b/lib/hipe/rtl/hipe_rtl_binary_construct.erl
index bc215e3abe..5b89d4946a 100644
--- a/lib/hipe/rtl/hipe_rtl_binary_construct.erl
+++ b/lib/hipe/rtl/hipe_rtl_binary_construct.erl
@@ -359,7 +359,8 @@ not_writable_code(Bin, SizeReg, Dst, Base, Offset, Unit,
 allocate_writable(Dst, Base, UsedBytes, TotBytes, TotSize) ->
   Zero = hipe_rtl:mk_imm(0),
   [NextLbl] = create_lbls(1),
-  [EndSubSize, EndSubBitSize, ProcBin] = create_regs(3),
+  [EndSubSize, EndSubBitSize] = create_regs(2),
+  [ProcBin] = create_unsafe_regs(1),
   [hipe_rtl:mk_call([Base], bs_allocate, [UsedBytes],
 		    hipe_rtl:label_name(NextLbl), [], not_remote),
    NextLbl,
@@ -586,12 +587,12 @@ const_init2(Size, Dst, Base, Offset, TrueLblName) ->
     false ->
       ByteSize = hipe_rtl:mk_new_reg(),
       [hipe_rtl:mk_gctest(?PROC_BIN_WORDSIZE+?SUB_BIN_WORDSIZE),
-       hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
        hipe_rtl:mk_move(ByteSize, hipe_rtl:mk_imm(Size)),
        hipe_rtl:mk_call([Base], bs_allocate, [ByteSize],
 			hipe_rtl:label_name(NextLbl), [], not_remote),
        NextLbl,
        hipe_tagscheme:create_refc_binary(Base, ByteSize, Dst),
+       hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
        hipe_rtl:mk_goto(TrueLblName)]
   end.
 
@@ -634,13 +635,12 @@ var_init2(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName
   Log2WordSize = hipe_rtl_arch:log2_word_size(),
   WordSize = hipe_rtl_arch:word_size(),
   [ContLbl, HeapLbl, REFCLbl, NextLbl] = create_lbls(4),
-  [USize, Tmp] = create_unsafe_regs(2),
+  [USize, Tmp] = create_regs(2),
   [get_word_integer(Size, USize, SystemLimitLblName, FalseLblName),
    hipe_rtl:mk_branch(USize, leu, hipe_rtl:mk_imm(?MAX_BINSIZE),
 		      hipe_rtl:label_name(ContLbl),
 		      SystemLimitLblName),
    ContLbl,
-   hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
    hipe_rtl:mk_branch(USize, leu, hipe_rtl:mk_imm(?MAX_HEAP_BIN_SIZE),
 		      hipe_rtl:label_name(HeapLbl),
 		      hipe_rtl:label_name(REFCLbl)),
@@ -650,6 +650,7 @@ var_init2(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName
    hipe_rtl:mk_alu(Tmp, Tmp, add, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE)),
    hipe_rtl:mk_gctest(Tmp),
    hipe_tagscheme:create_heap_binary(Base, USize, Dst),
+   hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
    hipe_rtl:mk_goto(TrueLblName),
    REFCLbl,
    hipe_rtl:mk_gctest(?PROC_BIN_WORDSIZE+?SUB_BIN_WORDSIZE),
@@ -657,6 +658,7 @@ var_init2(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName
 		    hipe_rtl:label_name(NextLbl), [], not_remote),
    NextLbl,
    hipe_tagscheme:create_refc_binary(Base, USize, Dst),
+   hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
    hipe_rtl:mk_goto(TrueLblName)].
 
 var_init_bits(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName) ->
@@ -863,7 +865,7 @@ get_base_offset_size(Binary, SrcBase, SrcOffset, SrcSize, FLName) ->
    JoinLbl,
    hipe_tagscheme:test_heap_binary(Orig, HeapLblName, REFCLblName),
    HeapLbl,
-   hipe_rtl:mk_alu(SrcBase, Orig, add, hipe_rtl:mk_imm(?HEAP_BIN_DATA-2)),
+   hipe_tagscheme:get_field_addr_from_term({heap_bin, {data, 0}}, Orig, SrcBase),
    hipe_rtl:mk_goto(EndLblName),
    REFCLbl,
    hipe_tagscheme:get_field_from_term({proc_bin,bytes}, Orig, SrcBase),
@@ -1210,6 +1212,12 @@ is_divisible(Dividend, Divisor, SuccLbl, FailLbl) ->
       [hipe_rtl:mk_branch(Dividend, 'and', Mask, eq, SuccLbl, FailLbl, 0.99)];
     false ->
       %% We need division, fall back to a primop
-      [hipe_rtl:mk_call([], is_divisible, [Dividend, hipe_rtl:mk_imm(Divisor)],
-			SuccLbl, FailLbl, not_remote)]
+      [Tmp] = create_regs(1),
+      RetLbl = hipe_rtl:mk_new_label(),
+      [hipe_rtl:mk_call([Tmp], is_divisible,
+                        [Dividend, hipe_rtl:mk_imm(Divisor)],
+                        hipe_rtl:label_name(RetLbl), [], not_remote),
+       RetLbl,
+       hipe_rtl:mk_branch(Tmp, ne, hipe_rtl:mk_imm(0),
+                          SuccLbl, FailLbl, 0.99)]
   end.
diff --git a/lib/hipe/rtl/hipe_rtl_binary_match.erl b/lib/hipe/rtl/hipe_rtl_binary_match.erl
index 362a52f8fe..4575213838 100644
--- a/lib/hipe/rtl/hipe_rtl_binary_match.erl
+++ b/lib/hipe/rtl/hipe_rtl_binary_match.erl
@@ -730,7 +730,7 @@ get_base(Orig,Base) ->
   [hipe_tagscheme:test_heap_binary(Orig, hipe_rtl:label_name(HeapLbl),
 				   hipe_rtl:label_name(REFCLbl)),
    HeapLbl,
-   hipe_rtl:mk_alu(Base, Orig, 'add', hipe_rtl:mk_imm(?HEAP_BIN_DATA-2)),
+   hipe_tagscheme:get_field_addr_from_term({heap_bin, {data, 0}}, Orig, Base),
    hipe_rtl:mk_goto(hipe_rtl:label_name(EndLbl)),
    REFCLbl,
    get_field_from_term({proc_bin, flags}, Orig, Flags),
@@ -740,7 +740,7 @@ get_base(Orig,Base) ->
    WritableLbl,
    hipe_rtl:mk_call([], emasculate_binary, [Orig], [], [], 'not_remote'),
    NotWritableLbl,
-   hipe_rtl:mk_load(Base, Orig, hipe_rtl:mk_imm(?PROC_BIN_BYTES-2)),
+   get_field_from_term({proc_bin, bytes}, Orig, Base),
    EndLbl].
 
 extract_matchstate_var(binsize, Ms) ->
@@ -842,12 +842,12 @@ make_dyn_prep(SizeReg, CCode) ->
 %%------------------------------------------------------------------------
 
 get_unaligned_int(Dst1, Size, Base, Offset, Shiftr, Type, TrueLblName) ->
-  [Reg] = create_regs(1),
+  [Reg] = create_gcsafe_regs(1),
   [get_maybe_unaligned_int_to_reg(Reg, Size, Base, Offset, Shiftr, Type),
    do_bignum_code(Size, Type, Reg, Dst1, TrueLblName)].
 
 get_maybe_unaligned_int_to_reg(Reg, Size, Base, Offset, Shiftr, Type) ->
-  [LowBits] = create_regs(1),
+  [LowBits] = create_gcsafe_regs(1),
   [AlignedLbl, UnAlignedLbl, EndLbl] = create_lbls(3),
   [hipe_rtl:mk_alub(LowBits, Offset, 'and', hipe_rtl:mk_imm(?LOW_BITS),
 		    eq, hipe_rtl:label_name(AlignedLbl), 
@@ -1001,7 +1001,7 @@ do_bignum_code(Size, {Signedness,_}, Src, Dst1, TrueLblName)
     end.
 
 signed_bignum(Dst1, Src, TrueLblName) ->
-  Tmp1 = hipe_rtl:mk_new_reg(),
+  Tmp1 = hipe_rtl:mk_new_reg_gcsafe(),
   BignumLabel = hipe_rtl:mk_new_label(),
   [hipe_tagscheme:realtag_fixnum(Dst1, Src),
    hipe_tagscheme:realuntag_fixnum(Tmp1, Dst1),
diff --git a/lib/hipe/rtl/hipe_rtl_cleanup_const.erl b/lib/hipe/rtl/hipe_rtl_cleanup_const.erl
index bfa6b9682e..00cc2bcb37 100644
--- a/lib/hipe/rtl/hipe_rtl_cleanup_const.erl
+++ b/lib/hipe/rtl/hipe_rtl_cleanup_const.erl
@@ -69,9 +69,9 @@ cleanup_instr([Const|Left], I, Acc) ->
   case I of
     X when is_record(X, fp_unop) orelse is_record(X, fp) ->
       Fdst = hipe_rtl:mk_new_fpreg(),
-      Fconv = hipe_tagscheme:unsafe_untag_float(Fdst, Dst),		  
+      Fconv = lists:flatten(hipe_tagscheme:unsafe_untag_float(Fdst, Dst)),
       NewI = hipe_rtl:subst_uses([{Const, Fdst}], I),
-      cleanup_instr(Left, NewI, Fconv ++ [Load|Acc]);
+      cleanup_instr(Left, NewI, lists:reverse(Fconv, [Load|Acc]));
     _ ->
       NewI = hipe_rtl:subst_uses([{Const, Dst}], I),
       cleanup_instr(Left, NewI, [Load|Acc])
diff --git a/lib/hipe/rtl/hipe_rtl_lcm.erl b/lib/hipe/rtl/hipe_rtl_lcm.erl
index 9dcdd05fb1..af39c9a0a4 100644
--- a/lib/hipe/rtl/hipe_rtl_lcm.erl
+++ b/lib/hipe/rtl/hipe_rtl_lcm.erl
@@ -182,42 +182,41 @@ delete_exprs(Code, _, _, []) ->
   Code;
 delete_exprs(Code, ExprMap, IdMap, [ExprId|Exprs]) ->
   Expr = expr_id_map_get_expr(IdMap, ExprId),
-  %% Perform a foldl that goes through the code and deletes all
-  %% occurences of the expression.
-  NewCode =
-    lists:reverse
-      (lists:foldl(fun(CodeExpr, Acc) ->
-                     case is_expr(CodeExpr) of
-                       true ->
-                         case expr_clear_dst(CodeExpr) =:= Expr of
-                           true ->
-                             pp_debug("  Deleting:         ", []),
-                             pp_debug_instr(CodeExpr),
-                             %% Lookup expression entry.
-                             Defines = 
-                               case expr_map_lookup(ExprMap, Expr) of
-                                 {value, {_, _, Defs}} -> 
-				   Defs;
-                                 none -> 
-                                   exit({?MODULE, expr_map_lookup,
-                                         "expression missing"})
-                               end,
-                             MoveCode = 
-                               mk_expr_move_instr(hipe_rtl:defines(CodeExpr),
-						  Defines),
-                             pp_debug("    Replacing with: ", []),
-                             pp_debug_instr(MoveCode),
-                             [MoveCode|Acc];
-                           false ->
-                             [CodeExpr|Acc]
-                         end;
-		       false ->
-                             [CodeExpr|Acc]
-                     end
-                   end, 
-		   [], Code)),
+  %% Lookup expression entry.
+  {value, {_, _, Defines}} = expr_map_lookup(ExprMap, Expr),
+  %% Go through the code and deletes all occurences of the expression.
+  NewCode = delete_expr(Code, Expr, Defines, []),
   delete_exprs(NewCode, ExprMap, IdMap, Exprs).
 
+delete_expr([], _Expr, _Defines, Acc) -> lists:reverse(Acc);
+delete_expr([CodeExpr|Code], Expr, Defines, Acc) ->
+  case exp_kill_expr(CodeExpr, [Expr]) of
+    [] -> % Expr was killed; deleting stops here
+      pp_debug("  Stopping before:  ", []),
+      pp_debug_instr(CodeExpr),
+      lists:reverse(Acc, [CodeExpr|Code]);
+    [Expr] ->
+      NewCodeExpr =
+        case is_expr(CodeExpr) of
+          true ->
+            case expr_clear_dst(CodeExpr) =:= Expr of
+              true ->
+                pp_debug("  Deleting:         ", []),
+                pp_debug_instr(CodeExpr),
+                MoveCode = mk_expr_move_instr(hipe_rtl:defines(CodeExpr),
+                                              Defines),
+                pp_debug("    Replacing with: ", []),
+                pp_debug_instr(MoveCode),
+                MoveCode;
+              false ->
+                CodeExpr
+            end;
+          false ->
+            CodeExpr
+        end,
+      delete_expr(Code, Expr, Defines, [NewCodeExpr|Acc])
+  end.
+
 %%=============================================================================
 %% Goes through the given list of expressions and inserts them at 
 %% appropriate places in the code.
@@ -226,13 +225,12 @@ insert_exprs(CFG, _, _, _, _, BetweenMap, []) ->
 insert_exprs(CFG, Pred, Succ, ExprMap, IdMap, BetweenMap, [ExprId|Exprs]) ->
   Expr = expr_id_map_get_expr(IdMap, ExprId),
   Instr = expr_map_get_instr(ExprMap, Expr),
-  case hipe_rtl_cfg:succ(CFG, Pred) of
-    [_] ->
+  case try_insert_expr_last(CFG, Pred, Instr) of
+    {ok, NewCFG} ->
       pp_debug("  Inserted last: ", []),
       pp_debug_instr(Instr),
-      NewCFG = insert_expr_last(CFG, Pred, Instr),
       insert_exprs(NewCFG, Pred, Succ, ExprMap, IdMap, BetweenMap, Exprs);
-    _ ->
+    not_safe ->
       case hipe_rtl_cfg:pred(CFG, Succ) of
         [_] ->
 	  pp_debug("  Inserted first: ", []),
@@ -252,25 +250,31 @@ insert_exprs(CFG, Pred, Succ, ExprMap, IdMap, BetweenMap, [ExprId|Exprs]) ->
 %% Recursively goes through the code in a block and returns a new block
 %% with the new code inserted second to last (assuming the last expression
 %% is a branch operation).
-insert_expr_last(CFG0, Label, Instr) ->
-  Code0 = hipe_bb:code(hipe_rtl_cfg:bb(CFG0, Label)),
-  %% FIXME: Use hipe_bb:butlast() instead?
-  Code1 = insert_expr_last_work(Label, Instr, Code0),
-  hipe_rtl_cfg:bb_add(CFG0, Label, hipe_bb:mk_bb(Code1)).
+try_insert_expr_last(CFG0, Label, Instr) ->
+  case hipe_rtl_cfg:succ(CFG0, Label) of
+    [_] ->
+      Code0 = hipe_bb:code(hipe_rtl_cfg:bb(CFG0, Label)),
+      case insert_expr_last_work(Instr, Code0) of
+        not_safe -> not_safe;
+        Code1 ->
+          {ok, hipe_rtl_cfg:bb_add(CFG0, Label, hipe_bb:mk_bb(Code1))}
+      end;
+    _ -> not_safe
+  end.
 
 %%=============================================================================
 %% Recursively goes through the code in a block and returns a new block
 %% with the new code inserted second to last (assuming the last expression
 %% is a branch operation).
-insert_expr_last_work(_, Instr, []) ->
-  %% This case should not happen since this means that block was completely 
-  %% empty when the function was called. For compatibility we insert it last.
-  [Instr];
-insert_expr_last_work(_, Instr, [Code1]) ->
+insert_expr_last_work(_Instr, [#call{}]) ->
+  %% Call instructions clobber all expressions; we musn't insert the expression
+  %% before it
+  not_safe;
+insert_expr_last_work(Instr, [Code1]) ->
   %% We insert the code next to last.
   [Instr, Code1];
-insert_expr_last_work(Label, Instr, [Code|Codes]) ->
-  [Code|insert_expr_last_work(Label, Instr, Codes)].
+insert_expr_last_work(Instr, [Code|Codes]) ->
+  [Code|insert_expr_last_work(Instr, Codes)].
 
 %%=============================================================================
 %% Inserts expression first in the block for the given label.
@@ -305,7 +309,8 @@ insert_expr_between(CFG0, BetweenMap, Pred, Succ, Instr) ->
     {value, Label} ->
       pp_debug("    Using existing new bb for edge (~w,~w) with label ~w~n", 
 	       [Pred, Succ, Label]),
-      {insert_expr_last(CFG0, Label, Instr), BetweenMap}
+      {ok, NewCfg} = try_insert_expr_last(CFG0, Label, Instr),
+      {NewCfg, BetweenMap}
   end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
diff --git a/lib/hipe/rtl/hipe_rtl_varmap.erl b/lib/hipe/rtl/hipe_rtl_varmap.erl
index 375a8f85c0..f34c66ab85 100644
--- a/lib/hipe/rtl/hipe_rtl_varmap.erl
+++ b/lib/hipe/rtl/hipe_rtl_varmap.erl
@@ -105,7 +105,7 @@ icode_var2rtl_var(Var, Map) ->
 	{reg, IsGcSafe} ->
 	  NewVar =
 	    case IsGcSafe of
-	      %% true -> hipe_rtl:mk_new_reg_gcsafe();
+              true -> hipe_rtl:mk_new_reg_gcsafe();
 	      false -> hipe_rtl:mk_new_reg()
 	    end,
 	  {NewVar, insert(Var, NewVar, Map)}
diff --git a/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl
new file mode 100644
index 0000000000..c3f20bfec1
--- /dev/null
+++ b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl
@@ -0,0 +1,88 @@
+%% -*- mode: erlang; erlang-indent-level: 2 -*-
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+-module(hipe_rtl_verify_gcsafe).
+
+-export([check/1]).
+
+-include("../flow/cfg.hrl").              %% needed for the specs
+-include("hipe_rtl.hrl").
+
+check(CFG) ->
+  Liveness = hipe_rtl_liveness:analyze(CFG),
+  put({?MODULE, 'fun'}, CFG#cfg.info#cfg_info.'fun'),
+  lists:foreach(
+    fun(Lb) ->
+        put({?MODULE, label}, Lb),
+        Liveout = hipe_rtl_liveness:liveout(Liveness, Lb),
+        BB = hipe_rtl_cfg:bb(CFG, Lb),
+        check_instrs(lists:reverse(hipe_bb:code(BB)), Liveout)
+    end, hipe_rtl_cfg:labels(CFG)),
+  erase({?MODULE, 'fun'}),
+  erase({?MODULE, label}),
+  erase({?MODULE, instr}),
+  ok.
+
+check_instrs([], _Livein) -> ok;
+check_instrs([I|Is], LiveOut) ->
+  Def = ordsets:from_list(hipe_rtl:defines(I)),
+  Use = ordsets:from_list(hipe_rtl:uses(I)),
+  LiveOver = ordsets:subtract(LiveOut, Def),
+  LiveIn = ordsets:union(LiveOver, Use),
+  case (hipe_rtl:is_call(I)
+        andalso not safe_primop(hipe_rtl:call_fun(I)))
+    orelse is_record(I, gctest)
+  of
+    false -> ok;
+    true ->
+      put({?MODULE, instr}, I),
+      lists:foreach(fun verify_live/1, LiveOver)
+  end,
+  check_instrs(Is, LiveIn).
+
+verify_live(T) ->
+  case hipe_rtl:is_reg(T) of
+    false -> ok;
+    true ->
+      case hipe_rtl:reg_is_gcsafe(T) of
+        true -> ok;
+        false ->
+          error({gcunsafe_live_over_call,
+                 get({?MODULE, 'fun'}),
+                 {label, get({?MODULE, label})},
+                 get({?MODULE, instr}),
+                 T})
+      end
+  end.
+
+%% Primops that can't gc
+%% Note: This information is essentially duplicated from hipe_bif_list.m4
+safe_primop(is_divisible) -> true;
+safe_primop(is_unicode) -> true;
+safe_primop(cmp_2) -> true;
+safe_primop(eq_2) -> true;
+safe_primop(bs_allocate) -> true;
+safe_primop(bs_reallocate) -> true;
+safe_primop(bs_utf8_size) -> true;
+safe_primop(bs_get_utf8) -> true;
+safe_primop(bs_utf16_size) -> true;
+safe_primop(bs_get_utf16) -> true;
+safe_primop(bs_validate_unicode_retract) -> true;
+safe_primop(bs_put_small_float) -> true;
+safe_primop(bs_put_bits) -> true;
+safe_primop(emasculate_binary) -> true;
+safe_primop(atomic_inc) -> true;
+%% Not noproc but manually verified
+safe_primop(bs_put_big_integer) -> true;
+safe_primop(_) -> false.
diff --git a/lib/hipe/rtl/hipe_tagscheme.erl b/lib/hipe/rtl/hipe_tagscheme.erl
index 68cbe75e85..737f0ec5e3 100644
--- a/lib/hipe/rtl/hipe_tagscheme.erl
+++ b/lib/hipe/rtl/hipe_tagscheme.erl
@@ -53,7 +53,8 @@
 -export([test_subbinary/3, test_heap_binary/3]).
 -export([create_heap_binary/3, create_refc_binary/3, create_refc_binary/4]).
 -export([create_matchstate/6, convert_matchstate/1, compare_matchstate/4]).
--export([get_field_from_term/3, get_field_from_pointer/3,
+-export([get_field_addr_from_term/3,
+	 get_field_from_term/3, get_field_from_pointer/3,
 	 set_field_from_term/3, set_field_from_pointer/3,
 	 extract_matchbuffer/2, extract_binary_bytes/2]).
 
@@ -76,6 +77,10 @@
 -define(TAG_PRIMARY_BOXED,  16#2).
 -define(TAG_PRIMARY_IMMED1, 16#3).
 
+%% Only when ?ERTS_USE_LITERAL_TAG =:= 1
+-define(TAG_PTR_MASK__,   16#7).
+-define(TAG_LITERAL_PTR,  16#4).
+
 -define(TAG_IMMED1_SIZE,  4).
 -define(TAG_IMMED1_MASK,  16#F).
 -define(TAG_IMMED1_PID,   ((16#0 bsl ?TAG_PRIMARY_SIZE) bor ?TAG_PRIMARY_IMMED1)).
@@ -157,6 +162,38 @@ tag_cons(Res, X) ->
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
+ptr_val(Res, X) ->
+  hipe_rtl:mk_alu(Res, X, 'and', hipe_rtl:mk_imm(bnot ?TAG_PTR_MASK__)).
+
+%% Returns {Base, Offset, Untag}. To be used like, for example:
+%% {Base, Offset, Untag} = untag_ptr(X, ?TAG_PRIMARY_BOXED),
+%% ...
+%% [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
+%%
+%% NB: Base might either be X or a new temp. It must thus not be modified.
+untag_ptr(X, Tag) ->
+  case ?ERTS_USE_LITERAL_TAG of
+    0 ->
+      {X, -Tag, []};
+    1 ->
+      Base = hipe_rtl:mk_new_reg(),
+      Untag = ptr_val(Base, X),
+      {Base, 0, Untag}
+  end.
+
+untag_ptr_nooffset(Dst, X, Tag) ->
+  %% We could just use ptr_val in all cases, but subtraction can use LEA on x86
+  %% and can be inlined into effective address computations on several
+  %% architectures.
+  case ?ERTS_USE_LITERAL_TAG of
+    0 ->
+      hipe_rtl:mk_alu(Dst, X, 'sub', hipe_rtl:mk_imm(Tag));
+    1 ->
+      ptr_val(Dst, X)
+  end.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
 %%% Operations to test if an object has a known type T.
 
 test_nil(X, TrueLab, FalseLab, Pred) ->
@@ -171,7 +208,8 @@ test_is_boxed(X, TrueLab, FalseLab, Pred) ->
   hipe_rtl:mk_branch(X, 'and', Mask, 'eq', TrueLab, FalseLab, Pred).
 
 get_header(Res, X) ->
-  hipe_rtl:mk_load(Res, X, hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED))).
+  {Base, Offset, Untag} = untag_ptr(X, ?TAG_PRIMARY_BOXED),
+  [Untag, hipe_rtl:mk_load(Res, Base, hipe_rtl:mk_imm(Offset))].
 
 mask_and_compare(X, Mask, Value, TrueLab, FalseLab, Pred) ->
   Tmp = hipe_rtl:mk_new_reg_gcsafe(),
@@ -617,21 +655,25 @@ test_either_immed(Arg1, Arg2, TrueLab, FalseLab) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 unsafe_car(Dst, Arg) ->
-  hipe_rtl:mk_load(Dst, Arg, hipe_rtl:mk_imm(-(?TAG_PRIMARY_LIST))).
+  {Base, Offset, Untag} = untag_ptr(Arg, ?TAG_PRIMARY_LIST),
+  [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
 
 unsafe_cdr(Dst, Arg) ->
+  {Base, Offset, Untag} = untag_ptr(Arg, ?TAG_PRIMARY_LIST),
   WordSize = hipe_rtl_arch:word_size(),
-  hipe_rtl:mk_load(Dst, Arg, hipe_rtl:mk_imm(-(?TAG_PRIMARY_LIST)+WordSize)).
+  [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset+WordSize))].
 
 unsafe_constant_element(Dst, Index, Tuple) ->	% Index is an immediate
   WordSize = hipe_rtl_arch:word_size(),
-  Offset = -(?TAG_PRIMARY_BOXED) + WordSize * hipe_rtl:imm_value(Index),
-  hipe_rtl:mk_load(Dst, Tuple, hipe_rtl:mk_imm(Offset)).
+  {Base, Offset0, Untag} = untag_ptr(Tuple, ?TAG_PRIMARY_BOXED),
+  Offset = Offset0 + WordSize * hipe_rtl:imm_value(Index),
+  [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
 
 unsafe_update_element(Tuple, Index, Value) ->   % Index is an immediate
   WordSize = hipe_rtl_arch:word_size(),
-  Offset = -(?TAG_PRIMARY_BOXED) + WordSize * hipe_rtl:imm_value(Index),
-  hipe_rtl:mk_store(Tuple, hipe_rtl:mk_imm(Offset), Value).
+  {Base, Offset0, Untag} = untag_ptr(Tuple, ?TAG_PRIMARY_BOXED),
+  Offset = Offset0 + WordSize * hipe_rtl:imm_value(Index),
+  [Untag, hipe_rtl:mk_store(Base, hipe_rtl:mk_imm(Offset), Value)].
 
 %%% wrong semantics
 %% unsafe_variable_element(Dst, Index, Tuple) -> % Index is an unknown fixnum
@@ -644,10 +686,12 @@ unsafe_update_element(Tuple, Index, Value) ->   % Index is an immediate
 %%     Tmp1 = hipe_rtl:mk_new_reg_gcsafe(),
 %%     Tmp2 = hipe_rtl:mk_new_reg_gcsafe(),
 %%     Shift = ?TAG_IMMED1_SIZE - 2,
-%%     OffAdj = (?TAG_IMMED1_SMALL bsr Shift) + ?TAG_PRIMARY_BOXED,
+%%     {Base, Off0, Untag} = untag_ptr(Tuple, ?TAG_PRIMARY_BOXED),
+%%     OffAdj = (?TAG_IMMED1_SMALL bsr Shift) - Off0,
 %%     [hipe_rtl:mk_alu(Tmp1, Index, 'srl', hipe_rtl:mk_imm(Shift)),
 %%      hipe_rtl:mk_alu(Tmp2, Tmp1, 'sub', hipe_rtl:mk_imm(OffAdj)),
-%%      hipe_rtl:mk_load(Dst, Tuple, Tmp2)].
+%%      Untag,
+%%      hipe_rtl:mk_load(Base, Tuple, Tmp2)].
 
 element(Dst, Index, Tuple, FailLabName, {tuple, A}, IndexInfo) ->
   FixnumOkLab = hipe_rtl:mk_new_label(),
@@ -660,7 +704,7 @@ element(Dst, Index, Tuple, FailLabName, {tuple, A}, IndexInfo) ->
       Offset = hipe_rtl:mk_new_reg_gcsafe(),
       Ptr = hipe_rtl:mk_new_reg(), % offset from Tuple
       [untag_fixnum(UIndex, Index),
-       hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)),
+       untag_ptr_nooffset(Ptr, Tuple, ?TAG_PRIMARY_BOXED),
        hipe_rtl:mk_alu(Offset, UIndex, 'sll', 
 		       hipe_rtl:mk_imm(hipe_rtl_arch:log2_word_size())),
        hipe_rtl:mk_load(Dst, Ptr, Offset)];
@@ -769,7 +813,7 @@ gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab) ->
    hipe_rtl:mk_branch(ZeroIndex, 'geu', Arity, FailLabName,
 		      hipe_rtl:label_name(IndexOkLab), 0.01),
    IndexOkLab,
-   hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)),
+   untag_ptr_nooffset(Ptr, Tuple, ?TAG_PRIMARY_BOXED),
    hipe_rtl:mk_alu(Offset, UIndex, 'sll',
                    hipe_rtl:mk_imm(hipe_rtl_arch:log2_word_size())),
    hipe_rtl:mk_load(Dst, Ptr, Offset)].
@@ -777,11 +821,13 @@ gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 unsafe_closure_element(Dst, Index, Closure) ->	% Index is an immediate
-  Offset = -(?TAG_PRIMARY_BOXED)    %% Untag
+  %% XXX: Can there even be closure literals?
+  {Base, Offset0, Untag} = untag_ptr(Closure, ?TAG_PRIMARY_BOXED),
+  Offset = Offset0                  %% Untag
     + ?EFT_ENV                      %% Field offset
                                     %% Index from 1 to N hence -1)
     + (hipe_rtl_arch:word_size() * (hipe_rtl:imm_value(Index)-1)),
-  hipe_rtl:mk_load(Dst, Closure, hipe_rtl:mk_imm(Offset)).
+  [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
 
 mk_fun_header() ->
   hipe_rtl:mk_imm(?HEADER_FUN).
@@ -790,7 +836,7 @@ tag_fun(Res, X) ->
   tag_boxed(Res, X).
 
 %% untag_fun(Res, X) ->
-%%   hipe_rtl:mk_alu(Res, X, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)).
+%%   untag_ptr_nooffset(Res, X, ?TAG_PRIMARY_BOXED).
 
 if_fun_get_arity_and_address(ArityReg, AddressReg, FunP, BadFunLab, Pred) ->
   %% EmuAddressPtrReg = hipe_rtl:mk_new_reg(),
@@ -801,15 +847,15 @@ if_fun_get_arity_and_address(ArityReg, AddressReg, FunP, BadFunLab, Pred) ->
   TrueLab0 = hipe_rtl:mk_new_label(),
   %% TrueLab1 = hipe_rtl:mk_new_label(),
   IsFunCode = test_closure(FunP, hipe_rtl:label_name(TrueLab0), BadFunLab, Pred),
+  {Base, Offset, Untag} = untag_ptr(FunP, ?TAG_PRIMARY_BOXED),
   GetArityCode =
     [TrueLab0,
      %% Funp->arity contains the arity
-     hipe_rtl:mk_load(ArityReg, FunP,
-		      hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED)+
-				      ?EFT_ARITY)),
-     hipe_rtl:mk_load(FEPtrReg, FunP,
-		      hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED)+
-				      ?EFT_FE)),
+     Untag,
+     hipe_rtl:mk_load(ArityReg, Base,
+		      hipe_rtl:mk_imm(Offset+?EFT_ARITY)),
+     hipe_rtl:mk_load(FEPtrReg, Base,
+		      hipe_rtl:mk_imm(Offset+?EFT_FE)),
      hipe_rtl:mk_load(AddressReg, FEPtrReg,
 		      hipe_rtl:mk_imm(?EFE_NATIVE_ADDRESS))],
   IsFunCode ++ GetArityCode.
@@ -927,20 +973,24 @@ test_subbinary(Binary, TrueLblName, FalseLblName) ->
 
 unsafe_load_float(DstLo, DstHi, Src) ->
   WordSize = hipe_rtl_arch:word_size(),
-  Offset1 = -(?TAG_PRIMARY_BOXED) + WordSize,
+  {Base, Offset0, Untag} = untag_ptr(Src, ?TAG_PRIMARY_BOXED),
+  Offset1 = Offset0 + WordSize,
   Offset2 = Offset1 + 4, %% This should really be 4 and not WordSize
   case hipe_rtl_arch:endianess() of
     little ->
-      [hipe_rtl:mk_load(DstLo, Src, hipe_rtl:mk_imm(Offset1), int32, unsigned),
-       hipe_rtl:mk_load(DstHi, Src, hipe_rtl:mk_imm(Offset2), int32, unsigned)];
+      [Untag,
+       hipe_rtl:mk_load(DstLo, Base, hipe_rtl:mk_imm(Offset1), int32, unsigned),
+       hipe_rtl:mk_load(DstHi, Base, hipe_rtl:mk_imm(Offset2), int32, unsigned)];
     big ->
-      [hipe_rtl:mk_load(DstHi, Src, hipe_rtl:mk_imm(Offset1), int32, unsigned),
-       hipe_rtl:mk_load(DstLo, Src, hipe_rtl:mk_imm(Offset2), int32, unsigned)]
+      [Untag,
+       hipe_rtl:mk_load(DstHi, Base, hipe_rtl:mk_imm(Offset1), int32, unsigned),
+       hipe_rtl:mk_load(DstLo, Base, hipe_rtl:mk_imm(Offset2), int32, unsigned)]
   end. 
 
 unsafe_untag_float(Dst, Src) ->
-  Offset = -(?TAG_PRIMARY_BOXED) + hipe_rtl_arch:word_size(),
-  [hipe_rtl:mk_fload(Dst, Src, hipe_rtl:mk_imm(Offset))].
+  {Base, Offset0, Untag} = untag_ptr(Src, ?TAG_PRIMARY_BOXED),
+  Offset = Offset0 + hipe_rtl_arch:word_size(),
+  [Untag, hipe_rtl:mk_fload(Dst, Base, hipe_rtl:mk_imm(Offset))].
 
 unsafe_tag_float(Dst, Src) ->
   {GetHPInsn, HP, PutHPInsn} = hipe_rtl_arch:heap_pointer(),
@@ -999,8 +1049,9 @@ get_one_word_pos_bignum(USize, Size, Fail) ->
 
 unsafe_get_one_word_pos_bignum(USize, Size) ->
   WordSize = hipe_rtl_arch:word_size(),
-  Imm = hipe_rtl:mk_imm(1*WordSize-?TAG_PRIMARY_BOXED),
-  [hipe_rtl:mk_load(USize, Size, Imm)].
+  {Base, Offset, Untag} = untag_ptr(Size, ?TAG_PRIMARY_BOXED),
+  Imm = hipe_rtl:mk_imm(1*WordSize+Offset),
+  [Untag, hipe_rtl:mk_load(USize, Base, Imm)].
 
 -spec bignum_sizeneed(non_neg_integer()) -> non_neg_integer().
 
@@ -1040,7 +1091,7 @@ create_matchstate(Max, BinSize, Base, Offset, Orig, Ms) ->
   SizeInWords = ((ByteSize div WordSize) - 1),
   Header = hipe_rtl:mk_imm(mk_header(SizeInWords, ?TAG_HEADER_BIN_MATCHSTATE)),
   [GetHPInsn,
-   hipe_rtl:mk_alu(Ms, HP, add, hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)),
+   tag_boxed(Ms, HP),
    set_field_from_term({matchstate,thing_word}, Ms, Header),
    set_field_from_term({matchstate,{matchbuffer,orig}}, Ms, Orig),
    set_field_from_term({matchstate,{matchbuffer,base}}, Ms, Base),
@@ -1078,7 +1129,10 @@ convert_matchstate(Ms) ->
    size_from_header(SizeInWords, Header),
    hipe_rtl:mk_alu(Hole, SizeInWords, sub, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE)),
    mk_var_header(BigIntHeader, Hole, ?TAG_HEADER_POS_BIG),
-   hipe_rtl:mk_store(Ms, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE*WordSize-?TAG_PRIMARY_BOXED),
+   %% Matchstates can't be literals; so untagging with ?TAG_PRIMARY_BOXED is
+   %% fine here
+   hipe_rtl:mk_store(Ms, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE*WordSize
+                                         -?TAG_PRIMARY_BOXED),
 		     BigIntHeader)].
 
 compare_matchstate(Max, Ms, LargeEnough, TooSmall) ->
@@ -1087,8 +1141,10 @@ compare_matchstate(Max, Ms, LargeEnough, TooSmall) ->
   SizeInWords = ((ByteSize div WordSize) - 1),
   Header = hipe_rtl:mk_imm(mk_header(SizeInWords, ?TAG_HEADER_BIN_MATCHSTATE)),
   RealHeader = hipe_rtl:mk_new_reg_gcsafe(),
-   [hipe_rtl:mk_load(RealHeader, Ms, hipe_rtl:mk_imm(-?TAG_PRIMARY_BOXED)),
-    hipe_rtl:mk_branch(RealHeader, ge, Header, LargeEnough, TooSmall)].
+  %% Matchstates can't be literals; so untagging with ?TAG_PRIMARY_BOXED is fine
+  %% here
+  [hipe_rtl:mk_load(RealHeader, Ms, hipe_rtl:mk_imm(-?TAG_PRIMARY_BOXED)),
+   hipe_rtl:mk_branch(RealHeader, ge, Header, LargeEnough, TooSmall)].
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%
@@ -1207,15 +1263,22 @@ get_field_size1({matchbuffer, base}) ->
 get_field_size1({matchbuffer, binsize}) ->  
   ?MB_SIZE_SIZE.
 
+get_field_addr_from_term(Struct, Term, Dst) ->
+  {Base, Offset0, Untag} = untag_ptr(Term, ?TAG_PRIMARY_BOXED),
+  Offset = hipe_rtl:mk_imm(get_field_offset(Struct) + Offset0),
+  [Untag, hipe_rtl:mk_alu(Dst, Base, add, Offset)].
+
 get_field_from_term(Struct, Term, Dst) ->
-  Offset = hipe_rtl:mk_imm(get_field_offset(Struct) - ?TAG_PRIMARY_BOXED),
+  {Base, Offset0, Untag} = untag_ptr(Term, ?TAG_PRIMARY_BOXED),
+  Offset = hipe_rtl:mk_imm(get_field_offset(Struct) + Offset0),
   Size = get_field_size(Struct),
-  hipe_rtl:mk_load(Dst, Term, Offset, Size, unsigned). 
+  [Untag, hipe_rtl:mk_load(Dst, Base, Offset, Size, unsigned)].
 
 set_field_from_term(Struct, Term, Value) ->
-  Offset = hipe_rtl:mk_imm(get_field_offset(Struct) - ?TAG_PRIMARY_BOXED),
+  {Base, Offset0, Untag} = untag_ptr(Term, ?TAG_PRIMARY_BOXED),
+  Offset = hipe_rtl:mk_imm(get_field_offset(Struct) + Offset0),
   Size = get_field_size(Struct),
-  hipe_rtl:mk_store(Term, Offset, Value, Size).
+  [Untag, hipe_rtl:mk_store(Base, Offset, Value, Size)].
 
 get_field_from_pointer(Struct, Term, Dst) ->
   Offset = hipe_rtl:mk_imm(get_field_offset(Struct)),
@@ -1229,6 +1292,8 @@ set_field_from_pointer(Struct, Term, Value) ->
   
 extract_matchbuffer(Mb, Ms) ->
   What = {matchstate, matchbuffer},
+  %% Matchstates can't be literals; so untagging with ?TAG_PRIMARY_BOXED is fine
+  %% here
   Offset = hipe_rtl:mk_imm(get_field_offset(What) - ?TAG_PRIMARY_BOXED),
   hipe_rtl:mk_alu(Mb, Ms, add, Offset).
 
diff --git a/lib/hipe/ssa/hipe_ssa.inc b/lib/hipe/ssa/hipe_ssa.inc
index c7c1a8e1d7..29e8b92266 100644
--- a/lib/hipe/ssa/hipe_ssa.inc
+++ b/lib/hipe/ssa/hipe_ssa.inc
@@ -463,20 +463,20 @@ updateStatementDefs([], Statement, Current, Acc) ->
 %%----------------------------------------------------------------------
 
 updateIndices(Current, Variable) ->
-  case ?CODE:is_var(Variable) of
-    true ->
-      NewVar = ?CODE:mk_new_var(),
-      {NewVar,gb_trees:enter(Variable, NewVar, Current)};
-    false ->
-      case is_fp_temp(Variable) of
-	true ->
-	  NewFVar = mk_new_fp_temp(),
-	  {NewFVar,gb_trees:enter(Variable, NewFVar, Current)};
-	false ->
-	  NewReg = ?CODE:mk_new_reg(),
-	  {NewReg,gb_trees:enter(Variable, NewReg, Current)}
-      end
-  end.
+  New =
+    case ?CODE:is_var(Variable) of
+      true -> ?CODE:mk_new_var();
+      false ->
+        case is_fp_temp(Variable) of
+          true -> mk_new_fp_temp();
+          false ->
+            case ?CODE:reg_is_gcsafe(Variable) of
+              true -> ?CODE:mk_new_reg_gcsafe();
+              false -> ?CODE:mk_new_reg()
+            end
+        end
+    end,
+  {New, gb_trees:enter(Variable, New, Current)}.
 
 %%----------------------------------------------------------------------
 %% Procedure : updateSuccPhi/4
diff --git a/lib/hipe/test/hipe_testsuite_driver.erl b/lib/hipe/test/hipe_testsuite_driver.erl
index ee9c57a908..8813af5dfc 100644
--- a/lib/hipe/test/hipe_testsuite_driver.erl
+++ b/lib/hipe/test/hipe_testsuite_driver.erl
@@ -161,7 +161,8 @@ run(TestCase, Dir, _OutDir) ->
     %% 		  end, DataFiles),
     %% try
     ok = TestCase:test(),
-    HiPEOpts = try TestCase:hipe_options() catch error:undef -> [] end,
+    HiPEOpts0 = try TestCase:hipe_options() catch error:undef -> [] end,
+    HiPEOpts = HiPEOpts0 ++ hipe_options(),
     {ok, TestCase} = hipe:c(TestCase, HiPEOpts),
     ok = TestCase:test(),
     {ok, TestCase} = hipe:c(TestCase, [o1|HiPEOpts]),
@@ -179,3 +180,6 @@ run(TestCase, Dir, _OutDir) ->
     %% 	lists:foreach(fun (DF) -> ok end, % = file:delete(DF) end,
     %% 		      [filename:join(OutDir, D) || D <- DataFiles])
     %% end.
+
+hipe_options() ->
+    [verify_gcsafe].