aboutsummaryrefslogtreecommitdiffstats
path: root/lib/hipe/rtl
diff options
context:
space:
mode:
Diffstat (limited to 'lib/hipe/rtl')
-rw-r--r--lib/hipe/rtl/Makefile2
-rw-r--r--lib/hipe/rtl/hipe_rtl.erl5
-rw-r--r--lib/hipe/rtl/hipe_rtl_binary_construct.erl22
-rw-r--r--lib/hipe/rtl/hipe_rtl_binary_match.erl10
-rw-r--r--lib/hipe/rtl/hipe_rtl_cleanup_const.erl4
-rw-r--r--lib/hipe/rtl/hipe_rtl_lcm.erl107
-rw-r--r--lib/hipe/rtl/hipe_rtl_varmap.erl2
-rw-r--r--lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl88
-rw-r--r--lib/hipe/rtl/hipe_tagscheme.erl141
9 files changed, 275 insertions, 106 deletions
diff --git a/lib/hipe/rtl/Makefile b/lib/hipe/rtl/Makefile
index 5abc9ec049..becdd0b7d8 100644
--- a/lib/hipe/rtl/Makefile
+++ b/lib/hipe/rtl/Makefile
@@ -50,7 +50,7 @@ HIPE_MODULES = hipe_rtl hipe_rtl_cfg \
hipe_rtl_ssa hipe_rtl_ssa_const_prop \
hipe_rtl_cleanup_const hipe_rtl_symbolic hipe_rtl_lcm \
hipe_rtl_ssapre hipe_rtl_binary hipe_rtl_ssa_avail_expr \
- hipe_rtl_arch hipe_tagscheme
+ hipe_rtl_arch hipe_tagscheme hipe_rtl_verify_gcsafe
else
HIPE_MODULES =
endif
diff --git a/lib/hipe/rtl/hipe_rtl.erl b/lib/hipe/rtl/hipe_rtl.erl
index 04c9728d5c..33027f3259 100644
--- a/lib/hipe/rtl/hipe_rtl.erl
+++ b/lib/hipe/rtl/hipe_rtl.erl
@@ -1740,7 +1740,10 @@ pp_reg(Dev, Arg) ->
true ->
pp_hard_reg(Dev, reg_index(Arg));
false ->
- io:format(Dev, "r~w", [reg_index(Arg)])
+ case reg_is_gcsafe(Arg) of
+ true -> io:format(Dev, "rs~w", [reg_index(Arg)]);
+ false -> io:format(Dev, "r~w", [reg_index(Arg)])
+ end
end.
pp_var(Dev, Arg) ->
diff --git a/lib/hipe/rtl/hipe_rtl_binary_construct.erl b/lib/hipe/rtl/hipe_rtl_binary_construct.erl
index bc215e3abe..5b89d4946a 100644
--- a/lib/hipe/rtl/hipe_rtl_binary_construct.erl
+++ b/lib/hipe/rtl/hipe_rtl_binary_construct.erl
@@ -359,7 +359,8 @@ not_writable_code(Bin, SizeReg, Dst, Base, Offset, Unit,
allocate_writable(Dst, Base, UsedBytes, TotBytes, TotSize) ->
Zero = hipe_rtl:mk_imm(0),
[NextLbl] = create_lbls(1),
- [EndSubSize, EndSubBitSize, ProcBin] = create_regs(3),
+ [EndSubSize, EndSubBitSize] = create_regs(2),
+ [ProcBin] = create_unsafe_regs(1),
[hipe_rtl:mk_call([Base], bs_allocate, [UsedBytes],
hipe_rtl:label_name(NextLbl), [], not_remote),
NextLbl,
@@ -586,12 +587,12 @@ const_init2(Size, Dst, Base, Offset, TrueLblName) ->
false ->
ByteSize = hipe_rtl:mk_new_reg(),
[hipe_rtl:mk_gctest(?PROC_BIN_WORDSIZE+?SUB_BIN_WORDSIZE),
- hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
hipe_rtl:mk_move(ByteSize, hipe_rtl:mk_imm(Size)),
hipe_rtl:mk_call([Base], bs_allocate, [ByteSize],
hipe_rtl:label_name(NextLbl), [], not_remote),
NextLbl,
hipe_tagscheme:create_refc_binary(Base, ByteSize, Dst),
+ hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
hipe_rtl:mk_goto(TrueLblName)]
end.
@@ -634,13 +635,12 @@ var_init2(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName
Log2WordSize = hipe_rtl_arch:log2_word_size(),
WordSize = hipe_rtl_arch:word_size(),
[ContLbl, HeapLbl, REFCLbl, NextLbl] = create_lbls(4),
- [USize, Tmp] = create_unsafe_regs(2),
+ [USize, Tmp] = create_regs(2),
[get_word_integer(Size, USize, SystemLimitLblName, FalseLblName),
hipe_rtl:mk_branch(USize, leu, hipe_rtl:mk_imm(?MAX_BINSIZE),
hipe_rtl:label_name(ContLbl),
SystemLimitLblName),
ContLbl,
- hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
hipe_rtl:mk_branch(USize, leu, hipe_rtl:mk_imm(?MAX_HEAP_BIN_SIZE),
hipe_rtl:label_name(HeapLbl),
hipe_rtl:label_name(REFCLbl)),
@@ -650,6 +650,7 @@ var_init2(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName
hipe_rtl:mk_alu(Tmp, Tmp, add, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE)),
hipe_rtl:mk_gctest(Tmp),
hipe_tagscheme:create_heap_binary(Base, USize, Dst),
+ hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
hipe_rtl:mk_goto(TrueLblName),
REFCLbl,
hipe_rtl:mk_gctest(?PROC_BIN_WORDSIZE+?SUB_BIN_WORDSIZE),
@@ -657,6 +658,7 @@ var_init2(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName
hipe_rtl:label_name(NextLbl), [], not_remote),
NextLbl,
hipe_tagscheme:create_refc_binary(Base, USize, Dst),
+ hipe_rtl:mk_move(Offset, hipe_rtl:mk_imm(0)),
hipe_rtl:mk_goto(TrueLblName)].
var_init_bits(Size, Dst, Base, Offset, TrueLblName, SystemLimitLblName, FalseLblName) ->
@@ -863,7 +865,7 @@ get_base_offset_size(Binary, SrcBase, SrcOffset, SrcSize, FLName) ->
JoinLbl,
hipe_tagscheme:test_heap_binary(Orig, HeapLblName, REFCLblName),
HeapLbl,
- hipe_rtl:mk_alu(SrcBase, Orig, add, hipe_rtl:mk_imm(?HEAP_BIN_DATA-2)),
+ hipe_tagscheme:get_field_addr_from_term({heap_bin, {data, 0}}, Orig, SrcBase),
hipe_rtl:mk_goto(EndLblName),
REFCLbl,
hipe_tagscheme:get_field_from_term({proc_bin,bytes}, Orig, SrcBase),
@@ -1210,6 +1212,12 @@ is_divisible(Dividend, Divisor, SuccLbl, FailLbl) ->
[hipe_rtl:mk_branch(Dividend, 'and', Mask, eq, SuccLbl, FailLbl, 0.99)];
false ->
%% We need division, fall back to a primop
- [hipe_rtl:mk_call([], is_divisible, [Dividend, hipe_rtl:mk_imm(Divisor)],
- SuccLbl, FailLbl, not_remote)]
+ [Tmp] = create_regs(1),
+ RetLbl = hipe_rtl:mk_new_label(),
+ [hipe_rtl:mk_call([Tmp], is_divisible,
+ [Dividend, hipe_rtl:mk_imm(Divisor)],
+ hipe_rtl:label_name(RetLbl), [], not_remote),
+ RetLbl,
+ hipe_rtl:mk_branch(Tmp, ne, hipe_rtl:mk_imm(0),
+ SuccLbl, FailLbl, 0.99)]
end.
diff --git a/lib/hipe/rtl/hipe_rtl_binary_match.erl b/lib/hipe/rtl/hipe_rtl_binary_match.erl
index 362a52f8fe..4575213838 100644
--- a/lib/hipe/rtl/hipe_rtl_binary_match.erl
+++ b/lib/hipe/rtl/hipe_rtl_binary_match.erl
@@ -730,7 +730,7 @@ get_base(Orig,Base) ->
[hipe_tagscheme:test_heap_binary(Orig, hipe_rtl:label_name(HeapLbl),
hipe_rtl:label_name(REFCLbl)),
HeapLbl,
- hipe_rtl:mk_alu(Base, Orig, 'add', hipe_rtl:mk_imm(?HEAP_BIN_DATA-2)),
+ hipe_tagscheme:get_field_addr_from_term({heap_bin, {data, 0}}, Orig, Base),
hipe_rtl:mk_goto(hipe_rtl:label_name(EndLbl)),
REFCLbl,
get_field_from_term({proc_bin, flags}, Orig, Flags),
@@ -740,7 +740,7 @@ get_base(Orig,Base) ->
WritableLbl,
hipe_rtl:mk_call([], emasculate_binary, [Orig], [], [], 'not_remote'),
NotWritableLbl,
- hipe_rtl:mk_load(Base, Orig, hipe_rtl:mk_imm(?PROC_BIN_BYTES-2)),
+ get_field_from_term({proc_bin, bytes}, Orig, Base),
EndLbl].
extract_matchstate_var(binsize, Ms) ->
@@ -842,12 +842,12 @@ make_dyn_prep(SizeReg, CCode) ->
%%------------------------------------------------------------------------
get_unaligned_int(Dst1, Size, Base, Offset, Shiftr, Type, TrueLblName) ->
- [Reg] = create_regs(1),
+ [Reg] = create_gcsafe_regs(1),
[get_maybe_unaligned_int_to_reg(Reg, Size, Base, Offset, Shiftr, Type),
do_bignum_code(Size, Type, Reg, Dst1, TrueLblName)].
get_maybe_unaligned_int_to_reg(Reg, Size, Base, Offset, Shiftr, Type) ->
- [LowBits] = create_regs(1),
+ [LowBits] = create_gcsafe_regs(1),
[AlignedLbl, UnAlignedLbl, EndLbl] = create_lbls(3),
[hipe_rtl:mk_alub(LowBits, Offset, 'and', hipe_rtl:mk_imm(?LOW_BITS),
eq, hipe_rtl:label_name(AlignedLbl),
@@ -1001,7 +1001,7 @@ do_bignum_code(Size, {Signedness,_}, Src, Dst1, TrueLblName)
end.
signed_bignum(Dst1, Src, TrueLblName) ->
- Tmp1 = hipe_rtl:mk_new_reg(),
+ Tmp1 = hipe_rtl:mk_new_reg_gcsafe(),
BignumLabel = hipe_rtl:mk_new_label(),
[hipe_tagscheme:realtag_fixnum(Dst1, Src),
hipe_tagscheme:realuntag_fixnum(Tmp1, Dst1),
diff --git a/lib/hipe/rtl/hipe_rtl_cleanup_const.erl b/lib/hipe/rtl/hipe_rtl_cleanup_const.erl
index bfa6b9682e..00cc2bcb37 100644
--- a/lib/hipe/rtl/hipe_rtl_cleanup_const.erl
+++ b/lib/hipe/rtl/hipe_rtl_cleanup_const.erl
@@ -69,9 +69,9 @@ cleanup_instr([Const|Left], I, Acc) ->
case I of
X when is_record(X, fp_unop) orelse is_record(X, fp) ->
Fdst = hipe_rtl:mk_new_fpreg(),
- Fconv = hipe_tagscheme:unsafe_untag_float(Fdst, Dst),
+ Fconv = lists:flatten(hipe_tagscheme:unsafe_untag_float(Fdst, Dst)),
NewI = hipe_rtl:subst_uses([{Const, Fdst}], I),
- cleanup_instr(Left, NewI, Fconv ++ [Load|Acc]);
+ cleanup_instr(Left, NewI, lists:reverse(Fconv, [Load|Acc]));
_ ->
NewI = hipe_rtl:subst_uses([{Const, Dst}], I),
cleanup_instr(Left, NewI, [Load|Acc])
diff --git a/lib/hipe/rtl/hipe_rtl_lcm.erl b/lib/hipe/rtl/hipe_rtl_lcm.erl
index 9dcdd05fb1..af39c9a0a4 100644
--- a/lib/hipe/rtl/hipe_rtl_lcm.erl
+++ b/lib/hipe/rtl/hipe_rtl_lcm.erl
@@ -182,42 +182,41 @@ delete_exprs(Code, _, _, []) ->
Code;
delete_exprs(Code, ExprMap, IdMap, [ExprId|Exprs]) ->
Expr = expr_id_map_get_expr(IdMap, ExprId),
- %% Perform a foldl that goes through the code and deletes all
- %% occurences of the expression.
- NewCode =
- lists:reverse
- (lists:foldl(fun(CodeExpr, Acc) ->
- case is_expr(CodeExpr) of
- true ->
- case expr_clear_dst(CodeExpr) =:= Expr of
- true ->
- pp_debug(" Deleting: ", []),
- pp_debug_instr(CodeExpr),
- %% Lookup expression entry.
- Defines =
- case expr_map_lookup(ExprMap, Expr) of
- {value, {_, _, Defs}} ->
- Defs;
- none ->
- exit({?MODULE, expr_map_lookup,
- "expression missing"})
- end,
- MoveCode =
- mk_expr_move_instr(hipe_rtl:defines(CodeExpr),
- Defines),
- pp_debug(" Replacing with: ", []),
- pp_debug_instr(MoveCode),
- [MoveCode|Acc];
- false ->
- [CodeExpr|Acc]
- end;
- false ->
- [CodeExpr|Acc]
- end
- end,
- [], Code)),
+ %% Lookup expression entry.
+ {value, {_, _, Defines}} = expr_map_lookup(ExprMap, Expr),
+ %% Go through the code and deletes all occurences of the expression.
+ NewCode = delete_expr(Code, Expr, Defines, []),
delete_exprs(NewCode, ExprMap, IdMap, Exprs).
+delete_expr([], _Expr, _Defines, Acc) -> lists:reverse(Acc);
+delete_expr([CodeExpr|Code], Expr, Defines, Acc) ->
+ case exp_kill_expr(CodeExpr, [Expr]) of
+ [] -> % Expr was killed; deleting stops here
+ pp_debug(" Stopping before: ", []),
+ pp_debug_instr(CodeExpr),
+ lists:reverse(Acc, [CodeExpr|Code]);
+ [Expr] ->
+ NewCodeExpr =
+ case is_expr(CodeExpr) of
+ true ->
+ case expr_clear_dst(CodeExpr) =:= Expr of
+ true ->
+ pp_debug(" Deleting: ", []),
+ pp_debug_instr(CodeExpr),
+ MoveCode = mk_expr_move_instr(hipe_rtl:defines(CodeExpr),
+ Defines),
+ pp_debug(" Replacing with: ", []),
+ pp_debug_instr(MoveCode),
+ MoveCode;
+ false ->
+ CodeExpr
+ end;
+ false ->
+ CodeExpr
+ end,
+ delete_expr(Code, Expr, Defines, [NewCodeExpr|Acc])
+ end.
+
%%=============================================================================
%% Goes through the given list of expressions and inserts them at
%% appropriate places in the code.
@@ -226,13 +225,12 @@ insert_exprs(CFG, _, _, _, _, BetweenMap, []) ->
insert_exprs(CFG, Pred, Succ, ExprMap, IdMap, BetweenMap, [ExprId|Exprs]) ->
Expr = expr_id_map_get_expr(IdMap, ExprId),
Instr = expr_map_get_instr(ExprMap, Expr),
- case hipe_rtl_cfg:succ(CFG, Pred) of
- [_] ->
+ case try_insert_expr_last(CFG, Pred, Instr) of
+ {ok, NewCFG} ->
pp_debug(" Inserted last: ", []),
pp_debug_instr(Instr),
- NewCFG = insert_expr_last(CFG, Pred, Instr),
insert_exprs(NewCFG, Pred, Succ, ExprMap, IdMap, BetweenMap, Exprs);
- _ ->
+ not_safe ->
case hipe_rtl_cfg:pred(CFG, Succ) of
[_] ->
pp_debug(" Inserted first: ", []),
@@ -252,25 +250,31 @@ insert_exprs(CFG, Pred, Succ, ExprMap, IdMap, BetweenMap, [ExprId|Exprs]) ->
%% Recursively goes through the code in a block and returns a new block
%% with the new code inserted second to last (assuming the last expression
%% is a branch operation).
-insert_expr_last(CFG0, Label, Instr) ->
- Code0 = hipe_bb:code(hipe_rtl_cfg:bb(CFG0, Label)),
- %% FIXME: Use hipe_bb:butlast() instead?
- Code1 = insert_expr_last_work(Label, Instr, Code0),
- hipe_rtl_cfg:bb_add(CFG0, Label, hipe_bb:mk_bb(Code1)).
+try_insert_expr_last(CFG0, Label, Instr) ->
+ case hipe_rtl_cfg:succ(CFG0, Label) of
+ [_] ->
+ Code0 = hipe_bb:code(hipe_rtl_cfg:bb(CFG0, Label)),
+ case insert_expr_last_work(Instr, Code0) of
+ not_safe -> not_safe;
+ Code1 ->
+ {ok, hipe_rtl_cfg:bb_add(CFG0, Label, hipe_bb:mk_bb(Code1))}
+ end;
+ _ -> not_safe
+ end.
%%=============================================================================
%% Recursively goes through the code in a block and returns a new block
%% with the new code inserted second to last (assuming the last expression
%% is a branch operation).
-insert_expr_last_work(_, Instr, []) ->
- %% This case should not happen since this means that block was completely
- %% empty when the function was called. For compatibility we insert it last.
- [Instr];
-insert_expr_last_work(_, Instr, [Code1]) ->
+insert_expr_last_work(_Instr, [#call{}]) ->
+ %% Call instructions clobber all expressions; we musn't insert the expression
+ %% before it
+ not_safe;
+insert_expr_last_work(Instr, [Code1]) ->
%% We insert the code next to last.
[Instr, Code1];
-insert_expr_last_work(Label, Instr, [Code|Codes]) ->
- [Code|insert_expr_last_work(Label, Instr, Codes)].
+insert_expr_last_work(Instr, [Code|Codes]) ->
+ [Code|insert_expr_last_work(Instr, Codes)].
%%=============================================================================
%% Inserts expression first in the block for the given label.
@@ -305,7 +309,8 @@ insert_expr_between(CFG0, BetweenMap, Pred, Succ, Instr) ->
{value, Label} ->
pp_debug(" Using existing new bb for edge (~w,~w) with label ~w~n",
[Pred, Succ, Label]),
- {insert_expr_last(CFG0, Label, Instr), BetweenMap}
+ {ok, NewCfg} = try_insert_expr_last(CFG0, Label, Instr),
+ {NewCfg, BetweenMap}
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
diff --git a/lib/hipe/rtl/hipe_rtl_varmap.erl b/lib/hipe/rtl/hipe_rtl_varmap.erl
index 375a8f85c0..f34c66ab85 100644
--- a/lib/hipe/rtl/hipe_rtl_varmap.erl
+++ b/lib/hipe/rtl/hipe_rtl_varmap.erl
@@ -105,7 +105,7 @@ icode_var2rtl_var(Var, Map) ->
{reg, IsGcSafe} ->
NewVar =
case IsGcSafe of
- %% true -> hipe_rtl:mk_new_reg_gcsafe();
+ true -> hipe_rtl:mk_new_reg_gcsafe();
false -> hipe_rtl:mk_new_reg()
end,
{NewVar, insert(Var, NewVar, Map)}
diff --git a/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl
new file mode 100644
index 0000000000..c3f20bfec1
--- /dev/null
+++ b/lib/hipe/rtl/hipe_rtl_verify_gcsafe.erl
@@ -0,0 +1,88 @@
+%% -*- mode: erlang; erlang-indent-level: 2 -*-
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+-module(hipe_rtl_verify_gcsafe).
+
+-export([check/1]).
+
+-include("../flow/cfg.hrl"). %% needed for the specs
+-include("hipe_rtl.hrl").
+
+check(CFG) ->
+ Liveness = hipe_rtl_liveness:analyze(CFG),
+ put({?MODULE, 'fun'}, CFG#cfg.info#cfg_info.'fun'),
+ lists:foreach(
+ fun(Lb) ->
+ put({?MODULE, label}, Lb),
+ Liveout = hipe_rtl_liveness:liveout(Liveness, Lb),
+ BB = hipe_rtl_cfg:bb(CFG, Lb),
+ check_instrs(lists:reverse(hipe_bb:code(BB)), Liveout)
+ end, hipe_rtl_cfg:labels(CFG)),
+ erase({?MODULE, 'fun'}),
+ erase({?MODULE, label}),
+ erase({?MODULE, instr}),
+ ok.
+
+check_instrs([], _Livein) -> ok;
+check_instrs([I|Is], LiveOut) ->
+ Def = ordsets:from_list(hipe_rtl:defines(I)),
+ Use = ordsets:from_list(hipe_rtl:uses(I)),
+ LiveOver = ordsets:subtract(LiveOut, Def),
+ LiveIn = ordsets:union(LiveOver, Use),
+ case (hipe_rtl:is_call(I)
+ andalso not safe_primop(hipe_rtl:call_fun(I)))
+ orelse is_record(I, gctest)
+ of
+ false -> ok;
+ true ->
+ put({?MODULE, instr}, I),
+ lists:foreach(fun verify_live/1, LiveOver)
+ end,
+ check_instrs(Is, LiveIn).
+
+verify_live(T) ->
+ case hipe_rtl:is_reg(T) of
+ false -> ok;
+ true ->
+ case hipe_rtl:reg_is_gcsafe(T) of
+ true -> ok;
+ false ->
+ error({gcunsafe_live_over_call,
+ get({?MODULE, 'fun'}),
+ {label, get({?MODULE, label})},
+ get({?MODULE, instr}),
+ T})
+ end
+ end.
+
+%% Primops that can't gc
+%% Note: This information is essentially duplicated from hipe_bif_list.m4
+safe_primop(is_divisible) -> true;
+safe_primop(is_unicode) -> true;
+safe_primop(cmp_2) -> true;
+safe_primop(eq_2) -> true;
+safe_primop(bs_allocate) -> true;
+safe_primop(bs_reallocate) -> true;
+safe_primop(bs_utf8_size) -> true;
+safe_primop(bs_get_utf8) -> true;
+safe_primop(bs_utf16_size) -> true;
+safe_primop(bs_get_utf16) -> true;
+safe_primop(bs_validate_unicode_retract) -> true;
+safe_primop(bs_put_small_float) -> true;
+safe_primop(bs_put_bits) -> true;
+safe_primop(emasculate_binary) -> true;
+safe_primop(atomic_inc) -> true;
+%% Not noproc but manually verified
+safe_primop(bs_put_big_integer) -> true;
+safe_primop(_) -> false.
diff --git a/lib/hipe/rtl/hipe_tagscheme.erl b/lib/hipe/rtl/hipe_tagscheme.erl
index 68cbe75e85..737f0ec5e3 100644
--- a/lib/hipe/rtl/hipe_tagscheme.erl
+++ b/lib/hipe/rtl/hipe_tagscheme.erl
@@ -53,7 +53,8 @@
-export([test_subbinary/3, test_heap_binary/3]).
-export([create_heap_binary/3, create_refc_binary/3, create_refc_binary/4]).
-export([create_matchstate/6, convert_matchstate/1, compare_matchstate/4]).
--export([get_field_from_term/3, get_field_from_pointer/3,
+-export([get_field_addr_from_term/3,
+ get_field_from_term/3, get_field_from_pointer/3,
set_field_from_term/3, set_field_from_pointer/3,
extract_matchbuffer/2, extract_binary_bytes/2]).
@@ -76,6 +77,10 @@
-define(TAG_PRIMARY_BOXED, 16#2).
-define(TAG_PRIMARY_IMMED1, 16#3).
+%% Only when ?ERTS_USE_LITERAL_TAG =:= 1
+-define(TAG_PTR_MASK__, 16#7).
+-define(TAG_LITERAL_PTR, 16#4).
+
-define(TAG_IMMED1_SIZE, 4).
-define(TAG_IMMED1_MASK, 16#F).
-define(TAG_IMMED1_PID, ((16#0 bsl ?TAG_PRIMARY_SIZE) bor ?TAG_PRIMARY_IMMED1)).
@@ -157,6 +162,38 @@ tag_cons(Res, X) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ptr_val(Res, X) ->
+ hipe_rtl:mk_alu(Res, X, 'and', hipe_rtl:mk_imm(bnot ?TAG_PTR_MASK__)).
+
+%% Returns {Base, Offset, Untag}. To be used like, for example:
+%% {Base, Offset, Untag} = untag_ptr(X, ?TAG_PRIMARY_BOXED),
+%% ...
+%% [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
+%%
+%% NB: Base might either be X or a new temp. It must thus not be modified.
+untag_ptr(X, Tag) ->
+ case ?ERTS_USE_LITERAL_TAG of
+ 0 ->
+ {X, -Tag, []};
+ 1 ->
+ Base = hipe_rtl:mk_new_reg(),
+ Untag = ptr_val(Base, X),
+ {Base, 0, Untag}
+ end.
+
+untag_ptr_nooffset(Dst, X, Tag) ->
+ %% We could just use ptr_val in all cases, but subtraction can use LEA on x86
+ %% and can be inlined into effective address computations on several
+ %% architectures.
+ case ?ERTS_USE_LITERAL_TAG of
+ 0 ->
+ hipe_rtl:mk_alu(Dst, X, 'sub', hipe_rtl:mk_imm(Tag));
+ 1 ->
+ ptr_val(Dst, X)
+ end.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
%%% Operations to test if an object has a known type T.
test_nil(X, TrueLab, FalseLab, Pred) ->
@@ -171,7 +208,8 @@ test_is_boxed(X, TrueLab, FalseLab, Pred) ->
hipe_rtl:mk_branch(X, 'and', Mask, 'eq', TrueLab, FalseLab, Pred).
get_header(Res, X) ->
- hipe_rtl:mk_load(Res, X, hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED))).
+ {Base, Offset, Untag} = untag_ptr(X, ?TAG_PRIMARY_BOXED),
+ [Untag, hipe_rtl:mk_load(Res, Base, hipe_rtl:mk_imm(Offset))].
mask_and_compare(X, Mask, Value, TrueLab, FalseLab, Pred) ->
Tmp = hipe_rtl:mk_new_reg_gcsafe(),
@@ -617,21 +655,25 @@ test_either_immed(Arg1, Arg2, TrueLab, FalseLab) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
unsafe_car(Dst, Arg) ->
- hipe_rtl:mk_load(Dst, Arg, hipe_rtl:mk_imm(-(?TAG_PRIMARY_LIST))).
+ {Base, Offset, Untag} = untag_ptr(Arg, ?TAG_PRIMARY_LIST),
+ [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
unsafe_cdr(Dst, Arg) ->
+ {Base, Offset, Untag} = untag_ptr(Arg, ?TAG_PRIMARY_LIST),
WordSize = hipe_rtl_arch:word_size(),
- hipe_rtl:mk_load(Dst, Arg, hipe_rtl:mk_imm(-(?TAG_PRIMARY_LIST)+WordSize)).
+ [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset+WordSize))].
unsafe_constant_element(Dst, Index, Tuple) -> % Index is an immediate
WordSize = hipe_rtl_arch:word_size(),
- Offset = -(?TAG_PRIMARY_BOXED) + WordSize * hipe_rtl:imm_value(Index),
- hipe_rtl:mk_load(Dst, Tuple, hipe_rtl:mk_imm(Offset)).
+ {Base, Offset0, Untag} = untag_ptr(Tuple, ?TAG_PRIMARY_BOXED),
+ Offset = Offset0 + WordSize * hipe_rtl:imm_value(Index),
+ [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
unsafe_update_element(Tuple, Index, Value) -> % Index is an immediate
WordSize = hipe_rtl_arch:word_size(),
- Offset = -(?TAG_PRIMARY_BOXED) + WordSize * hipe_rtl:imm_value(Index),
- hipe_rtl:mk_store(Tuple, hipe_rtl:mk_imm(Offset), Value).
+ {Base, Offset0, Untag} = untag_ptr(Tuple, ?TAG_PRIMARY_BOXED),
+ Offset = Offset0 + WordSize * hipe_rtl:imm_value(Index),
+ [Untag, hipe_rtl:mk_store(Base, hipe_rtl:mk_imm(Offset), Value)].
%%% wrong semantics
%% unsafe_variable_element(Dst, Index, Tuple) -> % Index is an unknown fixnum
@@ -644,10 +686,12 @@ unsafe_update_element(Tuple, Index, Value) -> % Index is an immediate
%% Tmp1 = hipe_rtl:mk_new_reg_gcsafe(),
%% Tmp2 = hipe_rtl:mk_new_reg_gcsafe(),
%% Shift = ?TAG_IMMED1_SIZE - 2,
-%% OffAdj = (?TAG_IMMED1_SMALL bsr Shift) + ?TAG_PRIMARY_BOXED,
+%% {Base, Off0, Untag} = untag_ptr(Tuple, ?TAG_PRIMARY_BOXED),
+%% OffAdj = (?TAG_IMMED1_SMALL bsr Shift) - Off0,
%% [hipe_rtl:mk_alu(Tmp1, Index, 'srl', hipe_rtl:mk_imm(Shift)),
%% hipe_rtl:mk_alu(Tmp2, Tmp1, 'sub', hipe_rtl:mk_imm(OffAdj)),
-%% hipe_rtl:mk_load(Dst, Tuple, Tmp2)].
+%% Untag,
+%% hipe_rtl:mk_load(Base, Tuple, Tmp2)].
element(Dst, Index, Tuple, FailLabName, {tuple, A}, IndexInfo) ->
FixnumOkLab = hipe_rtl:mk_new_label(),
@@ -660,7 +704,7 @@ element(Dst, Index, Tuple, FailLabName, {tuple, A}, IndexInfo) ->
Offset = hipe_rtl:mk_new_reg_gcsafe(),
Ptr = hipe_rtl:mk_new_reg(), % offset from Tuple
[untag_fixnum(UIndex, Index),
- hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)),
+ untag_ptr_nooffset(Ptr, Tuple, ?TAG_PRIMARY_BOXED),
hipe_rtl:mk_alu(Offset, UIndex, 'sll',
hipe_rtl:mk_imm(hipe_rtl_arch:log2_word_size())),
hipe_rtl:mk_load(Dst, Ptr, Offset)];
@@ -769,7 +813,7 @@ gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab) ->
hipe_rtl:mk_branch(ZeroIndex, 'geu', Arity, FailLabName,
hipe_rtl:label_name(IndexOkLab), 0.01),
IndexOkLab,
- hipe_rtl:mk_alu(Ptr, Tuple, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)),
+ untag_ptr_nooffset(Ptr, Tuple, ?TAG_PRIMARY_BOXED),
hipe_rtl:mk_alu(Offset, UIndex, 'sll',
hipe_rtl:mk_imm(hipe_rtl_arch:log2_word_size())),
hipe_rtl:mk_load(Dst, Ptr, Offset)].
@@ -777,11 +821,13 @@ gen_element_tail(Dst, Tuple, Arity, UIndex, FailLabName, IndexOkLab) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
unsafe_closure_element(Dst, Index, Closure) -> % Index is an immediate
- Offset = -(?TAG_PRIMARY_BOXED) %% Untag
+ %% XXX: Can there even be closure literals?
+ {Base, Offset0, Untag} = untag_ptr(Closure, ?TAG_PRIMARY_BOXED),
+ Offset = Offset0 %% Untag
+ ?EFT_ENV %% Field offset
%% Index from 1 to N hence -1)
+ (hipe_rtl_arch:word_size() * (hipe_rtl:imm_value(Index)-1)),
- hipe_rtl:mk_load(Dst, Closure, hipe_rtl:mk_imm(Offset)).
+ [Untag, hipe_rtl:mk_load(Dst, Base, hipe_rtl:mk_imm(Offset))].
mk_fun_header() ->
hipe_rtl:mk_imm(?HEADER_FUN).
@@ -790,7 +836,7 @@ tag_fun(Res, X) ->
tag_boxed(Res, X).
%% untag_fun(Res, X) ->
-%% hipe_rtl:mk_alu(Res, X, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)).
+%% untag_ptr_nooffset(Res, X, ?TAG_PRIMARY_BOXED).
if_fun_get_arity_and_address(ArityReg, AddressReg, FunP, BadFunLab, Pred) ->
%% EmuAddressPtrReg = hipe_rtl:mk_new_reg(),
@@ -801,15 +847,15 @@ if_fun_get_arity_and_address(ArityReg, AddressReg, FunP, BadFunLab, Pred) ->
TrueLab0 = hipe_rtl:mk_new_label(),
%% TrueLab1 = hipe_rtl:mk_new_label(),
IsFunCode = test_closure(FunP, hipe_rtl:label_name(TrueLab0), BadFunLab, Pred),
+ {Base, Offset, Untag} = untag_ptr(FunP, ?TAG_PRIMARY_BOXED),
GetArityCode =
[TrueLab0,
%% Funp->arity contains the arity
- hipe_rtl:mk_load(ArityReg, FunP,
- hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED)+
- ?EFT_ARITY)),
- hipe_rtl:mk_load(FEPtrReg, FunP,
- hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED)+
- ?EFT_FE)),
+ Untag,
+ hipe_rtl:mk_load(ArityReg, Base,
+ hipe_rtl:mk_imm(Offset+?EFT_ARITY)),
+ hipe_rtl:mk_load(FEPtrReg, Base,
+ hipe_rtl:mk_imm(Offset+?EFT_FE)),
hipe_rtl:mk_load(AddressReg, FEPtrReg,
hipe_rtl:mk_imm(?EFE_NATIVE_ADDRESS))],
IsFunCode ++ GetArityCode.
@@ -927,20 +973,24 @@ test_subbinary(Binary, TrueLblName, FalseLblName) ->
unsafe_load_float(DstLo, DstHi, Src) ->
WordSize = hipe_rtl_arch:word_size(),
- Offset1 = -(?TAG_PRIMARY_BOXED) + WordSize,
+ {Base, Offset0, Untag} = untag_ptr(Src, ?TAG_PRIMARY_BOXED),
+ Offset1 = Offset0 + WordSize,
Offset2 = Offset1 + 4, %% This should really be 4 and not WordSize
case hipe_rtl_arch:endianess() of
little ->
- [hipe_rtl:mk_load(DstLo, Src, hipe_rtl:mk_imm(Offset1), int32, unsigned),
- hipe_rtl:mk_load(DstHi, Src, hipe_rtl:mk_imm(Offset2), int32, unsigned)];
+ [Untag,
+ hipe_rtl:mk_load(DstLo, Base, hipe_rtl:mk_imm(Offset1), int32, unsigned),
+ hipe_rtl:mk_load(DstHi, Base, hipe_rtl:mk_imm(Offset2), int32, unsigned)];
big ->
- [hipe_rtl:mk_load(DstHi, Src, hipe_rtl:mk_imm(Offset1), int32, unsigned),
- hipe_rtl:mk_load(DstLo, Src, hipe_rtl:mk_imm(Offset2), int32, unsigned)]
+ [Untag,
+ hipe_rtl:mk_load(DstHi, Base, hipe_rtl:mk_imm(Offset1), int32, unsigned),
+ hipe_rtl:mk_load(DstLo, Base, hipe_rtl:mk_imm(Offset2), int32, unsigned)]
end.
unsafe_untag_float(Dst, Src) ->
- Offset = -(?TAG_PRIMARY_BOXED) + hipe_rtl_arch:word_size(),
- [hipe_rtl:mk_fload(Dst, Src, hipe_rtl:mk_imm(Offset))].
+ {Base, Offset0, Untag} = untag_ptr(Src, ?TAG_PRIMARY_BOXED),
+ Offset = Offset0 + hipe_rtl_arch:word_size(),
+ [Untag, hipe_rtl:mk_fload(Dst, Base, hipe_rtl:mk_imm(Offset))].
unsafe_tag_float(Dst, Src) ->
{GetHPInsn, HP, PutHPInsn} = hipe_rtl_arch:heap_pointer(),
@@ -999,8 +1049,9 @@ get_one_word_pos_bignum(USize, Size, Fail) ->
unsafe_get_one_word_pos_bignum(USize, Size) ->
WordSize = hipe_rtl_arch:word_size(),
- Imm = hipe_rtl:mk_imm(1*WordSize-?TAG_PRIMARY_BOXED),
- [hipe_rtl:mk_load(USize, Size, Imm)].
+ {Base, Offset, Untag} = untag_ptr(Size, ?TAG_PRIMARY_BOXED),
+ Imm = hipe_rtl:mk_imm(1*WordSize+Offset),
+ [Untag, hipe_rtl:mk_load(USize, Base, Imm)].
-spec bignum_sizeneed(non_neg_integer()) -> non_neg_integer().
@@ -1040,7 +1091,7 @@ create_matchstate(Max, BinSize, Base, Offset, Orig, Ms) ->
SizeInWords = ((ByteSize div WordSize) - 1),
Header = hipe_rtl:mk_imm(mk_header(SizeInWords, ?TAG_HEADER_BIN_MATCHSTATE)),
[GetHPInsn,
- hipe_rtl:mk_alu(Ms, HP, add, hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)),
+ tag_boxed(Ms, HP),
set_field_from_term({matchstate,thing_word}, Ms, Header),
set_field_from_term({matchstate,{matchbuffer,orig}}, Ms, Orig),
set_field_from_term({matchstate,{matchbuffer,base}}, Ms, Base),
@@ -1078,7 +1129,10 @@ convert_matchstate(Ms) ->
size_from_header(SizeInWords, Header),
hipe_rtl:mk_alu(Hole, SizeInWords, sub, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE)),
mk_var_header(BigIntHeader, Hole, ?TAG_HEADER_POS_BIG),
- hipe_rtl:mk_store(Ms, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE*WordSize-?TAG_PRIMARY_BOXED),
+ %% Matchstates can't be literals; so untagging with ?TAG_PRIMARY_BOXED is
+ %% fine here
+ hipe_rtl:mk_store(Ms, hipe_rtl:mk_imm(?SUB_BIN_WORDSIZE*WordSize
+ -?TAG_PRIMARY_BOXED),
BigIntHeader)].
compare_matchstate(Max, Ms, LargeEnough, TooSmall) ->
@@ -1087,8 +1141,10 @@ compare_matchstate(Max, Ms, LargeEnough, TooSmall) ->
SizeInWords = ((ByteSize div WordSize) - 1),
Header = hipe_rtl:mk_imm(mk_header(SizeInWords, ?TAG_HEADER_BIN_MATCHSTATE)),
RealHeader = hipe_rtl:mk_new_reg_gcsafe(),
- [hipe_rtl:mk_load(RealHeader, Ms, hipe_rtl:mk_imm(-?TAG_PRIMARY_BOXED)),
- hipe_rtl:mk_branch(RealHeader, ge, Header, LargeEnough, TooSmall)].
+ %% Matchstates can't be literals; so untagging with ?TAG_PRIMARY_BOXED is fine
+ %% here
+ [hipe_rtl:mk_load(RealHeader, Ms, hipe_rtl:mk_imm(-?TAG_PRIMARY_BOXED)),
+ hipe_rtl:mk_branch(RealHeader, ge, Header, LargeEnough, TooSmall)].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%
@@ -1207,15 +1263,22 @@ get_field_size1({matchbuffer, base}) ->
get_field_size1({matchbuffer, binsize}) ->
?MB_SIZE_SIZE.
+get_field_addr_from_term(Struct, Term, Dst) ->
+ {Base, Offset0, Untag} = untag_ptr(Term, ?TAG_PRIMARY_BOXED),
+ Offset = hipe_rtl:mk_imm(get_field_offset(Struct) + Offset0),
+ [Untag, hipe_rtl:mk_alu(Dst, Base, add, Offset)].
+
get_field_from_term(Struct, Term, Dst) ->
- Offset = hipe_rtl:mk_imm(get_field_offset(Struct) - ?TAG_PRIMARY_BOXED),
+ {Base, Offset0, Untag} = untag_ptr(Term, ?TAG_PRIMARY_BOXED),
+ Offset = hipe_rtl:mk_imm(get_field_offset(Struct) + Offset0),
Size = get_field_size(Struct),
- hipe_rtl:mk_load(Dst, Term, Offset, Size, unsigned).
+ [Untag, hipe_rtl:mk_load(Dst, Base, Offset, Size, unsigned)].
set_field_from_term(Struct, Term, Value) ->
- Offset = hipe_rtl:mk_imm(get_field_offset(Struct) - ?TAG_PRIMARY_BOXED),
+ {Base, Offset0, Untag} = untag_ptr(Term, ?TAG_PRIMARY_BOXED),
+ Offset = hipe_rtl:mk_imm(get_field_offset(Struct) + Offset0),
Size = get_field_size(Struct),
- hipe_rtl:mk_store(Term, Offset, Value, Size).
+ [Untag, hipe_rtl:mk_store(Base, Offset, Value, Size)].
get_field_from_pointer(Struct, Term, Dst) ->
Offset = hipe_rtl:mk_imm(get_field_offset(Struct)),
@@ -1229,6 +1292,8 @@ set_field_from_pointer(Struct, Term, Value) ->
extract_matchbuffer(Mb, Ms) ->
What = {matchstate, matchbuffer},
+ %% Matchstates can't be literals; so untagging with ?TAG_PRIMARY_BOXED is fine
+ %% here
Offset = hipe_rtl:mk_imm(get_field_offset(What) - ?TAG_PRIMARY_BOXED),
hipe_rtl:mk_alu(Mb, Ms, add, Offset).
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 22:17:01 +0000