%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
%% Purpose: Generate BEAM assembly code from the SSA format.
-module(beam_ssa_codegen).
-export([module/2]).
-export([classify_heap_need/2]). %Called from beam_ssa_pre_codegen.
-export_type([ssa_register/0]).
-include("beam_ssa.hrl").
-import(lists, [foldl/3,keymember/3,keysort/2,map/2,mapfoldl/3,
reverse/1,reverse/2,sort/1,splitwith/2,takewhile/2]).
-record(cg, {lcount=1 :: beam_label(), %Label counter
functable=#{} :: #{fa()=>beam_label()},
labels=#{} :: #{ssa_label()=>0|beam_label()},
used_labels=gb_sets:empty() :: gb_sets:set(ssa_label()),
regs=#{} :: #{beam_ssa:var_name()=>ssa_register()},
ultimate_fail=1 :: beam_label(),
catches=gb_sets:empty() :: gb_sets:set(ssa_label()),
fc_label=1 :: beam_label()
}).
-spec module(beam_ssa:b_module(), [compile:option()]) ->
{'ok',beam_asm:module_code()}.
module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, _Opts) ->
{Asm,St} = functions(Fs, {atom,Mod}),
{ok,{Mod,Es,Attrs,Asm,St#cg.lcount}}.
-record(need, {h=0 :: non_neg_integer(),
f=0 :: non_neg_integer()}).
-record(cg_blk, {anno=#{} :: anno(),
is=[] :: [instruction()],
last :: terminator()}).
-record(cg_set, {anno=#{} :: anno(),
dst :: b_var(),
op :: beam_ssa:op(),
args :: [beam_ssa:argument() | xreg()]}).
-record(cg_alloc, {anno=#{} :: anno(),
stack=none :: 'none' | pos_integer(),
words=#need{} :: #need{},
live :: 'undefined' | pos_integer(),
def_yregs=[] :: [yreg()]
}).
-record(cg_br, {bool :: beam_ssa:value(),
succ :: ssa_label(),
fail :: ssa_label()
}).
-record(cg_ret, {arg :: cg_value(),
dealloc=none :: 'none' | pos_integer()
}).
-record(cg_switch, {arg :: cg_value(),
fail :: ssa_label(),
list :: [sw_list_item()]
}).
-type fa() :: {beam_asm:function_name(),arity()}.
-type ssa_label() :: beam_ssa:label().
-type beam_label() :: beam_asm:label().
-type anno() :: beam_ssa:anno().
-type b_var() :: beam_ssa:b_var().
-type b_literal() :: beam_ssa:b_literal().
-type cg_value() :: beam_ssa:value() | xreg().
-type cg_set() :: #cg_set{}.
-type cg_alloc() :: #cg_alloc{}.
-type instruction() :: cg_set() | cg_alloc().
-type cg_br() :: #cg_br{}.
-type cg_ret() :: #cg_ret{}.
-type cg_switch() :: #cg_switch{}.
-type terminator() :: cg_br() | cg_ret() | cg_switch().
-type sw_list_item() :: {b_literal(),ssa_label()}.
-type reg_num() :: beam_asm:reg_num().
-type xreg() :: {'x',reg_num()}.
-type yreg() :: {'y',reg_num()}.
-type ssa_register() :: xreg() | yreg() | {'fr',reg_num()} | {'z',reg_num()}.
functions(Forms, AtomMod) ->
mapfoldl(fun (F, St) -> function(F, AtomMod, St) end,
#cg{lcount=1}, Forms).
function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) ->
#{func_info:={_,Name,Arity}} = Anno,
try
assert_exception_block(Blocks), %Assertion.
Regs = maps:get(registers, Anno),
St1 = St0#cg{labels=#{},used_labels=gb_sets:empty(),
regs=Regs},
{Fi,St2} = new_label(St1), %FuncInfo label
{Entry,St3} = local_func_label(Name, Arity, St2),
{Ult,St4} = new_label(St3), %Ultimate failure
Labels = (St4#cg.labels)#{0=>Entry,?EXCEPTION_BLOCK=>0},
St5 = St4#cg{labels=Labels,used_labels=gb_sets:singleton(Entry),
ultimate_fail=Ult},
{Body,St} = cg_fun(Blocks, St5#cg{fc_label=Fi}),
Asm = [{label,Fi},line(Anno),
{func_info,AtomMod,{atom,Name},Arity}] ++
add_parameter_annos(Body, Anno) ++
[{label,Ult},if_end],
Func = {function,Name,Arity,Entry,Asm},
{Func,St}
catch
Class:Error:Stack ->
io:fwrite("Function: ~w/~w\n", [Name,Arity]),
erlang:raise(Class, Error, Stack)
end.
assert_exception_block(Blocks) ->
%% Assertion: ?EXCEPTION_BLOCK must be a call erlang:error(badarg).
case Blocks of
#{?EXCEPTION_BLOCK:=Blk} ->
#b_blk{is=[#b_set{op=call,dst=Ret,
args=[#b_remote{mod=#b_literal{val=erlang},
name=#b_literal{val=error}},
#b_literal{val=badarg}]}],
last=#b_ret{arg=Ret}} = Blk,
ok;
#{} ->
%% ?EXCEPTION_BLOCK has been removed because it was never used.
ok
end.
add_parameter_annos([{label, _}=Entry | Body], Anno) ->
ParamInfo = maps:get(parameter_type_info, Anno, #{}),
Annos = maps:fold(
fun(K, V, Acc) when is_map_key(K, ParamInfo) ->
TypeInfo = maps:get(K, ParamInfo),
[{'%', {type_info, V, TypeInfo}} | Acc];
(_K, _V, Acc) ->
Acc
end, [], maps:get(registers, Anno)),
[Entry | sort(Annos)] ++ Body.
cg_fun(Blocks, St0) ->
Linear0 = linearize(Blocks),
St = collect_catch_labels(Linear0, St0),
Linear1 = need_heap(Linear0),
Linear2 = prefer_xregs(Linear1, St),
Linear3 = liveness(Linear2, St),
Linear4 = defined(Linear3, St),
Linear = opt_allocate(Linear4, St),
cg_linear(Linear, St).
%% collect_catch_labels(Linear, St) -> St.
%% Collect all catch labels (labels for blocks that begin
%% with 'landingpad' instructions) for later use.
collect_catch_labels(Linear, St) ->
Labels = collect_catch_labels_1(Linear),
St#cg{catches=gb_sets:from_list(Labels)}.
collect_catch_labels_1([{L,#cg_blk{is=[#cg_set{op=landingpad}|_]}}|Bs]) ->
[L|collect_catch_labels_1(Bs)];
collect_catch_labels_1([_|Bs]) ->
collect_catch_labels_1(Bs);
collect_catch_labels_1([]) -> [].
%% need_heap([{BlockLabel,Block]) -> [{BlockLabel,Block}].
%% Insert need_heap instructions in the instruction list. Try to be smart and
%% collect them together as much as possible.
need_heap(Bs0) ->
Bs1 = need_heap_allocs(Bs0, #{}),
{Bs,#need{h=0,f=0}} = need_heap_blks(reverse(Bs1), #need{}, []),
Bs.
need_heap_allocs([{L,#cg_blk{is=Is0,last=Terminator}=Blk0}|Bs], Counts0) ->
Next = next_block(Bs),
Successors = successors(Terminator),
Counts = foldl(fun(S, Cnts) ->
case Cnts of
#{S:=C} -> Cnts#{S:=C+1};
#{} when S =:= Next -> Cnts#{S=>1};
#{} -> Cnts#{S=>42}
end
end, Counts0, Successors),
case Counts of
#{L:=1} ->
[{L,Blk0}|need_heap_allocs(Bs, Counts)];
#{L:=_} ->
%% This block has multiple predecessors. Force an allocation
%% in this block so that the predecessors don't need to do
%% an allocation on behalf of this block.
Is = case need_heap_never(Is0) of
true -> Is0;
false -> [#cg_alloc{}|Is0]
end,
Blk = Blk0#cg_blk{is=Is},
[{L,Blk}|need_heap_allocs(Bs, Counts)];
#{} ->
[{L,Blk0}|need_heap_allocs(Bs, Counts)]
end;
need_heap_allocs([], _) -> [].
need_heap_never([#cg_alloc{}|_]) -> true;
need_heap_never([#cg_set{op=recv_next}|_]) -> true;
need_heap_never([#cg_set{op=wait}|_]) -> true;
need_heap_never(_) -> false.
need_heap_blks([{L,#cg_blk{is=Is0}=Blk0}|Bs], H0, Acc) ->
{Is1,H1} = need_heap_is(reverse(Is0), H0, []),
{Ns,H} = need_heap_terminator(Bs, L, H1),
Is = Ns ++ Is1,
Blk = Blk0#cg_blk{is=Is},
need_heap_blks(Bs, H, [{L,Blk}|Acc]);
need_heap_blks([], H, Acc) ->
{Acc,H}.
need_heap_is([#cg_alloc{words=Words}=Alloc0|Is], N, Acc) ->
Alloc = Alloc0#cg_alloc{words=add_heap_words(N, Words)},
need_heap_is(Is, #need{}, [Alloc|Acc]);
need_heap_is([#cg_set{anno=Anno,op=bs_init}=I0|Is], N, Acc) ->
Alloc = case need_heap_need(N) of
[#cg_alloc{words=Need}] -> alloc(Need);
[] -> 0
end,
I = I0#cg_set{anno=Anno#{alloc=>Alloc}},
need_heap_is(Is, #need{}, [I|Acc]);
need_heap_is([#cg_set{op=Op,args=Args}=I|Is], N, Acc) ->
case classify_heap_need(Op, Args) of
{put,Words} ->
%% Pass through adding to needed heap.
need_heap_is(Is, add_heap_words(N, Words), [I|Acc]);
put_float ->
need_heap_is(Is, add_heap_float(N), [I|Acc]);
neutral ->
need_heap_is(Is, N, [I|Acc]);
gc ->
need_heap_is(Is, #need{}, [I]++need_heap_need(N)++Acc)
end;
need_heap_is([], N, Acc) ->
{Acc,N}.
need_heap_terminator([{_,#cg_blk{last=#cg_br{succ=L,fail=L}}}|_], L, N) ->
%% Fallthrough.
{[],N};
need_heap_terminator([{_,#cg_blk{is=Is,last=#cg_br{succ=L}}}|_], L, N) ->
case need_heap_need(N) of
[] ->
{[],#need{}};
[_|_]=Alloc ->
%% If the preceding instructions are a binary construction,
%% hoist the allocation and incorporate into the bs_init
%% instruction.
case reverse(Is) of
[#cg_set{op=succeeded},#cg_set{op=bs_init}|_] ->
{[],N};
[#cg_set{op=bs_put}|_] ->
{[],N};
_ ->
%% Not binary construction. Must emit an allocation
%% instruction in this block.
{Alloc,#need{}}
end
end;
need_heap_terminator([{_,#cg_blk{}}|_], _, N) ->
{need_heap_need(N),#need{}};
need_heap_terminator([], _, H) ->
{need_heap_need(H),#need{}}.
need_heap_need(#need{h=0,f=0}) -> [];
need_heap_need(#need{}=N) -> [#cg_alloc{words=N}].
add_heap_words(#need{h=H1,f=F1}, #need{h=H2,f=F2}) ->
#need{h=H1+H2,f=F1+F2};
add_heap_words(#need{h=Heap}=N, Words) when is_integer(Words) ->
N#need{h=Heap+Words}.
add_heap_float(#need{f=F}=N) ->
N#need{f=F+1}.
%% classify_heap_need(Operation, Arguments) ->
%% gc | neutral | {put,Words} | put_float.
%% Classify the heap need for this instruction. The return
%% values have the following meaning.
%%
%% {put,Words} means that the instruction will use Words words to build
%% something on the heap.
%%
%% 'put_float' means that the instruction will build one floating point
%% number on the heap.
%%
%% 'gc' means that that the instruction can potentially do a GC or throw an
%% exception. That means that an allocation instruction for any building
%% must be placed after this instruction.
%%
%% 'neutral' means that the instruction does nothing to disturb the heap.
-spec classify_heap_need(beam_ssa:op(), [beam_ssa:value()]) ->
'gc' | 'neutral' |
{'put',non_neg_integer()} | 'put_float'.
classify_heap_need(put_list, _) ->
{put,2};
classify_heap_need(put_tuple_arity, [#b_literal{val=Words}]) ->
{put,Words+1};
classify_heap_need(put_tuple, Elements) ->
{put,length(Elements)+1};
classify_heap_need({bif,Name}, Args) ->
case is_gc_bif(Name, Args) of
false -> neutral;
true -> gc
end;
classify_heap_need({float,Op}, _Args) ->
case Op of
get -> put_float;
_ -> neutral
end;
classify_heap_need(Name, _Args) ->
classify_heap_need(Name).
%% classify_heap_need(Operation) -> gc | neutral.
%% Return either 'gc' or 'neutral'.
%%
%% 'gc' means that that the instruction can potentially do a GC or throw an
%% exception. That means that an allocation instruction for any building
%% must be placed after this instruction.
%%
%% 'neutral' means that the instruction does nothing to disturb the heap.
%%
%% Note: Only handle operations in this function that are not handled
%% by classify_heap_need/2.
classify_heap_need(bs_add) -> gc;
classify_heap_need(bs_get) -> gc;
classify_heap_need(bs_get_tail) -> gc;
classify_heap_need(bs_init) -> gc;
classify_heap_need(bs_init_writable) -> gc;
classify_heap_need(bs_match_string) -> gc;
classify_heap_need(bs_put) -> neutral;
classify_heap_need(bs_restore) -> neutral;
classify_heap_need(bs_save) -> neutral;
classify_heap_need(bs_get_position) -> gc;
classify_heap_need(bs_set_position) -> neutral;
classify_heap_need(bs_skip) -> gc;
classify_heap_need(bs_start_match) -> neutral;
classify_heap_need(bs_test_tail) -> neutral;
classify_heap_need(bs_utf16_size) -> neutral;
classify_heap_need(bs_utf8_size) -> neutral;
classify_heap_need(build_stacktrace) -> gc;
classify_heap_need(call) -> gc;
classify_heap_need(catch_end) -> gc;
classify_heap_need(copy) -> neutral;
classify_heap_need(extract) -> gc;
classify_heap_need(get_hd) -> neutral;
classify_heap_need(get_map_element) -> neutral;
classify_heap_need(get_tl) -> neutral;
classify_heap_need(get_tuple_element) -> neutral;
classify_heap_need(has_map_field) -> neutral;
classify_heap_need(is_nonempty_list) -> neutral;
classify_heap_need(is_tagged_tuple) -> neutral;
classify_heap_need(kill_try_tag) -> gc;
classify_heap_need(landingpad) -> gc;
classify_heap_need(make_fun) -> gc;
classify_heap_need(match_fail) -> gc;
classify_heap_need(new_try_tag) -> gc;
classify_heap_need(peek_message) -> gc;
classify_heap_need(put_map) -> gc;
classify_heap_need(put_tuple_elements) -> neutral;
classify_heap_need(raw_raise) -> gc;
classify_heap_need(recv_next) -> gc;
classify_heap_need(remove_message) -> neutral;
classify_heap_need(resume) -> gc;
classify_heap_need(set_tuple_element) -> gc;
classify_heap_need(succeeded) -> neutral;
classify_heap_need(timeout) -> gc;
classify_heap_need(wait) -> gc;
classify_heap_need(wait_timeout) -> gc.
%%%
%%% Because beam_ssa_pre_codegen has inserted 'copy' instructions to copy
%%% variables that must be saved on the stack, a value can for some time
%%% be in both an X register and a Y register.
%%%
%%% Here we will keep track of variables that have the same value and
%%% rewrite instructions to use the variable that refers to the X
%%% register instead of the Y register. That could improve performance,
%%% since the BEAM interpreter have more optimized instructions
%%% operating on X registers than on Y registers.
%%%
%%% 'call' and 'make_fun' are handled somewhat specially. If a value
%%% already is in the correct X register, the X register will always
%%% be used instead of the Y register. However, if there are one or more
%%% values in the wrong X registers, the X registers variables will be
%%% used only if that does not cause more 'move' instructions to be
%%% be emitted than if the Y register variables were used.
%%%
%%% Here are some examples. The first example shows how a 'move' from
%%% an Y register is eliminated:
%%%
%%% move x0 y1
%%% move y1 x0 %%Will be eliminated.
%%%
%%% call f/1
%%%
%%% Here is an example when x0 and x1 must be swapped to load the argument
%%% registers. Here the 'call' instruction will use the Y registers to
%%% avoid introducing an extra 'move' insruction:
%%%
%%% move x0 y0
%%% move x1 y1
%%%
%%% move y0 x1
%%% move y1 x0
%%%
%%% call f/2
%%%
%%% Using the X register to load the argument registers would need
%%% an extra 'move' instruction like this:
%%%
%%% move x0 y0
%%% move x1 y1
%%%
%%% move x1 x2
%%% move x0 x1
%%% move x2 x0
%%%
%%% call f/2
%%%
prefer_xregs(Linear, St) ->
prefer_xregs(Linear, St, #{0=>#{}}).
prefer_xregs([{L,#cg_blk{is=Is0,last=Last0}=Blk0}|Bs], St, Map0) ->
Copies0 = maps:get(L, Map0),
{Is,Copies} = prefer_xregs_is(Is0, St, Copies0, []),
Last = prefer_xregs_terminator(Last0, Copies, St),
Blk = Blk0#cg_blk{is=Is,last=Last},
Successors = successors(Last),
Map = prefer_xregs_successors(Successors, Copies, Map0),
[{L,Blk}|prefer_xregs(Bs, St, Map)];
prefer_xregs([], _St, _Map) -> [].
prefer_xregs_successors([L|Ls], Copies0, Map0) ->
case Map0 of
#{L:=Copies1} ->
Copies = merge_copies(Copies0, Copies1),
Map = Map0#{L:=Copies},
prefer_xregs_successors(Ls, Copies0, Map);
#{} ->
Map = Map0#{L=>Copies0},
prefer_xregs_successors(Ls, Copies0, Map)
end;
prefer_xregs_successors([], _, Map) -> Map.
prefer_xregs_is([#cg_alloc{}=I|Is], St, Copies0, Acc) ->
Copies = case I of
#cg_alloc{stack=none,words=#need{h=0,f=0}} ->
Copies0;
#cg_alloc{} ->
#{}
end,
prefer_xregs_is(Is, St, Copies, [I|Acc]);
prefer_xregs_is([#cg_set{op=copy,dst=Dst,args=[Src]}=I|Is], St, Copies0, Acc) ->
Copies1 = prefer_xregs_prune(I, Copies0, St),
Copies = case beam_args([Src,Dst], St) of
[Same,Same] -> Copies1;
[_,_] -> Copies1#{Dst=>Src}
end,
prefer_xregs_is(Is, St, Copies, [I|Acc]);
prefer_xregs_is([#cg_set{op=call,dst=Dst}=I0|Is], St, Copies, Acc) ->
I = prefer_xregs_call(I0, Copies, St),
prefer_xregs_is(Is, St, #{Dst=>{x,0}}, [I|Acc]);
prefer_xregs_is([#cg_set{op=make_fun,dst=Dst}=I0|Is], St, Copies, Acc) ->
I = prefer_xregs_call(I0, Copies, St),
prefer_xregs_is(Is, St, #{Dst=>{x,0}}, [I|Acc]);
prefer_xregs_is([#cg_set{op=set_tuple_element}=I|Is], St, Copies, Acc) ->
%% FIXME: HiPE translates the following code segment incorrectly:
%% {call_ext,3,{extfunc,erlang,setelement,3}}.
%% {move,{x,0},{y,3}}.
%% {set_tuple_element,{y,1},{y,3},1}.
%% Therefore, skip the translation of the arguments for set_tuple_element.
prefer_xregs_is(Is, St, Copies, [I|Acc]);
prefer_xregs_is([#cg_set{args=Args0}=I0|Is], St, Copies0, Acc) ->
Args = [do_prefer_xreg(A, Copies0, St) || A <- Args0],
I = I0#cg_set{args=Args},
Copies = prefer_xregs_prune(I, Copies0, St),
prefer_xregs_is(Is, St, Copies, [I|Acc]);
prefer_xregs_is([], _St, Copies, Acc) ->
{reverse(Acc),Copies}.
prefer_xregs_terminator(#cg_br{bool=Arg0}=I, Copies, St) ->
Arg = do_prefer_xreg(Arg0, Copies, St),
I#cg_br{bool=Arg};
prefer_xregs_terminator(#cg_ret{arg=Arg0}=I, Copies, St) ->
Arg = do_prefer_xreg(Arg0, Copies, St),
I#cg_ret{arg=Arg};
prefer_xregs_terminator(#cg_switch{arg=Arg0}=I, Copies, St) ->
Arg = do_prefer_xreg(Arg0, Copies, St),
I#cg_switch{arg=Arg}.
prefer_xregs_prune(#cg_set{anno=#{clobbers:=true}}, _, _) ->
#{};
prefer_xregs_prune(#cg_set{dst=Dst}, Copies, St) ->
DstReg = beam_arg(Dst, St),
F = fun(_, Alias) ->
beam_arg(Alias, St) =/= DstReg
end,
maps:filter(F, Copies).
%% prefer_xregs_call(Instruction, Copies, St) -> Instruction.
%% Given a 'call' or 'make_fun' instruction, minimize the number
%% of 'move' instructions to set up the argument registers.
%% Prefer using X registers over Y registers, unless that will
%% result in more 'move' instructions.
prefer_xregs_call(#cg_set{args=[_]}=I, _Copies, _St) ->
I;
prefer_xregs_call(#cg_set{args=[F|Args0]}=I, Copies, St) ->
case Args0 of
[A0] ->
%% Only one argument. Always prefer the X register
%% if available.
A = do_prefer_xreg(A0, Copies, St),
I#cg_set{args=[F,A]};
[_|_] ->
%% Two or more arguments. Try rewriting arguments in
%% two ways and see which way produces the least
%% number of 'move' instructions.
Args1 = prefer_xregs_call_1(Args0, Copies, 0, St),
Args2 = [do_prefer_xreg(A, Copies, St) || A <- Args0],
case {count_moves(Args1, St),count_moves(Args2, St)} of
{N1,N2} when N1 < N2 ->
%% There will be fewer 'move' instructions if
%% we keep using Y registers.
I#cg_set{args=[F|Args1]};
{_,_} ->
%% Always use the values in X registers.
I#cg_set{args=[F|Args2]}
end
end.
count_moves(Args, St) ->
length(setup_args(beam_args(Args, St))).
prefer_xregs_call_1([#b_var{}=A|As], Copies, X, St) ->
case {beam_arg(A, St),Copies} of
{{y,_},#{A:=Other}} ->
case beam_arg(Other, St) of
{x,X} ->
%% This value is already in the correct X register.
%% It is always benefical to use the X register variable.
[Other|prefer_xregs_call_1(As, Copies, X+1, St)];
_ ->
%% This value is another X register. Keep using
%% the Y register variable.
[A|prefer_xregs_call_1(As, Copies, X+1, St)]
end;
{_,_} ->
%% The value is not available in an X register.
[A|prefer_xregs_call_1(As, Copies, X+1, St)]
end;
prefer_xregs_call_1([A|As], Copies, X, St) ->
[A|prefer_xregs_call_1(As, Copies, X+1, St)];
prefer_xregs_call_1([], _, _, _) -> [].
do_prefer_xreg(#b_var{}=A, Copies, St) ->
case {beam_arg(A, St),Copies} of
{{y,_},#{A:=Copy}} ->
Copy;
{_,_} ->
A
end;
do_prefer_xreg(A, _, _) -> A.
merge_copies(Copies0, Copies1) when map_size(Copies0) =< map_size(Copies1) ->
maps:filter(fun(K, V) ->
case Copies1 of
#{K:=V} -> true;
#{} -> false
end
end, Copies0);
merge_copies(Copies0, Copies1) ->
merge_copies(Copies1, Copies0).
%%%
%%% Add annotations for the number of live registers.
%%%
liveness(Linear, #cg{regs=Regs}) ->
liveness(reverse(Linear), #{}, Regs, []).
liveness([{L,#cg_blk{is=Is0,last=Last0}=Blk0}|Bs], LiveMap0, Regs, Acc) ->
Successors = liveness_successors(Last0),
Live0 = ordsets:union([liveness_get(S, LiveMap0) || S <- Successors]),
Live1 = liveness_terminator(Last0, Live0),
{Is,Live} = liveness_is(reverse(Is0), Regs, Live1, []),
LiveMap = LiveMap0#{L=>Live},
Blk = Blk0#cg_blk{is=Is},
liveness(Bs, LiveMap, Regs, [{L,Blk}|Acc]);
liveness([], _LiveMap, _Regs, Acc) -> Acc.
liveness_get(S, LiveMap) ->
case LiveMap of
#{S:=Live} -> Live;
#{} -> []
end.
liveness_successors(Terminator) ->
successors(Terminator) -- [?EXCEPTION_BLOCK].
liveness_is([#cg_alloc{}=I0|Is], Regs, Live, Acc) ->
I = I0#cg_alloc{live=num_live(Live, Regs)},
liveness_is(Is, Regs, Live, [I|Acc]);
liveness_is([#cg_set{dst=Dst,args=Args}=I0|Is], Regs, Live0, Acc) ->
Live1 = liveness_clobber(I0, Live0, Regs),
I1 = liveness_yregs_anno(I0, Live1, Regs),
Live2 = liveness_args(Args, Live1),
Live = ordsets:del_element(Dst, Live2),
I = liveness_anno(I1, Live, Regs),
liveness_is(Is, Regs, Live, [I|Acc]);
liveness_is([], _, Live, Acc) ->
{Acc,Live}.
liveness_terminator(#cg_br{bool=Arg}, Live) ->
liveness_terminator_1(Arg, Live);
liveness_terminator(#cg_switch{arg=Arg}, Live) ->
liveness_terminator_1(Arg, Live);
liveness_terminator(#cg_ret{arg=Arg}, Live) ->
liveness_terminator_1(Arg, Live).
liveness_terminator_1(#b_var{}=V, Live) ->
ordsets:add_element(V, Live);
liveness_terminator_1(#b_literal{}, Live) ->
Live;
liveness_terminator_1(Reg, Live) ->
_ = verify_beam_register(Reg),
ordsets:add_element(Reg, Live).
liveness_args([#b_var{}=V|As], Live) ->
liveness_args(As, ordsets:add_element(V, Live));
liveness_args([#b_remote{mod=Mod,name=Name}|As], Live) ->
liveness_args([Mod,Name|As], Live);
liveness_args([A|As], Live) ->
case is_beam_register(A) of
true ->
liveness_args(As, ordsets:add_element(A, Live));
false ->
liveness_args(As, Live)
end;
liveness_args([], Live) -> Live.
liveness_anno(#cg_set{op=Op}=I, Live, Regs) ->
case need_live_anno(Op) of
true ->
NumLive = num_live(Live, Regs),
Anno = (I#cg_set.anno)#{live=>NumLive},
I#cg_set{anno=Anno};
false ->
I
end.
liveness_yregs_anno(#cg_set{op=Op,dst=Dst}=I, Live0, Regs) ->
case need_live_anno(Op) of
true ->
Live = ordsets:del_element(Dst, Live0),
LiveYregs = [V || V <- Live, is_yreg(V, Regs)],
Anno = (I#cg_set.anno)#{live_yregs=>LiveYregs},
I#cg_set{anno=Anno};
false ->
I
end.
liveness_clobber(#cg_set{anno=Anno}, Live, Regs) ->
case Anno of
#{clobbers:=true} ->
[R || R <- Live, is_yreg(R, Regs)];
_ ->
Live
end.
is_yreg(R, Regs) ->
case Regs of
#{R:={y,_}} -> true;
#{} -> false
end.
num_live(Live, Regs) ->
Rs = ordsets:from_list([get_register(V, Regs) || V <- Live]),
num_live_1(Rs, 0).
num_live_1([{x,X}|T], X) ->
num_live_1(T, X+1);
num_live_1([{x,_}|_]=T, X) ->
%% error({hole,{x,X},expected,Next});
num_live_1(T, X+1);
num_live_1([{y,_}|_], X) ->
X;
num_live_1([{z,_}|_], X) ->
X;
num_live_1([{fr,_}|T], X) ->
num_live_1(T, X);
num_live_1([], X) ->
X.
get_live(#cg_set{anno=#{live:=Live}}) ->
Live.
%% need_live_anno(Operation) -> true|false.
%% Return 'true' if the instruction needs a 'live' annotation with
%% the number live X registers, or 'false' otherwise.
need_live_anno(Op) ->
case Op of
{bif,_} -> true;
bs_get -> true;
bs_init -> true;
bs_get_position -> true;
bs_get_tail -> true;
bs_start_match -> true;
bs_skip -> true;
call -> true;
put_map -> true;
_ -> false
end.
%%%
%%% Add the following annotations for Y registers:
%%%
%%% def_yregs An ordset with variables that refer to live Y registers.
%%% That is, Y registers that that have been killed
%%% are not included. This annotation is added to all
%%% instructions that require Y registers to be initialized.
%%%
%%% kill_yregs This annotation is added to call instructions. It is
%%% an ordset containing variables referring to Y registers
%%% that will no longer be used after the call instruction.
%%%
defined(Linear, #cg{regs=Regs}) ->
def(Linear, #{}, Regs).
def([{L,#cg_blk{is=Is0,last=Last}=Blk0}|Bs], DefMap0, Regs) ->
Def0 = def_get(L, DefMap0),
{Is,Def,MaybeDef} = def_is(Is0, Regs, Def0, []),
DefMap = def_successors(Last, Def, MaybeDef, DefMap0),
Blk = Blk0#cg_blk{is=Is},
[{L,Blk}|def(Bs, DefMap, Regs)];
def([], _, _) -> [].
def_get(L, DefMap) ->
case DefMap of
#{L:=Def} -> Def;
#{} -> []
end.
def_is([#cg_alloc{anno=Anno0}=I0|Is], Regs, Def, Acc) ->
I = I0#cg_alloc{anno=Anno0#{def_yregs=>Def}},
def_is(Is, Regs, Def, [I|Acc]);
def_is([#cg_set{op=succeeded,args=[Var]}=I], Regs, Def, Acc) ->
%% Var will only be defined on the success branch of the `br`
%% for this block.
MaybeDef = def_add_yreg(Var, [], Regs),
{reverse(Acc, [I]),Def,MaybeDef};
def_is([#cg_set{op=kill_try_tag,args=[#b_var{}=Tag]}=I|Is], Regs, Def0, Acc) ->
Def = ordsets:del_element(Tag, Def0),
def_is(Is, Regs, Def, [I|Acc]);
def_is([#cg_set{op=catch_end,args=[#b_var{}=Tag|_]}=I|Is], Regs, Def0, Acc) ->
Def = ordsets:del_element(Tag, Def0),
def_is(Is, Regs, Def, [I|Acc]);
def_is([#cg_set{anno=Anno0,op=call,dst=Dst}=I0|Is],
Regs, Def0, Acc) ->
#{live_yregs:=LiveYregVars} = Anno0,
LiveRegs = gb_sets:from_list([maps:get(V, Regs) || V <- LiveYregVars]),
Kill0 = ordsets:subtract(Def0, LiveYregVars),
%% Kill0 is the set of variables that have just died. However, the registers
%% used for killed variables may have been reused, so we must check that the
%% registers to be killed are not used by other variables.
Kill = [K || K <- Kill0, not gb_sets:is_element(maps:get(K, Regs), LiveRegs)],
Anno = Anno0#{def_yregs=>Def0,kill_yregs=>Kill},
I = I0#cg_set{anno=Anno},
Def1 = ordsets:subtract(Def0, Kill),
Def = def_add_yreg(Dst, Def1, Regs),
def_is(Is, Regs, Def, [I|Acc]);
def_is([#cg_set{anno=Anno0,op={bif,Bif},dst=Dst,args=Args}=I0|Is],
Regs, Def0, Acc) ->
Arity = length(Args),
I = case is_gc_bif(Bif, Args) orelse not erl_bifs:is_safe(erlang, Bif, Arity) of
true ->
I0#cg_set{anno=Anno0#{def_yregs=>Def0}};
false ->
I0
end,
Def = def_add_yreg(Dst, Def0, Regs),
def_is(Is, Regs, Def, [I|Acc]);
def_is([#cg_set{anno=Anno0,dst=Dst}=I0|Is], Regs, Def0, Acc) ->
I = case need_y_init(I0) of
true ->
I0#cg_set{anno=Anno0#{def_yregs=>Def0}};
false ->
I0
end,
Def = def_add_yreg(Dst, Def0, Regs),
def_is(Is, Regs, Def, [I|Acc]);
def_is([], _, Def, Acc) ->
{reverse(Acc),Def,[]}.
def_add_yreg(Dst, Def, Regs) ->
case is_yreg(Dst, Regs) of
true -> ordsets:add_element(Dst, Def);
false -> Def
end.
def_successors(#cg_br{bool=#b_var{},succ=Succ,fail=Fail}, Def, MaybeDef, DefMap0) ->
DefMap = def_successors([Fail], ordsets:subtract(Def, MaybeDef), DefMap0),
def_successors([Succ], Def, DefMap);
def_successors(Last, Def, [], DefMap) ->
def_successors(successors(Last), Def, DefMap).
def_successors([S|Ss], Def0, DefMap) ->
case DefMap of
#{S:=Def1} ->
Def = ordsets:intersection(Def0, Def1),
def_successors(Ss, Def0, DefMap#{S:=Def});
#{} ->
def_successors(Ss, Def0, DefMap#{S=>Def0})
end;
def_successors([], _, DefMap) -> DefMap.
%% need_y_init(#cg_set{}) -> true|false.
%% Return true if this instructions needs initialized Y registers
%% (because the instruction may do a GC or cause an exception
%% so that the stack will be scanned), or false otherwise.
need_y_init(#cg_set{anno=#{clobbers:=Clobbers}}) -> Clobbers;
need_y_init(#cg_set{op=bs_get}) -> true;
need_y_init(#cg_set{op=bs_get_position}) -> true;
need_y_init(#cg_set{op=bs_get_tail}) -> true;
need_y_init(#cg_set{op=bs_init}) -> true;
need_y_init(#cg_set{op=bs_skip,args=[#b_literal{val=Type}|_]}) ->
case Type of
utf8 -> true;
utf16 -> true;
utf32 -> true;
_ -> false
end;
need_y_init(#cg_set{op=bs_start_match}) -> true;
need_y_init(#cg_set{op=put_map}) -> true;
need_y_init(#cg_set{}) -> false.
%% opt_allocate([{BlockLabel,Block}], #st{}) -> [BeamInstruction].
%% Update the def_yregs field of each #cg_alloc{} that allocates
%% a stack frame. #cg_alloc.def_yregs will list all Y registers
%% that will be initialized by the subsequent code (thus, the
%% listed Y registers don't require init/1 instructions).
opt_allocate(Linear, #cg{regs=Regs}) ->
opt_allocate_1(Linear, Regs).
opt_allocate_1([{L,#cg_blk{is=[#cg_alloc{stack=Stk}=I0|Is]}=Blk0}|Bs]=Bs0, Regs)
when is_integer(Stk) ->
%% Collect the variables that are initialized by copy
%% instruction in this block.
case ordsets:from_list(opt_allocate_defs(Is, Regs)) of
Yregs when length(Yregs) =:= Stk ->
%% Those copy instructions are sufficient to fully
%% initialize the stack frame.
I = I0#cg_alloc{def_yregs=Yregs},
[{L,Blk0#cg_blk{is=[I|Is]}}|opt_allocate_1(Bs, Regs)];
Yregs0 ->
%% Determine a conservative approximation of the Y
%% registers that are guaranteed to be initialized by all
%% successors of this block, and to it add the variables
%% initialized by copy instructions in this block.
Yregs1 = opt_alloc_def(Bs0, gb_sets:singleton(L), []),
Yregs = ordsets:union(Yregs0, Yregs1),
I = I0#cg_alloc{def_yregs=Yregs},
[{L,Blk0#cg_blk{is=[I|Is]}}|opt_allocate_1(Bs, Regs)]
end;
opt_allocate_1([B|Bs], Regs) ->
[B|opt_allocate_1(Bs, Regs)];
opt_allocate_1([], _) -> [].
opt_allocate_defs([#cg_set{op=copy,dst=Dst}|Is], Regs) ->
case is_yreg(Dst, Regs) of
true -> [Dst|opt_allocate_defs(Is, Regs)];
false -> []
end;
opt_allocate_defs(_, _Regs) -> [].
opt_alloc_def([{L,#cg_blk{is=Is,last=Last}}|Bs], Ws0, Def0) ->
case gb_sets:is_member(L, Ws0) of
false ->
opt_alloc_def(Bs, Ws0, Def0);
true ->
case opt_allocate_is(Is) of
none ->
Succ = successors(Last),
Ws = gb_sets:union(Ws0, gb_sets:from_list(Succ)),
opt_alloc_def(Bs, Ws, Def0);
Def1 when is_list(Def1) ->
Def = [Def1|Def0],
opt_alloc_def(Bs, Ws0, Def)
end
end;
opt_alloc_def([], _, Def) ->
ordsets:intersection(Def).
opt_allocate_is([#cg_set{anno=Anno}|Is]) ->
case Anno of
#{def_yregs:=Yregs} ->
Yregs;
#{} ->
opt_allocate_is(Is)
end;
opt_allocate_is([#cg_alloc{anno=#{def_yregs:=Yregs},stack=none}|_]) ->
Yregs;
opt_allocate_is([#cg_alloc{}|Is]) ->
opt_allocate_is(Is);
opt_allocate_is([]) -> none.
%%%
%%% Here follows the main code generation functions.
%%%
%% cg_linear([{BlockLabel,Block}]) -> [BeamInstruction].
%% Generate BEAM instructions.
cg_linear([{L,#cg_blk{anno=#{recv_set:=L}=Anno0}=B0}|Bs], St0) ->
Anno = maps:remove(recv_set, Anno0),
B = B0#cg_blk{anno=Anno},
{Is,St1} = cg_linear([{L,B}|Bs], St0),
{Fail,St} = use_block_label(L, St1),
{[{recv_set,Fail}|Is],St};
cg_linear([{L,#cg_blk{is=Is0,last=Last}}|Bs], St0) ->
Next = next_block(Bs),
St1 = new_block_label(L, St0),
{Is1,St2} = cg_block(Is0, Last, Next, St1),
{Is2,St} = cg_linear(Bs, St2),
{def_block_label(L, St)++Is1++Is2,St};
cg_linear([], St) -> {[],St}.
cg_block([#cg_set{op=recv_next}], #cg_br{succ=Lr0}, _Next, St0) ->
{Lr,St} = use_block_label(Lr0, St0),
{[{loop_rec_end,Lr}],St};
cg_block([#cg_set{op=wait}], #cg_br{succ=Lr0}, _Next, St0) ->
{Lr,St} = use_block_label(Lr0, St0),
{[{wait,Lr}],St};
cg_block(Is0, Last, Next, St0) ->
case Last of
#cg_br{succ=Next,fail=Next} ->
cg_block(Is0, none, St0);
#cg_br{succ=Same,fail=Same} when Same =:= ?EXCEPTION_BLOCK ->
%% An expression in this block *always* throws an exception, so we
%% terminate it with an 'if_end' to make sure the validator knows
%% that the following instructions won't actually be reached.
{Is,St} = cg_block(Is0, none, St0),
{Is++[if_end],St};
#cg_br{succ=Same,fail=Same} ->
{Fail,St1} = use_block_label(Same, St0),
{Is,St} = cg_block(Is0, none, St1),
{Is++[jump(Fail)],St};
#cg_br{bool=Bool,succ=Next,fail=Fail0} ->
{Fail,St1} = use_block_label(Fail0, St0),
{Is,St} = cg_block(Is0, {Bool,Fail}, St1),
{Is,St};
#cg_br{bool=Bool,succ=Succ0,fail=Fail0} ->
{[Succ,Fail],St1} = use_block_labels([Succ0,Fail0], St0),
{Is,St} = cg_block(Is0, {Bool,Fail}, St1),
{Is++[jump(Succ)],St};
#cg_ret{arg=Src0,dealloc=N} ->
Src = beam_arg(Src0, St0),
cg_block(Is0, {return,Src,N}, St0);
#cg_switch{} ->
cg_switch(Is0, Last, St0)
end.
cg_switch(Is0, Last, St0) ->
#cg_switch{arg=Src0,fail=Fail0,list=List0} = Last,
Src = beam_arg(Src0, St0),
{Fail1,St1} = use_block_label(Fail0, St0),
Fail = ensure_label(Fail1, St1),
{List1,St2} =
flatmapfoldl(fun({V,L}, S0) ->
{Lbl,S} = use_block_label(L, S0),
{[beam_arg(V, S),Lbl],S}
end, St1, List0),
{Is1,St} = cg_block(Is0, none, St2),
case reverse(Is1) of
[{bif,tuple_size,_,[Tuple],{z,_}=Src}|More] ->
List = map(fun({integer,Arity}) -> Arity;
({f,_}=F) -> F
end, List1),
Is = reverse(More, [{select_tuple_arity,Tuple,Fail,{list,List}}]),
{Is,St};
_ ->
SelectVal = {select_val,Src,Fail,{list,List1}},
{Is1 ++ [SelectVal],St}
end.
jump({f,_}=Fail) ->
{jump,Fail};
jump({catch_tag,Fail}) ->
{jump,Fail}.
bif_fail({f,_}=Fail) -> Fail;
bif_fail({catch_tag,_}) -> {f,0}.
next_block([]) -> none;
next_block([{Next,_}|_]) -> Next.
%% Certain instructions (such as get_map_element or is_nonempty_list)
%% are only used in guards and **must** have a non-zero label;
%% otherwise, the loader will refuse to load the
%% module. ensure_label/2 replaces a zero label with the "ultimate
%% failure" label to make the module loadable. The instruction that
%% have had the zero label replaced is **not** supposed to ever fail
%% and actually jump to the label.
ensure_label(Fail0, #cg{ultimate_fail=Lbl}) ->
case bif_fail(Fail0) of
{f,0} -> {f,Lbl};
{f,_}=Fail -> Fail
end.
cg_block([#cg_set{anno=#{recv_mark:=L}=Anno0}=I0|T], Context, St0) ->
Anno = maps:remove(recv_mark, Anno0),
I = I0#cg_set{anno=Anno},
{Is,St1} = cg_block([I|T], Context, St0),
{Fail,St} = use_block_label(L, St1),
{[{recv_mark,Fail}|Is],St};
cg_block([#cg_set{op=new_try_tag,dst=Tag,args=Args}], {Tag,Fail0}, St) ->
{catch_tag,Fail} = Fail0,
[Reg,{atom,Kind}] = beam_args([Tag|Args], St),
{[{Kind,Reg,Fail}],St};
cg_block([#cg_set{anno=Anno,op={bif,Name},dst=Dst0,args=Args0}=I,
#cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) ->
[Dst|Args] = beam_args([Dst0|Args0], St),
Line0 = call_line(body, {extfunc,erlang,Name,length(Args)}, Anno),
Fail = bif_fail(Fail0),
Line = case Fail of
{f,0} -> Line0;
{f,_} -> []
end,
case is_gc_bif(Name, Args) of
true ->
Live = get_live(I),
Kill = kill_yregs(Anno, St),
{Kill++Line++[{gc_bif,Name,Fail,Live,Args,Dst}],St};
false ->
{Line++[{bif,Name,Fail,Args,Dst}],St}
end;
cg_block([#cg_set{op={bif,tuple_size},dst=Arity0,args=[Tuple0]},
#cg_set{op={bif,'=:='},dst=Bool,args=[Arity0,#b_literal{val=Ar}]}=Eq],
{Bool,Fail}=Context, St0) ->
Tuple = beam_arg(Tuple0, St0),
case beam_arg(Arity0, St0) of
{z,_} ->
%% The size will only be used once. Combine to a test_arity instruction.
Test = {test,test_arity,ensure_label(Fail, St0),[Tuple,Ar]},
{[Test],St0};
Arity ->
%% The size will be used more than once. Must do an explicit
%% BIF call followed by the '==' test.
TupleSize = {bif,tuple_size,{f,0},[Tuple],Arity},
{Is,St} = cg_block([Eq], Context, St0),
{[TupleSize|Is],St}
end;
cg_block([#cg_set{op={bif,Name},dst=Dst0,args=Args0}]=Is0, {Dst0,Fail}, St0) ->
[Dst|Args] = beam_args([Dst0|Args0], St0),
case Dst of
{z,_} ->
%% The result of the BIF call will only be used once. Convert to
%% a test instruction.
{Test,St1} = bif_to_test(Name, Args, ensure_label(Fail, St0), St0),
{Test,St1};
_ ->
%% Must explicitly call the BIF since the result will be used
%% more than once.
{Is1,St1} = cg_block(Is0, none, St0),
{Is2,St} = cg_block([], {Dst0,Fail}, St1),
{Is1++Is2,St}
end;
cg_block([#cg_set{anno=Anno,op={bif,Name},dst=Dst0,args=Args0}=I|T],
Context, St0) ->
[Dst|Args] = beam_args([Dst0|Args0], St0),
{Is0,St} = cg_block(T, Context, St0),
case is_gc_bif(Name, Args) of
true ->
Line = call_line(body, {extfunc,erlang,Name,length(Args)}, Anno),
Live = get_live(I),
Kill = kill_yregs(Anno, St),
Is = Kill++Line++[{gc_bif,Name,{f,0},Live,Args,Dst}|Is0],
{Is,St};
false ->
Is = [{bif,Name,{f,0},Args,Dst}|Is0],
{Is,St}
end;
cg_block([#cg_set{op=bs_init,dst=Dst0,args=Args0,anno=Anno}=I,
#cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) ->
Fail = bif_fail(Fail0),
Line = line(Anno),
Alloc = map_get(alloc, Anno),
[#b_literal{val=Kind}|Args1] = Args0,
case Kind of
new ->
[Dst,Size,{integer,Unit}] = beam_args([Dst0|Args1], St),
Live = get_live(I),
{[Line|cg_bs_init(Dst, Size, Alloc, Unit, Live, Fail)],St};
private_append ->
[Dst,Src,Bits,{integer,Unit}] = beam_args([Dst0|Args1], St),
Flags = {field_flags,[]},
Is = [Line,{bs_private_append,Fail,Bits,Unit,Src,Flags,Dst}],
{Is,St};
append ->
[Dst,Src,Bits,{integer,Unit}] = beam_args([Dst0|Args1], St),
Flags = {field_flags,[]},
Live = get_live(I),
Is = [Line,{bs_append,Fail,Bits,Alloc,Live,Unit,Src,Flags,Dst}],
{Is,St}
end;
cg_block([#cg_set{anno=Anno,op=bs_start_match,dst=Ctx0,args=[Bin0]}=I,
#cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) ->
[Dst,Bin1] = beam_args([Ctx0,Bin0], St),
{Bin,Pre} = force_reg(Bin1, Dst),
Live = get_live(I),
%% num_slots is only set when using the old instructions.
case maps:find(num_slots, Anno) of
{ok, Slots} ->
Is = Pre ++ [{test,bs_start_match2,Fail,Live,[Bin,Slots],Dst}],
{Is,St};
error ->
Is = Pre ++ [{test,bs_start_match3,Fail,Live,[Bin],Dst}],
{Is,St}
end;
cg_block([#cg_set{op=bs_get}=Set,
#cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) ->
{cg_bs_get(Fail, Set, St),St};
cg_block([#cg_set{op=bs_match_string,args=[CtxVar,#b_literal{val=String}]},
#cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) ->
CtxReg = beam_arg(CtxVar, St),
Is = [{test,bs_match_string,Fail,[CtxReg,String]}],
{Is,St};
cg_block([#cg_set{dst=Dst0,op=landingpad,args=Args0}|T], Context, St0) ->
[Dst,{atom,Kind},Tag] = beam_args([Dst0|Args0], St0),
case Kind of
'catch' ->
cg_catch(Dst, T, Context, St0);
'try' ->
cg_try(Dst, Tag, T, Context, St0)
end;
cg_block([#cg_set{op=kill_try_tag,args=Args0}|Is], Context, St0) ->
[Reg] = beam_args(Args0, St0),
{Is0,St} = cg_block(Is, Context, St0),
{[{try_end,Reg}|Is0],St};
cg_block([#cg_set{op=catch_end,dst=Dst0,args=Args0}|Is], Context, St0) ->
[Dst,Reg,{x,0}] = beam_args([Dst0|Args0], St0),
{Is0,St} = cg_block(Is, Context, St0),
{[{catch_end,Reg}|copy({x,0}, Dst)++Is0],St};
cg_block([#cg_set{op=call}=I,
#cg_set{op=succeeded,dst=Bool}], {Bool,_Fail}, St) ->
%% A call in try/catch block.
cg_block([I], none, St);
cg_block([#cg_set{op=match_fail}=I,
#cg_set{op=succeeded,dst=Bool}], {Bool,_Fail}, St) ->
%% A match_fail instruction in a try/catch block.
cg_block([I], none, St);
cg_block([#cg_set{op=get_map_element,dst=Dst0,args=Args0},
#cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) ->
[Dst,Map,Key] = beam_args([Dst0|Args0], St),
Fail = ensure_label(Fail0, St),
{[{get_map_elements,Fail,Map,{list,[Key,Dst]}}],St};
cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=I,
#cg_set{op=succeeded,dst=Bool}], {Bool,Fail}, St) ->
[Dst|Args] = beam_args([Dst0|Args0], St),
{cg_test(Op, bif_fail(Fail), Args, Dst, I),St};
cg_block([#cg_set{op=bs_put,dst=Bool,args=Args0}], {Bool,Fail}, St) ->
Args = beam_args(Args0, St),
{cg_bs_put(bif_fail(Fail), Args),St};
cg_block([#cg_set{op=bs_test_tail,dst=Bool,args=Args0}], {Bool,Fail}, St) ->
[Ctx,{integer,Bits}] = beam_args(Args0, St),
{[{test,bs_test_tail2,bif_fail(Fail),[Ctx,Bits]}],St};
cg_block([#cg_set{op={float,checkerror},dst=Bool}], {Bool,Fail}, St) ->
{[{fcheckerror,bif_fail(Fail)}],St};
cg_block([#cg_set{op=is_tagged_tuple,dst=Bool,args=Args0}], {Bool,Fail}, St) ->
[Src,{integer,Arity},Tag] = beam_args(Args0, St),
{[{test,is_tagged_tuple,ensure_label(Fail, St),[Src,Arity,Tag]}],St};
cg_block([#cg_set{op=is_nonempty_list,dst=Bool,args=Args0}], {Bool,Fail}, St) ->
Args = beam_args(Args0, St),
{[{test,is_nonempty_list,ensure_label(Fail, St),Args}],St};
cg_block([#cg_set{op=has_map_field,dst=Bool,args=Args0}], {Bool,Fail}, St) ->
[Src,Key] = beam_args(Args0, St),
{[{test,has_map_fields,Fail,Src,{list,[Key]}}],St};
cg_block([#cg_set{op=call}=Call], {_Bool,_Fail}=Context, St0) ->
{Is0,St1} = cg_call(Call, body, none, St0),
{Is1,St} = cg_block([], Context, St1),
{Is0++Is1,St};
cg_block([#cg_set{op=call,dst=Dst0}=Call], Context, St) ->
Dst = beam_arg(Dst0, St),
case Context of
{return,Dst,_} ->
cg_call(Call, tail, Context, St);
_ ->
cg_call(Call, body, Context, St)
end;
cg_block([#cg_set{op=call}=Call|T], Context, St0) ->
{Is0,St1} = cg_call(Call, body, none, St0),
{Is1,St} = cg_block(T, Context, St1),
{Is0++Is1,St};
cg_block([#cg_set{op=make_fun,dst=Dst0,args=[Local|Args0]}|T],
Context, St0) ->
#b_local{name=#b_literal{val=Func},arity=Arity} = Local,
[Dst|Args] = beam_args([Dst0|Args0], St0),
{FuncLbl,St1} = local_func_label(Func, Arity, St0),
Is0 = setup_args(Args) ++
[{make_fun2,{f,FuncLbl},0,0,length(Args)}|copy({x,0}, Dst)],
{Is1,St} = cg_block(T, Context, St1),
{Is0++Is1,St};
cg_block([#cg_set{op=copy}|_]=T0, Context, St0) ->
{Is0,T} = cg_copy(T0, St0),
{Is1,St} = cg_block(T, Context, St0),
Is = Is0 ++ Is1,
case is_call(T) of
{yes,Arity} ->
{opt_call_moves(Is, Arity),St};
no ->
{Is,St}
end;
cg_block([#cg_set{op=match_fail,args=Args0,anno=Anno}], none, St) ->
Args = beam_args(Args0, St),
Is = cg_match_fail(Args, line(Anno), none),
{Is,St};
cg_block([#cg_set{op=match_fail,args=Args0,anno=Anno}|T], Context, St0) ->
FcLabel = case Context of
{return,_,none} ->
%% There is no stack frame. If this is a function_clause
%% exception, it is safe to jump to the label of the
%% func_info instruction.
St0#cg.fc_label;
_ ->
%% This is most probably not a function_clause.
%% If this is a function_clause exception
%% (rare), it is not safe to jump to the
%% func_info label.
none
end,
Args = beam_args(Args0, St0),
Is0 = cg_match_fail(Args, line(Anno), FcLabel),
{Is1,St} = cg_block(T, Context, St0),
{Is0++Is1,St};
cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=Set], none, St) ->
[Dst|Args] = beam_args([Dst0|Args0], St),
Is = cg_instr(Op, Args, Dst, Set),
{Is,St};
cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=Set|T], Context, St0) ->
[Dst|Args] = beam_args([Dst0|Args0], St0),
Is0 = cg_instr(Op, Args, Dst, Set),
{Is1,St} = cg_block(T, Context, St0),
{Is0++Is1,St};
cg_block([#cg_alloc{}=Alloc|T], Context, St0) ->
Is0 = cg_alloc(Alloc, St0),
{Is1,St} = cg_block(T, Context, St0),
{Is0++Is1,St};
cg_block([], {return,Arg,none}, St) ->
Is = copy(Arg, {x,0}) ++ [return],
{Is,St};
cg_block([], {return,Arg,N}, St) ->
Is = copy(Arg, {x,0}) ++ [{deallocate,N},return],
{Is,St};
cg_block([], none, St) ->
{[],St};
cg_block([], {Bool0,Fail}, St) ->
[Bool] = beam_args([Bool0], St),
{[{test,is_eq_exact,Fail,[Bool,{atom,true}]}],St}.
cg_copy(T0, St) ->
{Copies,T} = splitwith(fun(#cg_set{op=copy}) -> true;
(_) -> false
end, T0),
Moves0 = cg_copy_1(Copies, St),
Moves1 = [Move || {move,Src,Dst}=Move <- Moves0, Src =/= Dst],
Moves = order_moves(Moves1),
{Moves,T}.
cg_copy_1([#cg_set{dst=Dst0,args=Args}|T], St) ->
[Dst,Src] = beam_args([Dst0|Args], St),
Copies = cg_copy_1(T, St),
case keymember(Dst, 3, Copies) of
true ->
%% Will be overwritten. Don't generate a move instruction.
Copies;
false ->
[{move,Src,Dst}|Copies]
end;
cg_copy_1([], _St) -> [].
-define(IS_LITERAL(Val), (Val =:= nil orelse
element(1, Val) =:= integer orelse
element(1, Val) =:= float orelse
element(1, Val) =:= atom orelse
element(1, Val) =:= literal)).
bif_to_test('or', [V1,V2], {f,Lbl}=Fail, St0) when Lbl =/= 0 ->
{SuccLabel,St} = new_label(St0),
{[{test,is_eq_exact,{f,SuccLabel},[V1,{atom,false}]},
{test,is_eq_exact,Fail,[V2,{atom,true}]},
{label,SuccLabel}],St};
bif_to_test(Op, Args, Fail, St) ->
{bif_to_test(Op, Args, Fail),St}.
bif_to_test('and', [V1,V2], Fail) ->
[{test,is_eq_exact,Fail,[V1,{atom,true}]},
{test,is_eq_exact,Fail,[V2,{atom,true}]}];
bif_to_test('not', [Var], Fail) ->
[{test,is_eq_exact,Fail,[Var,{atom,false}]}];
bif_to_test(Name, Args, Fail) ->
[bif_to_test_1(Name, Args, Fail)].
bif_to_test_1(is_atom, [_]=Ops, Fail) ->
{test,is_atom,Fail,Ops};
bif_to_test_1(is_boolean, [_]=Ops, Fail) ->
{test,is_boolean,Fail,Ops};
bif_to_test_1(is_binary, [_]=Ops, Fail) ->
{test,is_binary,Fail,Ops};
bif_to_test_1(is_bitstring,[_]=Ops, Fail) ->
{test,is_bitstr,Fail,Ops};
bif_to_test_1(is_float, [_]=Ops, Fail) ->
{test,is_float,Fail,Ops};
bif_to_test_1(is_function, [_]=Ops, Fail) ->
{test,is_function,Fail,Ops};
bif_to_test_1(is_function, [_,_]=Ops, Fail) ->
{test,is_function2,Fail,Ops};
bif_to_test_1(is_integer, [_]=Ops, Fail) ->
{test,is_integer,Fail,Ops};
bif_to_test_1(is_list, [_]=Ops, Fail) ->
{test,is_list,Fail,Ops};
bif_to_test_1(is_map, [_]=Ops, Fail) ->
{test,is_map,Fail,Ops};
bif_to_test_1(is_number, [_]=Ops, Fail) ->
{test,is_number,Fail,Ops};
bif_to_test_1(is_pid, [_]=Ops, Fail) ->
{test,is_pid,Fail,Ops};
bif_to_test_1(is_port, [_]=Ops, Fail) ->
{test,is_port,Fail,Ops};
bif_to_test_1(is_reference, [_]=Ops, Fail) ->
{test,is_reference,Fail,Ops};
bif_to_test_1(is_tuple, [_]=Ops, Fail) ->
{test,is_tuple,Fail,Ops};
bif_to_test_1('=<', [A,B], Fail) ->
{test,is_ge,Fail,[B,A]};
bif_to_test_1('>', [A,B], Fail) ->
{test,is_lt,Fail,[B,A]};
bif_to_test_1('<', [_,_]=Ops, Fail) ->
{test,is_lt,Fail,Ops};
bif_to_test_1('>=', [_,_]=Ops, Fail) ->
{test,is_ge,Fail,Ops};
bif_to_test_1('==', [C,A], Fail) when ?IS_LITERAL(C) ->
{test,is_eq,Fail,[A,C]};
bif_to_test_1('==', [_,_]=Ops, Fail) ->
{test,is_eq,Fail,Ops};
bif_to_test_1('/=', [C,A], Fail) when ?IS_LITERAL(C) ->
{test,is_ne,Fail,[A,C]};
bif_to_test_1('/=', [_,_]=Ops, Fail) ->
{test,is_ne,Fail,Ops};
bif_to_test_1('=:=', [C,A], Fail) when ?IS_LITERAL(C) ->
{test,is_eq_exact,Fail,[A,C]};
bif_to_test_1('=:=', [_,_]=Ops, Fail) ->
{test,is_eq_exact,Fail,Ops};
bif_to_test_1('=/=', [C,A], Fail) when ?IS_LITERAL(C) ->
{test,is_ne_exact,Fail,[A,C]};
bif_to_test_1('=/=', [_,_]=Ops, Fail) ->
{test,is_ne_exact,Fail,Ops}.
opt_call_moves(Is0, Arity) ->
{Moves0,Is} = splitwith(fun({move,_,_}) -> true;
({kill,_}) -> true;
(_) -> false
end, Is0),
Moves = opt_call_moves_1(Moves0, Arity),
Moves ++ Is.
opt_call_moves_1([{move,Src,{x,_}=Tmp}=M1|[{kill,_}|_]=Is], Arity) ->
%% There could be a {move,Tmp,{x,0}} instruction after the
%% kill/1 instructions (moved to there by opt_move_to_x0/1).
case splitwith(fun({kill,_}) -> true;
(_) -> false
end, Is) of
{Kills,[{move,{x,_}=Tmp,{x,0}}=M2]} ->
%% The two move/2 instructions (M1 and M2) can be combined
%% to one. The question is, though, is it safe to place
%% them after the kill/1 instructions?
case is_killed(Src, Kills, Arity) of
true ->
%% Src (a Y register) is killed by one of the
%% kill/1 instructions. Thus M1 and M2
%% must be placed before the kill/1 instructions
%% (essentially undoing what opt_move_to_x0/1
%% did, which turned out to be a pessimization
%% in this case).
opt_call_moves_1([M1,M2|Kills], Arity);
false ->
%% Src is not killed by any of the kill/1
%% instructions. Thus it is safe to place
%% M1 and M2 after the kill/1 instructions.
opt_call_moves_1(Kills++[M1,M2], Arity)
end;
{_,_} ->
[M1|Is]
end;
opt_call_moves_1([{move,Src,{x,_}=Tmp}=M1,{move,Tmp,Dst}=M2|Is], Arity) ->
case is_killed(Tmp, Is, Arity) of
true ->
%% The X register Tmp is never used again. We can collapse
%% the two move instruction into one.
[{move,Src,Dst}|opt_call_moves_1(Is, Arity)];
false ->
[M1|opt_call_moves_1([M2|Is], Arity)]
end;
opt_call_moves_1([M|Ms], Arity) ->
[M|opt_call_moves_1(Ms, Arity)];
opt_call_moves_1([], _Arity) -> [].
is_killed(Y, [{kill,Y}|_], _) ->
true;
is_killed(R, [{kill,_}|Is], Arity) ->
is_killed(R, Is, Arity);
is_killed(R, [{move,R,_}|_], _) ->
false;
is_killed(R, [{move,_,R}|_], _) ->
true;
is_killed(R, [{move,_,_}|Is], Arity) ->
is_killed(R, Is, Arity);
is_killed({x,X}, [], Arity) ->
X >= Arity;
is_killed({y,_}, [], _) ->
false.
cg_alloc(#cg_alloc{stack=none,words=#need{h=0,f=0}}, _St) ->
[];
cg_alloc(#cg_alloc{stack=none,words=Need,live=Live}, _St) ->
[{test_heap,alloc(Need),Live}];
cg_alloc(#cg_alloc{stack=Stk,words=Need,live=Live,def_yregs=DefYregs},
#cg{regs=Regs}) when is_integer(Stk) ->
Alloc = alloc(Need),
All = [{y,Y} || Y <- lists:seq(0, Stk-1)],
Def = ordsets:from_list([maps:get(V, Regs) || V <- DefYregs]),
NeedInit = ordsets:subtract(All, Def),
NoZero = length(Def)*2 > Stk,
I = case {NoZero,Alloc} of
{true,0} -> {allocate,Stk,Live};
{true,_} -> {allocate_heap,Stk,Alloc,Live};
{false,0} -> {allocate_zero,Stk,Live};
{false,_} -> {allocate_heap_zero,Stk,Alloc,Live}
end,
[I|case NoZero of
true -> [{init,Y} || Y <- NeedInit];
false -> []
end].
alloc(#need{h=Words,f=0}) ->
Words;
alloc(#need{h=Words,f=Floats}) ->
{alloc,[{words,Words},{floats,Floats}]}.
is_call([#cg_set{op=call,args=[#b_var{}|Args]}|_]) ->
{yes,1+length(Args)};
is_call([#cg_set{op=call,args=[_|Args]}|_]) ->
{yes,length(Args)};
is_call([#cg_set{op=make_fun,args=[_|Args]}|_]) ->
{yes,length(Args)};
is_call(_) ->
no.
cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=[#b_local{}=Func0|Args0]},
Where, Context, St0) ->
[Dst|Args] = beam_args([Dst0|Args0], St0),
#b_local{name=Name0,arity=Arity} = Func0,
{atom,Name} = beam_arg(Name0, St0),
{FuncLbl,St} = local_func_label(Name, Arity, St0),
Line = call_line(Where, local, Anno),
Call = build_call(call, Arity, {f,FuncLbl}, Context, Dst),
Is = setup_args(Args, Anno, Context, St) ++ Line ++ Call,
case Anno of
#{ result_type := Info } ->
{Is ++ [{'%', {type_info, Dst, Info}}], St};
#{} ->
{Is, St}
end;
cg_call(#cg_set{anno=Anno0,op=call,dst=Dst0,args=[#b_remote{}=Func0|Args0]},
Where, Context, St) ->
[Dst|Args] = beam_args([Dst0|Args0], St),
#b_remote{mod=Mod0,name=Name0,arity=Arity} = Func0,
case {beam_arg(Mod0, St),beam_arg(Name0, St)} of
{{atom,Mod},{atom,Name}} ->
Func = {extfunc,Mod,Name,Arity},
Line = call_line(Where, Func, Anno0),
Call = build_call(call_ext, Arity, Func, Context, Dst),
Anno = case erl_bifs:is_exit_bif(Mod, Name, Arity) of
true ->
%% There is no need to kill Y registers
%% before calling an exit BIF.
maps:remove(kill_yregs, Anno0);
false ->
Anno0
end,
Is = setup_args(Args, Anno, Context, St) ++ Line ++ Call,
{Is,St};
{Mod,Name} ->
Apply = build_apply(Arity, Context, Dst),
Is = setup_args(Args++[Mod,Name], Anno0, Context, St) ++
[line(Anno0)] ++ Apply,
{Is,St}
end;
cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=Args0},
Where, Context, St) ->
[Dst,Func|Args] = beam_args([Dst0|Args0], St),
Line = call_line(Where, Func, Anno),
Arity = length(Args),
Call = build_call(call_fun, Arity, Func, Context, Dst),
Is = setup_args(Args++[Func], Anno, Context, St) ++ Line ++ Call,
{Is,St}.
cg_match_fail([{atom,function_clause}|Args], Line, Fc) ->
case Fc of
none ->
%% There is a stack frame (probably because of inlining).
%% Jumping to the func_info label is not allowed by
%% beam_validator. Rewrite the instruction as a call to
%% erlang:error/2.
make_fc(Args, Line);
_ ->
setup_args(Args) ++ [{jump,{f,Fc}}]
end;
cg_match_fail([{atom,Op}], Line, _Fc) ->
[Line,Op];
cg_match_fail([{atom,Op},Val], Line, _Fc) ->
[Line,{Op,Val}].
make_fc(Args, Line) ->
%% Recreate the original call to erlang:error/2.
Live = foldl(fun({x,X}, A) -> max(X+1, A);
(_, A) -> A
end, 0, Args),
TmpReg = {x,Live},
StkMoves = build_stk(reverse(Args), TmpReg, nil),
[{test_heap,2*length(Args),Live}|StkMoves] ++
[{move,{atom,function_clause},{x,0}},
Line,
{call_ext,2,{extfunc,erlang,error,2}}].
build_stk([V], _TmpReg, Tail) ->
[{put_list,V,Tail,{x,1}}];
build_stk([V|Vs], TmpReg, Tail) ->
I = {put_list,V,Tail,TmpReg},
[I|build_stk(Vs, TmpReg, TmpReg)];
build_stk([], _TmpReg, nil) ->
[{move,nil,{x,1}}].
build_call(call_fun, Arity, _Func, none, Dst) ->
[{call_fun,Arity}|copy({x,0}, Dst)];
build_call(call_fun, Arity, _Func, {return,Dst,N}, Dst) when is_integer(N) ->
[{call_fun,Arity},{deallocate,N},return];
build_call(call_fun, Arity, _Func, {return,Val,N}, _Dst) when is_integer(N) ->
[{call_fun,Arity},{move,Val,{x,0}},{deallocate,N},return];
build_call(call_ext, 2, {extfunc,erlang,'!',2}, none, Dst) ->
[send|copy({x,0}, Dst)];
build_call(call_ext, 2, {extfunc,erlang,'!',2}, {return,Dst,N}, Dst)
when is_integer(N) ->
[send,{deallocate,N},return];
build_call(Prefix, Arity, Func, {return,Dst,none}, Dst) ->
I = case Prefix of
call -> call_only;
call_ext -> call_ext_only
end,
[{I,Arity,Func}];
build_call(call_ext, Arity, {extfunc,Mod,Name,Arity}=Func, {return,_,none}, _Dst) ->
true = erl_bifs:is_exit_bif(Mod, Name, Arity), %Assertion.
[{call_ext_only,Arity,Func}];
build_call(Prefix, Arity, Func, {return,Dst,N}, Dst) when is_integer(N) ->
I = case Prefix of
call -> call_last;
call_ext -> call_ext_last
end,
[{I,Arity,Func,N}];
build_call(I, Arity, Func, {return,Val,N}, _Dst) when is_integer(N) ->
[{I,Arity,Func}|copy(Val, {x,0})++[{deallocate,N},return]];
build_call(I, Arity, Func, none, Dst) ->
[{I,Arity,Func}|copy({x,0}, Dst)].
build_apply(Arity, {return,Dst,N}, Dst) when is_integer(N) ->
[{apply_last,Arity,N}];
build_apply(Arity, {return,Val,N}, _Dst) when is_integer(N) ->
[{apply,Arity}|copy(Val, {x,0})++[{deallocate,N},return]];
build_apply(Arity, none, Dst) ->
[{apply,Arity}|copy({x,0}, Dst)].
cg_instr(bs_get_tail, [Src], Dst, Set) ->
Live = get_live(Set),
[{bs_get_tail,Src,Dst,Live}];
cg_instr(bs_get_position, [Ctx], Dst, Set) ->
Live = get_live(Set),
[{bs_get_position,Ctx,Dst,Live}];
cg_instr(put_map, [{atom,assoc},SrcMap|Ss], Dst, Set) ->
Live = get_live(Set),
[{put_map_assoc,{f,0},SrcMap,Dst,Live,{list,Ss}}];
cg_instr(Op, Args, Dst, _Set) ->
cg_instr(Op, Args, Dst).
cg_instr(bs_init_writable, Args, Dst) ->
setup_args(Args) ++ [bs_init_writable|copy({x,0}, Dst)];
cg_instr(bs_restore, [Ctx,Slot], _Dst) ->
case Slot of
{integer,N} ->
[{bs_restore2,Ctx,N}];
{atom,start} ->
[{bs_restore2,Ctx,Slot}]
end;
cg_instr(bs_save, [Ctx,Slot], _Dst) ->
{integer,N} = Slot,
[{bs_save2,Ctx,N}];
cg_instr(bs_set_position, [Ctx,Pos], _Dst) ->
[{bs_set_position,Ctx,Pos}];
cg_instr(build_stacktrace, Args, Dst) ->
setup_args(Args) ++ [build_stacktrace|copy({x,0}, Dst)];
cg_instr(set_tuple_element=Op, [New,Tuple,{integer,Index}], _Dst) ->
[{Op,New,Tuple,Index}];
cg_instr({float,clearerror}, [], _Dst) ->
[fclearerror];
cg_instr({float,get}, [Src], Dst) ->
[{fmove,Src,Dst}];
cg_instr({float,put}, [Src], Dst) ->
[{fmove,Src,Dst}];
cg_instr(get_hd=Op, [Src], Dst) ->
[{Op,Src,Dst}];
cg_instr(get_tl=Op, [Src], Dst) ->
[{Op,Src,Dst}];
cg_instr(get_tuple_element=Op, [Src,{integer,N}], Dst) ->
[{Op,Src,N,Dst}];
cg_instr(put_list=Op, [Hd,Tl], Dst) ->
[{Op,Hd,Tl,Dst}];
cg_instr(put_tuple, Elements, Dst) ->
[{put_tuple2,Dst,{list,Elements}}];
cg_instr(put_tuple_arity, [{integer,Arity}], Dst) ->
[{put_tuple,Arity,Dst}];
cg_instr(put_tuple_elements, Elements, _Dst) ->
[{put,E} || E <- Elements];
cg_instr(raw_raise, Args, Dst) ->
setup_args(Args) ++ [raw_raise|copy({x,0}, Dst)];
cg_instr(remove_message, [], _Dst) ->
[remove_message];
cg_instr(resume, [A,B], _Dst) ->
[{bif,raise,{f,0},[A,B],{x,0}}];
cg_instr(timeout, [], _Dst) ->
[timeout].
cg_test(bs_add=Op, Fail, [Src1,Src2,{integer,Unit}], Dst, _I) ->
[{Op,Fail,[Src1,Src2,Unit],Dst}];
cg_test(bs_skip, Fail, Args, _Dst, I) ->
cg_bs_skip(Fail, Args, I);
cg_test(bs_utf8_size=Op, Fail, [Src], Dst, _I) ->
[{Op,Fail,Src,Dst}];
cg_test(bs_utf16_size=Op, Fail, [Src], Dst, _I) ->
[{Op,Fail,Src,Dst}];
cg_test({float,convert}, Fail, [Src], Dst, _I) ->
{f,0} = Fail, %Assertion.
[{fconv,Src,Dst}];
cg_test({float,Op0}, Fail, Args, Dst, #cg_set{anno=Anno}) ->
Op = case Op0 of
'+' -> fadd;
'-' when length(Args) =:= 2 -> fsub;
'-' -> fnegate;
'*' -> fmul;
'/' -> fdiv
end,
[line(Anno),{bif,Op,Fail,Args,Dst}];
cg_test(peek_message, Fail, [], Dst, _I) ->
[{loop_rec,Fail,{x,0}}|copy({x,0}, Dst)];
cg_test(put_map, Fail, [{atom,exact},SrcMap|Ss], Dst, Set) ->
Live = get_live(Set),
[{put_map_exact,Fail,SrcMap,Dst,Live,{list,Ss}}];
cg_test(wait_timeout, Fail, [Timeout], _Dst, _) ->
case Timeout of
{atom,infinity} ->
[{wait,Fail}];
_ ->
[{wait_timeout,Fail,Timeout}]
end.
cg_bs_get(Fail, #cg_set{dst=Dst0,args=[#b_literal{val=Type}|Ss0]}=Set, St) ->
Op = case Type of
integer -> bs_get_integer2;
float -> bs_get_float2;
binary -> bs_get_binary2;
utf8 -> bs_get_utf8;
utf16 -> bs_get_utf16;
utf32 -> bs_get_utf32
end,
[Dst|Ss1] = beam_args([Dst0|Ss0], St),
Ss = case Ss1 of
[Ctx,{literal,Flags},Size,{integer,Unit}] ->
%% Plain integer/float/binary.
[Ctx,Size,Unit,field_flags(Flags, Set)];
[Ctx,{literal,Flags}] ->
%% Utf8/16/32.
[Ctx,field_flags(Flags, Set)]
end,
Live = get_live(Set),
[{test,Op,Fail,Live,Ss,Dst}].
cg_bs_skip(Fail, [{atom,Type}|Ss0], Set) ->
Op = case Type of
utf8 -> bs_skip_utf8;
utf16 -> bs_skip_utf16;
utf32 -> bs_skip_utf32;
_ -> bs_skip_bits2
end,
Live = get_live(Set),
Ss = case Ss0 of
[Ctx,{literal,Flags},Size,{integer,Unit}] ->
%% Plain integer/float/binary.
[Ctx,Size,Unit,field_flags(Flags, Set)];
[Ctx,{literal,Flags}] ->
%% Utf8/16/32.
[Ctx,Live,field_flags(Flags, Set)]
end,
case {Type,Ss} of
{binary,[_,{atom,all},1,_]} ->
[];
{binary,[R,{atom,all},U,_]} ->
[{test,bs_test_unit,Fail,[R,U]}];
{_,_} ->
[{test,Op,Fail,Ss}]
end.
field_flags(Flags, #cg_set{anno=#{location:={File,Line}}}) ->
{field_flags,[{anno,[Line,{file,File}]}|Flags]};
field_flags(Flags, _) ->
{field_flags,Flags}.
cg_bs_put(Fail, [{atom,Type},{literal,Flags}|Args]) ->
Op = case Type of
integer -> bs_put_integer;
float -> bs_put_float;
binary -> bs_put_binary;
utf8 -> bs_put_utf8;
utf16 -> bs_put_utf16;
utf32 -> bs_put_utf32
end,
case Args of
[Src,Size,{integer,Unit}] ->
[{Op,Fail,Size,Unit,{field_flags,Flags},Src}];
[Src] ->
[{Op,Fail,{field_flags,Flags},Src}]
end.
cg_bs_init(Dst, Size0, Alloc, Unit, Live, Fail) ->
Op = case Unit of
1 -> bs_init_bits;
8 -> bs_init2
end,
Size = cg_bs_init_size(Size0),
[{Op,Fail,Size,Alloc,Live,{field_flags,[]},Dst}].
cg_bs_init_size({x,_}=R) -> R;
cg_bs_init_size({y,_}=R) -> R;
cg_bs_init_size({integer,Int}) -> Int.
cg_catch(Agg, T0, Context, St0) ->
{Moves,T1} = cg_extract(T0, Agg, St0),
{T,St} = cg_block(T1, Context, St0),
{Moves++T,St}.
cg_try(Agg, Tag, T0, Context, St0) ->
{Moves0,T1} = cg_extract(T0, Agg, St0),
Moves = order_moves(Moves0),
[#cg_set{op=kill_try_tag}|T2] = T1,
{T,St} = cg_block(T2, Context, St0),
{[{try_case,Tag}|Moves++T],St}.
cg_extract([#cg_set{op=extract,dst=Dst0,args=Args0}|Is0], Agg, St) ->
[Dst,Agg,{integer,X}] = beam_args([Dst0|Args0], St),
{Ds,Is} = cg_extract(Is0, Agg, St),
case keymember(Dst, 3, Ds) of
true ->
%% This destination will be overwritten.
{Ds,Is};
false ->
{copy({x,X}, Dst)++Ds,Is}
end;
cg_extract(Is, _, _) ->
{[],Is}.
copy(Src, Src) -> [];
copy(Src, Dst) -> [{move,Src,Dst}].
force_reg({literal,_}=Lit, Reg) ->
{Reg,[{move,Lit,Reg}]};
force_reg({integer,_}=Lit, Reg) ->
{Reg,[{move,Lit,Reg}]};
force_reg({atom,_}=Lit, Reg) ->
{Reg,[{move,Lit,Reg}]};
force_reg({float,_}=Lit, Reg) ->
{Reg,[{move,Lit,Reg}]};
force_reg(nil=Lit, Reg) ->
{Reg,[{move,Lit,Reg}]};
force_reg({Kind,_}=R, _) when Kind =:= x; Kind =:= y ->
{R,[]}.
%% successors(Terminator) -> [Successor].
%% Return an ordset of all successors for the given terminator.
successors(#cg_br{succ=Succ,fail=Fail}) ->
ordsets:from_list([Succ,Fail]);
successors(#cg_switch{fail=Fail,list=List}) ->
ordsets:from_list([Fail|[Lbl || {_,Lbl} <- List]]);
successors(#cg_ret{}) -> [].
%% linearize(Blocks) -> [{BlockLabel,#cg_blk{}}].
%% Linearize the intermediate representation of the code. Also
%% translate blocks from the SSA records to internal record types
%% used only in this module.
linearize(Blocks) ->
Linear = beam_ssa:linearize(Blocks),
linearize_1(Linear, Blocks).
linearize_1([{?EXCEPTION_BLOCK,_}|Ls], Blocks) ->
linearize_1(Ls, Blocks);
linearize_1([{L,Block0}|Ls], Blocks) ->
Block = translate_block(L, Block0, Blocks),
[{L,Block}|linearize_1(Ls, Blocks)];
linearize_1([], _Blocks) -> [].
%% translate_block(BlockLabel, #b_blk{}, Blocks) -> #cg_blk{}.
%% Translate a block to the internal records used in this module.
%% Also eliminate phi nodes, replacing them with 'copy' instructions
%% in the predecessor blocks.
translate_block(L, #b_blk{anno=Anno,is=Is0,last=Last0}, Blocks) ->
Last = translate_terminator(Last0),
PhiCopies = translate_phis(L, Last, Blocks),
Is1 = translate_is(Is0, PhiCopies),
Is = case Anno of
#{frame_size:=Size} ->
Alloc = #cg_alloc{stack=Size},
[Alloc|Is1];
#{} -> Is1
end,
#cg_blk{anno=Anno,is=Is,last=Last}.
translate_is([#b_set{op=phi}|Is], Tail) ->
translate_is(Is, Tail);
translate_is([#b_set{anno=Anno0,op=Op,dst=Dst,args=Args}=I|Is], Tail) ->
Anno = case beam_ssa:clobbers_xregs(I) of
true -> Anno0#{clobbers=>true};
false -> Anno0
end,
[#cg_set{anno=Anno,op=Op,dst=Dst,args=Args}|translate_is(Is, Tail)];
translate_is([], Tail) -> Tail.
translate_terminator(#b_ret{anno=Anno,arg=Arg}) ->
Dealloc = case Anno of
#{deallocate:=N} -> N;
#{} -> none
end,
#cg_ret{arg=Arg,dealloc=Dealloc};
translate_terminator(#b_br{bool=#b_literal{val=true},succ=Succ}) ->
#cg_br{bool=#b_literal{val=true},succ=Succ,fail=Succ};
translate_terminator(#b_br{bool=#b_literal{val=false},fail=Fail}) ->
#cg_br{bool=#b_literal{val=true},succ=Fail,fail=Fail};
translate_terminator(#b_br{bool=Bool,succ=Succ,fail=Fail}) ->
#cg_br{bool=Bool,succ=Succ,fail=Fail};
translate_terminator(#b_switch{arg=Bool,fail=Fail,list=List}) ->
#cg_switch{arg=Bool,fail=Fail,list=List}.
translate_phis(L, #cg_br{succ=Target,fail=Target}, Blocks) ->
#b_blk{is=Is} = maps:get(Target, Blocks),
Phis = takewhile(fun(#b_set{op=phi}) -> true;
(#b_set{}) -> false
end, Is),
phi_copies(Phis, L);
translate_phis(_, _, _) -> [].
phi_copies([#b_set{dst=Dst,args=PhiArgs}|Sets], L) ->
CopyArgs = [V || {V,Target} <- PhiArgs, Target =:= L],
[#cg_set{op=copy,dst=Dst,args=CopyArgs}|phi_copies(Sets, L)];
phi_copies([], _) -> [].
%% opt_move_to_x0([Instruction]) -> [Instruction].
%% Simple peep-hole optimization to move a {move,Any,{x,0}} past
%% any kill up to the next call instruction. (To give the loader
%% an opportunity to combine the 'move' and the 'call' instructions.)
opt_move_to_x0(Moves) ->
opt_move_to_x0(Moves, []).
opt_move_to_x0([{move,_,{x,0}}=I|Is0], Acc0) ->
case move_past_kill(Is0, I, Acc0) of
impossible -> opt_move_to_x0(Is0, [I|Acc0]);
{Is,Acc} -> opt_move_to_x0(Is, Acc)
end;
opt_move_to_x0([I|Is], Acc) ->
opt_move_to_x0(Is, [I|Acc]);
opt_move_to_x0([], Acc) -> reverse(Acc).
move_past_kill([{kill,Src}|_], {move,Src,_}, _) ->
impossible;
move_past_kill([{kill,_}=I|Is], Move, Acc) ->
move_past_kill(Is, Move, [I|Acc]);
move_past_kill(Is, Move, Acc) ->
{Is,[Move|Acc]}.
%% setup_args(Args, Anno, Context) -> [Instruction].
%% setup_args(Args) -> [Instruction].
%% Set up X registers for a call.
setup_args(Args, Anno, none, St) ->
case {setup_args(Args),kill_yregs(Anno, St)} of
{Moves,[]} ->
Moves;
{Moves,Kills} ->
opt_move_to_x0(Moves ++ Kills)
end;
setup_args(Args, _, _, _) ->
setup_args(Args).
setup_args([]) ->
[];
setup_args([_|_]=Args) ->
Moves = gen_moves(Args, 0, []),
order_moves(Moves).
%% kill_yregs(Anno, #cg{}) -> [{kill,{y,Y}}].
%% Kill Y registers that will not be used again.
kill_yregs(#{kill_yregs:=Kill}, #cg{regs=Regs}) ->
ordsets:from_list([{kill,maps:get(V, Regs)} || V <- Kill]);
kill_yregs(#{}, #cg{}) -> [].
%% gen_moves(As, I, Acc)
%% Generate the basic move instruction to move the arguments
%% to their proper registers. The list will be sorted on
%% destinations. (I.e. the move to {x,0} will be first --
%% see the comment to order_moves/2.)
gen_moves([A|As], I, Acc) ->
gen_moves(As, I+1, copy(A, {x,I}) ++ Acc);
gen_moves([], _, Acc) ->
keysort(3, Acc).
%% order_moves([Move]) -> [Move]
%% Orders move instruction so that source registers are not
%% destroyed before they are used. If there are cycles
%% (such as {move,{x,0},{x,1}}, {move,{x,1},{x,1}}),
%% swap instructions will be used to break up the cycle.
%%
%% If possible, the first move of the input list is placed
%% last in the result list (to make the move to {x,0} occur
%% just before the call to allow the Beam loader to coalesce
%% the instructions).
order_moves(Ms) -> order_moves(Ms, []).
order_moves([{move,_,_}=M|Ms0], Acc0) ->
{Chain,Ms} = collect_chain(Ms0, [M]),
Acc = reverse(Chain, Acc0),
order_moves(Ms, Acc);
order_moves([], Acc) -> Acc.
collect_chain(Ms, Path) ->
collect_chain(Ms, Path, []).
collect_chain([{move,Src,Same}=M|Ms0], [{move,Same,_}|_]=Path, Others) ->
case keymember(Src, 3, Path) of
false ->
collect_chain(reverse(Others, Ms0), [M|Path], []);
true ->
%% There is a cycle.
{break_up_cycle(M, Path),reverse(Others, Ms0)}
end;
collect_chain([M|Ms], Path, Others) ->
collect_chain(Ms, Path, [M|Others]);
collect_chain([], Path, Others) ->
{Path,Others}.
break_up_cycle({move,Src,_Dst}=M, Path) ->
break_up_cycle_1(Src, [M|Path], []).
break_up_cycle_1(Dst, [{move,_Src,Dst}|Path], Acc) ->
reverse(Acc, Path);
break_up_cycle_1(Dst, [{move,S,D}|Path], Acc) ->
break_up_cycle_1(Dst, Path, [{swap,S,D}|Acc]).
%%%
%%% General utility functions.
%%%
verify_beam_register({x,_}=Reg) -> Reg.
is_beam_register({x,_}) -> true;
is_beam_register(_) -> false.
get_register(V, Regs) ->
case is_beam_register(V) of
true -> V;
false -> maps:get(V, Regs)
end.
beam_args(As, St) ->
[beam_arg(A, St) || A <- As].
beam_arg(#b_var{}=Name, #cg{regs=Regs}) ->
maps:get(Name, Regs);
beam_arg(#b_literal{val=Val}, _) ->
if
is_atom(Val) -> {atom,Val};
is_float(Val) -> {float,Val};
is_integer(Val) -> {integer,Val};
Val =:= [] -> nil;
true -> {literal,Val}
end;
beam_arg(Reg, _) ->
verify_beam_register(Reg).
new_block_label(L, St0) ->
{_Lbl,St} = label_for_block(L, St0),
St.
def_block_label(L, #cg{labels=Labels,used_labels=Used}) ->
Lbl = maps:get(L, Labels),
case gb_sets:is_member(Lbl, Used) of
false -> [];
true -> [{label,Lbl}]
end.
use_block_labels(Ls, St) ->
mapfoldl(fun use_block_label/2, St, Ls).
use_block_label(L, #cg{used_labels=Used,catches=Catches}=St0) ->
{Lbl,St} = label_for_block(L, St0),
case gb_sets:is_member(L, Catches) of
true ->
{{catch_tag,{f,Lbl}},
St#cg{used_labels=gb_sets:add(Lbl, Used)}};
false ->
{{f,Lbl},St#cg{used_labels=gb_sets:add(Lbl, Used)}}
end.
label_for_block(L, #cg{labels=Labels0}=St0) ->
case Labels0 of
#{L:=Lbl} ->
{Lbl,St0};
#{} ->
{Lbl,St} = new_label(St0),
Labels = Labels0#{L=>Lbl},
{Lbl,St#cg{labels=Labels}}
end.
%% local_func_label(Name, Arity, State) -> {Label,State'}
%% local_func_label({Name,Arity}, State) -> {Label,State'}
%% Get the function entry label for a local function.
local_func_label(Name, Arity, St) ->
local_func_label({Name,Arity}, St).
local_func_label(Key, #cg{functable=Map}=St0) ->
case Map of
#{Key := Label} ->
{Label,St0};
_ ->
{Label,St} = new_label(St0),
{Label,St#cg{functable=Map#{Key => Label}}}
end.
%% is_gc_bif(Name, Args) -> true|false.
%% Determines whether the BIF Name/Arity might do a GC.
-spec is_gc_bif(atom(), [beam_ssa:value()]) -> boolean().
is_gc_bif(hd, [_]) -> false;
is_gc_bif(tl, [_]) -> false;
is_gc_bif(self, []) -> false;
is_gc_bif(node, []) -> false;
is_gc_bif(node, [_]) -> false;
is_gc_bif(element, [_,_]) -> false;
is_gc_bif(get, [_]) -> false;
is_gc_bif(is_map_key, [_,_]) -> false;
is_gc_bif(map_get, [_,_]) -> false;
is_gc_bif(tuple_size, [_]) -> false;
is_gc_bif(Bif, Args) ->
Arity = length(Args),
not (erl_internal:bool_op(Bif, Arity) orelse
erl_internal:new_type_test(Bif, Arity) orelse
erl_internal:comp_op(Bif, Arity)).
%% new_label(St) -> {L,St}.
new_label(#cg{lcount=Next}=St) ->
{Next,St#cg{lcount=Next+1}}.
%% call_line(tail|body, Func, Anno) -> [] | [{line,...}].
%% Produce a line instruction if it will be needed by the
%% call to Func.
call_line(_Context, {extfunc,Mod,Name,Arity}, Anno) ->
case erl_bifs:is_safe(Mod, Name, Arity) of
false ->
%% The call could be to a BIF.
%% We'll need a line instruction in case the
%% BIF call fails.
[line(Anno)];
true ->
%% Call to a safe BIF. Since it cannot fail,
%% we don't need any line instruction here.
[]
end;
call_line(body, _, Anno) ->
[line(Anno)];
call_line(tail, local, _) ->
%% Tail-recursive call to a local function. A line
%% instruction will not be useful.
[];
call_line(tail, _, Anno) ->
%% Call to a fun.
[line(Anno)].
%% line(Le) -> {line,[] | {location,File,Line}}
%% Create a line instruction, containing information about
%% the current filename and line number. A line information
%% instruction should be placed before any operation that could
%% cause an exception.
line(#{location:={File,Line}}) ->
{line,[{location,File,Line}]};
line(#{}) ->
{line,[]}.
flatmapfoldl(F, Accu0, [Hd|Tail]) ->
{R,Accu1} = F(Hd, Accu0),
{Rs,Accu2} = flatmapfoldl(F, Accu1, Tail),
{R++Rs,Accu2};
flatmapfoldl(_, Accu, []) -> {[],Accu}.
