compiler: Introduce module-level type optimization

This commit lets the type optimization pass work across functions,
tracking return and argument types to eliminate redundant tests.

10 files changed, 367 insertions, 74 deletions

diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index aec66cb9a3..c4d9ea515a 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -36,6 +36,11 @@
 %% during compilation instead of at runtime, so do not perform this analysis.
 -compile([{hipe, [no_icode_range]}]).
 
+%% Module-level type optimization propagates the constants used when testing
+%% increment1/1 and increment2/1, which makes it test something completely
+%% different, so we're turning it off.
+-compile(no_module_opt).
+
 suite() ->
     [{ct_hooks,[ts_install_cth]},
      {timetrap, {minutes, 1}}].
diff --git a/lib/compiler/src/beam_ssa_codegen.erl b/lib/compiler/src/beam_ssa_codegen.erl
index fe1a0c8480..c2d5035b19 100644
--- a/lib/compiler/src/beam_ssa_codegen.erl
+++ b/lib/compiler/src/beam_ssa_codegen.erl
@@ -161,7 +161,7 @@ add_parameter_annos([{label, _}=Entry | Body], Anno) ->
            (_K, _V, Acc) ->
                 Acc
         end, [], maps:get(registers, Anno)),
-    [Entry | Annos] ++ Body.
+    [Entry | sort(Annos)] ++ Body.
 
 cg_fun(Blocks, St0) ->
     Linear0 = linearize(Blocks),
@@ -1449,7 +1449,12 @@ cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=[#b_local{}=Func0|Args0]},
     Line = call_line(Where, local, Anno),
     Call = build_call(call, Arity, {f,FuncLbl}, Context, Dst),
     Is = setup_args(Args, Anno, Context, St) ++ Line ++ Call,
-    {Is,St};
+    case Anno of
+        #{ result_type := Info } ->
+            {Is ++ [{'%', {type_info, Dst, Info}}], St};
+        #{} ->
+            {Is, St}
+    end;
 cg_call(#cg_set{anno=Anno0,op=call,dst=Dst0,args=[#b_remote{}=Func0|Args0]},
         Where, Context, St) ->
     [Dst|Args] = beam_args([Dst0|Args0], St),
@@ -1725,6 +1730,14 @@ copy(Src, Dst) -> [{move,Src,Dst}].
 
 force_reg({literal,_}=Lit, Reg) ->
     {Reg,[{move,Lit,Reg}]};
+force_reg({integer,_}=Lit, Reg) ->
+    {Reg,[{move,Lit,Reg}]};
+force_reg({atom,_}=Lit, Reg) ->
+    {Reg,[{move,Lit,Reg}]};
+force_reg({float,_}=Lit, Reg) ->
+    {Reg,[{move,Lit,Reg}]};
+force_reg(nil=Lit, Reg) ->
+    {Reg,[{move,Lit,Reg}]};
 force_reg({Kind,_}=R, _) when Kind =:= x; Kind =:= y ->
     {R,[]}.
 
diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl
index 93c5922547..3685c09a2b 100644
--- a/lib/compiler/src/beam_ssa_opt.erl
+++ b/lib/compiler/src/beam_ssa_opt.erl
@@ -39,7 +39,7 @@
 
 -include("beam_ssa_opt.hrl").
 
--import(lists, [append/1,foldl/3,keyfind/3,member/2,
+-import(lists, [append/1,duplicate/2,foldl/3,keyfind/3,member/2,
                 reverse/1,reverse/2,
                 splitwith/2,sort/1,takewhile/2,unzip/1]).
 
@@ -146,8 +146,8 @@ prologue_passes(Opts) ->
           ?PASS(ssa_opt_linearize),
           ?PASS(ssa_opt_tuple_size),
           ?PASS(ssa_opt_record),
-          ?PASS(ssa_opt_cse)                   %Helps the first type pass.
-          ],
+          ?PASS(ssa_opt_cse),                   %Helps the first type pass.
+          ?PASS(ssa_opt_type_start)],
     passes_1(Ps, Opts).
 
 %% These passes all benefit from each other (in roughly this order), so they
@@ -157,12 +157,13 @@ repeated_passes(Opts) ->
           ?PASS(ssa_opt_bs_puts),
           ?PASS(ssa_opt_dead),
           ?PASS(ssa_opt_cse),
-          ?PASS(ssa_opt_type)],                 %Must run after ssa_opt_dead to
+          ?PASS(ssa_opt_type_continue)],        %Must run after ssa_opt_dead to
                                                 %clean up phi nodes.
     passes_1(Ps, Opts).
 
 epilogue_passes(Opts) ->
-    Ps = [?PASS(ssa_opt_float),
+    Ps = [?PASS(ssa_opt_type_finish),
+          ?PASS(ssa_opt_float),
           ?PASS(ssa_opt_live),                  %One last time to clean up the
                                                 %mess left by the float pass.
           ?PASS(ssa_opt_bsm),
@@ -199,18 +200,21 @@ build_func_db(#b_module{body=Fs,exports=Exports}) ->
         throw:load_nif -> #{}
     end.
 
-fdb_1([#b_function{ bs=Bs }=F | Fs], Exports, FuncDb0) ->
+fdb_1([#b_function{ args=Args,bs=Bs }=F | Fs], Exports, FuncDb0) ->
     Id = get_func_id(F),
 
     #b_local{name=#b_literal{val=Name}, arity=Arity} = Id,
     Exported = gb_sets:is_element({Name, Arity}, Exports),
+    ArgTypes = duplicate(length(Args), #{}),
 
     FuncDb1 = case FuncDb0 of
                   %% We may have an entry already if someone's called us.
                   #{ Id := Info } ->
-                      FuncDb0#{ Id := Info#func_info{ exported=Exported }};
+                      FuncDb0#{ Id := Info#func_info{ exported=Exported,
+                                                      arg_types=ArgTypes }};
                   #{} ->
-                      FuncDb0#{ Id => #func_info{ exported=Exported }}
+                      FuncDb0#{ Id => #func_info{ exported=Exported,
+                                                  arg_types=ArgTypes }}
               end,
 
     FuncDb = beam_ssa:fold_rpo(fun(_L, #b_blk{is=Is}, FuncDb) ->
@@ -293,10 +297,18 @@ ssa_opt_dead({#st{ssa=Linear}=St, FuncDb}) ->
 ssa_opt_linearize({#st{ssa=Blocks}=St, FuncDb}) ->
     {St#st{ssa=beam_ssa:linearize(Blocks)}, FuncDb}.
 
-ssa_opt_type({#st{ssa=Linear0,args=Args}=St0, FuncDb}) ->
-    Linear = beam_ssa_type:opt(Linear0, Args),
+ssa_opt_type_start({#st{ssa=Linear0,args=Args,anno=Anno}=St0, FuncDb0}) ->
+    {Linear, FuncDb} = beam_ssa_type:opt_start(Linear0, Args, Anno, FuncDb0),
     {St0#st{ssa=Linear}, FuncDb}.
 
+ssa_opt_type_continue({#st{ssa=Linear0,args=Args,anno=Anno}=St0, FuncDb0}) ->
+    {Linear, FuncDb} = beam_ssa_type:opt_continue(Linear0, Args, Anno, FuncDb0),
+    {St0#st{ssa=Linear}, FuncDb}.
+
+ssa_opt_type_finish({#st{args=Args,anno=Anno0}=St0, FuncDb0}) ->
+    {Anno, FuncDb} = beam_ssa_type:opt_finish(Args, Anno0, FuncDb0),
+    {St0#st{anno=Anno}, FuncDb}.
+
 ssa_opt_blockify({#st{ssa=Linear}=St, FuncDb}) ->
     {St#st{ssa=maps:from_list(Linear)}, FuncDb}.
 
diff --git a/lib/compiler/src/beam_ssa_opt.hrl b/lib/compiler/src/beam_ssa_opt.hrl
index 21a45f562a..37711a6f48 100644
--- a/lib/compiler/src/beam_ssa_opt.hrl
+++ b/lib/compiler/src/beam_ssa_opt.hrl
@@ -27,11 +27,27 @@
 
          %% Whether the function is exported or not; some optimizations may
          %% need to be suppressed if it is.
-         exported = true :: boolean()}).
+         exported = true :: boolean(),
+
+         %% The inferred types of each argument (as opposed to parameter),
+         %% indexed by call site.
+         %%
+         %% This is more effective than the naive approach of joining into a
+         %% "parameter_type" as we go as it lets us narrow parameter types
+         %% without having to visit all callers on each pass, which helps a lot
+         %% when dealing with co-recursive functions.
+         arg_types = [] :: list(arg_type_map()),
+
+         %% The inferred return type of this function, this is either [type()]
+         %% or [] to note absence.
+         ret_type = [] :: list()}).
+
+-type arg_key() :: {CallerId :: func_id(),
+                    CallDst :: beam_ssa:b_var()}.
+-type arg_type_map() :: #{ arg_key() => term() }.
 
 %% Per-function metadata used by various optimization passes to perform
 %% module-level optimization. If a function is absent it means that
 %% module-level optimization has been turned off for said function.
-
 -type func_id() :: beam_ssa:b_local().
 -type func_info_db() :: #{ func_id() => #func_info{} }.
diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl
index fa1b7bb71e..03d193db4d 100644
--- a/lib/compiler/src/beam_ssa_pre_codegen.erl
+++ b/lib/compiler/src/beam_ssa_pre_codegen.erl
@@ -1031,7 +1031,7 @@ need_frame_1([#b_set{op=call,args=[Func|_]}|Is], Context) ->
     case Func of
         #b_remote{mod=#b_literal{val=Mod},
                   name=#b_literal{val=Name},
-                  arity=Arity} ->
+                  arity=Arity} when is_atom(Mod), is_atom(Name) ->
             case erl_bifs:is_exit_bif(Mod, Name, Arity) of
                 true ->
                     false;
@@ -1993,6 +1993,12 @@ reserve_zregs(Blocks, Intervals, Res) ->
         end,
     beam_ssa:fold_rpo(F, [0], Res, Blocks).
 
+reserve_zreg([#b_set{op=call,dst=Dst}],
+              #b_br{bool=Dst}, _ShortLived, A) ->
+    %% If type optimization has determined that the result of a call can be
+    %% used directly in a branch, we must avoid reserving a z register or code
+    %% generation will fail.
+    A;
 reserve_zreg([#b_set{op={bif,tuple_size},dst=Dst},
               #b_set{op={bif,'=:='},args=[Dst,Val]}], Last, ShortLived, A0) ->
     case {Val,Last} of
diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl
index fcfb7b86f6..38ea5e6914 100644
--- a/lib/compiler/src/beam_ssa_type.erl
+++ b/lib/compiler/src/beam_ssa_type.erl
@@ -19,19 +19,22 @@
 %%
 
 -module(beam_ssa_type).
--export([opt/2]).
+-export([opt_start/4, opt_continue/4, opt_finish/3]).
 
--include("beam_ssa.hrl").
+-include("beam_ssa_opt.hrl").
 -import(lists, [all/2,any/2,droplast/1,foldl/3,last/1,member/2,
                 partition/2,reverse/1,sort/1]).
 
 -define(UNICODE_INT, #t_integer{elements={0,16#10FFFF}}).
 
--record(d, {ds :: #{beam_ssa:b_var():=beam_ssa:b_set()},
-            ls :: #{beam_ssa:label():=type_db()},
-            once :: cerl_sets:set(beam_ssa:b_var()),
-            sub :: #{beam_ssa:b_var():=beam_ssa:value()}
-           }).
+-record(d,
+        {ds :: #{beam_ssa:b_var():=beam_ssa:b_set()},
+         ls :: #{beam_ssa:label():=type_db()},
+         once :: cerl_sets:set(beam_ssa:b_var()),
+         func_id :: func_id(),
+         func_db :: func_info_db(),
+         sub = #{} :: #{beam_ssa:b_var():=beam_ssa:value()},
+         ret_type = [] :: [type()]}).
 
 -define(ATOM_SET_SIZE, 5).
 
@@ -49,36 +52,155 @@
                 {'binary',pos_integer()} | 'cons' | 'float' | 'list' | 'map' | 'nil' |'number'.
 -type type_db() :: #{beam_ssa:var_name():=type()}.
 
--spec opt([{Label0,Block0}], Args) -> [{Label,Block}] when
-      Label0 :: beam_ssa:label(),
-      Block0 :: beam_ssa:b_blk(),
+-spec opt_start(Linear, Args, Anno, FuncDb) -> {Linear, FuncDb} when
+      Linear :: [{non_neg_integer(), beam_ssa:b_blk()}],
       Args :: [beam_ssa:b_var()],
-      Label :: beam_ssa:label(),
-      Block :: beam_ssa:b_blk().
-
-opt(Linear, Args) ->
-    UsedOnce = used_once(Linear, Args),
+      Anno :: beam_ssa:anno(),
+      FuncDb :: func_info_db().
+opt_start(Linear, Args, Anno, FuncDb) ->
+    %% This is the first run through the module, so our arg_types can be
+    %% incomplete as we may not have visited all call sites at least once.
     Ts = maps:from_list([{V,any} || #b_var{}=V <- Args]),
+    opt_continue_1(Linear, Args, get_func_id(Anno), Ts, FuncDb).
+
+-spec opt_continue(Linear, Args, Anno, FuncDb) -> {Linear, FuncDb} when
+      Linear :: [{non_neg_integer(), beam_ssa:b_blk()}],
+      Args :: [beam_ssa:b_var()],
+      Anno :: beam_ssa:anno(),
+      FuncDb :: func_info_db().
+opt_continue(Linear, Args, Anno, FuncDb) ->
+    Id = get_func_id(Anno),
+    case FuncDb of
+        #{ Id := #func_info{exported=false,arg_types=ArgTypes} } ->
+            %% This is a local function and we're guaranteed to have visited
+            %% every call site at least once, so we know that the parameter
+            %% types are at least as narrow as the join of all argument types.
+            Ts = join_arg_types(Args, ArgTypes, Anno),
+            opt_continue_1(Linear, Args, Id, Ts, FuncDb);
+        #{} ->
+            %% We can't infer the parameter types of exported functions, nor
+            %% the ones where module-level optimization is disabled, but
+            %% running the pass again could still help other functions.
+            Ts = maps:from_list([{V,any} || #b_var{}=V <- Args]),
+            opt_continue_1(Linear, Args, Id, Ts, FuncDb)
+    end.
+
+join_arg_types(Args, ArgTypes, Anno) ->
+    %% We suppress type optimization for parameters that have already been
+    %% optimized by another pass, as they may have done things we have no idea
+    %% how to interpret and running them over could generate incorrect code.
+    ParamTypes = maps:get(parameter_type_info, Anno, #{}),
+    Ts0 = join_arg_types_1(Args, ArgTypes, #{}),
+    maps:fold(fun(Arg, _V, Ts) ->
+                      maps:put(Arg, any, Ts)
+              end, Ts0, ParamTypes).
+
+join_arg_types_1([Arg | Args], [TM | TMs], Ts) when map_size(TM) =/= 0 ->
+    join_arg_types_1(Args, TMs, Ts#{ Arg => join(maps:values(TM))});
+join_arg_types_1([Arg | Args], [_TM | TMs], Ts) ->
+    join_arg_types_1(Args, TMs, Ts#{ Arg => any });
+join_arg_types_1([], [], Ts) ->
+    Ts.
+
+-spec opt_continue_1(Linear, Args, Id, Ts, FuncDb) -> Result when
+      Linear :: [{non_neg_integer(), beam_ssa:b_blk()}],
+      Args :: [beam_ssa:b_var()],
+      Id :: func_id(),
+      Ts :: type_db(),
+      FuncDb :: func_info_db(),
+      Result :: {Linear, FuncDb}.
+opt_continue_1(Linear0, Args, Id, Ts, FuncDb0) ->
+    UsedOnce = used_once(Linear0, Args),
     FakeCall = #b_set{op=call,args=[#b_remote{mod=#b_literal{val=unknown},
                                               name=#b_literal{val=unknown},
                                               arity=0}]},
     Defs = maps:from_list([{Var,FakeCall#b_set{dst=Var}} ||
                               #b_var{}=Var <- Args]),
-    D = #d{ds=Defs,ls=#{0=>Ts,?BADARG_BLOCK=>#{}},
-           once=UsedOnce,sub=#{}},
-    opt_1(Linear, D).
 
-opt_1([{L,Blk}|Bs], #d{ls=Ls}=D) ->
+    D = #d{ func_db=FuncDb0,
+            func_id=Id,
+            ds=Defs,
+            ls=#{0=>Ts,?BADARG_BLOCK=>#{}},
+            once=UsedOnce },
+
+    {Linear, FuncDb, NewRet} = opt_1(Linear0, D, []),
+
+    case FuncDb of
+        #{ Id := Entry0 } ->
+            Entry = Entry0#func_info{ret_type=NewRet},
+            {Linear, FuncDb#{ Id := Entry }};
+        #{} ->
+            %% Module-level optimizations have been turned off for this
+            %% function.
+            {Linear, FuncDb}
+    end.
+
+-spec opt_finish(Args, Anno, FuncDb) -> {Anno, FuncDb} when
+      Args :: [beam_ssa:b_var()],
+      Anno :: beam_ssa:anno(),
+      FuncDb :: func_info_db().
+opt_finish(Args, Anno, FuncDb) ->
+    Id = get_func_id(Anno),
+    case FuncDb of
+        #{ Id := #func_info{exported=false,arg_types=ArgTypes} } ->
+            ParamInfo0 = maps:get(parameter_type_info, Anno, #{}),
+            ParamInfo = opt_finish_1(Args, ArgTypes, ParamInfo0),
+            {Anno#{ parameter_type_info => ParamInfo }, FuncDb};
+        #{} ->
+            {Anno, FuncDb}
+    end.
+
+opt_finish_1([Arg | Args], [TypeMap | TypeMaps], ParamInfo)
+  when is_map_key(Arg, ParamInfo); %% See join_arg_types/3
+       map_size(TypeMap) =:= 0 ->
+    opt_finish_1(Args, TypeMaps, ParamInfo);
+opt_finish_1([Arg | Args], [TypeMap | TypeMaps], ParamInfo0) ->
+    case join(maps:values(TypeMap)) of
+        any ->
+            opt_finish_1(Args, TypeMaps, ParamInfo0);
+        JoinedType ->
+            JoinedType = verified_type(JoinedType),
+            ParamInfo = ParamInfo0#{ Arg => validator_anno(JoinedType) },
+            opt_finish_1(Args, TypeMaps, ParamInfo)
+    end;
+opt_finish_1([], [], ParamInfo) ->
+    ParamInfo.
+
+validator_anno(#t_tuple{size=Size,exact=Exact}) ->
+    beam_validator:type_anno(tuple, Size, Exact);
+validator_anno(#t_integer{elements={Same,Same}}) ->
+    beam_validator:type_anno(integer, Same);
+validator_anno(#t_integer{}) ->
+    beam_validator:type_anno(integer);
+validator_anno(float) ->
+    beam_validator:type_anno(float);
+validator_anno(#t_atom{elements=[Val]}) ->
+    beam_validator:type_anno(atom, Val);
+validator_anno(#t_atom{}=A) ->
+    case t_is_boolean(A) of
+        true -> beam_validator:type_anno(bool);
+        false -> beam_validator:type_anno(atom)
+    end;
+validator_anno(T) ->
+    beam_validator:type_anno(T).
+
+get_func_id(Anno) ->
+    #{func_info:={_Mod, Name, Arity}} = Anno,
+    #b_local{name=#b_literal{val=Name}, arity=Arity}.
+
+opt_1([{L,Blk}|Bs], #d{ls=Ls}=D, Acc) ->
     case Ls of
         #{L:=Ts} ->
-            opt_2(L, Blk, Bs, Ts, D);
+            opt_2(L, Blk, Bs, Ts, D, Acc);
         #{} ->
             %% This block is never reached. Discard it.
-            opt_1(Bs, D)
+            opt_1(Bs, D, Acc)
     end;
-opt_1([], #d{}) -> [].
+opt_1([], D, Acc) ->
+    #d{func_db=FuncDb,ret_type=NewRet} = D,
+    {reverse(Acc), FuncDb, NewRet}.
 
-opt_2(L, #b_blk{is=Is0}=Blk0, Bs, Ts, #d{sub=Sub}=D0) ->
+opt_2(L, #b_blk{is=Is0}=Blk0, Bs, Ts, #d{sub=Sub}=D0, Acc) ->
     case Is0 of
         [#b_set{op=call,dst=Dst,
                 args=[#b_remote{mod=#b_literal{val=Mod},
@@ -94,34 +216,43 @@ opt_2(L, #b_blk{is=Is0}=Blk0, Bs, Ts, #d{sub=Sub}=D0) ->
                     Ret = #b_ret{arg=Dst},
                     Blk = Blk0#b_blk{is=[I],last=Ret},
                     Ls = maps:remove(L, D0#d.ls),
-                    D = D0#d{ls=Ls},
-                    [{L,Blk}|opt_1(Bs, D)];
+
+                    %% We potentially lack a return value.
+                    RetType = join([none | D0#d.ret_type]),
+
+                    D = D0#d{ls=Ls,ret_type=[RetType]},
+                    opt_1(Bs, D, [{L,Blk} | Acc]);
                 false ->
-                    opt_3(L, Blk0, Bs, Ts, D0)
+                    opt_3(L, Blk0, Bs, Ts, D0, Acc)
             end;
         _ ->
-            opt_3(L, Blk0, Bs, Ts, D0)
+            opt_3(L, Blk0, Bs, Ts, D0, Acc)
     end.
 
 opt_3(L, #b_blk{is=Is0,last=Last0}=Blk0, Bs, Ts0,
-      #d{ds=Ds0,ls=Ls0,sub=Sub0}=D0) ->
-    {Is,Ts,Ds,Sub} = opt_is(Is0, Ts0, Ds0, Ls0, Sub0, []),
-    D1 = D0#d{ds=Ds,sub=Sub},
-    Last1 = simplify_terminator(Last0, Sub, Ts),
+      #d{ds=Ds0,ls=Ls0,sub=Sub0,func_db=Fdb0}=D0, Acc) ->
+    {Is,Ts,Ds,Fdb,Sub} = opt_is(Is0, Ts0, Ds0, Fdb0, Ls0, D0, Sub0, []),
+    D1 = D0#d{ds=Ds,sub=Sub,func_db=Fdb},
+    Last1 = simplify_terminator(Last0, Sub, Ts, Ds),
     Last = opt_terminator(Last1, Ts, Ds),
     D = update_successors(Last, Ts, D1),
     Blk = Blk0#b_blk{is=Is,last=Last},
-    [{L,Blk}|opt_1(Bs, D)].
+    opt_1(Bs, D, [{L,Blk} | Acc]).
 
-simplify_terminator(#b_br{bool=Bool}=Br, Sub, Ts) ->
+simplify_terminator(#b_br{bool=Bool}=Br, Sub, Ts, _Ds) ->
     Br#b_br{bool=simplify_arg(Bool, Sub, Ts)};
-simplify_terminator(#b_switch{arg=Arg}=Sw, Sub, Ts) ->
+simplify_terminator(#b_switch{arg=Arg}=Sw, Sub, Ts, _Ds) ->
     Sw#b_switch{arg=simplify_arg(Arg, Sub, Ts)};
-simplify_terminator(#b_ret{arg=Arg}=Ret, Sub, Ts) ->
-    Ret#b_ret{arg=simplify_arg(Arg, Sub, Ts)}.
+simplify_terminator(#b_ret{arg=Arg}=Ret, Sub, Ts, Ds) ->
+    %% Reducing the result of a call to a literal (fairly common for 'ok')
+    %% breaks tail call optimization.
+    case Ds of
+        #{ Arg := #b_set{op=call}} -> Ret;
+        #{} -> Ret#b_ret{arg=simplify_arg(Arg, Sub, Ts)}
+    end.
 
 opt_is([#b_set{op=phi,dst=Dst,args=Args0}=I0|Is],
-       Ts0, Ds0, Ls, Sub0, Acc) ->
+       Ts0, Ds0, Fdb, Ls, D, Sub0, Acc) ->
     %% Simplify the phi node by removing all predecessor blocks that no
     %% longer exists or no longer branches to this block.
     Args = [{simplify_arg(Arg, Sub0, Ts0),From} ||
@@ -132,28 +263,61 @@ opt_is([#b_set{op=phi,dst=Dst,args=Args0}=I0|Is],
             %% value or if the values are identical.
             [{Val,_}|_] = Args,
             Sub = Sub0#{Dst=>Val},
-            opt_is(Is, Ts0, Ds0, Ls, Sub, Acc);
+            opt_is(Is, Ts0, Ds0, Fdb, Ls, D, Sub, Acc);
         false ->
             I = I0#b_set{args=Args},
             Ts = update_types(I, Ts0, Ds0),
             Ds = Ds0#{Dst=>I},
-            opt_is(Is, Ts, Ds, Ls, Sub0, [I|Acc])
+            opt_is(Is, Ts, Ds, Fdb, Ls, D, Sub0, [I|Acc])
     end;
-opt_is([#b_set{op=succeeded,args=Args0,dst=Dst}=I],
-       Ts0, Ds0, Ls, Sub0, Acc) ->
-    Args = simplify_args(Args0, Sub0, Ts0),
-    Type = type(succeeded, Args, Ts0, Ds0),
-    case get_literal_from_type(Type) of
-        #b_literal{}=Lit ->
-            Sub = Sub0#{Dst=>Lit},
-            opt_is([], Ts0, Ds0, Ls, Sub, Acc);
-        none ->
+opt_is([#b_set{op=call,args=Args0,dst=Dst}=I0 | Is],
+       Ts0, Ds0, Fdb0, Ls, D, Sub, Acc) ->
+    Args = simplify_args(Args0, Sub, Ts0),
+    I1 = beam_ssa:normalize(I0#b_set{args=Args}),
+
+    %% This is a bit of a kludge; we know that any instruction whose return
+    %% type is 'none' will fail at runtime, but we don't yet have a way to cut
+    %% a block short so we move on like nothing nothing happened.
+    %%
+    %% This complicates argument type optimization as unreachable calls can
+    %% add types that will never occur, so we skip optimizing this call if
+    %% the type of any of its arguments is 'none'.
+    [_Callee | Rest] = Args,
+    case all(fun(Arg) -> get_type(Arg, Ts0) =/= none end, Rest) of
+        true ->
+            {Ts, Ds, Fdb, I} = opt_call(I1, D, Ts0, Ds0, Fdb0),
+            opt_is(Is, Ts, Ds, Fdb, Ls, D, Sub, [I|Acc]);
+        false ->
+            Ts = Ts0#{ Dst => any },
+            Ds = Ds0#{ Dst => I1 },
+            opt_is(Is, Ts, Ds, Fdb0, Ls, D, Sub, [I1|Acc])
+    end;
+opt_is([#b_set{op=succeeded,args=[Arg],dst=Dst}=I],
+       Ts0, Ds0, Fdb, Ls, D, Sub0, Acc) ->
+    case Ds0 of
+        #{ Arg := #b_set{op=call} } ->
+            %% The success check of a call is part of exception handling and
+            %% must not be optimized away. We still have to update its type
+            %% though.
             Ts = update_types(I, Ts0, Ds0),
             Ds = Ds0#{Dst=>I},
-            opt_is([], Ts, Ds, Ls, Sub0, [I|Acc])
+
+            opt_is([], Ts, Ds, Fdb, Ls, D, Sub0, [I|Acc]);
+        #{} ->
+            Args = simplify_args([Arg], Sub0, Ts0),
+            Type = type(succeeded, Args, Ts0, Ds0),
+            case get_literal_from_type(Type) of
+                #b_literal{}=Lit ->
+                    Sub = Sub0#{Dst=>Lit},
+                    opt_is([], Ts0, Ds0, Fdb, Ls, D, Sub, Acc);
+                none ->
+                    Ts = Ts0#{Dst=>Type},
+                    Ds = Ds0#{Dst=>I},
+                    opt_is([], Ts, Ds, Fdb, Ls, D, Sub0, [I|Acc])
+            end
     end;
 opt_is([#b_set{args=Args0,dst=Dst}=I0|Is],
-       Ts0, Ds0, Ls, Sub0, Acc) ->
+       Ts0, Ds0, Fdb, Ls, D, Sub0, Acc) ->
     Args = simplify_args(Args0, Sub0, Ts0),
     I1 = beam_ssa:normalize(I0#b_set{args=Args}),
     case simplify(I1, Ts0) of
@@ -161,23 +325,76 @@ opt_is([#b_set{args=Args0,dst=Dst}=I0|Is],
             I = beam_ssa:normalize(I2),
             Ts = update_types(I, Ts0, Ds0),
             Ds = Ds0#{Dst=>I},
-            opt_is(Is, Ts, Ds, Ls, Sub0, [I|Acc]);
+            opt_is(Is, Ts, Ds, Fdb, Ls, D, Sub0, [I|Acc]);
         #b_literal{}=Lit ->
             Sub = Sub0#{Dst=>Lit},
-            opt_is(Is, Ts0, Ds0, Ls, Sub, Acc);
+            opt_is(Is, Ts0, Ds0, Fdb, Ls, D, Sub, Acc);
         #b_var{}=Var ->
             case Is of
                 [#b_set{op=succeeded,dst=SuccDst,args=[Dst]}] ->
                     %% We must remove this 'succeeded' instruction.
                     Sub = Sub0#{Dst=>Var,SuccDst=>#b_literal{val=true}},
-                    opt_is([], Ts0, Ds0, Ls, Sub, Acc);
+                    opt_is([], Ts0, Ds0, Fdb, Ls, D, Sub, Acc);
                 _ ->
                     Sub = Sub0#{Dst=>Var},
-                    opt_is(Is, Ts0, Ds0, Ls, Sub, Acc)
+                    opt_is(Is, Ts0, Ds0, Fdb, Ls, D, Sub, Acc)
             end
     end;
-opt_is([], Ts, Ds, _Ls, Sub, Acc) ->
-    {reverse(Acc),Ts,Ds,Sub}.
+opt_is([], Ts, Ds, Fdb, _Ls, _D, Sub, Acc) ->
+    {reverse(Acc), Ts, Ds, Fdb, Sub}.
+
+opt_call(#b_set{dst=Dst,args=[#b_local{}=Callee|Args]}=I0, D, Ts0, Ds0, Fdb0) ->
+    {Ts, Ds, I} = opt_local_call(I0, Ts0, Ds0, Fdb0),
+    case Fdb0 of
+        #{ Callee := #func_info{exported=false,arg_types=ArgTypes0}=Info } ->
+            %% Update the argument types of *this exact call*, the types
+            %% will be joined later when the callee is optimized.
+            CallId = {D#d.func_id, Dst},
+            ArgTypes = update_arg_types(Args, ArgTypes0, CallId, Ts0),
+
+            Fdb = Fdb0#{ Callee => Info#func_info{arg_types=ArgTypes} },
+            {Ts, Ds, Fdb, I};
+        #{} ->
+            %% We can't narrow the argument types of exported functions as they
+            %% can receive anything as part of an external call.
+            {Ts, Ds, Fdb0, I}
+    end;
+opt_call(#b_set{dst=Dst}=I, _D, Ts0, Ds0, Fdb) ->
+    Ts = update_types(I, Ts0, Ds0),
+    Ds = Ds0#{ Dst => I },
+    {Ts, Ds, Fdb, I}.
+
+opt_local_call(#b_set{dst=Dst,args=[Id|_]}=I0, Ts0, Ds0, Fdb) ->
+    %% We skip propagating 'none' as we don't yet have a good way to cut a
+    %% block short.
+    Type = case Fdb of
+               #{ Id := #func_info{ret_type=[T]} } when T =/= none -> T;
+               #{} -> any
+           end,
+    I = case Type of
+            any -> I0;
+            _ -> beam_ssa:add_anno(result_type, validator_anno(Type), I0)
+        end,
+    Ts = Ts0#{ Dst => Type },
+    Ds = Ds0#{ Dst => I },
+    {Ts, Ds, I}.
+
+update_arg_types([Arg | Args], [TypeMap0 | TypeMaps], CallId, Ts) ->
+    %% Match contexts are treated as bitstrings when optimizing arguments, as
+    %% we don't yet support removing the "bs_start_match3" instruction.
+    NewType = case get_type(Arg, Ts) of
+                  #t_bs_match{} -> {binary, 1};
+                  Type -> Type
+              end,
+    PrevType = maps:get(CallId, TypeMap0, NewType),
+
+    %% The new type must be narrower than the old one.
+    true = meet(NewType, PrevType) =/= none,    %Assertion.
+
+    TypeMap = TypeMap0#{ CallId => NewType },
+    [TypeMap | update_arg_types(Args, TypeMaps, CallId, Ts)];
+update_arg_types([], [], _CallId, _Ts) ->
+    [].
 
 simplify(#b_set{op={bif,'and'},args=Args}=I, Ts) ->
     case is_safe_bool_op(Args, Ts) of
@@ -487,8 +704,18 @@ update_successors(#b_switch{arg=#b_var{}=V,fail=Fail,list=List}, Ts0, D0) ->
                         update_successor(S, Ts0#{V=>T}, A)
                 end,
             foldl(F, D, List)
-        end;
-update_successors(#b_ret{}, _Ts, D) -> D.
+    end;
+update_successors(#b_ret{arg=Arg}, Ts, D) ->
+    FuncId = D#d.func_id,
+    case D#d.ds of
+        #{ Arg := #b_set{op=call,args=[FuncId | _]} } ->
+            %% Returning a call to ourselves doesn't affect our own return
+            %% type.
+            D;
+        #{} ->
+            RetType = join([get_type(Arg, Ts) | D#d.ret_type]),
+            D#d{ret_type=[RetType]}
+    end.
 
 subtract_sw_list(V, List, Ts) ->
     Ts#{ V := sub_sw_list_1(get_type(V, Ts), List, Ts) }.
diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl
index 73c66e6efc..53d3cec2d7 100644
--- a/lib/compiler/src/compile.erl
+++ b/lib/compiler/src/compile.erl
@@ -268,6 +268,10 @@ expand_opt(r21, Os) ->
     [no_put_tuple2 | expand_opt(no_bsm3, Os)];
 expand_opt({debug_info_key,_}=O, Os) ->
     [encrypt_debug_info,O|Os];
+expand_opt(no_type_opt, Os) ->
+    [no_ssa_opt_type_start,
+     no_ssa_opt_type_continue,
+     no_ssa_opt_type_finish | Os];
 expand_opt(O, Os) -> [O|Os].
 
 expand_opt_before_21(Os) ->
diff --git a/lib/compiler/test/apply_SUITE.erl b/lib/compiler/test/apply_SUITE.erl
index 0f82a56fb7..2ee518b1a0 100644
--- a/lib/compiler/test/apply_SUITE.erl
+++ b/lib/compiler/test/apply_SUITE.erl
@@ -73,6 +73,7 @@ mfa(Config) when is_list(Config) ->
     {'EXIT',_} = (catch ?APPLY2(Mod, (id(bazzzzzz)), a, b)),
     {'EXIT',_} = (catch ?APPLY2({}, baz, a, b)),
     {'EXIT',_} = (catch ?APPLY2(?MODULE, [], a, b)),
+    {'EXIT',_} = (catch bad_literal_call(1)),
 
     ok = apply(Mod, foo, id([])),
     {[a,b|c]} = apply(Mod, bar, id([[a,b|c]])),
@@ -92,6 +93,13 @@ mfa(Config) when is_list(Config) ->
 
     apply(Mod, foo, []).
 
+%% The single call to this function with a literal argument caused type
+%% optimization to swap out the 'mod' field of a #b_remote{}, which was
+%% mishandled during code generation as it assumed that the module would always
+%% be an atom.
+bad_literal_call(I) ->
+    I:foo().
+
 foo() ->
     ok.
 
diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl
index c17d63cd60..7452466666 100644
--- a/lib/compiler/test/compile_SUITE.erl
+++ b/lib/compiler/test/compile_SUITE.erl
@@ -1249,7 +1249,8 @@ do_opt_guards_fun([]) -> [].
 is_exception(guard_SUITE, {'-complex_not/1-fun-4-',1}) -> true;
 is_exception(guard_SUITE, {'-complex_not/1-fun-5-',1}) -> true;
 is_exception(guard_SUITE, {bad_guards,1}) -> true;
-is_exception(guard_SUITE, {nested_not_2b,4}) -> true;
+is_exception(guard_SUITE, {nested_not_2b,6}) -> true; %% w/o type optimization
+is_exception(guard_SUITE, {nested_not_2b,2}) -> true; %% with type optimization
 is_exception(_, _) -> false.
 
 sys_pre_attributes(Config) ->
diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl
index 08f2e8fae9..26149e11e6 100644
--- a/lib/compiler/test/test_lib.erl
+++ b/lib/compiler/test/test_lib.erl
@@ -82,6 +82,7 @@ opt_opts(Mod) ->
                     (no_bsm3) -> true;
                     (no_bsm_opt) -> true;
                     (no_module_opt) -> true;
+                    (no_type_opt) -> true;
 		    (_) -> false
 		 end, Opts).