Eliminate crash in the beam_ssa_dead compiler pass

The compiler could crash in the beam_ssa_dead pass while compiling
complex nested `case` expressions. See the added test case for an
example and explanation.

https://bugs.erlang.org/browse/ERL-956

2 files changed, 74 insertions, 4 deletions

diff --git a/lib/compiler/src/beam_ssa_dead.erl b/lib/compiler/src/beam_ssa_dead.erl
index bb43a550ae..86f680c964 100644
--- a/lib/compiler/src/beam_ssa_dead.erl
+++ b/lib/compiler/src/beam_ssa_dead.erl
@@ -436,8 +436,22 @@ get_phi_arg([{Val,From}|_], From) -> Val;
 get_phi_arg([_|As], From) -> get_phi_arg(As, From).
 
 eval_terminator(#b_br{bool=#b_var{}=Bool}=Br, Bs, _St) ->
-    Val = get_value(Bool, Bs),
-    beam_ssa:normalize(Br#b_br{bool=Val});
+    case get_value(Bool, Bs) of
+        #b_literal{val=Val}=Lit ->
+            case is_boolean(Val) of
+                true ->
+                    beam_ssa:normalize(Br#b_br{bool=Lit});
+                false ->
+                    %% Non-boolean literal. This means that this `br`
+                    %% terminator will never actually be reached with
+                    %% these bindings. (There must be a previous two-way
+                    %% branch that branches the other way when Bool
+                    %% is bound to a non-boolean literal.)
+                    none
+            end;
+        #b_var{}=Var ->
+            beam_ssa:normalize(Br#b_br{bool=Var})
+    end;
 eval_terminator(#b_br{bool=#b_literal{}}=Br, _Bs, _St) ->
     beam_ssa:normalize(Br);
 eval_terminator(#b_switch{arg=Arg,fail=Fail,list=List}=Sw, Bs, St) ->
diff --git a/lib/compiler/test/beam_ssa_SUITE.erl b/lib/compiler/test/beam_ssa_SUITE.erl
index 15cf9bcbf3..a741ebbdf9 100644
--- a/lib/compiler/test/beam_ssa_SUITE.erl
+++ b/lib/compiler/test/beam_ssa_SUITE.erl
@@ -22,7 +22,8 @@
 -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1,
 	 init_per_group/2,end_per_group/2,
          calls/1,tuple_matching/1,recv/1,maps/1,
-         cover_ssa_dead/1,combine_sw/1,share_opt/1]).
+         cover_ssa_dead/1,combine_sw/1,share_opt/1,
+         beam_ssa_dead_crash/1]).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
@@ -37,7 +38,8 @@ groups() ->
        maps,
        cover_ssa_dead,
        combine_sw,
-       share_opt
+       share_opt,
+       beam_ssa_dead_crash
       ]}].
 
 init_per_suite(Config) ->
@@ -492,6 +494,60 @@ do_share_opt(A) ->
     end,
     receive after 1 -> ok end.
 
+beam_ssa_dead_crash(_Config) ->
+    not_A_B = do_beam_ssa_dead_crash(id(false), id(true)),
+    not_A_not_B = do_beam_ssa_dead_crash(false, false),
+    neither = do_beam_ssa_dead_crash(true, false),
+    neither = do_beam_ssa_dead_crash(true, true),
+    ok.
+
+do_beam_ssa_dead_crash(A, B) ->
+    %% beam_ssa_dead attempts to shortcut branches that branch other
+    %% branches. When a two-way branch is encountered, beam_ssa_dead
+    %% will simulate execution along both paths, in the hope that both
+    %% paths happens to end up in the same place.
+    %%
+    %% During the simulated execution of this function, the boolean
+    %% varible for a `br` instruction would be replaced with the
+    %% literal atom `nil`, which is not allowed, and would crash the
+    %% compiler. In practice, during the actual execution, control
+    %% would never be transferred to that `br` instruction when the
+    %% variable in question had the value `nil`.
+    %%
+    %% beam_ssa_dead has been updated to immediately abort the search
+    %% along the current path if there is an attempt to substitute a
+    %% non-boolean value into a `br` instruction.
+
+    case
+        case not A of
+            false ->
+                false;
+            true ->
+                B
+        end
+    of
+        V
+            when
+                V /= nil
+                andalso
+                V /= false ->
+            not_A_B;
+        _ ->
+            case
+                case not A of
+                    false ->
+                        false;
+                    true ->
+                        not B
+                end
+            of
+                true ->
+                    not_A_not_B;
+                false ->
+                    neither
+            end
+    end.
+
 
 %% The identity function.
 id(I) -> I.