Enhance beam_validator to check proper nesting of try/catch and catch

Every catch or try/catch must use a lower Y register number than any
enclosing catch or try/catch. That will ensure that when the stack
is scanned when an exception occurs, the innermost try/catch tag is
found first.

3 files changed, 98 insertions, 2 deletions

diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl
index 9de773542e..c30ab34ac7 100644
--- a/lib/compiler/src/beam_validator.erl
+++ b/lib/compiler/src/beam_validator.erl
@@ -1153,6 +1153,7 @@ set_type_y(Type, {y,Y}=Reg, #vst{current=#st{y=Ys0}=St}=Vst)
 	     {value,_} ->
 		 gb_trees:update(Y, Type, Ys0)
 	 end,
+    check_try_catch_tags(Type, Y, Ys0),
     Vst#vst{current=St#st{y=Ys}};
 set_type_y(Type, Reg, #vst{}) -> error({invalid_store,Reg,Type}).
 
@@ -1160,6 +1161,29 @@ set_catch_end({y,Y}, #vst{current=#st{y=Ys0}=St}=Vst) ->
     Ys = gb_trees:update(Y, initialized, Ys0),
     Vst#vst{current=St#st{y=Ys}}.
 
+check_try_catch_tags(Type, LastY, Ys) ->
+    case is_try_catch_tag(Type) of
+        false ->
+            ok;
+        true ->
+            %% Every catch or try/catch must use a lower Y register
+            %% number than any enclosing catch or try/catch. That will
+            %% ensure that when the stack is scanned when an
+            %% exception occurs, the innermost try/catch tag is found
+            %% first.
+            Bad = [{{y,Y},Tag} || {Y,Tag} <- gb_trees:to_list(Ys),
+                                  Y < LastY, is_try_catch_tag(Tag)],
+            case Bad of
+                [] ->
+                    ok;
+                [_|_] ->
+                    error({bad_try_catch_nesting,{y,LastY},Bad})
+            end
+    end.
+
+is_try_catch_tag({catchtag,_}) -> true;
+is_try_catch_tag({trytag,_}) -> true;
+is_try_catch_tag(_) -> false.
 
 is_reg_defined({x,_}=Reg, Vst) -> is_type_defined_x(Reg, Vst);
 is_reg_defined({y,_}=Reg, Vst) -> is_type_defined_y(Reg, Vst);
diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl
index 63a13281a8..b8fff7b100 100644
--- a/lib/compiler/test/beam_validator_SUITE.erl
+++ b/lib/compiler/test/beam_validator_SUITE.erl
@@ -33,7 +33,7 @@
 	 state_after_fault_in_catch/1,no_exception_in_catch/1,
 	 undef_label/1,illegal_instruction/1,failing_gc_guard_bif/1,
 	 map_field_lists/1,cover_bin_opt/1,
-	 val_dsetel/1,bad_tuples/1]).
+	 val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -62,7 +62,7 @@ groups() ->
        state_after_fault_in_catch,no_exception_in_catch,
        undef_label,illegal_instruction,failing_gc_guard_bif,
        map_field_lists,cover_bin_opt,val_dsetel,
-       bad_tuples]}].
+       bad_tuples,bad_try_catch_nesting]}].
 
 init_per_suite(Config) ->
     Config.
@@ -523,6 +523,14 @@ bad_tuples(Config) ->
 
     ok.
 
+bad_try_catch_nesting(Config) ->
+    Errors = do_val(bad_try_catch_nesting, Config),
+    [{{bad_try_catch_nesting,main,2},
+      {{'try',{y,2},{f,3}},
+       7,
+       {bad_try_catch_nesting,{y,2},[{{y,1},{trytag,[5]}}]}}}] = Errors,
+    ok.
+
 %%%-------------------------------------------------------------------------
 
 transform_remove(Remove, Module) ->
diff --git a/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S b/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S
new file mode 100644
index 0000000000..9f1b21a17b
--- /dev/null
+++ b/lib/compiler/test/beam_validator_SUITE_data/bad_try_catch_nesting.S
@@ -0,0 +1,64 @@
+{module, bad_try_catch_nesting}.  %% version = 0
+
+{exports, [{main,2},{module_info,0},{module_info,1}]}.
+
+{attributes, []}.
+
+{labels, 11}.
+
+
+{function, main, 2, 2}.
+  {label,1}.
+    {line,[{location,"bad_try_catch_nesting.erl",4}]}.
+    {func_info,{atom,bad_try_catch_nesting},{atom,main},2}.
+  {label,2}.
+    {allocate_zero,3,2}.
+    {'try',{y,1},{f,5}}.
+    {move,{x,1},{y,0}}.
+    {'try',{y,2},{f,3}}.
+    {line,[{location,"bad_try_catch_nesting.erl",7}]}.
+    {call_fun,0}.
+    {try_end,{y,2}}.
+    {jump,{f,4}}.
+  {label,3}.
+    {try_case,{y,2}}.
+    {test,is_ne_exact,{f,4},[{x,0},{atom,error}]}.
+    {line,[]}.
+    {bif,raise,{f,0},[{x,2},{x,1}],{x,0}}.
+  {label,4}.
+    {move,{y,0},{x,0}}.
+    {kill,{y,0}}.
+    {line,[{location,"bad_try_catch_nesting.erl",12}]}.
+    {call_fun,0}.
+    {try_end,{y,1}}.
+    {deallocate,3}.
+    return.
+  {label,5}.
+    {try_case,{y,1}}.
+    {test,is_eq_exact,{f,6},[{x,0},{atom,throw}]}.
+    {deallocate,3}.
+    return.
+  {label,6}.
+    {line,[]}.
+    {bif,raise,{f,0},[{x,2},{x,1}],{x,0}}.
+
+
+{function, module_info, 0, 8}.
+  {label,7}.
+    {line,[]}.
+    {func_info,{atom,bad_try_catch_nesting},{atom,module_info},0}.
+  {label,8}.
+    {move,{atom,bad_try_catch_nesting},{x,0}}.
+    {line,[]}.
+    {call_ext_only,1,{extfunc,erlang,get_module_info,1}}.
+
+
+{function, module_info, 1, 10}.
+  {label,9}.
+    {line,[]}.
+    {func_info,{atom,bad_try_catch_nesting},{atom,module_info},1}.
+  {label,10}.
+    {move,{x,0},{x,1}}.
+    {move,{atom,bad_try_catch_nesting},{x,0}}.
+    {line,[]}.
+    {call_ext_only,2,{extfunc,erlang,get_module_info,2}}.