beam_validator: Don't allow x(1023) to be used

In 45f469ca0890, the BEAM loader started to use x(1023) as scratch
register for some instructions. Therefore we should not allow
x(1023) to be used in code emitted by the compiler.

6 files changed, 24 insertions, 22 deletions

diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl
index 0b284299be..4e179daa0e 100644
--- a/lib/compiler/src/beam_block.erl
+++ b/lib/compiler/src/beam_block.erl
@@ -24,7 +24,6 @@
 
 -export([module/2]).
 -import(lists, [mapfoldl/3,reverse/1,reverse/2,foldl/3,member/2]).
--define(MAXREG, 1024).
 
 module({Mod,Exp,Attr,Fs0,Lc0}, _Opt) ->
     {Fs,Lc} = mapfoldl(fun function/2, Lc0, Fs0),
diff --git a/lib/compiler/src/beam_bool.erl b/lib/compiler/src/beam_bool.erl
index 14b6381230..c9e103eae9 100644
--- a/lib/compiler/src/beam_bool.erl
+++ b/lib/compiler/src/beam_bool.erl
@@ -25,8 +25,6 @@
 
 -import(lists, [reverse/1,reverse/2,foldl/3,mapfoldl/3,map/2]).
 
--define(MAXREG, 1024).
-
 -record(st,
 	{next,					%Next label number.
 	 ll					%Live regs at labels.
diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl
index 919ee3ee7d..7d66985791 100644
--- a/lib/compiler/src/beam_clean.erl
+++ b/lib/compiler/src/beam_clean.erl
@@ -141,7 +141,7 @@ renumber_labels([{bif,is_record,{f,_},
     renumber_labels(Is, Acc, St);
 renumber_labels([{test,is_record,{f,_}=Fail,
 		  [Term,{atom,Tag}=TagAtom,{integer,Arity}]}|Is0], Acc, St) ->
-    Tmp = {x,1023},
+    Tmp = {x,1022},
     Is = case is_record_tuple(Term, Tag, Arity) of
 	     yes ->
 		 Is0;
diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl
index 942d69a756..ab5fedf3bf 100644
--- a/lib/compiler/src/beam_validator.erl
+++ b/lib/compiler/src/beam_validator.erl
@@ -31,8 +31,6 @@
 
 -import(lists, [reverse/1,foldl/3,foreach/2,dropwhile/2]).
 
--define(MAXREG, 1024).
-
 %%-define(DEBUG, 1).
 -ifdef(DEBUG).
 -define(DBG_FORMAT(F, D), (io:format((F), (D)))).
@@ -970,9 +968,9 @@ get_fls(#vst{current=#st{fls=Fls}}) when is_atom(Fls) -> Fls.
 
 init_fregs() -> 0.
 
-set_freg({fr,Fr}, #vst{current=#st{f=Fregs0}=St}=Vst)
+set_freg({fr,Fr}=Freg, #vst{current=#st{f=Fregs0}=St}=Vst)
   when is_integer(Fr), 0 =< Fr ->
-    limit_check(Fr),
+    check_limit(Freg),
     Bit = 1 bsl Fr,
     if
 	Fregs0 band Bit =:= 0 ->
@@ -985,9 +983,10 @@ set_freg(Fr, _) -> error({bad_target,Fr}).
 assert_freg_set({fr,Fr}=Freg, #vst{current=#st{f=Fregs}})
   when is_integer(Fr), 0 =< Fr ->
     if
-	Fregs band (1 bsl Fr) =/= 0 ->
-	    limit_check(Fr);
-	true -> error({uninitialized_reg,Freg})
+	(Fregs bsr Fr) band 1 =:= 0 ->
+	    error({uninitialized_reg,Freg});
+	true ->
+	    ok
     end;
 assert_freg_set(Fr, _) -> error({bad_source,Fr}).
 
@@ -1066,16 +1065,16 @@ set_type(Type, {x,_}=Reg, Vst) -> set_type_reg(Type, Reg, Vst);
 set_type(Type, {y,_}=Reg, Vst) -> set_type_y(Type, Reg, Vst);
 set_type(_, _, #vst{}=Vst) -> Vst.
 
-set_type_reg(Type, {x,X}, #vst{current=#st{x=Xs}=St}=Vst) 
+set_type_reg(Type, {x,X}=Reg, #vst{current=#st{x=Xs}=St}=Vst)
   when is_integer(X), 0 =< X ->
-    limit_check(X),
+    check_limit(Reg),
     Vst#vst{current=St#st{x=gb_trees:enter(X, Type, Xs)}};
 set_type_reg(Type, Reg, Vst) ->
     set_type_y(Type, Reg, Vst).
 
 set_type_y(Type, {y,Y}=Reg, #vst{current=#st{y=Ys0}=St}=Vst)
   when is_integer(Y), 0 =< Y ->
-    limit_check(Y),
+    check_limit(Reg),
     Ys = case gb_trees:lookup(Y, Ys0) of
 	     none ->
 		 error({invalid_store,Reg,Type});
@@ -1591,9 +1590,15 @@ return_type_math(pow, 2) -> {float,[]};
 return_type_math(pi, 0) -> {float,[]};
 return_type_math(F, A) when is_atom(F), is_integer(A), A >= 0 -> term.
 
-limit_check(Num) when is_integer(Num), Num >= ?MAXREG ->
-    error(limit);
-limit_check(_) -> ok.
+check_limit({x,X}) when is_integer(X), X < 1023 ->
+    %% Note: x(1023) is reserved for use by the BEAM loader.
+    ok;
+check_limit({y,Y}) when is_integer(Y), Y < 1024 ->
+    ok;
+check_limit({fr,Fr}) when is_integer(Fr), Fr < 1024 ->
+    ok;
+check_limit(_) ->
+    error(limit).
 
 min(A, B) when is_integer(A), is_integer(B), A < B -> A;
 min(A, B) when is_integer(A), is_integer(B) -> B.
diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl
index 69391b15eb..d6deb4a730 100644
--- a/lib/compiler/test/beam_validator_SUITE.erl
+++ b/lib/compiler/test/beam_validator_SUITE.erl
@@ -107,13 +107,13 @@ xrange(Config) when is_list(Config) ->
 	  {{bif,'+',{f,0},[{x,-1},{x,1}],{x,0}},4,
 	   {uninitialized_reg,{x,-1}}}},
 	 {{t,sum_2,2},
-	  {{bif,'+',{f,0},[{x,0},{x,1024}],{x,0}},4,
-	   {uninitialized_reg,{x,1024}}}},
+	  {{bif,'+',{f,0},[{x,0},{x,1023}],{x,0}},4,
+	   {uninitialized_reg,{x,1023}}}},
 	 {{t,sum_3,2},
 	  {{bif,'+',{f,0},[{x,0},{x,1}],{x,-1}},4,
 	   {invalid_store,{x,-1},number}}},
 	 {{t,sum_4,2},
-	  {{bif,'+',{f,0},[{x,0},{x,1}],{x,1024}},4,limit}}] = Errors,
+	  {{bif,'+',{f,0},[{x,0},{x,1}],{x,1023}},4,limit}}] = Errors,
     ok.
 
 yrange(Config) when is_list(Config) ->
diff --git a/lib/compiler/test/beam_validator_SUITE_data/xrange.S b/lib/compiler/test/beam_validator_SUITE_data/xrange.S
index c6f20288f7..a76408dde3 100644
--- a/lib/compiler/test/beam_validator_SUITE_data/xrange.S
+++ b/lib/compiler/test/beam_validator_SUITE_data/xrange.S
@@ -20,7 +20,7 @@
   {label,3}.
     {func_info,{atom,t},{atom,sum_2},2}.
   {label,4}.
-    {bif,'+',{f,0},[{x,0},{x,1024}],{x,0}}.
+    {bif,'+',{f,0},[{x,0},{x,1023}],{x,0}}.
     {'%live',1}.
     return.
 
@@ -38,7 +38,7 @@
   {label,7}.
     {func_info,{atom,t},{atom,sum_4},2}.
   {label,8}.
-    {bif,'+',{f,0},[{x,0},{x,1}],{x,1024}}.
+    {bif,'+',{f,0},[{x,0},{x,1}],{x,1023}}.
     {'%live',1}.
     return.