Merge branch 'bjorn/compiler-bin-generators/OTP-8864' into dev

* bjorn/compiler-bin-generators/OTP-8864:
  core_lint: Enforce that tail segments only occur at the end
  Don't generate multiple tail segments in binary matching
  Factor out some of the code for binary generators
  Add tests for tail segments in binary generators

3 files changed, 102 insertions, 36 deletions

diff --git a/lib/compiler/src/core_lint.erl b/lib/compiler/src/core_lint.erl
index b633f568c9..097577b371 100644
--- a/lib/compiler/src/core_lint.erl
+++ b/lib/compiler/src/core_lint.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -116,7 +116,9 @@ format_error({duplicate_var,N,{F,A}}) ->
 format_error({unbound_var,N,{F,A}}) ->
     io_lib:format("unbound variable ~s in ~w/~w", [N,F,A]);
 format_error({undefined_function,{F1,A1},{F2,A2}}) ->
-    io_lib:format("undefined function ~w/~w in ~w/~w", [F1,A1,F2,A2]).
+    io_lib:format("undefined function ~w/~w in ~w/~w", [F1,A1,F2,A2]);
+format_error({tail_segment_not_at_end,{F,A}}) ->
+    io_lib:format("binary tail segment not at end in ~w/~w", [F,A]).
 
 -type ret() :: {'ok', [{module(), [warning(),...]}]}
              | {'error', [{module(), [error(),...]}],
@@ -450,7 +452,8 @@ pattern(#c_cons{hd=H,tl=T}, Def, Ps, St) ->
     pattern_list([H,T], Def, Ps, St);
 pattern(#c_tuple{es=Es}, Def, Ps, St) ->
     pattern_list(Es, Def, Ps, St);
-pattern(#c_binary{segments=Ss}, Def, Ps, St) ->
+pattern(#c_binary{segments=Ss}, Def, Ps, St0) ->
+    St = pat_bin_tail_check(Ss, St0),
     pat_bin(Ss, Def, Ps, St);
 pattern(#c_alias{var=V,pat=P}, Def, Ps, St0) ->
     {Vvs,St1} = variable(V, Ps, St0),
@@ -482,6 +485,19 @@ pat_segment(#c_bitstr{val=V,size=S,type=T}, Def0, Ps0, St0) ->
 pat_segment(_, Def, Ps, St) ->
     {Ps,Def,add_error({not_bs_pattern,St#lint.func}, St)}.
 
+%% pat_bin_tail_check([Elem], State) -> State.
+%%  There must be at most one tail segment (a size-less segment of
+%%  type binary) and it must occur at the end.
+
+pat_bin_tail_check([#c_bitstr{size=#c_literal{val=all}}], St) ->
+    %% Size-less field is OK at the end of the list of segments.
+    St;
+pat_bin_tail_check([#c_bitstr{size=#c_literal{val=all}}|_], St) ->
+    add_error({tail_segment_not_at_end,St#lint.func}, St);
+pat_bin_tail_check([_|Ss], St) ->
+    pat_bin_tail_check(Ss, St);
+pat_bin_tail_check([], St) -> St.
+
 %% pat_bit_expr(SizePat, Type, Defined, State) -> State.
 %%  Check the Size pattern, this is an input!  Because of optimizations,
 %%  we must allow any kind of constant and literal here.
diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl
index f6bb45787c..2da24b2908 100644
--- a/lib/compiler/src/v3_core.erl
+++ b/lib/compiler/src/v3_core.erl
@@ -892,25 +892,22 @@ lc_tq(Line, E, [{generate,Lg,P,G}|Qs0], Mc, St0) ->
 lc_tq(Line, E, [{b_generate,Lg,P,G}|Qs0], Mc, St0) ->
     {Gs,Qs1} =  splitwith(fun is_guard_test/1, Qs0),
     {Name,St1} = new_fun_name("blc", St0),
-    {Tname,St2} = new_var_name(St1),
-    LA = lineno_anno(Line, St2),
+    LA = lineno_anno(Line, St1),
     LAnno = #a{anno=LA},
-    HeadBinPattern = pattern(P,St2),
-    #c_binary{segments=Ps} = HeadBinPattern,
-    {EPs,St3} = emasculate_segments(Ps,St2),
-    Tail = #c_var{anno=LA,name=Tname},
-    TailSegment = #c_bitstr{val=Tail,size=#c_literal{val=all},
-			    unit=#c_literal{val=1},
-			    type=#c_literal{val=binary},
-			    flags=#c_literal{val=[big,unsigned]}},
-    Pattern = HeadBinPattern#c_binary{segments=Ps ++ [TailSegment]},
-    EPattern = HeadBinPattern#c_binary{segments=EPs ++ [TailSegment]},
+    HeadBinPattern = pattern(P, St1),
+    #c_binary{segments=Ps0} = HeadBinPattern,
+    {Ps,Tail,St2} = append_tail_segment(Ps0, St1),
+    {EPs,St3} = emasculate_segments(Ps, St2),
+    Pattern = HeadBinPattern#c_binary{segments=Ps},
+    EPattern = HeadBinPattern#c_binary{segments=EPs},
     {Arg,St4} = new_var(St3),
     {Guardc,St5} = lc_guard_tests(Gs, St4),	%These are always flat!
+    Tname = Tail#c_var.name,
     {Nc,[],St6} = expr({call,Lg,{atom,Lg,Name},[{var,Lg,Tname}]}, St5),
     {Bc,Bps,St7} = lc_tq(Line, E, Qs1, Nc, St6),
     {Gc,Gps,St10} = safe(G, St7),		%Will be a function argument!
     Fc = function_clause([Arg], LA, {Name,1}),
+    {TailSegList,_,St} = append_tail_segment([], St10),
     Cs = [#iclause{anno=#a{anno=[compiler_generated|LA]},
 		   pats=[Pattern],
 		   guard=Guardc,
@@ -922,14 +919,14 @@ lc_tq(Line, E, [{b_generate,Lg,P,G}|Qs0], Mc, St0) ->
 				 op=#c_var{anno=LA,name={Name,1}},
 				 args=[Tail]}]},
 	  #iclause{anno=LAnno,
-		   pats=[#c_binary{anno=LA, segments=[TailSegment]}],guard=[],
+		   pats=[#c_binary{anno=LA,segments=TailSegList}],guard=[],
 		   body=[Mc]}],
     Fun = #ifun{anno=LAnno,id=[],vars=[Arg],clauses=Cs,fc=Fc},
     {#iletrec{anno=LAnno,defs=[{{Name,1},Fun}],
 	      body=Gps ++ [#iapply{anno=LAnno,
 				   op=#c_var{anno=LA,name={Name,1}},
 				   args=[Gc]}]},
-     [],St10};
+     [],St};
 lc_tq(Line, E, [Fil0|Qs0], Mc, St0) ->
     %% Special case sequences guard tests.
     LA = lineno_anno(Line, St0),
@@ -1037,26 +1034,24 @@ bc_tq1(Line, E, [{b_generate,Lg,P,G}|Qs0], AccExpr, St0) ->
     {Gs,Qs1} =  splitwith(fun is_guard_test/1, Qs0),
     {Name,St1} = new_fun_name("lbc", St0),
     LA = lineno_anno(Line, St1),
-    {[Tail,AccVar],St2} = new_vars(LA, 2, St1),
+    {AccVar,St2} = new_var(LA, St1),
     LAnno = #a{anno=LA},
     HeadBinPattern = pattern(P, St2),
-    #c_binary{segments=Ps} = HeadBinPattern,
-    {EPs,St3} = emasculate_segments(Ps, St2),
-    TailSegment = #c_bitstr{val=Tail,size=#c_literal{val=all},
-			    unit=#c_literal{val=1},
-			    type=#c_literal{val=binary},
-			    flags=#c_literal{val=[big,unsigned]}},
-    Pattern = HeadBinPattern#c_binary{segments=Ps ++ [TailSegment]},
-    EPattern = HeadBinPattern#c_binary{segments=EPs ++ [TailSegment]},
-    {Arg,St4} = new_var(St3),
+    #c_binary{segments=Ps0} = HeadBinPattern,
+    {Ps,Tail,St3} = append_tail_segment(Ps0, St2),
+    {EPs,St4} = emasculate_segments(Ps, St3),
+    Pattern = HeadBinPattern#c_binary{segments=Ps},
+    EPattern = HeadBinPattern#c_binary{segments=EPs},
+    {Arg,St5} = new_var(St4),
     NewMore = {call,Lg,{atom,Lg,Name},[{var,Lg,Tail#c_var.name},
 				       {var,Lg,AccVar#c_var.name}]},
-    {Guardc,St5} = lc_guard_tests(Gs, St4),	%These are always flat!
-    {Bc,Bps,St6} = bc_tq1(Line, E, Qs1, AccVar, St5),
-    {Nc,Nps,St7} = expr(NewMore, St6),
-    {Gc,Gps,St8} = safe(G, St7),		%Will be a function argument!
+    {Guardc,St6} = lc_guard_tests(Gs, St5),	%These are always flat!
+    {Bc,Bps,St7} = bc_tq1(Line, E, Qs1, AccVar, St6),
+    {Nc,Nps,St8} = expr(NewMore, St7),
+    {Gc,Gps,St9} = safe(G, St8),	 %Will be a function argument!
     Fc = function_clause([Arg,AccVar], LA, {Name,2}),
     Body = Bps ++ Nps ++ [#iset{var=AccVar,arg=Bc},Nc],
+    {TailSegList,_,St} = append_tail_segment([], St9),
     Cs = [#iclause{anno=LAnno,
 		   pats=[Pattern,AccVar],
 		   guard=Guardc,
@@ -1066,7 +1061,7 @@ bc_tq1(Line, E, [{b_generate,Lg,P,G}|Qs0], AccExpr, St0) ->
 		   guard=[],
 		   body=Nps ++ [Nc]},
 	  #iclause{anno=LAnno,
-		   pats=[#c_binary{anno=LA,segments=[TailSegment]},AccVar],
+		   pats=[#c_binary{anno=LA,segments=TailSegList},AccVar],
 		   guard=[],
 		   body=[AccVar]}],
     Fun = #ifun{anno=LAnno,id=[],vars=[Arg,AccVar],clauses=Cs,fc=Fc},
@@ -1074,7 +1069,7 @@ bc_tq1(Line, E, [{b_generate,Lg,P,G}|Qs0], AccExpr, St0) ->
 	      body=Gps ++ [#iapply{anno=LAnno,
 				   op=#c_var{anno=LA,name={Name,2}},
 				   args=[Gc,AccExpr]}]},
-     [],St8};
+     [],St};
 bc_tq1(Line, E, [Fil0|Qs0], AccVar, St0) ->
     %% Special case sequences guard tests.
     LA = lineno_anno(Line, St0),
@@ -1120,6 +1115,29 @@ bc_tq1(_, {bin,Bl,Elements}, [], AccVar, St0) ->
     %%Anno = Anno0#a{anno=[compiler_generated|A]},
     {set_anno(E, Anno),Pre,St}.
 
+append_tail_segment(Segs, St) ->
+    app_tail_seg(Segs, St, []).
+
+app_tail_seg([#c_bitstr{val=Var0,size=#c_literal{val=all}}=Seg0]=L,
+	     St0, Acc) ->
+    case Var0 of
+	#c_var{name='_'} ->
+	    {Var,St} = new_var(St0),
+	    Seg = Seg0#c_bitstr{val=Var},
+	    {reverse(Acc, [Seg]),Var,St};
+	#c_var{} ->
+	    {reverse(Acc, L),Var0,St0}
+    end;
+app_tail_seg([H|T], St, Acc) ->
+    app_tail_seg(T, St, [H|Acc]);
+app_tail_seg([], St0, Acc) ->
+    {Var,St} = new_var(St0),
+    Tail = #c_bitstr{val=Var,size=#c_literal{val=all},
+		     unit=#c_literal{val=1},
+		     type=#c_literal{val=binary},
+		     flags=#c_literal{val=[unsigned,big]}},
+    {reverse(Acc, [Tail]),Var,St}.
+
 emasculate_segments(Segs, St) ->
     emasculate_segments(Segs, St, []).
 
diff --git a/lib/compiler/test/bs_bincomp_SUITE.erl b/lib/compiler/test/bs_bincomp_SUITE.erl
index a64a5d590b..74f69893af 100644
--- a/lib/compiler/test/bs_bincomp_SUITE.erl
+++ b/lib/compiler/test/bs_bincomp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -24,7 +24,7 @@
 -export([all/1,
 	 byte_aligned/1,bit_aligned/1,extended_byte_aligned/1,
 	 extended_bit_aligned/1,mixed/1,filters/1,trim_coverage/1,
-	 nomatch/1,sizes/1]).
+	 nomatch/1,sizes/1,tail/1]).
 
 -include("test_server.hrl").
 
@@ -32,7 +32,7 @@ all(suite) ->
     test_lib:recompile(?MODULE),
     [byte_aligned,bit_aligned,extended_byte_aligned,
      extended_bit_aligned,mixed,filters,trim_coverage,
-     nomatch,sizes].
+     nomatch,sizes,tail].
 
 
 byte_aligned(Config) when is_list(Config) ->
@@ -270,6 +270,38 @@ sizes(Config) when is_list(Config) ->
     ?line cs_end(),
     ok.
 
+tail(Config) when is_list(Config) ->
+    ?line [] = tail_1(<<0:7>>),
+    ?line [0] = tail_1(<<0>>),
+    ?line [0] = tail_1(<<0:12>>),
+    ?line [0,0] = tail_1(<<0:20>>),
+
+    ?line [] = tail_2(<<0:7>>),
+    ?line [42] = tail_2(<<0>>),
+    ?line [] = tail_2(<<0:12>>),
+    ?line [42,42] = tail_2(<<0,1>>),
+
+    ?line <<>> = tail_3(<<0:7>>),
+    ?line <<42>> = tail_3(<<0>>),
+    ?line <<42>> = tail_3(<<0:12>>),
+    ?line <<42,42>> = tail_3(<<0:20>>),
+
+    ?line [] = tail_4(<<0:15>>),
+    ?line [7] = tail_4(<<7,8>>),
+    ?line [9] = tail_4(<<9,17:12>>),
+    ok.
+
+tail_1(Bits) ->
+    [X || <<X:8/integer, _/bits>> <= Bits].
+
+tail_2(Bits) ->
+    [42 || <<_:8/integer, _/bytes>> <= Bits].
+
+tail_3(Bits) ->
+    << <<42>> || <<_:8/integer, _/bits>> <= Bits >>.
+
+tail_4(Bits) ->
+    [X || <<X:8/integer, Tail/bits>> <= Bits, bit_size(Tail) >= 8].
 
 
 cs_init() ->