Generalize bit string comprehensions

The expression in a bit string comprehension is limited to a
literal bit string expression. That is, the following code
is legal:

  << <<X>> || X <- List >>

but not this code:

  << foo(X) || X <- List >>

The limitation is annoying. For one thing, tools that transform
the abstract format must be careful not to produce code such as:

  << begin
      %% Some instrumentation code.
      <<X>>
     end || X <- List >>

One reason for the limitation could be that we'll get
reduce/reduce conflicts if we try to allow an arbitrary
expression in a bit string comprehension:

  binary_comprehension -> '<<' expr '||' lc_exprs '>>' :
	{bc,?anno('$1'),'$2','$4'}.

Unfortunately, there does not seem to be an easy way to work
around that problem. The best we can do is to allow 'expr_max'
expressions (as in the binary syntax):

  binary_comprehension -> '<<' expr_max '||' lc_exprs '>>' :
	{bc,?anno('$1'),'$2','$4'}.

That will work, but functions calls must be enclosed in
parentheses:

  << (foo(X)) || X <- List >>

6 files changed, 106 insertions, 23 deletions

diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl
index 68c9f964d8..830dd9973a 100644
--- a/lib/compiler/src/v3_core.erl
+++ b/lib/compiler/src/v3_core.erl
@@ -1080,13 +1080,39 @@ bc_tq1(Line, E, [#igen{anno=GAnno,ceps=Ceps,
 bc_tq1(Line, E, [#ifilter{}=Filter|Qs], Mc, St) ->
     filter_tq(Line, E, Filter, Mc, St, Qs, fun bc_tq1/5);
 bc_tq1(_, {bin,Bl,Elements}, [], AccVar, St0) ->
-    {E,Pre,St} = expr({bin,Bl,[{bin_element,Bl,
-				{var,Bl,AccVar#c_var.name},
-				{atom,Bl,all},
-				[binary,{unit,1}]}|Elements]}, St0),
+    bc_tq_build(Bl, [], AccVar, Elements, St0);
+bc_tq1(Line, E0, [], AccVar, St0) ->
+    BsFlags = [binary,{unit,1}],
+    BsSize = {atom,Line,all},
+    {E1,Pre0,St1} = safe(E0, St0),
+    case E1 of
+	#c_var{name=VarName} ->
+	    Var = {var,Line,VarName},
+	    Els = [{bin_element,Line,Var,BsSize,BsFlags}],
+	    bc_tq_build(Line, Pre0, AccVar, Els, St1);
+	#c_literal{val=Val} when is_bitstring(Val) ->
+	    Bits = bit_size(Val),
+	    <<Int0:Bits>> = Val,
+	    Int = {integer,Line,Int0},
+	    Sz = {integer,Line,Bits},
+	    Els = [{bin_element,Line,Int,Sz,[integer,{unit,1},big]}],
+	    bc_tq_build(Line, Pre0, AccVar, Els, St1);
+	_ ->
+	    %% Any other safe (cons, tuple, literal) is not a
+	    %% bitstring. Force the evaluation to fail (and
+	    %% generate a warning).
+	    Els = [{bin_element,Line,{atom,Line,bad_value},BsSize,BsFlags}],
+	    bc_tq_build(Line, Pre0, AccVar, Els, St1)
+    end.
+
+bc_tq_build(Line, Pre0, #c_var{name=AccVar}, Elements0, St0) ->
+    Elements = [{bin_element,Line,{var,Line,AccVar},{atom,Line,all},
+		 [binary,{unit,1}]}|Elements0],
+    {E,Pre,St} = expr({bin,Line,Elements}, St0),
     #a{anno=A} = Anno0 = get_anno(E),
     Anno = Anno0#a{anno=[compiler_generated,single_use|A]},
-    {set_anno(E, Anno),Pre,St}.
+    {set_anno(E, Anno),Pre0++Pre,St}.
+
 
 %% filter_tq(Line, Expr, Filter, Mc, State, [Qualifier], TqFun) ->
 %%     {Case,[PreExpr],State}.
@@ -1307,7 +1333,9 @@ bc_elem_size({bin,_,El}, St0) ->
 	    Vs = [V || {_,#c_var{name=V}} <- Vars0],
 	    {E,Pre,St} = bc_mul_pairs(F, #c_literal{val=Bits}, [], St0),
 	    {E,Pre,Vs,St}
-    end.
+    end;
+bc_elem_size(_, _) ->
+    throw(impossible).
 
 bc_elem_size_1([{bin_element,_,_,{integer,_,N},Flags}|Es], Bits, Vars) ->
     {unit,U} = keyfind(unit, 1, Flags),
diff --git a/lib/compiler/test/bs_bincomp_SUITE.erl b/lib/compiler/test/bs_bincomp_SUITE.erl
index 93bce5c1f6..5e5f6e2169 100644
--- a/lib/compiler/test/bs_bincomp_SUITE.erl
+++ b/lib/compiler/test/bs_bincomp_SUITE.erl
@@ -26,7 +26,7 @@
 	 init_per_group/2,end_per_group/2,
 	 byte_aligned/1,bit_aligned/1,extended_byte_aligned/1,
 	 extended_bit_aligned/1,mixed/1,filters/1,trim_coverage/1,
-	 nomatch/1,sizes/1]).
+	 nomatch/1,sizes/1,general_expressions/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
@@ -36,7 +36,7 @@ all() ->
     test_lib:recompile(?MODULE),
     [byte_aligned, bit_aligned, extended_byte_aligned,
      extended_bit_aligned, mixed, filters, trim_coverage,
-     nomatch, sizes].
+     nomatch, sizes, general_expressions].
 
 groups() -> 
     [].
@@ -295,6 +295,48 @@ sizes(Config) when is_list(Config) ->
     cs_end(),
     ok.
 
+-define(BAD(E),   {'EXIT',{badarg,_}} = (catch << (E) || _ <- [1,2,3] >>)).
+-define(BAD_V(E), {'EXIT',{badarg,_}} = (catch << (E) || I <- [1,2,3] >>)).
+
+general_expressions(_) ->
+    <<1,2,3>> = << begin <<1,2,3>> end || _ <- [1] >>,
+    <<"abc">> = << begin <<"abc">> end || _ <- [1] >>,
+    <<1,2,3>> = << begin
+		       I = <<(I0+1)>>,
+		       id(I)
+		   end || <<I0>> <= <<0,1,2>> >>,
+    <<1,2,3>> = << I || I <- [<<1,2>>,<<3>>] >>,
+    <<1,2,3>> = << (id(<<I>>)) || I <- [1,2,3] >>,
+    <<2,4>> = << case I rem 2 of
+		     0 -> <<I>>;
+		     1 -> <<>>
+		 end || I <- [1,2,3,4,5] >>,
+    <<2,3,4,5,6,7>> = << << (id(<<J>>)) || J <- [2*I,2*I+1] >> ||
+			  I <- [1,2,3] >>,
+    <<1,2,2,3,4,4>> = << if
+			     I rem 2 =:= 0 -> <<I,I>>;
+			     true -> <<I>>
+			 end || I <- [1,2,3,4] >>,
+    self() ! <<42>>,
+    <<42>> = << receive B -> B end || _ <- [1] >>,
+    <<10,5,3>> = << try
+			<<(10 div I)>>
+		    catch _:_ ->
+			    <<>>
+		    end || I <- [0,1,2,3] >>,
+
+    %% Failing expressions.
+    ?BAD(bad_atom),
+    ?BAD(42),
+    ?BAD(42.0),
+    ?BAD_V({ok,I}),
+    ?BAD_V([I]),
+    ?BAD_V(fun() -> I end),
+
+    ok.
+
+-undef(BAD).
+
 cs_init() ->
     erts_debug:set_internal_state(available_internal_state, true),
     ok.
diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl
index b1c574ea60..6f8e5e8449 100644
--- a/lib/stdlib/src/erl_parse.yrl
+++ b/lib/stdlib/src/erl_parse.yrl
@@ -311,7 +311,7 @@ bit_size_expr -> expr_max : '$1'.
 
 list_comprehension -> '[' expr '||' lc_exprs ']' :
 	{lc,?anno('$1'),'$2','$4'}.
-binary_comprehension -> '<<' binary '||' lc_exprs '>>' :
+binary_comprehension -> '<<' expr_max '||' lc_exprs '>>' :
 	{bc,?anno('$1'),'$2','$4'}.
 lc_exprs -> lc_expr : ['$1'].
 lc_exprs -> lc_expr ',' lc_exprs : ['$1'|'$3'].
diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl
index 3732b0fc85..87de31919f 100644
--- a/lib/tools/src/cover.erl
+++ b/lib/tools/src/cover.erl
@@ -2002,9 +2002,7 @@ munge_expr({lc,Line,Expr,Qs}, Vars) ->
     {MungedQs, Vars3} = munge_qualifiers(Qs, Vars2),
     {{lc,Line,MungedExpr,MungedQs}, Vars3};
 munge_expr({bc,Line,Expr,Qs}, Vars) ->
-    {bin,BLine,[{bin_element,EL,Val,Sz,TSL}|Es]} = Expr,
-    Expr2 = {bin,BLine,[{bin_element,EL,Val,Sz,TSL}|Es]},
-    {MungedExpr,Vars2} = munge_expr(Expr2, Vars),
+    {MungedExpr,Vars2} = munge_expr(?BLOCK1(Expr), Vars),
     {MungedQs, Vars3} = munge_qualifiers(Qs, Vars2),
     {{bc,Line,MungedExpr,MungedQs}, Vars3};
 munge_expr({block,Line,Body}, Vars) ->
diff --git a/lib/tools/test/cover_SUITE.erl b/lib/tools/test/cover_SUITE.erl
index b9a9cd5be4..d4b346c407 100644
--- a/lib/tools/test/cover_SUITE.erl
+++ b/lib/tools/test/cover_SUITE.erl
@@ -29,7 +29,7 @@ all() ->
     NoStartStop = [eif,otp_5305,otp_5418,otp_7095,otp_8273,
                    otp_8340,otp_8188,compile_beam_opts,eep37,
 		   analyse_no_beam, line_0, compile_beam_no_file,
-                   otp_13277],
+                   otp_13277, otp_13289],
     StartStop = [start, compile, analyse, misc, stop,
 		 distribution, reconnect, die_and_reconnect,
 		 dont_reconnect_after_stop, stop_node_after_disconnect,
@@ -1253,7 +1253,7 @@ otp_8340(doc) ->
     ["OTP-8340. Bug."];
 otp_8340(suite) -> [];
 otp_8340(Config) when is_list(Config) ->
-    ?line [{{t,1},1},{{t,4},1}] =
+    [{{t,1},1},{{t,2},1},{{t,4},1}] =
         analyse_expr(<<"<< \n"
                        " <<3:2, \n"
                        "   SeqId:62>> \n"
@@ -1547,10 +1547,8 @@ comprehension_8188(Cf) ->
                        "    true]. \n" % 2
                        "  two() -> 2">>, Cf), % 1
 
-    %% The template cannot have a counter since it is not allowed to
-    %% be a block.
     ?line [{{t,1},1},
-           %% {{t,2},2},
+           {{t,2},2},
            {{t,3},1},
            {{t,4},1},
            {{t,5},0},
@@ -1560,7 +1558,7 @@ comprehension_8188(Cf) ->
            {{t,13},2},
            {{t,14},2}] = 
         analyse_expr(<<"<< \n" % 1
-                       " << (X*2) >> || \n" % 2 (now: 0)
+                       " << (X*2) >> || \n" % 2
                        "    <<X>> <= << (case two() of\n"
                        "                     2 -> 1;\n" % 1
                        "                     _ -> 2\n" % 0
@@ -1575,7 +1573,7 @@ comprehension_8188(Cf) ->
                        "two() -> 2">>, Cf),
 
     ?line [{{t,1},1},
-           %% {{t,2},4},
+           {{t,2},4},
            {{t,4},1},
            {{t,6},1},
            {{t,7},0},
@@ -1584,7 +1582,7 @@ comprehension_8188(Cf) ->
            {{t,12},4},
            {{t,13},1}] =
         analyse_expr(<<"<< \n" % 1
-                       " << (2)\n" % 4 (now: 0)
+                       " << (2)\n" % 4
                        "     :(8) >> || \n"
                        "    <<X>> <= << 1,\n" % 1
                        "                (case two() of \n"
@@ -1766,6 +1764,21 @@ otp_13277(Config) ->
     ?line ok = file:delete(File),
     ok.
 
+%% Test general expressions in a binary comprehension.
+otp_13289(Config) ->
+    Test = <<"-module(t).
+              -export([t/0]).
+
+              t() ->
+                  << (id(<<I>>)) || I <- [1,2,3] >>.
+
+              id(I) -> I.
+             ">>,
+    File = cc_mod(t, Test, Config),
+    <<1,2,3>> = t:t(),
+    ok = file:delete(File),
+    ok.
+
 %%--Auxiliary------------------------------------------------------------
 
 analyse_expr(Expr, Config) ->
diff --git a/system/doc/reference_manual/expressions.xml b/system/doc/reference_manual/expressions.xml
index e98fcbcbb9..75486488e9 100644
--- a/system/doc/reference_manual/expressions.xml
+++ b/system/doc/reference_manual/expressions.xml
@@ -1556,9 +1556,11 @@ end</pre>
     <p>Bit string comprehensions are written with
     the following syntax:</p>
     <pre>
-&lt;&lt; BitString || Qualifier1,...,QualifierN &gt;&gt;</pre>
-    <p>Here, <c>BitString</c> is a bit string expression and each
-      <c>Qualifier</c> is either a generator, a bit string generator or a filter.</p>
+&lt;&lt; BitStringExpr || Qualifier1,...,QualifierN &gt;&gt;</pre>
+    <p><c>BitStringExpr</c> is an expression that evalutes to a bit
+    string. If <c>BitStringExpr</c> is a function call, it must be
+    enclosed in parentheses.  Each <c>Qualifier</c> is either a
+    generator, a bit string generator or a filter.</p>
     <list type="bulleted">
  <item>A <em>generator</em> is written as:      <br></br>
       &nbsp;&nbsp;<c><![CDATA[Pattern <- ListExpr]]></c>.      <br></br>