Merge branch 'nox/illegal-bitstring-gen-pattern/OTP-11186'

* nox/illegal-bitstring-gen-pattern/OTP-11186:
  Bootstrap added
  Simplify v3_core's translation of bit string generators
  Forbid unsized fields in patterns of binary generators

6 files changed, 67 insertions, 77 deletions

diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl
index d6fdcb2b21..8159b42f2f 100644
--- a/lib/compiler/src/v3_core.erl
+++ b/lib/compiler/src/v3_core.erl
@@ -1133,28 +1133,13 @@ bc_tq1(_, {bin,Bl,Elements}, [], AccVar, St0) ->
     %%Anno = Anno0#a{anno=[compiler_generated|A]},
     {set_anno(E, Anno),Pre,St}.
 
-append_tail_segment(Segs, St) ->
-    app_tail_seg(Segs, St, []).
-
-app_tail_seg([#c_bitstr{val=Var0,size=#c_literal{val=all}}=Seg0]=L,
-	     St0, Acc) ->
-    case Var0 of
-	#c_var{name='_'} ->
-	    {Var,St} = new_var(St0),
-	    Seg = Seg0#c_bitstr{val=Var},
-	    {reverse(Acc, [Seg]),Var,St};
-	#c_var{} ->
-	    {reverse(Acc, L),Var0,St0}
-    end;
-app_tail_seg([H|T], St, Acc) ->
-    app_tail_seg(T, St, [H|Acc]);
-app_tail_seg([], St0, Acc) ->
+append_tail_segment(Segs, St0) ->
     {Var,St} = new_var(St0),
     Tail = #c_bitstr{val=Var,size=#c_literal{val=all},
 		     unit=#c_literal{val=1},
 		     type=#c_literal{val=binary},
 		     flags=#c_literal{val=[unsigned,big]}},
-    {reverse(Acc, [Tail]),Var,St}.
+    {Segs++[Tail],Var,St}.
 
 emasculate_segments(Segs, St) ->
     emasculate_segments(Segs, St, []).
diff --git a/lib/compiler/test/bs_bincomp_SUITE.erl b/lib/compiler/test/bs_bincomp_SUITE.erl
index f6d8b1c532..4450405695 100644
--- a/lib/compiler/test/bs_bincomp_SUITE.erl
+++ b/lib/compiler/test/bs_bincomp_SUITE.erl
@@ -25,7 +25,7 @@
 	 init_per_group/2,end_per_group/2,
 	 byte_aligned/1,bit_aligned/1,extended_byte_aligned/1,
 	 extended_bit_aligned/1,mixed/1,filters/1,trim_coverage/1,
-	 nomatch/1,sizes/1,tail/1]).
+	 nomatch/1,sizes/1]).
 
 -include_lib("test_server/include/test_server.hrl").
 
@@ -35,7 +35,7 @@ all() ->
     test_lib:recompile(?MODULE),
     [byte_aligned, bit_aligned, extended_byte_aligned,
      extended_bit_aligned, mixed, filters, trim_coverage,
-     nomatch, sizes, tail].
+     nomatch, sizes].
 
 groups() -> 
     [].
@@ -290,40 +290,6 @@ sizes(Config) when is_list(Config) ->
     ?line cs_end(),
     ok.
 
-tail(Config) when is_list(Config) ->
-    ?line [] = tail_1(<<0:7>>),
-    ?line [0] = tail_1(<<0>>),
-    ?line [0] = tail_1(<<0:12>>),
-    ?line [0,0] = tail_1(<<0:20>>),
-
-    ?line [] = tail_2(<<0:7>>),
-    ?line [42] = tail_2(<<0>>),
-    ?line [] = tail_2(<<0:12>>),
-    ?line [42,42] = tail_2(<<0,1>>),
-
-    ?line <<>> = tail_3(<<0:7>>),
-    ?line <<42>> = tail_3(<<0>>),
-    ?line <<42>> = tail_3(<<0:12>>),
-    ?line <<42,42>> = tail_3(<<0:20>>),
-
-    ?line [] = tail_4(<<0:15>>),
-    ?line [7] = tail_4(<<7,8>>),
-    ?line [9] = tail_4(<<9,17:12>>),
-    ok.
-
-tail_1(Bits) ->
-    [X || <<X:8/integer, _/bits>> <= Bits].
-
-tail_2(Bits) ->
-    [42 || <<_:8/integer, _/bytes>> <= Bits].
-
-tail_3(Bits) ->
-    << <<42>> || <<_:8/integer, _/bits>> <= Bits >>.
-
-tail_4(Bits) ->
-    [X || <<X:8/integer, Tail/bits>> <= Bits, bit_size(Tail) >= 8].
-
-
 cs_init() ->
     erts_debug:set_internal_state(available_internal_state, true),
     ok.
diff --git a/lib/dialyzer/test/small_SUITE_data/src/bin_compr.erl b/lib/dialyzer/test/small_SUITE_data/src/bin_compr.erl
deleted file mode 100644
index 8c2497ed21..0000000000
--- a/lib/dialyzer/test/small_SUITE_data/src/bin_compr.erl
+++ /dev/null
@@ -1,16 +0,0 @@
-%%% -*- erlang-indent-level: 2 -*-
-%%%------------------------------------------------------------------------
-%%% File    : bin_compr.erl
-%%% Purpose : Test case which crashes in dialyzer_dataflow:bind_bin_segs/5.
-%%%------------------------------------------------------------------------
-
--module(bin_compr).
-
--export([bc/1]).
-
-%% The binary comprehension below is stupid: it consumes the whole
-%% bitstr in one go and produces a [666] result provided Bits is a
-%% bitstr of at least 8 bits. Still, this is a valid Erlang program
-%% and dialyzer's analysis should not crash on it.
-bc(Bits) ->
-  [666 || <<_:8/integer, _/bits>> <= Bits].
diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl
index 08b8541014..573b05e4e2 100644
--- a/lib/stdlib/src/erl_lint.erl
+++ b/lib/stdlib/src/erl_lint.erl
@@ -281,6 +281,8 @@ format_error(utf_bittype_size_or_unit) ->
     "neither size nor unit must be given for segments of type utf8/utf16/utf32";
 format_error({bad_bitsize,Type}) ->
     io_lib:format("bad ~s bit size", [Type]);
+format_error(unsized_binary_in_bin_gen_pattern) ->
+    "binary fields without size are not allowed in patterns of bit string generators";
 %% --- behaviours ---
 format_error({conflicting_behaviours,{Name,Arity},B,FirstL,FirstB}) ->
     io_lib:format("conflicting behaviours - callback ~w/~w required by both '~p' "
@@ -2878,7 +2880,8 @@ lc_quals([{generate,_Line,P,E} | Qs], Vt0, Uvt0, St0) ->
     {Vt,Uvt,St} = handle_generator(P,E,Vt0,Uvt0,St0),
     lc_quals(Qs, Vt, Uvt, St);
 lc_quals([{b_generate,_Line,P,E} | Qs], Vt0, Uvt0, St0) ->
-    {Vt,Uvt,St} = handle_generator(P,E,Vt0,Uvt0,St0),
+    St1 = handle_bitstring_gen_pat(P,St0),
+    {Vt,Uvt,St} = handle_generator(P,E,Vt0,Uvt0,St1),
     lc_quals(Qs, Vt, Uvt, St);
 lc_quals([F|Qs], Vt, Uvt, St0) ->
     {Fvt,St1} = case is_guard_test2(F, St0#lint.records) of
@@ -2906,6 +2909,22 @@ handle_generator(P,E,Vt,Uvt,St0) ->
     Vt3 = vtupdate(vtsubtract(Vt2, Binvt), Binvt),
     {Vt3,NUvt,St5}.
 
+handle_bitstring_gen_pat({bin,_,Segments=[_|_]},St) ->
+    case lists:last(Segments) of
+        {bin_element,Line,{var,_,_},default,Flags} when is_list(Flags) ->
+            case member(binary, Flags) orelse member(bits, Flags)
+                                       orelse member(bitstring, Flags) of
+                true ->
+                    add_error(Line, unsized_binary_in_bin_gen_pattern, St);
+                false ->
+                    St
+            end;
+        _ ->
+            St
+    end;
+handle_bitstring_gen_pat(_,St) ->
+    St.
+
 %% fun_clauses(Clauses, ImportVarTable, State) ->
 %%      {UsedVars, State}.
 %%  Fun's cannot export any variables.
diff --git a/lib/stdlib/src/eval_bits.erl b/lib/stdlib/src/eval_bits.erl
index e49cbc1fd1..75fe2c00c7 100644
--- a/lib/stdlib/src/eval_bits.erl
+++ b/lib/stdlib/src/eval_bits.erl
@@ -192,7 +192,7 @@ bin_gen_field({bin_element,Line,VE,Size0,Options0},
         make_bit_type(Line, Size0, Options0),
     V = erl_eval:partial_eval(VE),
     NewV = coerce_to_float(V, Type),
-    match_check_size(Mfun, Size1, BBs0),
+    match_check_size(Mfun, Size1, BBs0, false),
     {value, Size, _BBs} = Efun(Size1, BBs0),
     bin_gen_field1(Bin, Type, Size, Unit, Sign, Endian, NewV, Bs0, BBs0, Mfun).
 
@@ -380,20 +380,25 @@ make_bit_type(_Line, Size, Type0) -> %Size evaluates to an integer or 'all'
         {error,Reason} -> error(Reason)
     end.
 
-match_check_size(Mfun, {var,_,V}, Bs) ->
+match_check_size(Mfun, Size, Bs) ->
+    match_check_size(Mfun, Size, Bs, true).
+
+match_check_size(Mfun, {var,_,V}, Bs, _AllowAll) ->
     case Mfun(binding, {V,Bs}) of
         {value,_} -> ok;
 	unbound -> throw(invalid) % or, rather, error({unbound,V})
     end;
-match_check_size(_, {atom,_,all}, _Bs) ->
+match_check_size(_, {atom,_,all}, _Bs, true) ->
     ok;
-match_check_size(_, {atom,_,undefined}, _Bs) ->
+match_check_size(_, {atom,_,all}, _Bs, false) ->
+    throw(invalid);
+match_check_size(_, {atom,_,undefined}, _Bs, _AllowAll) ->
     ok;
-match_check_size(_, {integer,_,_}, _Bs) ->
+match_check_size(_, {integer,_,_}, _Bs, _AllowAll) ->
     ok;
-match_check_size(_, {value,_,_}, _Bs) ->
+match_check_size(_, {value,_,_}, _Bs, _AllowAll) ->
     ok;	%From the debugger.
-match_check_size(_, _, _Bs) ->
+match_check_size(_, _, _Bs, _AllowAll) ->
     throw(invalid).
 
 %% error(Reason) -> exception thrown
diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl
index f8345559c4..b1236ceedf 100644
--- a/lib/stdlib/test/erl_lint_SUITE.erl
+++ b/lib/stdlib/test/erl_lint_SUITE.erl
@@ -48,6 +48,7 @@
 	  unused_function/1,
 	  unsafe_vars/1,unsafe_vars2/1,
 	  unsafe_vars_try/1,
+	  unsized_binary_in_bin_gen_pattern/1,
 	  guard/1, otp_4886/1, otp_4988/1, otp_5091/1, otp_5276/1, otp_5338/1,
 	  otp_5362/1, otp_5371/1, otp_7227/1, otp_5494/1, otp_5644/1, otp_5878/1,
 	  otp_5917/1, otp_6585/1, otp_6885/1, otp_10436/1,
@@ -80,6 +81,7 @@ all() ->
     [{group, unused_vars_warn}, export_vars_warn,
      shadow_vars, unused_import, unused_function,
      unsafe_vars, unsafe_vars2, unsafe_vars_try, guard,
+     unsized_binary_in_bin_gen_pattern,
      otp_4886, otp_4988, otp_5091, otp_5276, otp_5338,
      otp_5362, otp_5371, otp_7227, otp_5494, otp_5644,
      otp_5878, otp_5917, otp_6585, otp_6885, otp_10436, export_all,
@@ -1079,6 +1081,35 @@ unsafe_vars_try(Config) when is_list(Config) ->
         ?line [] = run(Config, Ts),
     ok.
 
+unsized_binary_in_bin_gen_pattern(doc) ->
+    "Unsized binary fields are forbidden in patterns of bit string generators";
+unsized_binary_in_bin_gen_pattern(suite) -> [];
+unsized_binary_in_bin_gen_pattern(Config) when is_list(Config) ->
+    Ts = [{unsized_binary_in_bin_gen_pattern,
+	   <<"t({bc,binary,Bin}) ->
+		  << <<X,Tail/binary>> || <<X,Tail/binary>> <= Bin >>;
+	      t({bc,bits,Bin}) ->
+		  << <<X,Tail/bits>> || <<X,Tail/bits>> <= Bin >>;
+	      t({bc,bitstring,Bin}) ->
+		  << <<X,Tail/bits>> || <<X,Tail/bitstring>> <= Bin >>;
+	      t({lc,binary,Bin}) ->
+		  [ {X,Tail} || <<X,Tail/binary>> <= Bin ];
+	      t({lc,bits,Bin}) ->
+		  [ {X,Tail} || <<X,Tail/bits>> <= Bin ];
+	      t({lc,bitstring,Bin}) ->
+		  [ {X,Tail} || <<X,Tail/bitstring>> <= Bin ].">>,
+	   [],
+	   {errors,
+	    [{2,erl_lint,unsized_binary_in_bin_gen_pattern},
+	     {4,erl_lint,unsized_binary_in_bin_gen_pattern},
+	     {6,erl_lint,unsized_binary_in_bin_gen_pattern},
+	     {8,erl_lint,unsized_binary_in_bin_gen_pattern},
+	     {10,erl_lint,unsized_binary_in_bin_gen_pattern},
+	     {12,erl_lint,unsized_binary_in_bin_gen_pattern}],
+	     []}}],
+    [] = run(Config, Ts),
+    ok.
+
 guard(doc) ->
     "OTP-4670. Guards, is_record in particular.";
 guard(suite) -> [];