[stdlib] Fix bugs in eval_bits

Unicode related.

2 files changed, 103 insertions, 9 deletions

diff --git a/lib/stdlib/src/eval_bits.erl b/lib/stdlib/src/eval_bits.erl
index f40904df1c..e49cbc1fd1 100644
--- a/lib/stdlib/src/eval_bits.erl
+++ b/lib/stdlib/src/eval_bits.erl
@@ -2,7 +2,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2013. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -67,7 +67,8 @@ expr_grp([], Bs0, _Lf, Acc) ->
     {value,Acc,Bs0}.
 
 eval_field({bin_element, _, {string, _, S}, default, default}, Bs0, _Fun) ->
-    {list_to_binary(S),Bs0};
+    Latin1 = [C band 16#FF || C <- S],
+    {list_to_binary(Latin1),Bs0};
 eval_field({bin_element, Line, {string, _, S}, Size0, Options0}, Bs, _Fun) ->
     {_Size,[Type,_Unit,_Sign,Endian]} = 
         make_bit_type(Line, Size0, Options0),
@@ -162,8 +163,10 @@ bin_gen([], Bin, _Bs0, _BBs0, _Mfun, _Efun, false) ->
   
 bin_gen_field({bin_element,_,{string,_,S},default,default},
               Bin, Bs, BBs, _Mfun, _Efun) ->
-    Bits = list_to_binary(S),
-    Size = byte_size(Bits),
+    Bits = try list_to_binary(S)
+           catch _:_ -> <<>>
+           end,
+    Size = length(S),
     case Bin of
         <<Bits:Size/binary,Rest/bitstring>> ->
             {match,Bs,BBs,Rest};
@@ -172,16 +175,42 @@ bin_gen_field({bin_element,_,{string,_,S},default,default},
         _ ->
             done
     end;
+bin_gen_field({bin_element,Line,{string,SLine,S},Size0,Options0},
+              Bin0, Bs0, BBs0, Mfun, Efun) ->
+    {Size1, [Type,{unit,Unit},Sign,Endian]} =
+        make_bit_type(Line, Size0, Options0),
+    match_check_size(Mfun, Size1, BBs0),
+    {value, Size, _BBs} = Efun(Size1, BBs0),
+    F = fun(C, Bin, Bs, BBs) ->
+                bin_gen_field1(Bin, Type, Size, Unit, Sign, Endian,
+                               {integer,SLine,C}, Bs, BBs, Mfun)
+        end,
+    bin_gen_field_string(S, Bin0, Bs0, BBs0, F);
 bin_gen_field({bin_element,Line,VE,Size0,Options0}, 
               Bin, Bs0, BBs0, Mfun, Efun) ->
     {Size1, [Type,{unit,Unit},Sign,Endian]} = 
         make_bit_type(Line, Size0, Options0),
     V = erl_eval:partial_eval(VE),
+    NewV = coerce_to_float(V, Type),
     match_check_size(Mfun, Size1, BBs0),
     {value, Size, _BBs} = Efun(Size1, BBs0),
+    bin_gen_field1(Bin, Type, Size, Unit, Sign, Endian, NewV, Bs0, BBs0, Mfun).
+
+bin_gen_field_string([], Rest, Bs, BBs, _F) ->
+    {match,Bs,BBs,Rest};
+bin_gen_field_string([C|Cs], Bin0, Bs0, BBs0, Fun) ->
+    case Fun(C, Bin0, Bs0, BBs0) of
+        {match,Bs,BBs,Rest} ->
+            bin_gen_field_string(Cs, Rest, Bs, BBs, Fun);
+        {nomatch,Rest} ->
+            {nomatch,Rest};
+        done ->
+            done
+    end.
+
+bin_gen_field1(Bin, Type, Size, Unit, Sign, Endian, NewV, Bs0, BBs0, Mfun) ->
     case catch get_value(Bin, Type, Size, Unit, Sign, Endian) of
         {Val,<<_/bitstring>>=Rest} ->
-            NewV = coerce_to_float(V, Type),
             case catch Mfun(match, {NewV,Val,Bs0}) of
                 {match,Bs} ->
                     BBs = add_bin_binding(Mfun, NewV, Bs, BBs0),
@@ -223,20 +252,41 @@ match_bits_1([F|Fs], Bits0, Bs0, BBs0, Mfun, Efun) ->
 
 match_field_1({bin_element,_,{string,_,S},default,default},
               Bin, Bs, BBs, _Mfun, _Efun) ->
-    Bits = list_to_binary(S),
+    Bits = list_to_binary(S), % fails if there are characters > 255
     Size = byte_size(Bits),
     <<Bits:Size/binary,Rest/binary-unit:1>> = Bin,
     {Bs,BBs,Rest};
+match_field_1({bin_element,Line,{string,SLine,S},Size0,Options0},
+              Bin0, Bs0, BBs0, Mfun, Efun) ->
+    {Size1, [Type,{unit,Unit},Sign,Endian]} =
+        make_bit_type(Line, Size0, Options0),
+    Size2 = erl_eval:partial_eval(Size1),
+    match_check_size(Mfun, Size2, BBs0),
+    {value, Size, _BBs} = Efun(Size2, BBs0),
+    F = fun(C, Bin, Bs, BBs) ->
+                match_field(Bin, Type, Size, Unit, Sign, Endian,
+                            {integer,SLine,C}, Bs, BBs, Mfun)
+        end,
+    match_field_string(S, Bin0, Bs0, BBs0, F);
 match_field_1({bin_element,Line,VE,Size0,Options0}, 
               Bin, Bs0, BBs0, Mfun, Efun) ->
     {Size1, [Type,{unit,Unit},Sign,Endian]} = 
         make_bit_type(Line, Size0, Options0),
     V = erl_eval:partial_eval(VE),
+    NewV = coerce_to_float(V, Type),
     Size2 = erl_eval:partial_eval(Size1),
     match_check_size(Mfun, Size2, BBs0),
     {value, Size, _BBs} = Efun(Size2, BBs0),
+    match_field(Bin, Type, Size, Unit, Sign, Endian, NewV, Bs0, BBs0, Mfun).
+
+match_field_string([], Rest, Bs, BBs, _Fun) ->
+    {Bs,BBs,Rest};
+match_field_string([C|Cs], Bin0, Bs0, BBs0, Fun) ->
+    {Bs,BBs,Bin} = Fun(C, Bin0, Bs0, BBs0),
+    match_field_string(Cs, Bin, Bs, BBs, Fun).
+
+match_field(Bin, Type, Size, Unit, Sign, Endian, NewV, Bs0, BBs0, Mfun) ->
     {Val,Rest} = get_value(Bin, Type, Size, Unit, Sign, Endian),
-    NewV = coerce_to_float(V, Type),
     {match,Bs} = Mfun(match, {NewV,Val,Bs0}),
     BBs = add_bin_binding(Mfun, NewV, Bs, BBs0),
     {Bs,BBs,Rest}.
diff --git a/lib/stdlib/test/erl_eval_SUITE.erl b/lib/stdlib/test/erl_eval_SUITE.erl
index 04d49770cb..d3c91c7326 100644
--- a/lib/stdlib/test/erl_eval_SUITE.erl
+++ b/lib/stdlib/test/erl_eval_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2012. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2013. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -37,6 +37,7 @@
          otp_6977/1,
 	 otp_7550/1,
          otp_8133/1,
+         otp_10622/1,
          funs/1,
 	 try_catch/1,
 	 eval_expr_5/1,
@@ -79,7 +80,7 @@ all() ->
      pattern_expr, match_bin, guard_3, guard_4, lc,
      simple_cases, unary_plus, apply_atom, otp_5269,
      otp_6539, otp_6543, otp_6787, otp_6977, otp_7550,
-     otp_8133, funs, try_catch, eval_expr_5, zero_width].
+     otp_8133, otp_10622, funs, try_catch, eval_expr_5, zero_width].
 
 groups() -> 
     [].
@@ -960,6 +961,7 @@ otp_8133(Config) when is_list(Config) ->
                   E = fun(N) -> 
                               if 
                                   is_integer(N) -> <<N/integer>>; 
+
                                   true -> erlang:error(foo) 
                               end 
                       end,
@@ -980,6 +982,48 @@ otp_8133(Config) when is_list(Config) ->
             ok),
     ok.
 
+otp_10622(doc) ->
+    ["OTP-10622. Bugs."];
+otp_10622(suite) ->
+    [];
+otp_10622(Config) when is_list(Config) ->
+    check(fun() -> <<0>> = <<"\x{400}">> end,
+          "<<0>> = <<\"\\x{400}\">>. ",
+          <<0>>),
+    check(fun() -> <<"\x{aa}ff"/utf8>> = <<"\x{aa}ff"/utf8>> end,
+          "<<\"\\x{aa}ff\"/utf8>> = <<\"\\x{aa}ff\"/utf8>>. ",
+          <<"�\xaaff">>),
+    %% The same bug as last example:
+    check(fun() -> case <<"foo"/utf8>> of
+                       <<"foo"/utf8>> -> true
+                   end
+          end,
+          "case <<\"foo\"/utf8>> of <<\"foo\"/utf8>> -> true end.",
+          true),
+    check(fun() -> <<"\x{400}"/utf8>> = <<"\x{400}"/utf8>> end,
+          "<<\"\\x{400}\"/utf8>> = <<\"\\x{400}\"/utf8>>. ",
+          <<208,128>>),
+    error_check("<<\"\\x{aaa}\">> = <<\"\\x{aaa}\">>.",
+                {badmatch,<<"\xaa">>}),
+
+    check(fun() -> [a || <<"\x{aaa}">> <= <<2703:16>>] end,
+          "[a || <<\"\\x{aaa}\">> <= <<2703:16>>]. ",
+          []),
+    check(fun() -> [a || <<"\x{aa}"/utf8>> <= <<"\x{aa}"/utf8>>] end,
+          "[a || <<\"\\x{aa}\"/utf8>> <= <<\"\\x{aa}\"/utf8>>]. ",
+          [a]),
+    check(fun() -> [a || <<"\x{aa}x"/utf8>> <= <<"\x{aa}y"/utf8>>] end,
+          "[a || <<\"\\x{aa}x\"/utf8>> <= <<\"\\x{aa}y\"/utf8>>]. ",
+          []),
+    check(fun() -> [a || <<"\x{aaa}">> <= <<"\x{aaa}">>] end,
+          "[a || <<\"\\x{aaa}\">> <= <<\"\\x{aaa}\">>]. ",
+          []),
+    check(fun() -> [a || <<"\x{aaa}"/utf8>> <= <<"\x{aaa}"/utf8>>] end,
+          "[a || <<\"\\x{aaa}\"/utf8>> <= <<\"\\x{aaa}\"/utf8>>]. ",
+          [a]),
+
+    ok.
+
 funs(doc) ->
     ["Simple cases, just to cover some code."];
 funs(suite) ->