Fix a problem with HiPE-compiled Yecc parsers

1 files changed, 129 insertions, 86 deletions

diff --git a/lib/parsetools/src/yeccparser.erl b/lib/parsetools/src/yeccparser.erl
index 415547b4ce..63127802ee 100644
--- a/lib/parsetools/src/yeccparser.erl
+++ b/lib/parsetools/src/yeccparser.erl
@@ -38,16 +38,16 @@ line_of(Token) ->
 
 -type yecc_ret() :: {'error', _} | {'ok', _}.
 
--spec parse(_) -> yecc_ret().
+-spec parse(Tokens :: list()) -> yecc_ret().
 parse(Tokens) ->
-    yeccpars0(Tokens, false).
+    yeccpars0(Tokens, {no_func, no_line}, 0, [], []).
 
--spec parse_and_scan({function() | {atom(), atom()}, [_]} | {atom(), atom(), [_]}) ->
-            yecc_ret().
+-spec parse_and_scan({function() | {atom(), atom()}, [_]}
+                     | {atom(), atom(), [_]}) -> yecc_ret().
 parse_and_scan({F, A}) -> % Fun or {M, F}
-    yeccpars0([], {F, A});
+    yeccpars0([], {{F, A}, no_line}, 0, [], []);
 parse_and_scan({M, F, A}) ->
-    yeccpars0([], {{M, F}, A}).
+    yeccpars0([], {{{M, F}, A}, no_line}, 0, [], []).
 
 -spec format_error(any()) -> [char() | list()].
 format_error(Message) ->
@@ -60,54 +60,58 @@ format_error(Message) ->
 
 %% To be used in grammar files to throw an error message to the parser
 %% toplevel. Doesn't have to be exported!
--compile({nowarn_unused_function,{return_error,2}}).
+-compile({nowarn_unused_function, return_error/2}).
 -spec return_error(integer(), any()) -> no_return().
 return_error(Line, Message) ->
     throw({error, {Line, ?MODULE, Message}}).
 
--define(CODE_VERSION, "1.3").
+-define(CODE_VERSION, "1.4").
 
-yeccpars0(Tokens, MFA) ->
-    try yeccpars1(Tokens, MFA, 0, [], [])
+yeccpars0(Tokens, Tzr, State, States, Vstack) ->
+    try yeccpars1(Tokens, Tzr, State, States, Vstack)
     catch 
         error: Error ->
             Stacktrace = erlang:get_stacktrace(),
             try yecc_error_type(Error, Stacktrace) of
-                {syntax_error, Token} ->
-                    yeccerror(Token);
-                {missing_in_goto_table=Tag, Symbol, State} ->
-                    Desc = {Symbol, State, Tag},
+                Desc ->
                     erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc},
-                                Stacktrace)
+                                 Stacktrace)
             catch _:_ -> erlang:raise(error, Error, Stacktrace)
             end;
-        throw: {error, {_Line, ?MODULE, _M}} = Error -> 
-            Error % probably from return_error/2
+        %% Probably thrown from return_error/2:
+        throw: {error, {_Line, ?MODULE, _M}} = Error ->
+            Error
     end.
 
-yecc_error_type(function_clause, [{?MODULE,F,[State,_,_,_,Token,_,_]} | _]) ->
+yecc_error_type(function_clause, [{?MODULE,F,ArityOrArgs} | _]) ->
     case atom_to_list(F) of
-        "yeccpars2" ++ _ ->
-            {syntax_error, Token};
         "yeccgoto_" ++ SymbolL ->
             {ok,[{atom,_,Symbol}],_} = erl_scan:string(SymbolL),
-            {missing_in_goto_table, Symbol, State}
+            State = case ArityOrArgs of
+                        [S,_,_,_,_,_,_] -> S;
+                        _ -> state_is_unknown
+                    end,
+            {Symbol, State, missing_in_goto_table}
     end.
 
-yeccpars1([Token | Tokens], Tokenizer, State, States, Vstack) ->
-    yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, 
-              Tokenizer);
-yeccpars1([], {F, A}, State, States, Vstack) ->
+yeccpars1([Token | Tokens], Tzr, State, States, Vstack) ->
+    yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, Tzr);
+yeccpars1([], {{F, A},_Line}, State, States, Vstack) ->
     case apply(F, A) of
-        {ok, Tokens, _Endline} ->
-            yeccpars1(Tokens, {F, A}, State, States, Vstack);
-        {eof, _Endline} ->
-            yeccpars1([], false, State, States, Vstack);
+        {ok, Tokens, Endline} ->
+            yeccpars1(Tokens, {{F, A}, Endline}, State, States, Vstack);
+        {eof, Endline} ->
+            yeccpars1([], {no_func, Endline}, State, States, Vstack);
         {error, Descriptor, _Endline} ->
             {error, Descriptor}
     end;
-yeccpars1([], false, State, States, Vstack) ->
-    yeccpars2(State, '$end', States, Vstack, {'$end', 999999}, [], false).
+yeccpars1([], {no_func, no_line}, State, States, Vstack) ->
+    Line = 999999,
+    yeccpars2(State, '$end', States, Vstack, yecc_end(Line), [],
+              {no_func, Line});
+yeccpars1([], {no_func, Endline}, State, States, Vstack) ->
+    yeccpars2(State, '$end', States, Vstack, yecc_end(Endline), [],
+              {no_func, Endline}).
 
 %% yeccpars1/7 is called from generated code.
 %%
@@ -115,48 +119,73 @@ yeccpars1([], false, State, States, Vstack) ->
 %% yeccpars1/7 can be found by parsing the file without following
 %% include directives. yecc will otherwise assume that an old
 %% yeccpre.hrl is included (one which defines yeccpars1/5).
-yeccpars1(State1, State, States, Vstack, Stack1, [Token | Tokens], 
-          Tokenizer) ->
+yeccpars1(State1, State, States, Vstack, Token0, [Token | Tokens], Tzr) ->
     yeccpars2(State, element(1, Token), [State1 | States],
-              [Stack1 | Vstack], Token, Tokens, Tokenizer);
-yeccpars1(State1, State, States, Vstack, Stack1, [], {F, A}) ->
-    case apply(F, A) of
-        {ok, Tokens, _Endline} ->
-            yeccpars1(State1, State, States, Vstack, Stack1, Tokens, {F, A});
-        {eof, _Endline} ->
-            yeccpars1(State1, State, States, Vstack, Stack1, [], false);
-        {error, Descriptor, _Endline} ->
-            {error, Descriptor}
-    end;
-yeccpars1(State1, State, States, Vstack, Stack1, [], false) ->
-    yeccpars2(State, '$end', [State1 | States], [Stack1 | Vstack],
-              {'$end', 999999}, [], false).
+              [Token0 | Vstack], Token, Tokens, Tzr);
+yeccpars1(State1, State, States, Vstack, Token0, [], {{_F,_A}, _Line}=Tzr) ->
+    yeccpars1([], Tzr, State, [State1 | States], [Token0 | Vstack]);
+yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, no_line}) ->
+    Line = yecctoken_end_location(Token0),
+    yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack],
+              yecc_end(Line), [], {no_func, Line});
+yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, Line}) ->
+    yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack],
+              yecc_end(Line), [], {no_func, Line}).
+
+%% For internal use only.
+yecc_end({Line,_Column}) ->
+    {'$end', Line};
+yecc_end(Line) ->
+    {'$end', Line}.
+
+yecctoken_end_location(Token) ->
+    try
+        {text, Str} = erl_scan:token_info(Token, text),
+        {line, Line} = erl_scan:token_info(Token, line),
+        Parts = re:split(Str, "\n"),
+        Dline = length(Parts) - 1,
+        Yline = Line + Dline,
+        case erl_scan:token_info(Token, column) of
+            {column, Column} ->
+                Col = byte_size(lists:last(Parts)),
+                {Yline, Col + if Dline =:= 0 -> Column; true -> 1 end};
+            undefined ->
+                Yline
+        end
+    catch _:_ ->
+        yecctoken_location(Token)
+    end.
 
-% For internal use only.
 yeccerror(Token) ->
-    Text = case catch erl_scan:token_info(Token, text) of
-               {text, Txt} -> Txt;
-               _ -> yecctoken2string(Token)
-           end,
-    Location = case catch erl_scan:token_info(Token, location) of
-                   {location, Loc} -> Loc;
-                   _ -> element(2, Token)
-               end,
+    Text = yecctoken_to_string(Token),
+    Location = yecctoken_location(Token),
     {error, {Location, ?MODULE, ["syntax error before: ", Text]}}.
 
+yecctoken_to_string(Token) ->
+    case catch erl_scan:token_info(Token, text) of
+        {text, Txt} -> Txt;
+        _ -> yecctoken2string(Token)
+    end.
+
+yecctoken_location(Token) ->
+    case catch erl_scan:token_info(Token, location) of
+        {location, Loc} -> Loc;
+        _ -> element(2, Token)
+    end.
+
 yecctoken2string({atom, _, A}) -> io_lib:write(A);
 yecctoken2string({integer,_,N}) -> io_lib:write(N);
 yecctoken2string({float,_,F}) -> io_lib:write(F);
 yecctoken2string({char,_,C}) -> io_lib:write_char(C);
 yecctoken2string({var,_,V}) -> io_lib:format("~s", [V]);
 yecctoken2string({string,_,S}) -> io_lib:write_unicode_string(S);
-yecctoken2string({reserved_symbol, _, A}) -> io_lib:format("~w", [A]);
-yecctoken2string({_Cat, _, Val}) -> io_lib:format("~w", [Val]);
+yecctoken2string({reserved_symbol, _, A}) -> io_lib:write(A);
+yecctoken2string({_Cat, _, Val}) -> io_lib:write(Val);
 yecctoken2string({dot, _}) -> "'.'";
 yecctoken2string({'$end', _}) ->
     [];
 yecctoken2string({Other, _}) when is_atom(Other) ->
-    io_lib:format("~w", [Other]);
+    io_lib:write(Other);
 yecctoken2string(Other) ->
     io_lib:write(Other).
 
@@ -164,7 +193,7 @@ yecctoken2string(Other) ->
 
 
 
--file("yeccparser.erl", 168).
+-file("yeccparser.erl", 196).
 
 yeccpars2(0=S, Cat, Ss, Stack, T, Ts, Tzr) ->
  yeccpars2_0(S, Cat, Ss, Stack, T, Ts, Tzr);
@@ -248,7 +277,9 @@ yeccpars2_0(S, integer, Ss, Stack, T, Ts, Tzr) ->
 yeccpars2_0(S, reserved_word, Ss, Stack, T, Ts, Tzr) ->
  yeccpars1(S, 8, Ss, Stack, T, Ts, Tzr);
 yeccpars2_0(S, var, Ss, Stack, T, Ts, Tzr) ->
- yeccpars1(S, 9, Ss, Stack, T, Ts, Tzr).
+ yeccpars1(S, 9, Ss, Stack, T, Ts, Tzr);
+yeccpars2_0(_, _, _, _, T, _, _) ->
+ yeccerror(T).
 
 yeccpars2_1(S, atom, Ss, Stack, T, Ts, Tzr) ->
  yeccpars1(S, 6, Ss, Stack, T, Ts, Tzr);
@@ -267,10 +298,14 @@ yeccpars2_2(_S, Cat, Ss, Stack, T, Ts, Tzr) ->
  yeccgoto_grammar(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr).
 
 yeccpars2_3(S, '->', Ss, Stack, T, Ts, Tzr) ->
- yeccpars1(S, 10, Ss, Stack, T, Ts, Tzr).
+ yeccpars1(S, 10, Ss, Stack, T, Ts, Tzr);
+yeccpars2_3(_, _, _, _, T, _, _) ->
+ yeccerror(T).
 
-yeccpars2_4(_S, '$end', _Ss, Stack,  _T, _Ts, _Tzr) ->
- {ok, hd(Stack)}.
+yeccpars2_4(_S, '$end', _Ss, Stack, _T, _Ts, _Tzr) ->
+ {ok, hd(Stack)};
+yeccpars2_4(_, _, _, _, T, _, _) ->
+ yeccerror(T).
 
 yeccpars2_5(_S, Cat, Ss, Stack, T, Ts, Tzr) ->
  yeccgoto_grammar(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr).
@@ -317,7 +352,9 @@ yeccpars2_13(_S, Cat, Ss, Stack, T, Ts, Tzr) ->
  yeccgoto_symbols(hd(Nss), Cat, Nss, NewStack, T, Ts, Tzr).
 
 yeccpars2_14(S, dot, Ss, Stack, T, Ts, Tzr) ->
- yeccpars1(S, 29, Ss, Stack, T, Ts, Tzr).
+ yeccpars1(S, 29, Ss, Stack, T, Ts, Tzr);
+yeccpars2_14(_, _, _, _, T, _, _) ->
+ yeccerror(T).
 
 yeccpars2_15(S, '->', Ss, Stack, T, Ts, Tzr) ->
  yeccpars1(S, 18, Ss, Stack, T, Ts, Tzr);
@@ -338,7 +375,9 @@ yeccpars2_15(S, reserved_word, Ss, Stack, T, Ts, Tzr) ->
 yeccpars2_15(S, string, Ss, Stack, T, Ts, Tzr) ->
  yeccpars1(S, 26, Ss, Stack, T, Ts, Tzr);
 yeccpars2_15(S, var, Ss, Stack, T, Ts, Tzr) ->
- yeccpars1(S, 27, Ss, Stack, T, Ts, Tzr).
+ yeccpars1(S, 27, Ss, Stack, T, Ts, Tzr);
+yeccpars2_15(_, _, _, _, T, _, _) ->
+ yeccerror(T).
 
 yeccpars2_16(_S, Cat, Ss, Stack, T, Ts, Tzr) ->
  [_|Nss] = Ss,
@@ -414,10 +453,14 @@ yeccpars2_29(_S, Cat, Ss, Stack, T, Ts, Tzr) ->
  yeccgoto_rule(hd(Nss), Cat, Nss, NewStack, T, Ts, Tzr).
 
 yeccpars2_30(S, dot, Ss, Stack, T, Ts, Tzr) ->
- yeccpars1(S, 35, Ss, Stack, T, Ts, Tzr).
+ yeccpars1(S, 35, Ss, Stack, T, Ts, Tzr);
+yeccpars2_30(_, _, _, _, T, _, _) ->
+ yeccerror(T).
 
 yeccpars2_31(S, dot, Ss, Stack, T, Ts, Tzr) ->
- yeccpars1(S, 34, Ss, Stack, T, Ts, Tzr).
+ yeccpars1(S, 34, Ss, Stack, T, Ts, Tzr);
+yeccpars2_31(_, _, _, _, T, _, _) ->
+ yeccerror(T).
 
 yeccpars2_32(S, string, Ss, Stack, T, Ts, Tzr) ->
  yeccpars1(S, 32, Ss, Stack, T, Ts, Tzr);
@@ -486,7 +529,7 @@ yeccgoto_tokens(15=_S, Cat, Ss, Stack, T, Ts, Tzr) ->
 yeccgoto_tokens(17=_S, Cat, Ss, Stack, T, Ts, Tzr) ->
  yeccpars2_28(_S, Cat, Ss, Stack, T, Ts, Tzr).
 
--compile({inline,{yeccpars2_6_,1}}).
+-compile({inline,yeccpars2_6_/1}).
 -file("yeccgramm.yrl", 44).
 yeccpars2_6_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -494,7 +537,7 @@ yeccpars2_6_(__Stack0) ->
    symbol ( __1 )
   end | __Stack].
 
--compile({inline,{yeccpars2_7_,1}}).
+-compile({inline,yeccpars2_7_/1}).
 -file("yeccgramm.yrl", 45).
 yeccpars2_7_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -502,7 +545,7 @@ yeccpars2_7_(__Stack0) ->
    symbol ( __1 )
   end | __Stack].
 
--compile({inline,{yeccpars2_8_,1}}).
+-compile({inline,yeccpars2_8_/1}).
 -file("yeccgramm.yrl", 46).
 yeccpars2_8_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -510,7 +553,7 @@ yeccpars2_8_(__Stack0) ->
    symbol ( __1 )
   end | __Stack].
 
--compile({inline,{yeccpars2_9_,1}}).
+-compile({inline,yeccpars2_9_/1}).
 -file("yeccgramm.yrl", 43).
 yeccpars2_9_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -518,14 +561,14 @@ yeccpars2_9_(__Stack0) ->
    symbol ( __1 )
   end | __Stack].
 
--compile({inline,{yeccpars2_11_,1}}).
+-compile({inline,yeccpars2_11_/1}).
 -file("yeccgramm.yrl", 40).
 yeccpars2_11_(__Stack0) ->
  [begin
    { erlang_code , [ { atom , 0 , '$undefined' } ] }
   end | __Stack0].
 
--compile({inline,{yeccpars2_12_,1}}).
+-compile({inline,yeccpars2_12_/1}).
 -file("yeccgramm.yrl", 35).
 yeccpars2_12_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -533,7 +576,7 @@ yeccpars2_12_(__Stack0) ->
    [ __1 ]
   end | __Stack].
 
--compile({inline,{yeccpars2_13_,1}}).
+-compile({inline,yeccpars2_13_/1}).
 -file("yeccgramm.yrl", 36).
 yeccpars2_13_(__Stack0) ->
  [__2,__1 | __Stack] = __Stack0,
@@ -541,7 +584,7 @@ yeccpars2_13_(__Stack0) ->
    [ __1 | __2 ]
   end | __Stack].
 
--compile({inline,{yeccpars2_16_,1}}).
+-compile({inline,yeccpars2_16_/1}).
 -file("yeccgramm.yrl", 39).
 yeccpars2_16_(__Stack0) ->
  [__2,__1 | __Stack] = __Stack0,
@@ -549,7 +592,7 @@ yeccpars2_16_(__Stack0) ->
    { erlang_code , __2 }
   end | __Stack].
 
--compile({inline,{yeccpars2_17_,1}}).
+-compile({inline,yeccpars2_17_/1}).
 -file("yeccgramm.yrl", 41).
 yeccpars2_17_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -557,7 +600,7 @@ yeccpars2_17_(__Stack0) ->
    [ __1 ]
   end | __Stack].
 
--compile({inline,{yeccpars2_18_,1}}).
+-compile({inline,yeccpars2_18_/1}).
 -file("yeccgramm.yrl", 55).
 yeccpars2_18_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -565,7 +608,7 @@ yeccpars2_18_(__Stack0) ->
    { '->' , line_of ( __1 ) }
   end | __Stack].
 
--compile({inline,{yeccpars2_19_,1}}).
+-compile({inline,yeccpars2_19_/1}).
 -file("yeccgramm.yrl", 56).
 yeccpars2_19_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -573,7 +616,7 @@ yeccpars2_19_(__Stack0) ->
    { ':' , line_of ( __1 ) }
   end | __Stack].
 
--compile({inline,{yeccpars2_24_,1}}).
+-compile({inline,yeccpars2_24_/1}).
 -file("yeccgramm.yrl", 53).
 yeccpars2_24_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -581,7 +624,7 @@ yeccpars2_24_(__Stack0) ->
    { value_of ( __1 ) , line_of ( __1 ) }
   end | __Stack].
 
--compile({inline,{yeccpars2_25_,1}}).
+-compile({inline,yeccpars2_25_/1}).
 -file("yeccgramm.yrl", 54).
 yeccpars2_25_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -589,7 +632,7 @@ yeccpars2_25_(__Stack0) ->
    { value_of ( __1 ) , line_of ( __1 ) }
   end | __Stack].
 
--compile({inline,{yeccpars2_28_,1}}).
+-compile({inline,yeccpars2_28_/1}).
 -file("yeccgramm.yrl", 42).
 yeccpars2_28_(__Stack0) ->
  [__2,__1 | __Stack] = __Stack0,
@@ -597,7 +640,7 @@ yeccpars2_28_(__Stack0) ->
    [ __1 | __2 ]
   end | __Stack].
 
--compile({inline,{yeccpars2_29_,1}}).
+-compile({inline,yeccpars2_29_/1}).
 -file("yeccgramm.yrl", 33).
 yeccpars2_29_(__Stack0) ->
  [__5,__4,__3,__2,__1 | __Stack] = __Stack0,
@@ -605,7 +648,7 @@ yeccpars2_29_(__Stack0) ->
    { rule , [ __1 | __3 ] , __4 }
   end | __Stack].
 
--compile({inline,{yeccpars2_32_,1}}).
+-compile({inline,yeccpars2_32_/1}).
 -file("yeccgramm.yrl", 37).
 yeccpars2_32_(__Stack0) ->
  [__1 | __Stack] = __Stack0,
@@ -613,7 +656,7 @@ yeccpars2_32_(__Stack0) ->
    [ __1 ]
   end | __Stack].
 
--compile({inline,{yeccpars2_33_,1}}).
+-compile({inline,yeccpars2_33_/1}).
 -file("yeccgramm.yrl", 38).
 yeccpars2_33_(__Stack0) ->
  [__2,__1 | __Stack] = __Stack0,
@@ -621,7 +664,7 @@ yeccpars2_33_(__Stack0) ->
    [ __1 | __2 ]
   end | __Stack].
 
--compile({inline,{yeccpars2_34_,1}}).
+-compile({inline,yeccpars2_34_/1}).
 -file("yeccgramm.yrl", 32).
 yeccpars2_34_(__Stack0) ->
  [__3,__2,__1 | __Stack] = __Stack0,
@@ -629,7 +672,7 @@ yeccpars2_34_(__Stack0) ->
    { __1 , __2 }
   end | __Stack].
 
--compile({inline,{yeccpars2_35_,1}}).
+-compile({inline,yeccpars2_35_/1}).
 -file("yeccgramm.yrl", 31).
 yeccpars2_35_(__Stack0) ->
  [__3,__2,__1 | __Stack] = __Stack0,