diff options
Diffstat (limited to 'lib/parsetools')
| -rw-r--r-- | lib/parsetools/doc/src/notes.xml | 22 | ||||
| -rw-r--r-- | lib/parsetools/include/yeccpre.hrl | 24 | ||||
| -rw-r--r-- | lib/parsetools/src/leex.erl | 53 | ||||
| -rw-r--r-- | lib/parsetools/src/yecc.erl | 42 | ||||
| -rw-r--r-- | lib/parsetools/src/yeccparser.erl | 215 | ||||
| -rw-r--r-- | lib/parsetools/test/yecc_SUITE.erl | 74 | ||||
| -rw-r--r-- | lib/parsetools/vsn.mk | 2 |
7 files changed, 284 insertions, 148 deletions
diff --git a/lib/parsetools/doc/src/notes.xml b/lib/parsetools/doc/src/notes.xml index 8a6f2c2714..63c37bc27d 100644 --- a/lib/parsetools/doc/src/notes.xml +++ b/lib/parsetools/doc/src/notes.xml @@ -30,6 +30,28 @@ </header> <p>This document describes the changes made to the Parsetools application.</p> +<section><title>Parsetools 2.0.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Yecc failed to report reduce/reduce conflicts where + one of the reductions involved the root symbol. This bug + has been fixed. (Thanks to Manolis Papadakis.)</p> + <p> + Own Id: OTP-8483</p> + </item> + <item> + <p>A bug introduced in Parsetools 1.4.4 (R12B-2) has been + fixed. (Thanks to Manolis Papadakis.)</p> + <p> + Own Id: OTP-8486</p> + </item> + </list> + </section> + +</section> + <section><title>Parsetools 2.0.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/parsetools/include/yeccpre.hrl b/lib/parsetools/include/yeccpre.hrl index 33a103d95f..39dea0552d 100644 --- a/lib/parsetools/include/yeccpre.hrl +++ b/lib/parsetools/include/yeccpre.hrl @@ -26,8 +26,8 @@ parse(Tokens) -> yeccpars0(Tokens, {no_func, no_line}, 0, [], []). --spec parse_and_scan({function() | {atom(), atom()}, [_]} | {atom(), atom(), [_]}) -> - yecc_ret(). +-spec parse_and_scan({function() | {atom(), atom()}, [_]} + | {atom(), atom(), [_]}) -> yecc_ret(). parse_and_scan({F, A}) -> % Fun or {M, F} yeccpars0([], {{F, A}, no_line}, 0, [], []); parse_and_scan({M, F, A}) -> @@ -44,7 +44,7 @@ format_error(Message) -> %% To be used in grammar files to throw an error message to the parser %% toplevel. Doesn't have to be exported! --compile({nowarn_unused_function,{return_error,2}}). +-compile({nowarn_unused_function, return_error/2}). -spec return_error(integer(), any()) -> no_return(). return_error(Line, Message) -> throw({error, {Line, ?MODULE, Message}}). @@ -57,10 +57,7 @@ yeccpars0(Tokens, Tzr, State, States, Vstack) -> error: Error -> Stacktrace = erlang:get_stacktrace(), try yecc_error_type(Error, Stacktrace) of - {syntax_error, Token} -> - yeccerror(Token); - {missing_in_goto_table=Tag, Symbol, State} -> - Desc = {Symbol, State, Tag}, + Desc -> erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, Stacktrace) catch _:_ -> erlang:raise(error, Error, Stacktrace) @@ -70,13 +67,15 @@ yeccpars0(Tokens, Tzr, State, States, Vstack) -> Error end. -yecc_error_type(function_clause, [{?MODULE,F,[State,_,_,_,Token,_,_]} | _]) -> +yecc_error_type(function_clause, [{?MODULE,F,ArityOrArgs} | _]) -> case atom_to_list(F) of - "yeccpars2" ++ _ -> - {syntax_error, Token}; "yeccgoto_" ++ SymbolL -> {ok,[{atom,_,Symbol}],_} = erl_scan:string(SymbolL), - {missing_in_goto_table, Symbol, State} + State = case ArityOrArgs of + [S,_,_,_,_,_,_] -> S; + _ -> state_is_unknown + end, + {Symbol, State, missing_in_goto_table} end. yeccpars1([Token | Tokens], Tzr, State, States, Vstack) -> @@ -141,11 +140,13 @@ yecctoken_end_location(Token) -> yecctoken_location(Token) end. +-compile({nowarn_unused_function, yeccerror/1}). yeccerror(Token) -> Text = yecctoken_to_string(Token), Location = yecctoken_location(Token), {error, {Location, ?MODULE, ["syntax error before: ", Text]}}. +-compile({nowarn_unused_function, yecctoken_to_string/1}). yecctoken_to_string(Token) -> case catch erl_scan:token_info(Token, text) of {text, Txt} -> Txt; @@ -158,6 +159,7 @@ yecctoken_location(Token) -> _ -> element(2, Token) end. +-compile({nowarn_unused_function, yecctoken2string/1}). yecctoken2string({atom, _, A}) -> io_lib:write(A); yecctoken2string({integer,_,N}) -> io_lib:write(N); yecctoken2string({float,_,F}) -> io_lib:write(F); diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl index fd494eaf06..d0b4b9efe7 100644 --- a/lib/parsetools/src/leex.erl +++ b/lib/parsetools/src/leex.erl @@ -35,7 +35,7 @@ -export([compile/3,file/1,file/2,format_error/1]). -import(lists, [member/2,reverse/1,sort/1,delete/2, - keysearch/3,keysort/2,keydelete/3,keyfind/3, + keysort/2,keydelete/3,keyfind/3, map/2,foldl/3,foreach/2,flatmap/2]). -import(string, [substr/2,substr/3,span/2]). -import(ordsets, [is_element/2,add_element/2,union/2]). @@ -182,18 +182,18 @@ options(Options0, [Key|Keys], L) when is_list(Options0) -> false -> Options0 end, - V = case keysearch(Key, 1, Options) of - {value, {Key, Filename0}} when Key =:= includefile; - Key =:= scannerfile -> + V = case lists:keyfind(Key, 1, Options) of + {Key, Filename0} when Key =:= includefile; + Key =:= scannerfile -> case is_filename(Filename0) of no -> badarg; Filename -> {ok,[{Key,Filename}]} end; - {value,{Key,Bool}} when Bool; not Bool -> - {ok,[{Key, Bool}]}; - {value,{Key, _}} -> + {Key, Bool} = KB when is_boolean(Bool) -> + {ok, [KB]}; + {Key, _} -> badarg; false -> {ok,[{Key,default_option(Key)}]} @@ -231,8 +231,7 @@ atom_option(verbose) -> {verbose,true}; atom_option(Key) -> Key. is_filename(T) -> - try filename:flatten(T) of - Filename -> Filename + try filename:flatten(T) catch error: _ -> no end. @@ -320,10 +319,10 @@ filenames(File, Opts, St0) -> St1 = St0#leex{xfile=Xfile, opts=Opts, module=Module}, - {value,{includefile,Ifile0}} = keysearch(includefile, 1, Opts), + {includefile,Ifile0} = lists:keyfind(includefile, 1, Opts), Ifile = inc_file_name(Ifile0), %% Test for explicit scanner file. - {value,{scannerfile,Ofile}} = keysearch(scannerfile, 1, Opts), + {scannerfile,Ofile} = lists:keyfind(scannerfile, 1, Opts), if Ofile =:= [] -> St1#leex{efile=filename:join(Dir, Efile), @@ -495,7 +494,7 @@ parse_rule(S, Line, Atoks, Ms, N, St) -> end. var_used(Name, Toks) -> - case keyfind(Name, 3, Toks) of + case lists:keyfind(Name, 3, Toks) of {var,_,Name} -> true; %It's the var we want _ -> false end. @@ -629,7 +628,7 @@ re_seq(Cs0, Sn0, St) -> {Rs,Sn1,Cs1} -> {{seq,Rs},Sn1,Cs1} end. -re_seq1([C|_]=Cs0, Sn0, St) when C /= $|, C /= $) -> +re_seq1([C|_]=Cs0, Sn0, St) when C =/= $|, C =/= $) -> {L,Sn1,Cs1} = re_repeat(Cs0, Sn0, St), {Rs,Sn2,Cs2} = re_seq1(Cs1, Sn1, St), {[L|Rs],Sn2,Cs2}; @@ -751,9 +750,9 @@ re_char_class("[:" ++ Cs0, Cc, #leex{posix=true}=St) -> {Pcl,":]" ++ Cs1} -> re_char_class(Cs1, [{posix,Pcl}|Cc], St); {_,Cs1} -> parse_error({posix_cc,string_between(Cs0, Cs1)}) end; -re_char_class([C1|Cs0], Cc, St) when C1 /= $] -> +re_char_class([C1|Cs0], Cc, St) when C1 =/= $] -> case re_char(C1, Cs0) of - {Cf,[$-,C2|Cs1]} when C2 /= $] -> + {Cf,[$-,C2|Cs1]} when C2 =/= $] -> case re_char(C2, Cs1) of {Cl,Cs2} when Cf < Cl -> re_char_class(Cs2, [{range,Cf,Cl}|Cc], St); @@ -998,7 +997,7 @@ pack_crs([{C1,C2},{C3,C4}|Crs]) when C2 >= C3, C2 < C4 -> %% C1 C2 %% C3 C4 pack_crs([{C1,C4}|Crs]); -pack_crs([{C1,C2},{C3,C4}|Crs]) when C2 + 1 == C3 -> +pack_crs([{C1,C2},{C3,C4}|Crs]) when C2 + 1 =:= C3 -> %% C1 C2 %% C3 C4 pack_crs([{C1,C4}|Crs]); @@ -1055,7 +1054,7 @@ build_dfa(Set, Us, N, Ts, Ms, NFA) -> %% List of all transition sets. Crs0 = [Cr || S <- Set, {Crs,_St} <- (element(S, NFA))#nfa_state.edges, - Crs /= epsilon, % Not an epsilon transition + Crs =/= epsilon, % Not an epsilon transition Cr <- Crs ], Crs1 = lists:usort(Crs0), % Must remove duplicates! %% Build list of disjoint test ranges. @@ -1072,7 +1071,7 @@ disjoint_crs([{_C1,C2}=Cr1,{C3,_C4}=Cr2|Crs]) when C2 < C3 -> %% C1 C2 %% C3 C4 [Cr1|disjoint_crs([Cr2|Crs])]; -disjoint_crs([{C1,C2},{C3,C4}|Crs]) when C1 == C3 -> +disjoint_crs([{C1,C2},{C3,C4}|Crs]) when C1 =:= C3 -> %% C1 C2 %% C3 C4 [{C1,C2}|disjoint_crs(add_element({C2+1,C4}, Crs))]; @@ -1080,7 +1079,7 @@ disjoint_crs([{C1,C2},{C3,C4}|Crs]) when C1 < C3, C2 >= C3, C2 < C4 -> %% C1 C2 %% C3 C4 [{C1,C3-1}|disjoint_crs(union([{C3,C2},{C2+1,C4}], Crs))]; -disjoint_crs([{C1,C2},{C3,C4}|Crs]) when C1 < C3, C2 == C4 -> +disjoint_crs([{C1,C2},{C3,C4}|Crs]) when C1 < C3, C2 =:= C4 -> %% C1 C2 %% C3 C4 [{C1,C3-1}|disjoint_crs(add_element({C3,C4}, Crs))]; @@ -1093,7 +1092,7 @@ disjoint_crs([]) -> []. build_dfa([Cr|Crs], Set, Us, N, Ts, Ms, NFA) -> case eclosure(move(Set, Cr, NFA), NFA) of - S when S /= [] -> + S when S =/= [] -> case dfa_state_exist(S, Us, Ms) of {yes,T} -> build_dfa(Crs, Set, Us, N, store(Cr, T, Ts), Ms, NFA); @@ -1110,11 +1109,11 @@ build_dfa([], _, Us, N, Ts, _, _) -> %% dfa_state_exist(Set, Unmarked, Marked) -> {yes,State} | no. dfa_state_exist(S, Us, Ms) -> - case keysearch(S, #dfa_state.nfa, Us) of - {value,#dfa_state{no=T}} -> {yes,T}; + case lists:keyfind(S, #dfa_state.nfa, Us) of + #dfa_state{no=T} -> {yes,T}; false -> - case keysearch(S, #dfa_state.nfa, Ms) of - {value,#dfa_state{no=T}} -> {yes,T}; + case lists:keyfind(S, #dfa_state.nfa, Ms) of + #dfa_state{no=T} -> {yes,T}; false -> no end end. @@ -1129,7 +1128,7 @@ eclosure(Sts, NFA) -> eclosure(Sts, NFA, []). eclosure([St|Sts], NFA, Ec) -> #nfa_state{edges=Es} = element(St, NFA), eclosure([ N || {epsilon,N} <- Es, - not is_element(N, Ec) ] ++ Sts, + not is_element(N, Ec) ] ++ Sts, NFA, add_element(St, Ec)); eclosure([], _, Ec) -> Ec. @@ -1137,7 +1136,7 @@ move(Sts, Cr, NFA) -> %% io:fwrite("move1: ~p\n", [{Sts,Cr}]), [ St || N <- Sts, {Crs,St} <- (element(N, NFA))#nfa_state.edges, - Crs /= epsilon, % Not an epsilon transition + Crs =/= epsilon, % Not an epsilon transition in_crs(Cr, Crs) ]. in_crs({C1,C2}, [{C3,C4}|_Crs]) when C1 >= C3, C2 =< C4 -> true; @@ -1436,7 +1435,7 @@ pack_trans([{{$\n,Cl},S}|Trs], Pt) -> pack_trans([{{Cf,Cl},S}|Trs], Pt) when Cf < $\n, Cl > $\n -> pack_trans([{{Cf,$\n-1},S},{{$\n+1,Cl},S}|Trs], [{$\n,S}|Pt]); %% Small ranges become singletons. -pack_trans([{{Cf,Cl},S}|Trs], Pt) when Cl == Cf + 1 -> +pack_trans([{{Cf,Cl},S}|Trs], Pt) when Cl =:= Cf + 1 -> pack_trans(Trs, [{Cf,S},{Cl,S}|Pt]); pack_trans([Tr|Trs], Pt) -> % The default uninteresting case pack_trans(Trs, Pt ++ [Tr]); diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl index b8b2b2308c..4119e2631b 100644 --- a/lib/parsetools/src/yecc.erl +++ b/lib/parsetools/src/yecc.erl @@ -1582,6 +1582,11 @@ find_action_conflicts2(Rs, Cxt0) -> find_reduce_reduce([R], Cxt) -> {R, Cxt}; +find_reduce_reduce([accept=A, #reduce{}=R | Rs], Cxt0) -> + Confl = conflict(R, A, Cxt0), + St = conflict_error(Confl, Cxt0#cxt.yecc), + Cxt = Cxt0#cxt{yecc = St}, + find_reduce_reduce([R | Rs], Cxt); find_reduce_reduce([#reduce{head = Categ1, prec = {P1, _}}=R1, #reduce{head = Categ2, prec = {P2, _}}=R2 | Rs], Cxt0) -> #cxt{res = Res0, yecc = St0} = Cxt0, @@ -1773,6 +1778,8 @@ add_conflict(Conflict, St) -> case Conflict of {Symbol, StateN, _, {reduce, _, _, _}} -> St#yecc{reduce_reduce = [{StateN,Symbol} |St#yecc.reduce_reduce]}; + {Symbol, StateN, _, {accept, _}} -> + St#yecc{reduce_reduce = [{StateN,Symbol} |St#yecc.reduce_reduce]}; {Symbol, StateN, _, {shift, _, _}} -> St#yecc{shift_reduce = [{StateN,Symbol} | St#yecc.shift_reduce]}; {_Symbol, _StateN, {one_level_up, _, _}, _Confl} -> @@ -1791,6 +1798,8 @@ conflict(#reduce{rule_nmbr = RuleNmbr1}, NewAction, Cxt) -> #cxt{terminal = Symbol, state_n = N, yecc = St} = Cxt, {R1, RuleLine1, RuleN1} = rule(RuleNmbr1, St), Confl = case NewAction of + accept -> + {accept, St#yecc.rootsymbol}; #reduce{rule_nmbr = RuleNmbr2} -> {R2, RuleLine2, RuleN2} = rule(RuleNmbr2, St), {reduce, R2, RuleN2, RuleLine2}; @@ -1830,7 +1839,10 @@ format_conflict({Symbol, N, Reduce, Confl}) -> {shift, NewState, Sym} -> io_lib:fwrite(<<" shift to state ~w, adding right " "sisters to ~s.">>, - [NewState, format_symbol(Sym)]) + [NewState, format_symbol(Sym)]); + {accept, Rootsymbol} -> + io_lib:fwrite(<<" reduce to rootsymbol ~s.">>, + [format_symbol(Rootsymbol)]) end, [S1, S2, S3]. @@ -1863,8 +1875,12 @@ format_conflict({Symbol, N, Reduce, Confl}) -> %% - "__Stack" has been substituted for "Stack"; %% - several states can share yeccpars2_S_cont(), which reduces code size; %% - instead if calling lists:nthtail() matching code is emitted. +%% +%% "1.4", parsetools-2.0.4: +%% - yeccerror() is called when a syntax error is found (as in version 1.1). +%% - the include file yeccpre.hrl has been changed. --define(CODE_VERSION, "1.3"). +-define(CODE_VERSION, "1.4"). -define(YECC_BUG(M, A), iolist_to_binary([" erlang:error({yecc_bug,\"",?CODE_VERSION,"\",", io_lib:fwrite(M, A), "}).\n\n"])). @@ -1994,14 +2010,16 @@ output_actions(St0, StateJumps, StateInfo) -> %% Not all the clauses of the dispatcher function yeccpars2() can %% be reached. Only when shifting, that is, calling yeccpars1(), %% will yeccpars2() be called. - Y2CL = [NewState || {_State,{Actions,_J}} <- StateJumps, - {_LA, #shift{state = NewState}} <- Actions], + Y2CL = [NewState || {_State,{Actions,J}} <- StateJumps, + {_LA, #shift{state = NewState}} <- + (Actions + ++ [A || {_Tag,_To,Part} <- [J], A <- Part])], Y2CS = ordsets:from_list([0 | Y2CL]), Y2S = ordsets:from_list([S || {S,_} <- StateJumps]), NY2CS = ordsets:subtract(Y2S, Y2CS), Sel = [{S,true} || S <- ordsets:to_list(Y2CS)] ++ [{S,false} || S <- ordsets:to_list(NY2CS)], - + SelS = [{State,Called} || {{State,_JActions}, {State,Called}} <- lists:zip(StateJumps, lists:keysort(1, Sel))], @@ -2078,7 +2096,7 @@ output_action(St0, State, Terminal, #shift{state = NewState}, IsFirst, _SI) -> output_action(St0, State, Terminal, accept, IsFirst, _SI) -> St10 = delim(St0, IsFirst), St = fwrite(St10, - <<"yeccpars2_~w(_S, ~s, _Ss, Stack, _T, _Ts, _Tzr) ->\n">>, + <<"yeccpars2_~w(_S, ~s, _Ss, Stack, _T, _Ts, _Tzr) ->\n">>, [State, quoted_atom(Terminal)]), fwrite(St, <<" {ok, hd(Stack)}">>, []); output_action(St, _State, _Terminal, nonassoc, _IsFirst, _SI) -> @@ -2092,13 +2110,11 @@ output_call_to_includefile(NewState, St) -> fwrite(St, <<" yeccpars1(S, ~w, Ss, Stack, T, Ts, Tzr)">>, [NewState]). -output_state_actions_fini(State, #yecc{includefile_version = {1,1}}=St0) -> - %% Backward compatibility. +output_state_actions_fini(State, St0) -> + %% Backward compatible. St10 = delim(St0, false), St = fwrite(St10, <<"yeccpars2_~w(_, _, _, _, T, _, _) ->\n">>, [State]), - fwrite(St, <<" yeccerror(T).\n\n">>, []); -output_state_actions_fini(_State, St) -> - fwrite(St, <<".\n\n">>, []). + fwrite(St, <<" yeccerror(T).\n\n">>, []). output_reduce(St0, State, Terminal0, #reduce{rule_nmbr = RuleNmbr, @@ -2402,7 +2418,7 @@ include1(Line, Inport, Outport, Nmbr_of_lines) -> include1(io:get_line(Inport, ''), Inport, Outport, Nmbr_of_lines + Incr). includefile_version([]) -> - {1,2}; + {1,4}; includefile_version(Includefile) -> case epp:open(Includefile, []) of {ok, Epp} -> @@ -2418,7 +2434,7 @@ includefile_version(Includefile) -> parse_file(Epp) -> case epp:parse_erl_form(Epp) of {ok, {function,_Line,yeccpars1,7,_Clauses}} -> - {1,2}; + {1,4}; {eof,_Line} -> {1,1}; _Form -> diff --git a/lib/parsetools/src/yeccparser.erl b/lib/parsetools/src/yeccparser.erl index 415547b4ce..63127802ee 100644 --- a/lib/parsetools/src/yeccparser.erl +++ b/lib/parsetools/src/yeccparser.erl @@ -38,16 +38,16 @@ line_of(Token) -> -type yecc_ret() :: {'error', _} | {'ok', _}. --spec parse(_) -> yecc_ret(). +-spec parse(Tokens :: list()) -> yecc_ret(). parse(Tokens) -> - yeccpars0(Tokens, false). + yeccpars0(Tokens, {no_func, no_line}, 0, [], []). --spec parse_and_scan({function() | {atom(), atom()}, [_]} | {atom(), atom(), [_]}) -> - yecc_ret(). +-spec parse_and_scan({function() | {atom(), atom()}, [_]} + | {atom(), atom(), [_]}) -> yecc_ret(). parse_and_scan({F, A}) -> % Fun or {M, F} - yeccpars0([], {F, A}); + yeccpars0([], {{F, A}, no_line}, 0, [], []); parse_and_scan({M, F, A}) -> - yeccpars0([], {{M, F}, A}). + yeccpars0([], {{{M, F}, A}, no_line}, 0, [], []). -spec format_error(any()) -> [char() | list()]. format_error(Message) -> @@ -60,54 +60,58 @@ format_error(Message) -> %% To be used in grammar files to throw an error message to the parser %% toplevel. Doesn't have to be exported! --compile({nowarn_unused_function,{return_error,2}}). +-compile({nowarn_unused_function, return_error/2}). -spec return_error(integer(), any()) -> no_return(). return_error(Line, Message) -> throw({error, {Line, ?MODULE, Message}}). --define(CODE_VERSION, "1.3"). +-define(CODE_VERSION, "1.4"). -yeccpars0(Tokens, MFA) -> - try yeccpars1(Tokens, MFA, 0, [], []) +yeccpars0(Tokens, Tzr, State, States, Vstack) -> + try yeccpars1(Tokens, Tzr, State, States, Vstack) catch error: Error -> Stacktrace = erlang:get_stacktrace(), try yecc_error_type(Error, Stacktrace) of - {syntax_error, Token} -> - yeccerror(Token); - {missing_in_goto_table=Tag, Symbol, State} -> - Desc = {Symbol, State, Tag}, + Desc -> erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, - Stacktrace) + Stacktrace) catch _:_ -> erlang:raise(error, Error, Stacktrace) end; - throw: {error, {_Line, ?MODULE, _M}} = Error -> - Error % probably from return_error/2 + %% Probably thrown from return_error/2: + throw: {error, {_Line, ?MODULE, _M}} = Error -> + Error end. -yecc_error_type(function_clause, [{?MODULE,F,[State,_,_,_,Token,_,_]} | _]) -> +yecc_error_type(function_clause, [{?MODULE,F,ArityOrArgs} | _]) -> case atom_to_list(F) of - "yeccpars2" ++ _ -> - {syntax_error, Token}; "yeccgoto_" ++ SymbolL -> {ok,[{atom,_,Symbol}],_} = erl_scan:string(SymbolL), - {missing_in_goto_table, Symbol, State} + State = case ArityOrArgs of + [S,_,_,_,_,_,_] -> S; + _ -> state_is_unknown + end, + {Symbol, State, missing_in_goto_table} end. -yeccpars1([Token | Tokens], Tokenizer, State, States, Vstack) -> - yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, - Tokenizer); -yeccpars1([], {F, A}, State, States, Vstack) -> +yeccpars1([Token | Tokens], Tzr, State, States, Vstack) -> + yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, Tzr); +yeccpars1([], {{F, A},_Line}, State, States, Vstack) -> case apply(F, A) of - {ok, Tokens, _Endline} -> - yeccpars1(Tokens, {F, A}, State, States, Vstack); - {eof, _Endline} -> - yeccpars1([], false, State, States, Vstack); + {ok, Tokens, Endline} -> + yeccpars1(Tokens, {{F, A}, Endline}, State, States, Vstack); + {eof, Endline} -> + yeccpars1([], {no_func, Endline}, State, States, Vstack); {error, Descriptor, _Endline} -> {error, Descriptor} end; -yeccpars1([], false, State, States, Vstack) -> - yeccpars2(State, '$end', States, Vstack, {'$end', 999999}, [], false). +yeccpars1([], {no_func, no_line}, State, States, Vstack) -> + Line = 999999, + yeccpars2(State, '$end', States, Vstack, yecc_end(Line), [], + {no_func, Line}); +yeccpars1([], {no_func, Endline}, State, States, Vstack) -> + yeccpars2(State, '$end', States, Vstack, yecc_end(Endline), [], + {no_func, Endline}). %% yeccpars1/7 is called from generated code. %% @@ -115,48 +119,73 @@ yeccpars1([], false, State, States, Vstack) -> %% yeccpars1/7 can be found by parsing the file without following %% include directives. yecc will otherwise assume that an old %% yeccpre.hrl is included (one which defines yeccpars1/5). -yeccpars1(State1, State, States, Vstack, Stack1, [Token | Tokens], - Tokenizer) -> +yeccpars1(State1, State, States, Vstack, Token0, [Token | Tokens], Tzr) -> yeccpars2(State, element(1, Token), [State1 | States], - [Stack1 | Vstack], Token, Tokens, Tokenizer); -yeccpars1(State1, State, States, Vstack, Stack1, [], {F, A}) -> - case apply(F, A) of - {ok, Tokens, _Endline} -> - yeccpars1(State1, State, States, Vstack, Stack1, Tokens, {F, A}); - {eof, _Endline} -> - yeccpars1(State1, State, States, Vstack, Stack1, [], false); - {error, Descriptor, _Endline} -> - {error, Descriptor} - end; -yeccpars1(State1, State, States, Vstack, Stack1, [], false) -> - yeccpars2(State, '$end', [State1 | States], [Stack1 | Vstack], - {'$end', 999999}, [], false). + [Token0 | Vstack], Token, Tokens, Tzr); +yeccpars1(State1, State, States, Vstack, Token0, [], {{_F,_A}, _Line}=Tzr) -> + yeccpars1([], Tzr, State, [State1 | States], [Token0 | Vstack]); +yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, no_line}) -> + Line = yecctoken_end_location(Token0), + yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack], + yecc_end(Line), [], {no_func, Line}); +yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, Line}) -> + yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack], + yecc_end(Line), [], {no_func, Line}). + +%% For internal use only. +yecc_end({Line,_Column}) -> + {'$end', Line}; +yecc_end(Line) -> + {'$end', Line}. + +yecctoken_end_location(Token) -> + try + {text, Str} = erl_scan:token_info(Token, text), + {line, Line} = erl_scan:token_info(Token, line), + Parts = re:split(Str, "\n"), + Dline = length(Parts) - 1, + Yline = Line + Dline, + case erl_scan:token_info(Token, column) of + {column, Column} -> + Col = byte_size(lists:last(Parts)), + {Yline, Col + if Dline =:= 0 -> Column; true -> 1 end}; + undefined -> + Yline + end + catch _:_ -> + yecctoken_location(Token) + end. -% For internal use only. yeccerror(Token) -> - Text = case catch erl_scan:token_info(Token, text) of - {text, Txt} -> Txt; - _ -> yecctoken2string(Token) - end, - Location = case catch erl_scan:token_info(Token, location) of - {location, Loc} -> Loc; - _ -> element(2, Token) - end, + Text = yecctoken_to_string(Token), + Location = yecctoken_location(Token), {error, {Location, ?MODULE, ["syntax error before: ", Text]}}. +yecctoken_to_string(Token) -> + case catch erl_scan:token_info(Token, text) of + {text, Txt} -> Txt; + _ -> yecctoken2string(Token) + end. + +yecctoken_location(Token) -> + case catch erl_scan:token_info(Token, location) of + {location, Loc} -> Loc; + _ -> element(2, Token) + end. + yecctoken2string({atom, _, A}) -> io_lib:write(A); yecctoken2string({integer,_,N}) -> io_lib:write(N); yecctoken2string({float,_,F}) -> io_lib:write(F); yecctoken2string({char,_,C}) -> io_lib:write_char(C); yecctoken2string({var,_,V}) -> io_lib:format("~s", [V]); yecctoken2string({string,_,S}) -> io_lib:write_unicode_string(S); -yecctoken2string({reserved_symbol, _, A}) -> io_lib:format("~w", [A]); -yecctoken2string({_Cat, _, Val}) -> io_lib:format("~w", [Val]); +yecctoken2string({reserved_symbol, _, A}) -> io_lib:write(A); +yecctoken2string({_Cat, _, Val}) -> io_lib:write(Val); yecctoken2string({dot, _}) -> "'.'"; yecctoken2string({'$end', _}) -> []; yecctoken2string({Other, _}) when is_atom(Other) -> - io_lib:format("~w", [Other]); + io_lib:write(Other); yecctoken2string(Other) -> io_lib:write(Other). @@ -164,7 +193,7 @@ yecctoken2string(Other) -> --file("yeccparser.erl", 168). +-file("yeccparser.erl", 196). yeccpars2(0=S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_0(S, Cat, Ss, Stack, T, Ts, Tzr); @@ -248,7 +277,9 @@ yeccpars2_0(S, integer, Ss, Stack, T, Ts, Tzr) -> yeccpars2_0(S, reserved_word, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 8, Ss, Stack, T, Ts, Tzr); yeccpars2_0(S, var, Ss, Stack, T, Ts, Tzr) -> - yeccpars1(S, 9, Ss, Stack, T, Ts, Tzr). + yeccpars1(S, 9, Ss, Stack, T, Ts, Tzr); +yeccpars2_0(_, _, _, _, T, _, _) -> + yeccerror(T). yeccpars2_1(S, atom, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 6, Ss, Stack, T, Ts, Tzr); @@ -267,10 +298,14 @@ yeccpars2_2(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_grammar(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). yeccpars2_3(S, '->', Ss, Stack, T, Ts, Tzr) -> - yeccpars1(S, 10, Ss, Stack, T, Ts, Tzr). + yeccpars1(S, 10, Ss, Stack, T, Ts, Tzr); +yeccpars2_3(_, _, _, _, T, _, _) -> + yeccerror(T). -yeccpars2_4(_S, '$end', _Ss, Stack, _T, _Ts, _Tzr) -> - {ok, hd(Stack)}. +yeccpars2_4(_S, '$end', _Ss, Stack, _T, _Ts, _Tzr) -> + {ok, hd(Stack)}; +yeccpars2_4(_, _, _, _, T, _, _) -> + yeccerror(T). yeccpars2_5(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_grammar(hd(Ss), Cat, Ss, Stack, T, Ts, Tzr). @@ -317,7 +352,9 @@ yeccpars2_13(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_symbols(hd(Nss), Cat, Nss, NewStack, T, Ts, Tzr). yeccpars2_14(S, dot, Ss, Stack, T, Ts, Tzr) -> - yeccpars1(S, 29, Ss, Stack, T, Ts, Tzr). + yeccpars1(S, 29, Ss, Stack, T, Ts, Tzr); +yeccpars2_14(_, _, _, _, T, _, _) -> + yeccerror(T). yeccpars2_15(S, '->', Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 18, Ss, Stack, T, Ts, Tzr); @@ -338,7 +375,9 @@ yeccpars2_15(S, reserved_word, Ss, Stack, T, Ts, Tzr) -> yeccpars2_15(S, string, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 26, Ss, Stack, T, Ts, Tzr); yeccpars2_15(S, var, Ss, Stack, T, Ts, Tzr) -> - yeccpars1(S, 27, Ss, Stack, T, Ts, Tzr). + yeccpars1(S, 27, Ss, Stack, T, Ts, Tzr); +yeccpars2_15(_, _, _, _, T, _, _) -> + yeccerror(T). yeccpars2_16(_S, Cat, Ss, Stack, T, Ts, Tzr) -> [_|Nss] = Ss, @@ -414,10 +453,14 @@ yeccpars2_29(_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_rule(hd(Nss), Cat, Nss, NewStack, T, Ts, Tzr). yeccpars2_30(S, dot, Ss, Stack, T, Ts, Tzr) -> - yeccpars1(S, 35, Ss, Stack, T, Ts, Tzr). + yeccpars1(S, 35, Ss, Stack, T, Ts, Tzr); +yeccpars2_30(_, _, _, _, T, _, _) -> + yeccerror(T). yeccpars2_31(S, dot, Ss, Stack, T, Ts, Tzr) -> - yeccpars1(S, 34, Ss, Stack, T, Ts, Tzr). + yeccpars1(S, 34, Ss, Stack, T, Ts, Tzr); +yeccpars2_31(_, _, _, _, T, _, _) -> + yeccerror(T). yeccpars2_32(S, string, Ss, Stack, T, Ts, Tzr) -> yeccpars1(S, 32, Ss, Stack, T, Ts, Tzr); @@ -486,7 +529,7 @@ yeccgoto_tokens(15=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccgoto_tokens(17=_S, Cat, Ss, Stack, T, Ts, Tzr) -> yeccpars2_28(_S, Cat, Ss, Stack, T, Ts, Tzr). --compile({inline,{yeccpars2_6_,1}}). +-compile({inline,yeccpars2_6_/1}). -file("yeccgramm.yrl", 44). yeccpars2_6_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -494,7 +537,7 @@ yeccpars2_6_(__Stack0) -> symbol ( __1 ) end | __Stack]. --compile({inline,{yeccpars2_7_,1}}). +-compile({inline,yeccpars2_7_/1}). -file("yeccgramm.yrl", 45). yeccpars2_7_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -502,7 +545,7 @@ yeccpars2_7_(__Stack0) -> symbol ( __1 ) end | __Stack]. --compile({inline,{yeccpars2_8_,1}}). +-compile({inline,yeccpars2_8_/1}). -file("yeccgramm.yrl", 46). yeccpars2_8_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -510,7 +553,7 @@ yeccpars2_8_(__Stack0) -> symbol ( __1 ) end | __Stack]. --compile({inline,{yeccpars2_9_,1}}). +-compile({inline,yeccpars2_9_/1}). -file("yeccgramm.yrl", 43). yeccpars2_9_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -518,14 +561,14 @@ yeccpars2_9_(__Stack0) -> symbol ( __1 ) end | __Stack]. --compile({inline,{yeccpars2_11_,1}}). +-compile({inline,yeccpars2_11_/1}). -file("yeccgramm.yrl", 40). yeccpars2_11_(__Stack0) -> [begin { erlang_code , [ { atom , 0 , '$undefined' } ] } end | __Stack0]. --compile({inline,{yeccpars2_12_,1}}). +-compile({inline,yeccpars2_12_/1}). -file("yeccgramm.yrl", 35). yeccpars2_12_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -533,7 +576,7 @@ yeccpars2_12_(__Stack0) -> [ __1 ] end | __Stack]. --compile({inline,{yeccpars2_13_,1}}). +-compile({inline,yeccpars2_13_/1}). -file("yeccgramm.yrl", 36). yeccpars2_13_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, @@ -541,7 +584,7 @@ yeccpars2_13_(__Stack0) -> [ __1 | __2 ] end | __Stack]. --compile({inline,{yeccpars2_16_,1}}). +-compile({inline,yeccpars2_16_/1}). -file("yeccgramm.yrl", 39). yeccpars2_16_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, @@ -549,7 +592,7 @@ yeccpars2_16_(__Stack0) -> { erlang_code , __2 } end | __Stack]. --compile({inline,{yeccpars2_17_,1}}). +-compile({inline,yeccpars2_17_/1}). -file("yeccgramm.yrl", 41). yeccpars2_17_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -557,7 +600,7 @@ yeccpars2_17_(__Stack0) -> [ __1 ] end | __Stack]. --compile({inline,{yeccpars2_18_,1}}). +-compile({inline,yeccpars2_18_/1}). -file("yeccgramm.yrl", 55). yeccpars2_18_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -565,7 +608,7 @@ yeccpars2_18_(__Stack0) -> { '->' , line_of ( __1 ) } end | __Stack]. --compile({inline,{yeccpars2_19_,1}}). +-compile({inline,yeccpars2_19_/1}). -file("yeccgramm.yrl", 56). yeccpars2_19_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -573,7 +616,7 @@ yeccpars2_19_(__Stack0) -> { ':' , line_of ( __1 ) } end | __Stack]. --compile({inline,{yeccpars2_24_,1}}). +-compile({inline,yeccpars2_24_/1}). -file("yeccgramm.yrl", 53). yeccpars2_24_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -581,7 +624,7 @@ yeccpars2_24_(__Stack0) -> { value_of ( __1 ) , line_of ( __1 ) } end | __Stack]. --compile({inline,{yeccpars2_25_,1}}). +-compile({inline,yeccpars2_25_/1}). -file("yeccgramm.yrl", 54). yeccpars2_25_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -589,7 +632,7 @@ yeccpars2_25_(__Stack0) -> { value_of ( __1 ) , line_of ( __1 ) } end | __Stack]. --compile({inline,{yeccpars2_28_,1}}). +-compile({inline,yeccpars2_28_/1}). -file("yeccgramm.yrl", 42). yeccpars2_28_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, @@ -597,7 +640,7 @@ yeccpars2_28_(__Stack0) -> [ __1 | __2 ] end | __Stack]. --compile({inline,{yeccpars2_29_,1}}). +-compile({inline,yeccpars2_29_/1}). -file("yeccgramm.yrl", 33). yeccpars2_29_(__Stack0) -> [__5,__4,__3,__2,__1 | __Stack] = __Stack0, @@ -605,7 +648,7 @@ yeccpars2_29_(__Stack0) -> { rule , [ __1 | __3 ] , __4 } end | __Stack]. --compile({inline,{yeccpars2_32_,1}}). +-compile({inline,yeccpars2_32_/1}). -file("yeccgramm.yrl", 37). yeccpars2_32_(__Stack0) -> [__1 | __Stack] = __Stack0, @@ -613,7 +656,7 @@ yeccpars2_32_(__Stack0) -> [ __1 ] end | __Stack]. --compile({inline,{yeccpars2_33_,1}}). +-compile({inline,yeccpars2_33_/1}). -file("yeccgramm.yrl", 38). yeccpars2_33_(__Stack0) -> [__2,__1 | __Stack] = __Stack0, @@ -621,7 +664,7 @@ yeccpars2_33_(__Stack0) -> [ __1 | __2 ] end | __Stack]. --compile({inline,{yeccpars2_34_,1}}). +-compile({inline,yeccpars2_34_/1}). -file("yeccgramm.yrl", 32). yeccpars2_34_(__Stack0) -> [__3,__2,__1 | __Stack] = __Stack0, @@ -629,7 +672,7 @@ yeccpars2_34_(__Stack0) -> { __1 , __2 } end | __Stack]. --compile({inline,{yeccpars2_35_,1}}). +-compile({inline,yeccpars2_35_/1}). -file("yeccgramm.yrl", 31). yeccpars2_35_(__Stack0) -> [__3,__2,__1 | __Stack] = __Stack0, diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index b5da414f7b..93949a074a 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -44,7 +44,7 @@ empty/1, prec/1, yeccpre/1, lalr/1, old_yecc/1, other_examples/1, bugs/1, - otp_5369/1, otp_6362/1, otp_7945/1, + otp_5369/1, otp_6362/1, otp_7945/1, otp_8483/1, otp_8486/1, improvements/1, otp_7292/1, otp_7969/1]). @@ -298,8 +298,8 @@ syntax(Config) when is_list(Config) -> {_,[{L1,_,{undefined_function,{yeccpars2_2_,1}}}, {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], []} = compile:file(Parserfile1, [basic_validation,return]), - ?line L1 = 24 + SzYeccPre, - ?line L2 = 31 + SzYeccPre + ?line L1 = 28 + SzYeccPre, + ?line L2 = 35 + SzYeccPre end(), %% Bad macro in action. OTP-7224. @@ -316,8 +316,8 @@ syntax(Config) when is_list(Config) -> {_,[{L1,_,{undefined_function,{yeccpars2_2_,1}}}, {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], []} = compile:file(Parserfile1, [basic_validation,return]), - ?line L1 = 24 + SzYeccPre, - ?line L2 = 31 + SzYeccPre + ?line L1 = 28 + SzYeccPre, + ?line L2 = 35 + SzYeccPre end(), %% Check line numbers. OTP-7224. @@ -1284,7 +1284,7 @@ other_examples(Config) when is_list(Config) -> ok. bugs(suite) -> - [otp_5369, otp_6362, otp_7945]. + [otp_5369, otp_6362, otp_7945, otp_8483, otp_8486]. otp_5369(doc) -> "OTP-5369. A bug in parse_and_scan reported on erlang questions."; @@ -1486,6 +1486,60 @@ otp_7945(Config) when is_list(Config) -> ?line {error,_} = erl_parse:parse([{atom,3,foo},{'.',2,9,9}]), ok. +otp_8483(doc) -> + "OTP-8483. reduce/accept conflict"; +otp_8483(suite) -> []; +otp_8483(Config) when is_list(Config) -> + Dir = ?privdir, + Input = filename:join(Dir, "bug.yrl"), + + Bug1 = <<"Nonterminals elem seq. + Terminals 'foo'. + Rootsymbol elem. + elem -> 'foo'. + elem -> seq. + seq -> elem. + seq -> seq elem.">>, + ?line ok = file:write_file(Input, Bug1), + Ret = [return, {report, true}], + ?line {error,[{_,[{none,yecc,{conflict,_}}, + {none,yecc,{conflict,_}}, + {none,yecc,{conflict,_}}]}], + [{_,[{none,yecc,{conflicts,1,3}}]}]} = + yecc:file(Input, Ret), + file:delete(Input), + ok. + +otp_8486(doc) -> + "OTP-8486."; +otp_8486(suite) -> []; +otp_8486(Config) when is_list(Config) -> + Ts = [{otp_8486,<<" + Nonterminals boolean command. + Terminals '(' ')' if then else true and or skip while do. + Rootsymbol command. + Left 100 or. + Left 200 and. + boolean -> '(' boolean ')' : '$2'. + boolean -> 'true' : b. + boolean -> boolean 'and' boolean : {a,'$1','$3'}. + boolean -> boolean 'or' boolean : {o,'$1','$3'}. + command -> 'skip' : s. + command -> 'if' boolean 'then' command 'else' command : + {i,'$2','$4','$6'}. + command -> 'while' boolean 'do' command : {w,'$2','$4'}. + + Erlang code. + -export([t/0]). + t() -> + {ok,{i,{o,b,b},s,s}} = + parse([{'if',1},{'true',1},{'or',1},{'true',1}, + {'then',1},{'skip',1},{'else',1},{'skip',1}]), + ok. + ">>,default,ok}], + ?line run(Config, Ts), + ok. + improvements(suite) -> [otp_7292, otp_7969]. @@ -1530,8 +1584,8 @@ otp_7292(Config) when is_list(Config) -> {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], [{_,[{16,_,{unused_function,{foo,0}}}]}]} = compile:file(Parserfile1, [basic_validation, return]), - ?line L1 = 34 + SzYeccPre, - ?line L2 = 41 + SzYeccPre + ?line L1 = 38 + SzYeccPre, + ?line L2 = 45 + SzYeccPre end(), YeccPre = filename:join(Dir, "yeccpre.hrl"), @@ -1548,8 +1602,8 @@ otp_7292(Config) when is_list(Config) -> {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], [{_,[{16,_,{unused_function,{foo,0}}}]}]} = compile:file(Parserfile1, [basic_validation, return]), - ?line L1 = 33 + SzYeccPre, - ?line L2 = 40 + SzYeccPre + ?line L1 = 37 + SzYeccPre, + ?line L2 = 44 + SzYeccPre end(), file:delete(YeccPre), diff --git a/lib/parsetools/vsn.mk b/lib/parsetools/vsn.mk index b1354e89d8..46915baed6 100644 --- a/lib/parsetools/vsn.mk +++ b/lib/parsetools/vsn.mk @@ -1 +1 @@ -PARSETOOLS_VSN = 2.0.2 +PARSETOOLS_VSN = 2.0.4 |