diff options
Diffstat (limited to 'lib/parsetools')
| -rw-r--r-- | lib/parsetools/doc/src/leex.xml | 38 | ||||
| -rw-r--r-- | lib/parsetools/doc/src/notes.xml | 22 | ||||
| -rw-r--r-- | lib/parsetools/include/yeccpre.hrl | 39 | ||||
| -rw-r--r-- | lib/parsetools/src/yecc.erl | 49 | ||||
| -rw-r--r-- | lib/parsetools/src/yeccparser.erl | 39 | ||||
| -rw-r--r-- | lib/parsetools/test/Makefile | 78 | ||||
| -rw-r--r-- | lib/parsetools/test/leex_SUITE.erl | 909 | ||||
| -rw-r--r-- | lib/parsetools/test/parsetools.spec | 1 | ||||
| -rw-r--r-- | lib/parsetools/test/yecc_SUITE.erl | 1795 | ||||
| -rw-r--r-- | lib/parsetools/vsn.mk | 2 |
10 files changed, 2884 insertions, 88 deletions
diff --git a/lib/parsetools/doc/src/leex.xml b/lib/parsetools/doc/src/leex.xml index c113b586df..12abfd244f 100644 --- a/lib/parsetools/doc/src/leex.xml +++ b/lib/parsetools/doc/src/leex.xml @@ -281,7 +281,7 @@ NAME = VALUE</code> <Regexp> : <Erlang code>.</code> <p>The <Regexp> must occur at the start of a line and not - include any blanks; use <c>\\t</c> and <c>\\s</c> to include TAB + include any blanks; use <c>\t</c> and <c>\s</c> to include TAB and SPACE characters in the regular expression. If <Regexp> matches then the corresponding <Erlang code> is evaluated to generate a token. With the Erlang code the following predefined @@ -344,7 +344,7 @@ D = [0-9] {D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}. -{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? : +{D}+\.{D}+((E|e)(\+|\-)?{D}+)? : {token,{float,TokenLine,list_to_float(TokenChars)}}.</code> <p>The Erlang code in the "Erlang code." section is written into @@ -367,7 +367,7 @@ D = [0-9] <tag><c>c</c></tag> <item><p>Matches the non-metacharacter c.</p> </item> - <tag><c>\\c</c></tag> + <tag><c>\c</c></tag> <item><p>Matches the escape sequence or literal character c.</p> </item> <tag><c>.</c></tag> @@ -410,33 +410,33 @@ D = [0-9] <p>The escape sequences allowed are the same as for Erlang strings:</p> <taglist> - <tag><c>\\b</c></tag> + <tag><c>\b</c></tag> <item><p>Backspace.</p></item> - <tag><c>\\f</c></tag> + <tag><c>\f</c></tag> <item><p>Form feed.</p></item> - <tag><c>\\n</c></tag> + <tag><c>\n</c></tag> <item><p>Newline (line feed).</p></item> - <tag><c>\\r</c></tag> + <tag><c>\r</c></tag> <item><p>Carriage return.</p></item> - <tag><c>\\t</c></tag> + <tag><c>\t</c></tag> <item><p>Tab.</p></item> - <tag><c>\\e</c></tag> + <tag><c>\e</c></tag> <item><p>Escape.</p></item> - <tag><c>\\v</c></tag> + <tag><c>\v</c></tag> <item><p>Vertical tab.</p></item> - <tag><c>\\s</c></tag> + <tag><c>\s</c></tag> <item><p>Space.</p></item> - <tag><c>\\d</c></tag> + <tag><c>\d</c></tag> <item><p>Delete.</p></item> - <tag><c>\\ddd</c></tag> + <tag><c>\ddd</c></tag> <item><p>The octal value <c>ddd</c>.</p></item> - <tag><c>\\xhh</c></tag> + <tag><c>\xhh</c></tag> <item><p>The hexadecimal value <c>hh</c>.</p></item> - <tag><c>\\x{h...}</c></tag> + <tag><c>\x{h...}</c></tag> <item><p>The hexadecimal value <c>h...</c>.</p></item> - <tag><c>\\c</c></tag> - <item><p>Any other character literally, for example <c>\\\\</c> for - backslash, <c>\\"</c> for <c>"</c>.</p> + <tag><c>\c</c></tag> + <item><p>Any other character literally, for example <c>\\</c> for + backslash, <c>\"</c> for <c>"</c>.</p> </item> </taglist> @@ -446,7 +446,7 @@ Atoms [a-z][0-9a-zA-Z_]* Variables [A-Z_][0-9a-zA-Z_]* -Floats (\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)?</code> +Floats (\+|-)?[0-9]+\.[0-9]+((E|e)(\+|-)?[0-9]+)?</code> <note><p>Anchoring a regular expression with <c>^</c> and <c>$</c> is not implemented in the current version of Leex and just diff --git a/lib/parsetools/doc/src/notes.xml b/lib/parsetools/doc/src/notes.xml index 2947517717..8a6f2c2714 100644 --- a/lib/parsetools/doc/src/notes.xml +++ b/lib/parsetools/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2009</year> + <year>1997</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Parsetools Release Notes</title> @@ -30,6 +30,22 @@ </header> <p>This document describes the changes made to the Parsetools application.</p> +<section><title>Parsetools 2.0.2</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Cleanups suggested by tidier and modernization of types + and specs.</p> + <p> + Own Id: OTP-8455</p> + </item> + </list> + </section> + +</section> + <section><title>Parsetools 2.0.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/parsetools/include/yeccpre.hrl b/lib/parsetools/include/yeccpre.hrl index 2ffa13d6a7..33a103d95f 100644 --- a/lib/parsetools/include/yeccpre.hrl +++ b/lib/parsetools/include/yeccpre.hrl @@ -1,52 +1,51 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % The parser generator will insert appropriate declarations before this line.% --type(yecc_ret() :: {'error', _} | {'ok', _}). +-type yecc_ret() :: {'error', _} | {'ok', _}. -spec parse(Tokens :: list()) -> yecc_ret(). parse(Tokens) -> yeccpars0(Tokens, {no_func, no_line}, 0, [], []). --spec(parse_and_scan/1 :: - ({function() | {atom(), atom()}, [_]} | {atom(), atom(), [_]}) -> - yecc_ret()). +-spec parse_and_scan({function() | {atom(), atom()}, [_]} | {atom(), atom(), [_]}) -> + yecc_ret(). parse_and_scan({F, A}) -> % Fun or {M, F} yeccpars0([], {{F, A}, no_line}, 0, [], []); parse_and_scan({M, F, A}) -> yeccpars0([], {{{M, F}, A}, no_line}, 0, [], []). --spec(format_error/1 :: (any()) -> [char() | list()]). +-spec format_error(any()) -> [char() | list()]. format_error(Message) -> case io_lib:deep_char_list(Message) of - true -> - Message; - _ -> - io_lib:write(Message) + true -> + Message; + _ -> + io_lib:write(Message) end. -% To be used in grammar files to throw an error message to the parser -% toplevel. Doesn't have to be exported! --compile({nowarn_unused_function, return_error/2}). --spec(return_error/2 :: (integer(), any()) -> no_return()). +%% To be used in grammar files to throw an error message to the parser +%% toplevel. Doesn't have to be exported! +-compile({nowarn_unused_function,{return_error,2}}). +-spec return_error(integer(), any()) -> no_return(). return_error(Line, Message) -> throw({error, {Line, ?MODULE, Message}}). @@ -85,7 +84,7 @@ yeccpars1([Token | Tokens], Tzr, State, States, Vstack) -> yeccpars1([], {{F, A},_Line}, State, States, Vstack) -> case apply(F, A) of {ok, Tokens, Endline} -> - yeccpars1(Tokens, {{F, A}, Endline}, State, States, Vstack); + yeccpars1(Tokens, {{F, A}, Endline}, State, States, Vstack); {eof, Endline} -> yeccpars1([], {no_func, Endline}, State, States, Vstack); {error, Descriptor, _Endline} -> @@ -118,7 +117,7 @@ yeccpars1(State1, State, States, Vstack, Token0, [], {no_func, Line}) -> yeccpars2(State, '$end', [State1 | States], [Token0 | Vstack], yecc_end(Line), [], {no_func, Line}). -% For internal use only. +%% For internal use only. yecc_end({Line,_Column}) -> {'$end', Line}; yecc_end(Line) -> diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl index f4d76f471d..b8b2b2308c 100644 --- a/lib/parsetools/src/yecc.erl +++ b/lib/parsetools/src/yecc.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% %% Yacc like LALR-1 parser generator for Erlang. @@ -30,8 +30,8 @@ -import(lists, [append/1, append/2, concat/1, delete/2, filter/2, flatmap/2, foldl/3, foldr/3, foreach/2, keydelete/3, - keysearch/3, keysort/2, last/1, map/2, member/2, - reverse/1, sort/1, usort/1]). + keysort/2, last/1, map/2, member/2, reverse/1, + sort/1, usort/1]). -include("erl_compile.hrl"). -include("ms_transform.hrl"). @@ -297,18 +297,18 @@ options(Options0, [Key | Keys], L) when is_list(Options0) -> false -> Options0 end, - V = case keysearch(Key, 1, Options) of - {value, {Key, Filename0}} when Key =:= includefile; - Key =:= parserfile -> + V = case lists:keyfind(Key, 1, Options) of + {Key, Filename0} when Key =:= includefile; + Key =:= parserfile -> case is_filename(Filename0) of no -> badarg; Filename -> {ok, [{Key, Filename}]} end; - {value, {Key, Bool}} when Bool =:= true; Bool =:= false -> - {ok, [{Key, Bool}]}; - {value, {Key, _}} -> + {Key, Bool} = KB when is_boolean(Bool) -> + {ok, [KB]}; + {Key, _} -> badarg; false -> {ok, [{Key, default_option(Key)}]} @@ -348,8 +348,7 @@ atom_option(verbose) -> {verbose, true}; atom_option(Key) -> Key. is_filename(T) -> - try filename:flatten(T) of - Filename -> Filename + try filename:flatten(T) catch error: _ -> no end. @@ -366,8 +365,8 @@ shorten_filename(Name0) -> start(Infilex, Options) -> Infile = assure_extension(Infilex, ".yrl"), - {value, {_, Outfilex0}} = keysearch(parserfile, 1, Options), - {value, {_, Includefilex}} = keysearch(includefile, 1, Options), + {_, Outfilex0} = lists:keyfind(parserfile, 1, Options), + {_, Includefilex} = lists:keyfind(includefile, 1, Options), Outfilex = case Outfilex0 of [] -> filename:rootname(Infilex, ".yrl"); _ -> Outfilex0 @@ -715,14 +714,14 @@ names(Symbols) -> map(fun(Symbol) -> Symbol#symbol.name end, Symbols). symbol_line(Name, St) -> - {value, #symbol{line = Line}} = symbol_search(Name, St#yecc.all_symbols), + #symbol{line = Line} = symbol_find(Name, St#yecc.all_symbols), Line. symbol_member(Symbol, Symbols) -> - symbol_search(Symbol#symbol.name, Symbols) =/= false. + symbol_find(Symbol#symbol.name, Symbols) =/= false. -symbol_search(Name, Symbols) -> - keysearch(Name, #symbol.name, Symbols). +symbol_find(Name, Symbols) -> + lists:keyfind(Name, #symbol.name, Symbols). states_and_goto_table(St0) -> St1 = create_symbol_table(St0), @@ -876,8 +875,8 @@ add_warnings(SymNames, W0, St0) -> check_rhs([#symbol{name = '$empty'}], St) -> St; check_rhs(Rhs, St0) -> - case symbol_search('$empty', Rhs) of - {value, #symbol{line = Line}} -> + case symbol_find('$empty', Rhs) of + #symbol{line = Line} -> add_error(Line, illegal_empty, St0); false -> foldl(fun(Sym, St) -> @@ -1096,10 +1095,10 @@ compute_states2([{Sym,Seed} | Seeds], N, Try, CurrState, StateTab, Tables) -> compute_states2(Seeds, N, Try, CurrState, StateTab, Tables); {merge, M, NewCurrent} -> %% io:fwrite(<<"Merging with state ~w\n">>, [M]), - Try1 = case keysearch(M, 1, Try) of + Try1 = case lists:keyfind(M, 1, Try) of false -> [{M, NewCurrent} | Try]; - {value, {_, OldCurrent}} -> + {_, OldCurrent} -> case ordsets:is_subset(NewCurrent, OldCurrent) of true -> Try; diff --git a/lib/parsetools/src/yeccparser.erl b/lib/parsetools/src/yeccparser.erl index 80a6bbce0e..415547b4ce 100644 --- a/lib/parsetools/src/yeccparser.erl +++ b/lib/parsetools/src/yeccparser.erl @@ -16,53 +16,52 @@ line_of(Token) -> -file("/clearcase/otp/erts/lib/parsetools/include/yeccpre.hrl", 0). %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % The parser generator will insert appropriate declarations before this line.% --type(yecc_ret() :: {'error', _} | {'ok', _}). +-type yecc_ret() :: {'error', _} | {'ok', _}. --spec(parse/1 :: (_) -> yecc_ret()). +-spec parse(_) -> yecc_ret(). parse(Tokens) -> yeccpars0(Tokens, false). --spec(parse_and_scan/1 :: - ({function() | {atom(), atom()}, [_]} | {atom(), atom(), [_]}) -> - yecc_ret()). +-spec parse_and_scan({function() | {atom(), atom()}, [_]} | {atom(), atom(), [_]}) -> + yecc_ret(). parse_and_scan({F, A}) -> % Fun or {M, F} yeccpars0([], {F, A}); parse_and_scan({M, F, A}) -> yeccpars0([], {{M, F}, A}). --spec(format_error/1 :: (any()) -> [char() | list()]). +-spec format_error(any()) -> [char() | list()]. format_error(Message) -> case io_lib:deep_char_list(Message) of - true -> - Message; - _ -> - io_lib:write(Message) + true -> + Message; + _ -> + io_lib:write(Message) end. -% To be used in grammar files to throw an error message to the parser -% toplevel. Doesn't have to be exported! +%% To be used in grammar files to throw an error message to the parser +%% toplevel. Doesn't have to be exported! -compile({nowarn_unused_function,{return_error,2}}). --spec(return_error/2 :: (integer(), any()) -> no_return()). +-spec return_error(integer(), any()) -> no_return(). return_error(Line, Message) -> throw({error, {Line, ?MODULE, Message}}). @@ -101,7 +100,7 @@ yeccpars1([Token | Tokens], Tokenizer, State, States, Vstack) -> yeccpars1([], {F, A}, State, States, Vstack) -> case apply(F, A) of {ok, Tokens, _Endline} -> - yeccpars1(Tokens, {F, A}, State, States, Vstack); + yeccpars1(Tokens, {F, A}, State, States, Vstack); {eof, _Endline} -> yeccpars1([], false, State, States, Vstack); {error, Descriptor, _Endline} -> @@ -123,7 +122,7 @@ yeccpars1(State1, State, States, Vstack, Stack1, [Token | Tokens], yeccpars1(State1, State, States, Vstack, Stack1, [], {F, A}) -> case apply(F, A) of {ok, Tokens, _Endline} -> - yeccpars1(State1, State, States, Vstack, Stack1, Tokens, {F, A}); + yeccpars1(State1, State, States, Vstack, Stack1, Tokens, {F, A}); {eof, _Endline} -> yeccpars1(State1, State, States, Vstack, Stack1, [], false); {error, Descriptor, _Endline} -> diff --git a/lib/parsetools/test/Makefile b/lib/parsetools/test/Makefile new file mode 100644 index 0000000000..19354b87b2 --- /dev/null +++ b/lib/parsetools/test/Makefile @@ -0,0 +1,78 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2005-2009. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# +include $(ERL_TOP)/make/target.mk +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +MODULES = \ + leex_SUITE \ + yecc_SUITE + +ERL_FILES= $(MODULES:%=%.erl) + +TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) +INSTALL_PROGS= $(TARGET_FILES) + +EMAKEFILE=Emakefile + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/parsetools_test + +# ---------------------------------------------------- +# FLAGS +# ---------------------------------------------------- +ERL_MAKE_FLAGS += +ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/test_server/include + +EBIN = . + +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- +.PHONY: make_emakefile + +make_emakefile: + $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES)\ + > $(EMAKEFILE) + +tests debug opt: make_emakefile + erl $(ERL_MAKE_FLAGS) -make + +clean: + rm -f $(EMAKEFILE) + rm -f $(TARGET_FILES) + rm -f core + +docs: + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_spec: opt + +release_tests_spec: make_emakefile + $(INSTALL_DIR) $(RELSYSDIR) + $(INSTALL_DATA) parsetools.spec $(EMAKEFILE) $(ERL_FILES) $(RELSYSDIR) + chmod -f -R u+w $(RELSYSDIR) + # @tar cf - *_SUITE_data | (cd $(RELSYSDIR); tar xf -) + +release_docs_spec: diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl new file mode 100644 index 0000000000..069f780b5e --- /dev/null +++ b/lib/parsetools/test/leex_SUITE.erl @@ -0,0 +1,909 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(leex_SUITE). + +%-define(debug, true). + +-include_lib("stdlib/include/erl_compile.hrl"). +-include_lib("kernel/include/file.hrl"). + +-ifdef(debug). +-define(line, put(line, ?LINE), ). +-define(config(X,Y), foo). +-define(datadir, "leex_SUITE_data"). +-define(privdir, "leex_SUITE_priv"). +-define(t, test_server). +-else. +-include("test_server.hrl"). +-define(datadir, ?config(data_dir, Config)). +-define(privdir, ?config(priv_dir, Config)). +-endif. + +-export([all/1, init_per_testcase/2, fin_per_testcase/2]). + +-export([checks/1, + file/1, compile/1, syntax/1, + examples/1, + pt/1, man/1, ex/1, ex2/1, not_yet/1]). + +% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +init_per_testcase(_Case, Config) -> + ?line Dog = ?t:timetrap(?default_timeout), + [{watchdog, Dog} | Config]. + +fin_per_testcase(_Case, Config) -> + Dog = ?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +all(suite) -> [checks, examples]. + +checks(suite) -> + [file, compile, syntax]. + +file(doc) -> + "Bad files and options."; +file(suite) -> []; +file(Config) when is_list(Config) -> + Dir = ?privdir, + Ret = [return, {report, false}], + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file("not_a_file", Ret), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file("not_a_file", [{return,true}]), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file("not_a_file", [{report,false},return_errors]), + ?line error = leex:file("not_a_file"), + ?line error = leex:file("not_a_file", [{return,false},report]), + ?line error = leex:file("not_a_file", [return_warnings,{report,false}]), + + Filename = filename:join(Dir, "file.xrl"), + file:delete(Filename), + + ?line {'EXIT', {badarg, _}} = (catch leex:file({foo})), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {parserfile,{foo}})), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {includefile,{foo}})), + + ?line {'EXIT', {badarg, _}} = (catch leex:file(Filename, no_option)), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, [return | report])), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {return,foo})), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, includefile)), + + Mini = <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" + "Erlang code.\n">>, + ?line ok = file:write_file(Filename, Mini), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [{scannerfile,"//"} | Ret]), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [{includefile,"//"} | Ret]), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [{includefile,"/ /"} | Ret]), + + LeexPre = filename:join(Dir, "leexinc.hrl"), + ?line ok = file:write_file(LeexPre, <<"syntax error.\n">>), + PreErrors = run_test(Config, Mini, LeexPre), + ?line {errors, + [{1,_,["syntax error before: ","error"]}, + {3,_,undefined_module}], + []} = + extract(LeexPre, PreErrors), + file:delete(LeexPre), + + Ret2 = [return, report_errors, report_warnings, verbose], + Scannerfile = filename:join(Dir, "file.erl"), + ?line ok = file:write_file(Scannerfile, <<"nothing">>), + ?line unwritable(Scannerfile), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, Ret2), + ?line writable(Scannerfile), + file:delete(Scannerfile), + + Dotfile = filename:join(Dir, "file.dot"), + ?line ok = file:write_file(Dotfile, <<"nothing">>), + ?line unwritable(Dotfile), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [dfa_graph | Ret2]), + ?line writable(Dotfile), + file:delete(Dotfile), + + file:delete(Filename), + ok. + +compile(doc) -> + "Check of compile/3."; +compile(suite) -> []; +compile(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + Scannerfile = filename:join(Dir, "file.erl"), + Mini = <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" + "Erlang code.\n">>, + ?line ok = file:write_file(Filename, Mini), + ?line error = leex:compile(Filename, "//", #options{}), + ?line ok = leex:compile(Filename, Scannerfile, #options{}), + file:delete(Scannerfile), + file:delete(Filename), + ok. + +syntax(doc) -> + "Syntax checks."; +syntax(suite) -> []; +syntax(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + Ret = [return, {report, true}], + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "%% comment\n" + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n + ">>), + ?line {error,[{_,[{7,leex,missing_code}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : \n">>), + ?line {error,[{_,[{5,leex,missing_code}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[] :">>), + ?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : .\n" + "[] : ">>), + ?line {error,[{_,[{5,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[] : .\n">>), + ?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ ">>), + ?line {error,[{_,[{5,leex,bad_rule}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ ; ">>), + ?line {error,[{_,[{4,leex,bad_rule}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[] : '99\n">>), + ?line {error,[{_,[{4,erl_scan,_}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n">>), + ?line {error,[{_,[{3,leex,empty_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "Erlang code.\n">>), + ?line {error,[{_,[{4,leex,empty_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n">>), + ?line {error,[{_,[{2,leex,missing_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Erlang code.\n">>), + ?line {error,[{_,[{3,leex,missing_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"">>), + %% This is a weird line: + ?line {error,[{_,[{0,leex,missing_defs}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Rules.\n">>), + ?line {error,[{_,[{1,leex,missing_defs}]}],[]} = leex:file(Filename, Ret), + + %% Check that correct line number is used in messages. + ErlFile = filename:join(Dir, "file.erl"), + Ret1 = [{scannerfile,ErlFile}|Ret], + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,\n" + " {word,TokenLine,TokenChars,\n" + " DDDD}}.\n" % unbound + "Erlang code.\n" + "an error.\n">>), % syntax error + ?line {ok, _, []} = leex:file(Filename, Ret1), + ?line {error, + [{_,[{8,_,["syntax error before: ","error"]}]}, + {_,[{6,_,{unbound_var,'DDDD'}}]}], + []} = + compile:file(ErlFile, [basic_validation, return]), + + %% Ignored characters + ?line ok = file:write_file(Filename, + <<"Definitions. D = [0-9]\n" + "Rules. [a-z] : .\n" + "1 : skip_token.\n" + "Erlang code. f() -> a.\n">>), + ?line {ok,_,[{_, + [{1,leex,ignored_characters}, + {2,leex,ignored_characters}, + {4,leex,ignored_characters}]}]} = + leex:file(Filename, Ret), + + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+\\ : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\"}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+\\x : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{illegal_char,"\\x"}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+\\x{ : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\x{"}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[^ab : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"["}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "(a : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"("}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[b-a] : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{char_class,"b-a"}}}]}],[]} = + leex:file(Filename, Ret), + + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "\\x{333333333333333333333333} : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp, + {illegal_char, + "\\x{333333333333333333333333}"}}}]}],[]} = + leex:file(Filename, Ret), + ok. + +examples(suite) -> + [pt,man,ex,ex2,not_yet]. + +pt(doc) -> + "Pushing back characters."; +pt(suite) -> []; +pt(Config) when is_list(Config) -> + %% Needs more testing... + Ts = [{pt_1, + <<"Definitions.\n" + "D = [0-9]\n" + "L = [a-z]\n" + + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" + "abc{D}+ : {skip_token,\"sture\" ++ string:substr(TokenChars, 4)}.\n" + "{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + "\\s : .\n" + "\\r\\n : {end_token,{crlf,TokenLine}}.\n" + + "Erlang code.\n" + "-export([t/0]).\n" + "t() -> + {ok,[{word,1,\"sture\"},{integer,1,123}],1} = + string(\"abc123\"), ok. ">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +man(doc) -> + "Examples from the manpage."; +man(suite) -> []; +man(Config) when is_list(Config) -> + Ts = [{man_1, + <<"Definitions.\n" + "Rules.\n" + "[a-z][0-9a-zA-Z_]* :\n" + " {token,{atom,TokenLine,list_to_atom(TokenChars)}}.\n" + "[A-Z_][0-9a-zA-Z_]* :\n" + " {token,{var,TokenLine,list_to_atom(TokenChars)}}.\n" + "(\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)? : \n" + " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" + "\\s : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{float,1,3.14},{atom,1,atom},{var,1,'V314'}],1} =\n" + " string(\"3.14atom V314\"),\n" + " ok.\n">>, + default, + ok}, + + {man_2, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{D}+ :\n" + " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + "{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? :\n" + " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" + "\\s : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{float,1,3.14},{integer,1,314}],1} = \n" + " string(\"3.14 314\"),\n" + " ok.\n">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +ex(doc) -> + "Examples."; +ex(suite) -> []; +ex(Config) when is_list(Config) -> + Ts = [{ex_1, + <<"Definitions.\n" + "D = [0-543-705-982]\n" + "Rules.\n" + "{D}+ :\n" + " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + "[^235]+ :\n" + " {token,{list_to_atom(TokenChars),TokenLine}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{integer,1,12},{' c\\na',1},{integer,2,34},{b789a,2}],2} =\n" + " string(\"12 c\\na34b789a\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_2, + <<"Definitions.\n" + "L = [a-z]\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,chars}.\n" + "zyx{D}+ : {token,zyx}.\n" + "\\s : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[chars,zyx],1} = string(\"abcdef zyx123\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_3, + <<"Definitions.\n" + "NL = [\\n]\n" + "Rules.\n" + "{NL}* : {token,newlines}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[],1} = string(\"\"), ok.\n">>, % string("a") would loop... + default, + ok}, + + {ex_4, + <<"Definitions.\n" + "SP1 = [\\n-\\s]\n" + "SP0 = [\\000-\\n]\n" + "Rules.\n" + "{SP0}+ : {token,{small,TokenChars}}.\n" + "{SP1}+ : {token,{big,TokenChars}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " string(\"\\x00\\n\\s\\n\\n\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_5, + <<"Definitions.\n" + "L = [a-z]\n" + "W = [\\s\\b\\n\\r\\t\\e\\v\\d\\f]\n" + "Rules.\n" + "\\[{L}+(,{L}+)*\\] : {token,{list,TokenChars}}.\n" + "\"{L}+\" : {token,{string,TokenChars}}.\n" + "\\$. : {token,{char,TokenChars}}.\n" + "{W}+ : {token,{white,TokenChars}}.\n" + "ff\\f+ : {token,{form,TokenChars}}.\n" + "\\$\\^+\\\\+ : {token,{other,TokenChars}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{white,\"\\b\\f\"}],1} = string(\"\\b\\f\"),\n" + " {ok,[{form,\"ff\\f\"}],1} = string(\"ff\\f\"),\n" + " {ok,[{string,\"\\\"foo\\\"\"}],1} = string(\"\\\"foo\\\"\"),\n" + " {ok,[{char,\"$.\"}],1} = string(\"$\\.\"),\n" + " {ok,[{list,\"[a,b,c]\"}],1} = string(\"[a,b,c]\"),\n" + " {ok,[{other,\"$^\\\\\"}],1} = string(\"$^\\\\\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_6, + <<"Definitions.\n" + "L = [a-z]\n" + "Rules.\n" + "L}+ : {token,{TokenChars,#r.f}}.\n" + "Erlang code.\n" + "-record(r, {f}).\n" + "-export([t/0]).\n" + "t() ->\n" + " string(\"abc\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_7, %% Assumes regexp can handle \x + <<"Definitions.\n" + "H1 = \\x11\\x{ab}\n" + "H2 = [\\x{30}\\x{ac}]\n" + "Rules.\n" + "{H1}{H2}+ : {token,{hex,TokenChars}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{hex,[17,171,48,172]}],1} =\n" + " string(\"\\x{11}\\xab0\\xac\"),\n" + " ok.\n">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +ex2(doc) -> + "More examples."; +ex2(suite) -> []; +ex2(Config) when is_list(Config) -> + Xrl = + <<" +%%% File : erlang_scan.xrl +%%% Author : Robert Virding +%%% Purpose : Tkoen definitions for Erlang. + +Definitions. +O = [0-7] +D = [0-9] +H = [0-9a-fA-F] +U = [A-Z] +L = [a-z] +A = ({U}|{L}|{D}|_|@) +WS = ([\\000-\\s]|%.*) + +Rules. +{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? : + {token,{float,TokenLine,list_to_float(TokenChars)}}. +{D}+#{H}+ : base(TokenLine, TokenChars). +{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}. +{L}{A}* : Atom = list_to_atom(TokenChars), + {token,case reserved_word(Atom) of + true -> {Atom,TokenLine}; + false -> {atom,TokenLine,Atom} + end}. +'(\\\\\\^.|\\\\.|[^'])*' : + %% Strip quotes. + S = lists:sublist(TokenChars, 2, TokenLen - 2), + case catch list_to_atom(string_gen(S)) of + {'EXIT',_} -> {error,\"illegal atom \" ++ TokenChars}; + Atom -> {token,{atom,TokenLine,Atom}} + end. +({U}|_){A}* : {token,{var,TokenLine,list_to_atom(TokenChars)}}. +\"(\\\\\\^.|\\\\.|[^\"])*\" : + %% Strip quotes. + S = lists:sublist(TokenChars, 2, TokenLen - 2), + {token,{string,TokenLine,string_gen(S)}}. +\\$(\\\\{O}{O}{O}|\\\\\\^.|\\\\.|.) : + {token,{char,TokenLine,cc_convert(TokenChars)}}. +-> : {token,{'->',TokenLine}}. +:- : {token,{':-',TokenLine}}. +\\|\\| : {token,{'||',TokenLine}}. +<- : {token,{'<-',TokenLine}}. +\\+\\+ : {token,{'++',TokenLine}}. +-- : {token,{'--',TokenLine}}. +=/= : {token,{'=/=',TokenLine}}. +== : {token,{'==',TokenLine}}. +=:= : {token,{'=:=',TokenLine}}. +/= : {token,{'/=',TokenLine}}. +>= : {token,{'>=',TokenLine}}. +=< : {token,{'=<',TokenLine}}. +<= : {token,{'<=',TokenLine}}. +<< : {token,{'<<',TokenLine}}. +>> : {token,{'>>',TokenLine}}. +:: : {token,{'::',TokenLine}}. +[]()[}{|!?/;:,.*+#<>=-] : + {token,{list_to_atom(TokenChars),TokenLine}}. +\\.{WS} : {end_token,{dot,TokenLine}}. +{WS}+ : skip_token. + +Erlang code. + +-export([reserved_word/1]). + +%% reserved_word(Atom) -> Bool +%% return 'true' if Atom is an Erlang reserved word, else 'false'. + +reserved_word('after') -> true; +reserved_word('begin') -> true; +reserved_word('case') -> true; +reserved_word('try') -> true; +reserved_word('cond') -> true; +reserved_word('catch') -> true; +reserved_word('andalso') -> true; +reserved_word('orelse') -> true; +reserved_word('end') -> true; +reserved_word('fun') -> true; +reserved_word('if') -> true; +reserved_word('let') -> true; +reserved_word('of') -> true; +reserved_word('query') -> true; +reserved_word('receive') -> true; +reserved_word('when') -> true; +reserved_word('bnot') -> true; +reserved_word('not') -> true; +reserved_word('div') -> true; +reserved_word('rem') -> true; +reserved_word('band') -> true; +reserved_word('and') -> true; +reserved_word('bor') -> true; +reserved_word('bxor') -> true; +reserved_word('bsl') -> true; +reserved_word('bsr') -> true; +reserved_word('or') -> true; +reserved_word('xor') -> true; +reserved_word('spec') -> true; +reserved_word(_) -> false. + +base(L, Cs) -> + H = string:chr(Cs, $#), + case list_to_integer(string:substr(Cs, 1, H-1)) of + B when B > 16 -> {error,\"illegal base\"}; + B -> + case base(string:substr(Cs, H+1), B, 0) of + error -> {error,\"illegal based number\"}; + N -> {token,{integer,L,N}} + end + end. + +base([C|Cs], Base, SoFar) when C >= $0, C =< $9, C < Base + $0 -> + Next = SoFar * Base + (C - $0), + base(Cs, Base, Next); +base([C|Cs], Base, SoFar) when C >= $a, C =< $f, C < Base + $a - 10 -> + Next = SoFar * Base + (C - $a + 10), + base(Cs, Base, Next); +base([C|Cs], Base, SoFar) when C >= $A, C =< $F, C < Base + $A - 10 -> + Next = SoFar * Base + (C - $A + 10), + base(Cs, Base, Next); +base([_|_], _, _) -> error; %Unknown character +base([], _, N) -> N. + +cc_convert([$$,$\\\\|Cs]) -> + hd(string_escape(Cs)); +cc_convert([$$,C]) -> C. + +string_gen([$\\\\|Cs]) -> + string_escape(Cs); +string_gen([C|Cs]) -> + [C|string_gen(Cs)]; +string_gen([]) -> []. + +string_escape([O1,O2,O3|S]) when + O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> + [(O1*8 + O2)*8 + O3 - 73*$0|string_gen(S)]; +string_escape([$^,C|Cs]) -> + [C band 31|string_gen(Cs)]; +string_escape([C|Cs]) when C >= $\\000, C =< $\\s -> + string_gen(Cs); +string_escape([C|Cs]) -> + [escape_char(C)|string_gen(Cs)]. + +escape_char($n) -> $\\n; %\\n = LF +escape_char($r) -> $\\r; %\\r = CR +escape_char($t) -> $\\t; %\\t = TAB +escape_char($v) -> $\\v; %\\v = VT +escape_char($b) -> $\\b; %\\b = BS +escape_char($f) -> $\\f; %\\f = FF +escape_char($e) -> $\\e; %\\e = ESC +escape_char($s) -> $\\s; %\\s = SPC +escape_char($d) -> $\\d; %\\d = DEL +escape_char(C) -> C. + ">>, + Dir = ?privdir, + XrlFile = filename:join(Dir, "erlang_scan.xrl"), + ?line ok = file:write_file(XrlFile, Xrl), + ErlFile = filename:join(Dir, "erlang_scan.erl"), + ?line {ok, _} = leex:file(XrlFile, []), + ?line {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), + code:purge(erlang_scan), + AbsFile = filename:rootname(ErlFile, ".erl"), + code:load_abs(AbsFile, erlang_scan), + + F = fun(Cont, Chars, Location) -> + erlang_scan:tokens(Cont, Chars, Location) + end, + F1 = fun(Cont, Chars, Location) -> + erlang_scan:token(Cont, Chars, Location) + end, + fun() -> + S = "ab cd. ", + {ok, Ts, 1} = scan_tokens_1(S, F, 1), + {ok, Ts, 1} = scan_token_1(S, F1, 1), + {ok, Ts, 1} = scan_tokens(S, F, 1), + {ok, Ts, 1} = erlang_scan:string(S, 1) + end(), + fun() -> + S = "'ab\n cd'. ", + {ok, Ts, 2} = scan_tokens_1(S, F, 1), + {ok, Ts, 2} = scan_token_1(S, F1, 1), + {ok, Ts, 2} = scan_tokens(S, F, 1), + {ok, Ts, 2} = erlang_scan:string(S, 1) + end(), + fun() -> + S = "99. ", + {ok, Ts, 1} = scan_tokens_1(S, F, 1), + {ok, Ts, 1} = scan_token_1(S, F1, 1), + {ok, Ts, 1} = scan_tokens(S, F, 1), + {ok, Ts, 1} = erlang_scan:string(S, 1) + end(), + {ok,[{integer,1,99},{dot,1}],1} = erlang_scan:string("99. "), + fun() -> + Atom = "'" ++ lists:duplicate(1000,$a) ++ "'", + S = Atom ++ ". ", + Reason = "illegal atom " ++ Atom, + Err = {error,{1,erlang_scan,{user,Reason}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[]} = scan_token_1(S, F1, 1), + {done,Err,[]} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa}. ", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,[]} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa} + 1. 34", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa} \x{bbb}. 34", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa} 18#34. 34", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa}"++eof, + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,eof} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,eof} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + ok. + +scan_tokens(String, Fun, Location) -> + scan_tokens(String, Fun, Location, []). + +scan_tokens(String, Fun, Location, Rs) -> + case Fun([], String, Location) of + {done, {error,_,_}, _} = Error -> + Error; + {done, {ok,Ts,End}, ""} -> + {ok, lists:append(lists:reverse([Ts|Rs])), End}; + {done, {ok,Ts,End}, Rest} -> + scan_tokens(Rest, Fun, End, [Ts|Rs]) + end. + +scan_tokens_1(String, Fun, Location) -> + scan_tokens_1({more, []}, String, Fun, Location, []). + +scan_tokens_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) -> + Error; +scan_tokens_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) -> + {ok,lists:append(lists:reverse([Ts|Rs])),End}; +scan_tokens_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) -> + scan_tokens_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]); +scan_tokens_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> + R = Fun(Cont, [C], Loc), + scan_tokens_1(R, Cs, Fun, Loc, Rs); +scan_tokens_1({more, Cont}, eof, Fun, Loc, Rs) -> + R = Fun(Cont, eof, Loc), + scan_tokens_1(R, eof, Fun, Loc, Rs). + +scan_token_1(String, Fun, Location) -> + scan_token_1({more, []}, String, Fun, Location, []). + +scan_token_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) -> + Error; +scan_token_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) -> + {ok,lists:reverse([Ts|Rs]),End}; +scan_token_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) -> + scan_token_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]); +scan_token_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> + R = Fun(Cont, [C], Loc), + scan_token_1(R, Cs, Fun, Loc, Rs). + +%% End of ex2 + +not_yet(doc) -> + "Not yet implemented."; +not_yet(suite) -> []; +not_yet(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + Ret = [return, {report, true}], + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "Rules.\n" + "$ : .\n" + "Erlang code.\n">>), + ?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "Rules.\n" + "^ : .\n" + "Erlang code.\n">>), + ?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + + ok. + +unwritable(Fname) -> + {ok, Info} = file:read_file_info(Fname), + Mode = Info#file_info.mode - 8#00200, + ok = file:write_file_info(Fname, Info#file_info{mode = Mode}). + +writable(Fname) -> + {ok, Info} = file:read_file_info(Fname), + Mode = Info#file_info.mode bor 8#00200, + ok = file:write_file_info(Fname, Info#file_info{mode = Mode}). + +run(Config, Tests) -> + F = fun({N,P,Pre,E}) -> + case catch run_test(Config, P, Pre) of + E -> + ok; + Bad -> + ?t:format("~nTest ~p failed. Expected~n ~p~n" + "but got~n ~p~n", [N, E, Bad]), + fail() + end + end, + lists:foreach(F, Tests). + +run_test(Config, Def, Pre) -> + %% io:format("testing ~s~n", [binary_to_list(Def)]), + DefFile = 'leex_test.xrl', + Filename = 'leex_test.erl', + DataDir = ?privdir, + XrlFile = filename:join(DataDir, DefFile), + ErlFile = filename:join(DataDir, Filename), + Opts = [return, warn_unused_vars,{outdir,DataDir}], + ok = file:write_file(XrlFile, Def), + LOpts = [return, {report, false} | + case Pre of + default -> + []; + _ -> + [{includefile,Pre}] + end], + XOpts = [verbose, dfa_graph], % just to get some code coverage... + LRet = leex:file(XrlFile, XOpts ++ LOpts), + case LRet of + {ok, _Outfile, _LWs} -> + CRet = compile:file(ErlFile, Opts), + case CRet of + {ok, _M, _Ws} -> + AbsFile = filename:rootname(ErlFile, ".erl"), + Mod = leex_test, + code:purge(Mod), + code:load_abs(AbsFile, Mod), + Mod:t(); + %% warnings(ErlFile, Ws); + {error, [{ErlFile,Es}], []} -> {error, Es, []}; + {error, [{ErlFile,Es}], [{ErlFile,Ws}]} -> {error, Es, Ws}; + Error -> Error + end; + {error, [{XrlFile,LEs}], []} -> {error, LEs, []}; + {error, [{XrlFile,LEs}], [{XrlFile,LWs}]} -> {error, LEs, LWs}; + LError -> LError + end. + +extract(File, {error, Es, Ws}) -> + {errors, extract(File, Es), extract(File, Ws)}; +extract(File, Ts) -> + lists:append([T || {F, T} <- Ts, F =:= File]). + +fail() -> + ?t:fail(). diff --git a/lib/parsetools/test/parsetools.spec b/lib/parsetools/test/parsetools.spec new file mode 100644 index 0000000000..5b34633378 --- /dev/null +++ b/lib/parsetools/test/parsetools.spec @@ -0,0 +1 @@ +{topcase, {dir, "../parsetools_test"}}. diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl new file mode 100644 index 0000000000..b5da414f7b --- /dev/null +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -0,0 +1,1795 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(yecc_SUITE). + +%-define(debug, true). + +-include_lib("stdlib/include/erl_compile.hrl"). + +-ifdef(debug). +-define(line, put(line, ?LINE), ). +-define(config(X,Y), foo). +-define(datadir, "yecc_SUITE_data"). +-define(privdir, "yecc_SUITE_priv"). +-define(t, test_server). +-else. +-include("test_server.hrl"). +-define(datadir, ?config(data_dir, Config)). +-define(privdir, ?config(priv_dir, Config)). +-endif. + +-export([all/1, init_per_testcase/2, fin_per_testcase/2]). + +-export([app_test/1, + checks/1, + file/1, syntax/1, compile/1, rules/1, expect/1, + conflicts/1, + examples/1, + empty/1, prec/1, yeccpre/1, lalr/1, old_yecc/1, + other_examples/1, + bugs/1, + otp_5369/1, otp_6362/1, otp_7945/1, + improvements/1, + otp_7292/1, otp_7969/1]). + +% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +init_per_testcase(_Case, Config) -> + ?line Dog = ?t:timetrap(?default_timeout), + [{watchdog, Dog} | Config]. + +fin_per_testcase(_Case, Config) -> + Dog = ?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +all(suite) -> [app_test, checks, examples, bugs, improvements]. + +app_test(doc) -> + ["Tests the applications consistency."]; +app_test(suite) -> + []; +app_test(Config) when is_list(Config) -> + ?line ok=?t:app_test(parsetools), + ok. + +checks(suite) -> + [file, syntax, compile, rules, expect, conflicts]. + +file(doc) -> + "Bad files and options."; +file(suite) -> []; +file(Config) when is_list(Config) -> + Dir = ?privdir, + Ret = [return, {report, false}], + ?line {error,[{_,[{none,yecc,{file_error,_}}]}],[]} = + yecc:file("not_a_file", Ret), + ?line {error,[{_,[{none,yecc,{file_error,_}}]}],[]} = + yecc:file("not_a_file", [{return,true}]), + ?line {error,[{_,[{none,yecc,{file_error,_}}]}],[]} = + yecc:file("not_a_file", [{report,false},return_errors]), + ?line error = yecc:file("not_a_file"), + ?line error = yecc:file("not_a_file", [{return,false},report]), + ?line error = yecc:file("not_a_file", [return_warnings,{report,false}]), + Filename = filename:join(Dir, "file.yrl"), + file:delete(Filename), + + ?line {'EXIT', {badarg, _}} = (catch yecc:file({foo})), + ?line {'EXIT', {badarg, _}} = + (catch yecc:file(Filename, {parserfile,{foo}})), + ?line {'EXIT', {badarg, _}} = + (catch yecc:file(Filename, {includefile,{foo}})), + + ?line {'EXIT', {badarg, _}} = (catch yecc:file(Filename, no_option)), + ?line {'EXIT', {badarg, _}} = + (catch yecc:file(Filename, [return | report])), + ?line {'EXIT', {badarg, _}} = + (catch yecc:file(Filename, {return,foo})), + ?line {'EXIT', {badarg, _}} = + (catch yecc:file(Filename, includefile)), + + Mini = <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + nt -> t.">>, + ?line ok = file:write_file(Filename, Mini), + ?line {error,[{_,[{none,yecc,{file_error,_}}]}],[]} = + yecc:file(Filename, [{parserfile,"//"} | Ret]), + + ?line {error,[{_,[{none,yecc,{file_error,_}}]}],[]} = + yecc:file(Filename, [{includefile,"//"} | Ret]), + ?line {error,[{_,[{none,yecc,{file_error,_}}]}],[]} = + yecc:file(Filename, [{includefile,"/ /"} | Ret]), + + YeccPre = filename:join(Dir, "yeccpre.hrl"), + ?line ok = file:write_file(YeccPre, <<"syntax error. ">>), + PreErrors1 = run_test(Config, Mini, YeccPre), + ?line {errors,[_],[]} = extract(YeccPre, PreErrors1), + ?line ok = file:write_file(YeccPre, my_yeccpre()), + ?line {'EXIT', {undef,_}} = (catch run_test(Config, Mini, YeccPre)), + + MiniCode = <<" + Nonterminals nt. + Terminals t. + Rootsymbol nt. + nt -> t. + Erlang code. + ">>, + ?line {'EXIT', {undef,_}} = (catch run_test(Config, MiniCode, YeccPre)), + + file:delete(YeccPre), + file:delete(Filename), + + ok. + +syntax(doc) -> + "Syntax checks."; +syntax(suite) -> []; +syntax(Config) when is_list(Config) -> + Dir = ?privdir, + %% Report errors. Very simple test of format_error/1. + Ret = [return, {report, true}], + Filename = filename:join(Dir, "file.yrl"), + Parserfile1 = filename:join(Dir, "a file"), + + ?line ok = file:write_file(Filename, <<"">>), + ?line {error,[{_,[{none,yecc,no_grammar_rules}, + {none,yecc,nonterminals_missing}, + {none,yecc,rootsymbol_missing}, + {none,yecc,terminals_missing}]}],[]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename, <<"Nonterminals">>), + ?line {error,[{_,[{_,yecc,{error,yeccparser,_}}]}],[]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename, <<"Nonterminals nt.">>), + ?line {error,[{_,[{none,yecc,no_grammar_rules}, + {none,yecc,rootsymbol_missing}, + {none,yecc,terminals_missing}]}],[]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename, <<"Nonterminals nt. Terminals t.">>), + ?line {error,[{_,[{none,yecc,no_grammar_rules}, + {none,yecc,rootsymbol_missing}]}],[]} = + yecc:file(Filename, Ret), + + %% Nonterminals and terminals not disjoint. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt 't t'. Terminals t 't t'. Rootsymbol nt.">>), + ?line {error,[{_,[{1,yecc,{symbol_terminal_and_nonterminal,'t t'}}, + {none,yecc,no_grammar_rules}]}], + []} = yecc:file(Filename, Ret), + + %% Rootsymbol is not a nonterminal. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. + Rootsymbol t. nt -> t.">>), + ?line {error,[{_,[{2,yecc,{bad_rootsymbol,t}}]}],[]} = + yecc:file(Filename, Ret), + + %% Rootsymbol is not a nonterminal. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. + Rootsymbol t. nt -> t.">>), + ?line {error,[{_,[{2,yecc,{bad_rootsymbol,t}}]}],[]} = + yecc:file(Filename, Ret), + + %% Endsymbol is a nonterminal. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol nt. + Endsymbol nt. + nt -> t.">>), + ?line {error,[{_,[{2,yecc,{endsymbol_is_nonterminal,nt}}]}],[]} = + yecc:file(Filename, Ret), + + %% Endsymbol is a terminal. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol nt. + Endsymbol t. + nt -> t.">>), + ?line {error,[{_,[{2,yecc,{endsymbol_is_terminal,t}}]}],[]} = + yecc:file(Filename, Ret), + + %% No grammar rules. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol nt. Endsymbol e.">>), + ?line {error,[{_,[{none,yecc,no_grammar_rules}]}],[]} = + yecc:file(Filename, Ret), + + %% Bad declaration. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol nt. Endsymbol e. + nt -> t. e e.">>), + ?line {ok,_,[{_,[{2,yecc,bad_declaration}]}]} = + yecc:file(Filename, Ret), + + %% Bad declaration. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. + Rootsymbol nt nt. Rootsymbol nt. Endsymbol e. + nt -> t.">>), + ?line {ok,_,[{_,[{2,yecc,bad_declaration}]}]} = + yecc:file(Filename, Ret), + + %% Syntax error found by yeccparser. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol nt. Endsymbol e. + a - a.">>), + ?line {error,[{_,[{2,yecc,{error,_yeccparser,_}}]}],[]} = + yecc:file(Filename, Ret), + + %% Syntax error: unknown nonterminal. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol nt. Endsymbol e. + 'unknown ' -> t.">>), + ?line {error,[{_,[{2,yecc,{undefined_nonterminal,'unknown '}}]}],[]} = + yecc:file(Filename, Ret), + + %% Undefined rhs symbols. Note quotes in output. + ?line ok = file:write_file(Filename, + <<"Nonterminals Nonterminals nt. + Terminals t Terminals. + Rootsymbol nt. + Endsymbol e. + nt -> Nonterminals. + Nonterminals -> Terminals receive foo 45 + '17' 'a b'.">>), + ?line {error,[{_,[{6,yecc,{undefined_symbol,45}}, + {6,yecc,{undefined_symbol,foo}}, + {6,yecc,{undefined_symbol,'receive'}}, + {7,yecc,{undefined_symbol,'17'}}, + {7,yecc,{undefined_symbol,'a b'}}]}],[]} = + yecc:file(Filename, Ret), + + %% '$empty' used early, before Terminals. OK. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. + nt -> '$empty'. + Terminals t. + Rootsymbol nt. + Endsymbol e.">>), + ?line {ok,_,[{_,[{3,yecc,{unused_terminal,t}}]}]} = + yecc:file(Filename, Ret), + + %% Illegal use of '$empty'. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt nt2. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> t. + nt2 -> t '$empty'.">>), + ?line {error,[{_,[{6,yecc,illegal_empty}]}],[]} = + yecc:file(Filename, Ret), + + ParserFile3 = [{parserfile, Parserfile1}], + + %% Bad Erlang expression in action. Changed in OTP-7224. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> t : a bad code.">>), + ?line {ok, _, []} = yecc:file(Filename, ParserFile3 ++ Ret), + + SzYeccPre = yeccpre_size(), + %% Note: checking the line numbers. Changes when yeccpre.hrl changes. + fun() -> + ?line {error,[{_,[{5,_,["syntax error before: ","bad"]}]}, + {_,[{L1,_,{undefined_function,{yeccpars2_2_,1}}}, + {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], + []} = compile:file(Parserfile1, [basic_validation,return]), + ?line L1 = 24 + SzYeccPre, + ?line L2 = 31 + SzYeccPre + end(), + + %% Bad macro in action. OTP-7224. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> t : ?F(3).">>), + ?line {ok, _, []} = yecc:file(Filename, ParserFile3 ++ Ret), + %% Note: checking the line numbers. Changes when yeccpre.hrl changes. + fun() -> + ?line {error,[{_,[{5,_,{undefined,'F',1}}]}, + {_,[{L1,_,{undefined_function,{yeccpars2_2_,1}}}, + {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], + []} = compile:file(Parserfile1, [basic_validation,return]), + ?line L1 = 24 + SzYeccPre, + ?line L2 = 31 + SzYeccPre + end(), + + %% Check line numbers. OTP-7224. + ?line ok = file:write_file(Filename, + <<"Terminals t. + Nonterminals nt. + Rootsymbol nt. + Endsymbol e. + nt -> t : ?F(3). + Erlang code. + -define(F(X), X). + t() -> + bad().">>), + ?line {ok, _, []} = yecc:file(Filename, ParserFile3 ++ Ret), + ?line {error,[{_,[{9,_,{undefined_function,{bad,0}}}]}], + [{_,[{8,_,{unused_function,{t,0}}}]}]} + = compile:file(Parserfile1, [basic_validation, return]), + + %% Terminals defined before nonterminals. (One of many checks...) + %% Used to give an error message, but now allowed. + ?line ok = file:write_file(Filename, + <<"Terminals t. + Nonterminals nt. + Rootsymbol nt. + Endsymbol e. + nt -> t. + Erlang code.">>), + ?line {ok, _, []} = yecc:file(Filename, Ret), + + %% Precedence with swapped arguments. Bad declaration. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> t. + Right t. + Left nt 100.">>), + ?line {ok,_,[{_,[{6,yecc,bad_declaration},{7,yecc,bad_declaration}]}]} = + yecc:file(Filename, Ret), + + %% Precedence with unknown operator. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> t. + Unary 100 '-'.">>), + ?line {error,[{_,[{6,yecc,{precedence_op_is_unknown,'-'}}]}],[]} = + yecc:file(Filename, Ret), + + %% Precedence with endsymbol operator. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> t. + Unary 100 e.">>), + ?line {error,[{_,[{6,yecc,{precedence_op_is_endsymbol,e}}]}],[]} = + yecc:file(Filename, Ret), + + %% Duplicated precedence. + ?line ok = file:write_file(Filename, <<" + Nonterminals nt. + Terminals t '+'. + Rootsymbol nt. + nt -> t '+' nt. + Left 100 '+'. + Right 200 t. + Left 200 '+'. + Left 200 '+'. + Right 200 t.">>), + ?line {error,[{_,[{8,yecc,{duplicate_precedence,'+'}}, + {9,yecc,{duplicate_precedence,'+'}}, + {10,yecc,{duplicate_precedence,t}}]}], + []} = yecc:file(Filename, Ret), + + %% Duplicated nonterminal. + ?line ok = file:write_file(Filename, + <<"Nonterminals 'n t' 'n t'. Terminals t. + Rootsymbol 'n t'. 'n t' -> t.">>), + ?line {ok, _, [{_,[{1,yecc,{duplicate_nonterminal,'n t'}}]}]} = + yecc:file(Filename, Ret), + ?line {ok, _, [{_,[{1,yecc,{duplicate_nonterminal,'n t'}}]}]} = + yecc:file(Filename, [return_warnings, {report, false}]), + ?line {ok, _} = yecc:file(Filename), + ?line {ok, _} = + yecc:file(Filename, [{report,false}, {return_warnings,false}]), + + %% Duplicated terminal. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals 't t' 't t'. + Rootsymbol nt. nt -> 't t'.">>), + ?line {ok, _, [{_,[{1,yecc,{duplicate_terminal,'t t'}}]}]} = + yecc:file(Filename, Ret), + + %% Two Nonterminals declarations. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. + Nonterminals nt2. + Rootsymbol nt2. nt -> t. nt2 -> nt.">>), + ?line {error,[{_,[{2,yecc,{duplicate_declaration,'Nonterminals'}}]}], + []} = yecc:file(Filename, Ret), + + %% Three Terminals declarations. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. + Terminals t1. + Terminals t1. + Rootsymbol nt. nt -> t t1.">>), + ?line {error,[{_,[{2,yecc,{duplicate_declaration,'Terminals'}}, + {3,yecc,{duplicate_declaration,'Terminals'}}]}], + []} = yecc:file(Filename, Ret), + + %% Two root symbols. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol t. + Rootsymbol nt. nt -> t.">>), + ?line {error,[{_,[{2,yecc,{duplicate_declaration,'Rootsymbol'}}]}],[]} = + yecc:file(Filename, Ret), + + %% Two end symbols. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol t. + Endsymbol e. + Endsymbol e. nt -> t.">>), + ?line {error,[{_,[{3,yecc,{duplicate_declaration,'Endsymbol'}}]}],[]} = + yecc:file(Filename, Ret), + + %% Two end symbols. + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol t. + Expect 1. + Expect 0. + Endsymbol e. nt -> t.">>), + ?line {error,[{_,[{3,yecc,{duplicate_declaration,'Expect'}}]}],[]} = + yecc:file(Filename, Ret), + + %% Some words should not be used. + ?line ok = file:write_file(Filename, <<" + Terminals '$empty' '$end'. + Nonterminals '$undefined'. + Rootsymbol '$undefined'. + Endsymbol '$end'. + '$undefined' -> '$empty'. + ">>), + ?line {error,[{_,[{2,yecc,{reserved,'$empty'}}, + {2,yecc,{reserved,'$end'}}, + {3,yecc,{reserved,'$undefined'}}, + {5,yecc,{endsymbol_is_terminal,'$end'}}]}],[]} = + yecc:file(Filename, Ret), + + %% Undefined pseudo variable. + ?line ok = file:write_file(Filename,<<" + Nonterminals nt. + Terminals t. + Rootsymbol nt. + nt -> t : '$2'. + ">>), + ?line {error,[{_,[{5,yecc,{undefined_pseudo_variable,'$2'}}]}],[]} = + yecc:file(Filename, Ret), + + %% Space in module name. + ?line ok = file:write_file(Filename, <<" + Nonterminals list. + Terminals element. + Rootsymbol list. + + list -> element : {single, '$1'}. + list -> list element : {pair, '$1', '$2'}. + + Erlang code. + + -export([t/0]). + + t() -> + L = [{element, 1}, {element, 2}, {element, 3}], + {ok,_} = parse(L), + ok. + ">>), + + Parserfile2 = filename:join(Dir, "a \"file\""), + %% The parser (yeccgramm.yrl) allows many symbol names... + ?line ok = file:write_file(Filename, <<" + Nonterminals Nonterminals Rootsymbol ':' '->'. + Terminals Terminals. + Rootsymbol Rootsymbol. + Endsymbol e. + Rootsymbol -> Nonterminals ':' '->'. + Nonterminals -> Terminals. + ':' -> '->'. + '->' -> Terminals. + ">>), + ?line {ok, _} = yecc:file(Filename, [{parserfile, Parserfile1}]), + ?line {ok, _} = yecc:file(Filename, [{parserfile, Parserfile1},time]), + ?line {ok,[]} = compile:file(Parserfile1, [basic_validation]), + + Quotes = <<" + Nonterminals + tail try 17 42 Unused ' unused ' '42' 'fnutt\\'' receive. + + Terminals + ']' '|' ',' '\"hi\"' 'there\\'' 'you\\'' ACCEPT. + + Rootsymbol 17. + + Endsymbol '$end'. + + 17 -> try : '$1':more([\"hi \\\" there\", '.', 3.14, % $3 , % '$1' is a module + -104, ' ', ' a', '->', $a, $\t]), '$1'. + + try -> tail 'you\\'' 'fnutt\\''. + + 'fnutt\\'' -> 'you\\'' ACCEPT. + + '42' -> tail. + + tail -> ']' : + (fun(Strange) -> foo end)(apa). + tail -> '|' try ']' : + (fun(Strange) -> foo; + (_) -> 'when' % warning + end)('ap a', foo), + %% This line intentionally left blank. + R = #rec{}, + A = R#rec.a, + <<\"hi\">> = <<\"hi\">>, + there. + tail -> ',' try tail : {cons,line('$2'),'$2','$3'}. + + + Erlang code. + + -export([t/0]). + + -record(rec, {a}). + + just(Some, Code) -> + true. + + more(Code) -> + io:format(\"~p~n\", [Code]), true. + + line(_) -> + 17. + + t() -> + ok. % compiled successfully... + + ">>, + + ?line ok = file:write_file(Filename, Quotes), + ?line {ok,_,[{_, + [{3,yecc,{unused_nonterminal,42}}, + {3,yecc,{unused_nonterminal,' unused '}}, + {3,yecc,{unused_nonterminal,'42'}}, + {3,yecc,{unused_nonterminal,'Unused'}}, + {3,yecc,{unused_nonterminal,'receive'}}, + {6,yecc,{unused_terminal,'"hi"'}}, + {6,yecc,{unused_terminal,'there\''}}]}]} = + yecc:file(Filename, Ret), + + Ts = [{quotes, Quotes, default, ok}], + ?line run(Config, Ts), + + %% Non-terminal has no rules, but is unused. + ?line ok = file:write_file(Filename,<<" + Nonterminals nt bad. + Terminals t. + Rootsymbol nt. + + nt -> t : something. + ">>), + ?line {ok,_,[{_,[{2,yecc,{unused_nonterminal,bad}}]}]} = + yecc:file(Filename, Ret), + + %% Non-terminal has no rules and is used. + ?line ok = file:write_file(Filename,<<" + Nonterminals nt bad. + Terminals t. + Rootsymbol nt. + + nt -> t bad : something. + ">>), + ?line {error,[{_,[{2,yecc,{missing_syntax_rule,bad}}]}],[]} = + yecc:file(Filename, Ret), + + %% Warning in Erlang code. With and without file attributes. + ?line ok = file:write_file(Filename,<<" + Nonterminals nt. + Terminals t. + Rootsymbol nt. + + nt -> t : t(17). + + Erlang code. + + t(A) -> + b. + ">>), + ?line {ok, _, []} = + yecc:file(Filename, [file_attributes | Ret]), + Opts = [basic_validation, return], + Erlfile = filename:join(Dir, "file.erl"), + ?line {ok,[],[{_,[{10,_,_}]}]} = compile:file(Erlfile, Opts), + ?line {ok, _, []} = + yecc:file(Filename, [{file_attributes,false} | Ret]), + ?line {ok,[],[{_,[{4,_,_}]}]} = compile:file(Erlfile, Opts), + + file:delete(Parserfile1 ++ ".erl"), + file:delete(Parserfile2 ++ ".erl"), + file:delete(Filename), + + ok. + +compile(doc) -> + "Check of compile/3."; +compile(suite) -> []; +compile(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.yrl"), + Parserfile = filename:join(Dir, "file.erl"), + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + nt -> t.">>), + ?line error = yecc:compile(Filename, "//", #options{}), + ?line ok = yecc:compile(Filename, Parserfile, #options{}), + file:delete(Parserfile), + file:delete(Filename), + ok. + +rules(doc) -> + "Check of rules."; +rules(suite) -> []; +rules(Config) when is_list(Config) -> + Dir = ?privdir, + Ret = [return, {report, false}], + Filename = filename:join(Dir, "file.yrl"), + + ?line ok = file:write_file(Filename, + <<"Nonterminals nt. Terminals t. Rootsymbol nt. + nt -> t. nt -> t.">>), + ?line {error,[{_,[{none,yecc,{conflict,_}}]}], + [{_,[{none,yecc,{conflicts,0,1}}]}]} = + yecc:file(Filename, [return, report]), + + ?line ok = file:write_file(Filename, <<" + Nonterminals A B E. + Terminals c d f x y. + Rootsymbol A. + + A -> B c d. + A -> E c f. + B -> x y. + E -> x y. + ">>), + ?line {error,[{_,[{none,yecc,{conflict,_}}]}], + [{_,[{none,yecc,{conflicts,0,1}}]}]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename,<<" + Terminals t. + Nonterminals nt. + Rootsymbol nt. + nt -> '$empty' : '$1'. + nt -> t. + ">>), + ?line {error,[{_,[{5,yecc,{undefined_pseudo_variable,'$1'}}]}],[]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename,<<" + Terminals t. + Nonterminals nt. + Rootsymbol nt. + nt -> '$empty' : '$0'. + nt -> t. + ">>), + ?line {error,[{_,[{5,yecc,{undefined_pseudo_variable,'$0'}}]}],[]} = + yecc:file(Filename, Ret), + + file:delete(Filename), + + %% No precedences. + Ts = [{rules_1,<<" + Nonterminals exp. + Terminals number '*' '+' '(' ')'. + Rootsymbol exp. + + exp -> exp '+' exp : {plus,'$1','$3'}. + exp -> exp '*' exp : {times,'$1','$3'}. + exp -> number : element(2, '$1'). + exp -> '(' exp ')' : '$2'. + + Erlang code. + + -export([t/0]). + + t() -> + {ok, {times, 1, {plus, 3, 5}}} = + parse([{number,1}, {'*',2}, {number,3}, + {'+',4}, {number,5}]), + ok. + ">>, + default, + ok}], + ?line run(Config, Ts), + ok. + +examples(suite) -> + [empty, prec, yeccpre, lalr, old_yecc, other_examples]. + +expect(doc) -> + "Check of expect."; +expect(suite) -> []; +expect(Config) when is_list(Config) -> + Dir = ?privdir, + Ret = [return, {report, true}], + Filename = filename:join(Dir, "file.yrl"), + + ?line ok = file:write_file(Filename, <<" + Nonterminals e. + Terminals i t else. + Rootsymbol e. + Expect a. + + e -> i e t e. + e -> i e t e else e. + ">>), + ?line {error,[{_,[{5,yecc,{bad_expect,a}}]}],[]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename, <<" + Nonterminals e. + Terminals i t else. + Rootsymbol e. + Expect 1. + + e -> i e t e. + e -> i e t e else e. + ">>), + ?line {ok, _, []} = yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename, <<" + Nonterminals e. + Terminals i t else. + Rootsymbol e. + Expect 2. + + e -> i e t e. + e -> i e t e else e. + ">>), + ?line {ok, _, [{_,[{none,yecc,{conflicts,1,0}}]}]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename, <<" + Nonterminals e. + Terminals i t else. + Rootsymbol e. + Expect -2. + + e -> i e t e. + e -> i e t e else e. + ">>), + ?line {error,[{_,[{5,yecc,{error,_yeccparser,_}}]}],[]} = + yecc:file(Filename, Ret), + + %% States N. An undocumented declaration used for testing. + ?line ok = file:write_file(Filename, <<" + Nonterminals nt. + Terminals t. + Rootsymbol nt. + States 100. + nt -> t.">>), + ?line {ok,_,[{_, [{none,yecc,{n_states,100,3}}]}]} = + yecc:file(Filename, Ret), + + ?line ok = file:write_file(Filename, <<" + Nonterminals nt. + Terminals t. + Rootsymbol nt. + States bad. + nt -> t.">>), + ?line {error,[{_,[{5,yecc,{bad_states,bad}}]}],[]} = + yecc:file(Filename, Ret), + + file:delete(Filename), + ok. + +conflicts(doc) -> + "Shift/reduce and reduce/reduce conflicts."; +conflicts(suite) -> []; +conflicts(Config) when is_list(Config) -> + Dir = ?privdir, + Ret = [return, {report, true}], + Filename = filename:join(Dir, "file.yrl"), + + ?line ok = file:write_file(Filename, <<" + Nonterminals S List Tuple Elements Element. + Terminals '{' '}' '[' ']' ',' nil e. + Rootsymbol S. + + S -> '$empty' : empty. + S -> List : '$1'. + + List -> '$empty' : []. + List -> nil : []. + List -> '[' Tuple ']' : {list, '$2'}. + + Tuple -> '$empty' : {}. + Tuple -> '{' '}' : {}. + Tuple -> '{' Elements '}' : {tuple, '$2'}. + + Elements -> '$empty' : none. + Elements -> Elements ',' Element : {elements, '$3', '$1'}. + Elements -> Element : '$1'. + + Element -> List : '$1'. + Element -> Tuple : '$1'. + Element -> e : '$1'. + ">>), + ?line {error,[{_,_}],[{_,[{none,yecc,{conflicts,1,5}}]}]} = + yecc:file(Filename, Ret), + + %% Can easily be resolved (but don't do it!). + ?line ok = file:write_file(Filename, <<" + Nonterminals S List Tuple Elements Element. + Terminals '{' '}' '[' ']' ',' nil e. + Rootsymbol S. + + Right 100 List. + + S -> '$empty' : empty. + S -> List : '$1'. + + List -> '$empty' : []. + List -> nil : []. + List -> '[' Tuple ']' : {list, '$2'}. + + Tuple -> '$empty' : {}. + Tuple -> '{' '}' : {}. + Tuple -> '{' Elements '}' : {tuple, '$2'}. + + Elements -> '$empty' : none. + Elements -> Elements ',' Element : {elements, '$3', '$1'}. + Elements -> Element : '$1'. + + Element -> List : '$1'. + Element -> Tuple : '$1'. + Element -> e : '$1'. + ">>), + ?line {ok, _, []} = + yecc:file(Filename, Ret), + + file:delete(Filename), + ok. + +empty(doc) -> + "'$empty'."; +empty(suite) -> []; +empty(Config) when is_list(Config) -> + Ts = [{empty_1, <<" + Nonterminals nt. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> '$empty': nothing. + nt -> t : something. + + Erlang code. + + -export([t/0]). + + t() -> + {ok, nothing} = parse([{e,2}]), + {ok, something} = parse([{t,1},{e,2}]), + ok.">>, + default, + ok}, + {empty_2, <<" + %% There used to be a bug when it came to the left + %% corners. In this example rule 2 is a prefix of rule 1 + %% such that the rule 2 generates $empty but rule 1 does + %% not. The old buggy code seemingly worked well too. + Nonterminals top A B C D. + Terminals t. + Rootsymbol top. + + top -> A B C D : {'$1','$2','$3','$4'}. + top -> A B C : {'$1','$2','$3'}. + + A -> '$empty' : e. + B -> '$empty' : e. + C -> '$empty' : e. + + D -> t : t. + + Erlang code. + + -export([t/0]). + + t() -> + {ok,{e,e,e,t}} = parse([{t, 1}]), + {ok,{e,e,e}} = parse([]), + ok. + ">>, + default, + ok}], + ?line run(Config, Ts), + ok. + +prec(doc) -> + "Precedence."; +prec(suite) -> []; +prec(Config) when is_list(Config) -> + Dir = ?privdir, + Ret = [return, {report, false}], + Filename = filename:join(Dir, "file.yrl"), + + %% Don't know what 'Unary' actually means, but this is how it has + %% always worked... + ?line ok = file:write_file(Filename, <<" + Nonterminals E. + Terminals '*' '+' '=' integer. + Rootsymbol E. + + E -> E '=' E : {op, '=', '$1', '$3'}. + E -> E '*' E : {op, '*', '$1', '$3'}. + E -> E '+' E : {op, '+', '$1', '$3'}. + E -> integer : '$1'. + + Unary 100 '='. + Left 200 '+'. + Left 400 '*'. + ">>), + ?line {error,[{_,[{none,yecc,{conflict,_}}]}], + [{_,[{none,yecc,{conflicts,1,0}}]}]} = + yecc:file(Filename, Ret), + + Ts = [{prec_1, <<" + Nonterminals E Expr Uminus. + Terminals '=' '+' '*' '-' '(' ')' integer var dot. + Rootsymbol Expr. + + Expr -> E dot : '$1'. + E -> var '=' E : {match, line_of('$1'), '$1', '$3'}. + E -> E '+' E : {op, line_of('$1'), '+', '$1', '$3'}. + E -> E '-' E : {op, line_of('$1'), '-', '$1', '$3'}. + E -> E '*' E : {op, line_of('$1'), '*', '$1', '$3'}. + E -> Uminus : '$1'. + E -> '(' E ')' : '$2'. + E -> integer : '$1'. + + Uminus -> '-' E : {op, line_of('$1'), '-', '$2'}. + + Right 200 '='. + Left 300 '+'. + Left 300 '-'. + Left 400 '*'. + Unary 500 Uminus. + + Erlang code. + + -export([t/0, t/1]). + + line_of(Token) -> + element(2, Token). + + t() -> + {ok, -56} = t(\"A = (4 + 3) * - 8. \"), + {ok, 28} = t(\"A = 4 + B=3 * 8. \"), + {ok, 28} = t(\"4 - 3 * - 8. \"), + {ok, -20} = t(\"4 - 3 * 8. \"), + {ok, 2} = t(\"1 - - 1.\"), + ok. + + t(S) -> + case erl_scan:string(S) of + {ok, Tokens, _Line} -> + case parse(Tokens) of + {ok, Expr} -> + case catch erl_eval:expr(Expr, []) of + {value, Value, _Bs} -> + {ok, Value}; + {'EXIT', Reason} -> + {error, Reason} + end; + Error -> + Error + end; + Error -> + Error + end. + ">>, + default, + ok}, + + {prec_2, <<" + Nonterminals E uminus. + Terminals '*' '-' integer. + Rootsymbol E. + + E -> E '-' E : {op, '-', '$1', '$3'}. + E -> E '*' E : {op, '*', '$1', '$3'}. + E -> uminus: '$1'. + E -> integer : '$1'. + + uminus -> '-' E : {op, '-', '$2'}. + + Left 200 '-'. + Left 400 '*'. + Unary 500 uminus. + + Erlang code. + + -export([t/0]). + + t() -> + %% This used to be parsed -(4 * 3), but that has been corrected: + {ok,{op,'*',{op,'-',{integer,1,4}},{integer,3,12}}} = + parse([{'-',0},{integer,1,4},{'*',2},{integer,3,12}]), + + {ok,{op,'-',{op,'-',{integer,1,4}},{integer,3,12}}} = + parse([{'-',0},{integer,1,4},{'-',2},{integer,3,12}]), + {ok,{op,'*',{op,'-',{op,'-',{integer,2,4}}},{integer,4,12}}} = + parse([{'-',0},{'-',1},{integer,2,4},{'*',3},{integer,4,12}]), + {ok,{op,'-',{integer,1,4},{op,'-',{integer,4,12}}}} = + parse([{integer,1,4},{'-',2},{'-',3},{integer,4,12}]), + ok. + ">>, + default, + ok}, + {prec_3, <<" + Nonterminals nt. + Terminals '(' ')' t op. + Rootsymbol nt. + + Nonassoc 100 op. + + nt -> nt op nt : {'$2', '$1', '$3'}. + nt -> '(' nt ')' : '$2'. + nt -> t : '$1'. + + Erlang code. + + -export([t/0]). + + t() -> + {error,{4,yecc_test,[\"syntax error before: \",\"op\"]}} = + parse([{t,1},{op,2},{t,3},{op,4},{t,5}]), + {ok,{{op,2},{t,1},{{op,5},{t,4},{t,6}}}} = + parse([{t,1},{op,2},{'(',3},{t,4},{op,5},{t,6},{')',7}]), + ok. + ">>, + default, + ok}, + + {prec_4, <<" + Nonterminals E. + Terminals '-' '+' '=' id. + Rootsymbol E. + + E -> E '=' E : {op, '=', '$1', '$3'}. + E -> E '+' E : {op, '+', '$1', '$3'}. + E -> '-' E : {op, '-', '$2'}. + E -> id : '$1'. + + Nonassoc 100 '='. + Right 200 '+' '-'. + + Erlang code. + + -export([t/0]). + + t() -> + {ok,{op,'=',{id,1},{op,'-',{op,'+',{id,4},{id,6}}}}} = + parse([{id,1},{'=',2},{'-',3},{id,4},{'+',5},{id,6}]), + ok. + + ">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +yeccpre(doc) -> + "Errors etc. in actions, handled by yeccpre. parse_and_scan."; +yeccpre(suite) -> []; +yeccpre(Config) when is_list(Config) -> + Ts = [{error_1, <<" + Nonterminals list. + Terminals element e. + Rootsymbol list. + + list -> element : {single, '$1'}. + list -> list element : throw(error). + list -> e : return_error(element(2, '$1'), bad_element). + + Erlang code. + + -export([t/0]). + + t() -> + error = (catch parse([{element,1},{element,2}])), + {error, {2, _, Mess}} = parse([{element,1},{e,2}]), + ok. + ">>, + default, + ok}, + + {error_2, <<" + Nonterminals list. + Terminals element. + Rootsymbol list. + + list -> element. + + Erlang code. + + -export([t/0, scan/1]). + + scan(How) -> + case How of + thrown -> throw(thrown); + error -> erlang:error(error); + exit -> exit(exit) + end. + + t() -> + try parse_and_scan({yecc_test, scan, [thrown]}) + catch throw: thrown -> + ok + end, + try parse_and_scan({yecc_test, scan, [error]}) + catch error: error -> + ok + end, + try parse_and_scan({{yecc_test, scan}, [exit]}) + catch exit: exit -> + ok + end, + try parse_and_scan({fun scan/1, [thrown]}) + catch throw: thrown -> + ok + end, + ok. + ">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +lalr(doc) -> + "Examples of grammars that are LALR(1) but not SLR(1)."; +lalr(suite) -> []; +lalr(Config) when is_list(Config) -> + Ts = [{lalr_1, <<" + %% http://inst.cs.berkeley.edu/~cs164/lectures/slide14a.pdf + %% http://pages.cpsc.ucalgary.ca/~robin/class/411/LR.1.html + %% b d . c. Shift or reduce? + Nonterminals S A. + Terminals a b c d. + Rootsymbol S. + S -> A a : {r1,'$1','$2'}. + S -> b A c : {r2,'$1','$2','$3'}. + S -> d c : {r3,'$1','$2'}. + S -> b d a : {r4,'$1','$2','$3'}. + + A -> d : {r5,'$1'}. + + Erlang code. + + -export([t/0]). + + t() -> + %% Reduce! + {ok,{r2,{b,1},{r5,{d,2}},{c,3}}} = parse([{b,1},{d,2},{c,3}]), + ok. + ">>, + default, + ok}, + {lalr_2, <<" + %% http://www.cs.pitt.edu/~mock/cs2210/lectures/lecture5.pdf + Nonterminals S L R. + Terminals '*' id '='. + Rootsymbol S. + S -> L '=' R : {r1,'$1','$3'}. + S -> R : {r2,'$1'}. + + L -> '*' R : {r3,'$2'}. + L -> id : {r4,'$1'}. + + R -> L : {r5,'$1'}. + + Erlang code. + + -export([t/0]). + + t() -> + {ok,{r1,{r3,{r5,{r4,{id,1}}}},{r5,{r4,{id,3}}}}} = + parse([{'*',0},{id,1},{'=',2},{id,3}]), + ok. + ">>, + default, + ok}], + ?line run(Config, Ts), + ok. + +old_yecc(doc) -> + "The old interface yecc:yecc/2,3,4."; +old_yecc(suite) -> []; +old_yecc(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.yrl"), + Parserfile = filename:join(Dir, "file.erl"), + Mini = <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + nt -> t.">>, + ?line ok = file:write_file(Filename, Mini), + ?line {_, _} = yecc:yecc(Filename, Parserfile), + ?line {_, _} = yecc:yecc(Filename, Parserfile, true), + ?line {_, _} = yecc:yecc(Filename, Parserfile, true), + TE = process_flag(trap_exit, true), + ?line {'EXIT', error} = + (catch yecc:yecc(Filename, Parserfile, false, Parserfile)), + _ = process_flag(trap_exit, TE), + ok. + +other_examples(doc) -> + "Misc examples."; +other_examples(suite) -> []; +other_examples(Config) when is_list(Config) -> + Ts = [{empty, <<" + %% G1 from the 1974 article. + + Nonterminals LIST ELEMENT. + Terminals ',' a b. + Rootsymbol LIST. + + LIST -> LIST ',' ELEMENT : {'$1', '$3'}. + LIST -> ELEMENT : '$1'. + + ELEMENT -> a : '$1'. + ELEMENT -> b : '$1'. + + Erlang code. + + -export([t/0]). + + t() -> + L = [{a, 1}, {',', 2}, {b, 3}], + {ok,{{a,1},{b,3}}} = parse(L), + ok. + ">>, + default, + ok}], + ?line run(Config, Ts), + ok. + +bugs(suite) -> + [otp_5369, otp_6362, otp_7945]. + +otp_5369(doc) -> + "OTP-5369. A bug in parse_and_scan reported on erlang questions."; +otp_5369(suite) -> []; +otp_5369(Config) when is_list(Config) -> + Ts = [{otp_5369,<<" + Nonterminals list. + Terminals element. + Rootsymbol list. + + list -> element : {single, '$1'}. + list -> list element : {pair, '$1', '$2'}. + + Erlang code. + + -export([t/0, scan/1]). + + scan(Lexer) -> + Lexer ! {scan, self()}, + receive + {ok, Lexer, [Token], Position} -> + {ok, [Token], Position}; + {eof, Lexer, Position} -> + {eof, Position} + end. + + make_scanner(L) -> + spawn_link(fun() -> + loop(L) + end). + + loop([]) -> + receive + {scan, Pid} -> + Pid ! {eof, self(), 1000} + end; + loop([Token = {_, P}|T]) -> + receive + {scan, Pid} -> + Pid ! {ok, self(), [Token], P}, + loop(T) + end. + + t(L) -> + case parse_and_scan({yecc_test, scan, [make_scanner(L)]}) of + {ok, _Result} -> + ok; + {error, {_LineNumber, Module, Message}} -> + _M = apply(Module, format_error, [Message]), + not_ok + end. + + t() -> + not_ok = t([]), %% This test has been added afterwards. + %% OTP-5369 did not fix this bug! + L = [{element, 1}, + {element, 2}, + {element, 3}], + t(L). + ">>, + default, + ok}], + ?line run(Config, Ts), + ok. + +otp_6362(doc) -> + "OTP-6362. A precedence declaration bug reported on erlang questions."; +otp_6362(suite) -> []; +otp_6362(Config) when is_list(Config) -> + Ts = [{otp_6362_1,<<" + Nonterminals cmp compare expr. + Terminals 'string' '>' '='. + Rootsymbol compare. + Nonassoc 250 cmp. + compare -> expr cmp expr : {cmp, '$2', '$1', '$3'}. + compare -> expr : {cmp, '==', '$1', {string, \"TRUE\"}}. + cmp -> '>' '=' : '>='. + cmp -> '>' : '>'. + expr -> 'string' : '$1'. + + Erlang code. + + -export([t/0]). + + t() -> + {ok,{cmp,'>',{string,1},{string,3}}} = + parse([{string,1}, {'>', 2}, {string,3}]), + ok. + ">>, + default, + ok}], + ?line run(Config, Ts), + + Dir = ?privdir, + %% Report errors. Very simple test of format_error/1. + Ret = [return, {report, true}], + Filename = filename:join(Dir, "file.yrl"), + + %% An error introduced due to this ticket. Terminals can be + %% assigned conflicting precedences, which cannot be resolved. + ?line ok = file:write_file(Filename,<<" + Nonterminals cmp compare expr fopp. + Terminals string '>' '='. + Rootsymbol compare. + Nonassoc 250 cmp. + Left 300 '>'. + + compare -> expr cmp expr : {cmp, '$2', '$1', '$3'}. + compare -> expr fopp expr : {cmp, '$2', '$1', '$3'}. + compare -> expr '=' expr : '$1'. + compare -> expr : {cmp, '==', '$1', {string, foo}}. + cmp -> '>' '=' : '>='. + cmp -> '>' : '>'. + expr -> string : '$1'. + + fopp -> '>' '=' : '>='. + fopp -> '>' : '>'.">>), + + Ret = [return, {report, true}], + ?line {error,[{_,[{none,yecc,{conflict,_}}]}],[]} = + yecc:file(Filename, Ret), + file:delete(Filename), + ok. + +my_yeccpre() -> + %% Version 1.1. + <<"parse(Tokens) -> + yeccpars0(Tokens, false). + + parse_and_scan({F, A}) -> % Fun or {M, F} + yeccpars0([], {F, A}); + parse_and_scan({M, F, A}) -> + yeccpars0([], {{M, F}, A}). + + format_error(Message) -> + case io_lib:deep_char_list(Message) of + true -> + Message; + _ -> + io_lib:write(Message) + end. + + % To be used in grammar files to throw an error message to the parser + % toplevel. Doesn't have to be exported! + -compile({nowarn_unused_function,{return_error,2}}). + return_error(Line, Message) -> + throw({error, {Line, yecc_test, Message}}). + + yeccpars0(Tokens, MFA) -> + try yeccpars1(Tokens, MFA, 0, [], []) + catch + throw: {error, {_Line, yecc_test, _M}} = Error -> + Error % probably from return_error/1 + end. + + % Don't change yeccpars1/6 too much, it is called recursively by yeccpars2/8! + yeccpars1([Token | Tokens], Tokenizer, State, States, Vstack) -> + yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, + Tokenizer); + yeccpars1([], {F, A}, State, States, Vstack) -> + case apply(F, A) of + {ok, Tokens, _Endline} -> + yeccpars1(Tokens, {F, A}, State, States, Vstack); + {eof, _Endline} -> + yeccpars1([], false, State, States, Vstack); + {error, Descriptor, _Endline} -> + {error, Descriptor} + end; + yeccpars1([], false, State, States, Vstack) -> + yeccpars2(State, '$end', States, Vstack, {'$end', 999999}, [], false). + + % For internal use only. + yeccerror(Token) -> + {error, + {element(2, Token), yecc_test, + [\"syntax error before: \", yecctoken2string(Token)]}}. + + yecctoken2string({atom, _, A}) -> io_lib:write(A); + yecctoken2string({integer,_,N}) -> io_lib:write(N); + yecctoken2string({float,_,F}) -> io_lib:write(F); + yecctoken2string({char,_,C}) -> io_lib:write_char(C); + yecctoken2string({var,_,V}) -> io_lib:format('~s', [V]); + yecctoken2string({string,_,S}) -> io_lib:write_string(S); + yecctoken2string({reserved_symbol, _, A}) -> io_lib:format('~w', [A]); + yecctoken2string({_Cat, _, Val}) -> io_lib:format('~w', [Val]); + yecctoken2string({'dot', _}) -> io_lib:format('~w', ['.']); + yecctoken2string({'$end', _}) -> + []; + yecctoken2string({Other, _}) when is_atom(Other) -> + io_lib:format('~w', [Other]); + yecctoken2string(Other) -> + io_lib:write(Other). +">>. + +otp_7945(doc) -> + "OTP-7945. A bug introduced in R13A."; +otp_7945(suite) -> []; +otp_7945(Config) when is_list(Config) -> + ?line {error,_} = erl_parse:parse([{atom,3,foo},{'.',2,9,9}]), + ok. + +improvements(suite) -> + [otp_7292, otp_7969]. + +otp_7292(doc) -> + "OTP-7292. Header declarations for edoc."; +otp_7292(suite) -> []; +otp_7292(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.yrl"), + Parserfile1 = filename:join(Dir, "a file"), + + Contents = <<"Nonterminals nt. + Terminals t. + Rootsymbol nt. + Endsymbol e. + nt -> t : a bad code. + + Header \"%% copyright bla bla bla\" + \"%% @private\" \"%% foo\" + \"%% @author X.Y.\". + + Erlang code. + + %% @private + %% etc. + + foo() -> + bar. ">>, + + %% Check that correct line number is used in messages. + ?line ok = file:write_file(Filename, Contents), + ParserFile3 = [{parserfile, Parserfile1}], + Ret = [return, {report, true}], + ?line {ok, _, []} = yecc:file(Filename, ParserFile3 ++ Ret), + + %% Note: checking the line numbers. Changes when yeccpre.hrl changes. + fun() -> + SzYeccPre = yeccpre_size(), + ?line {error, + [{_,[{5,_,["syntax error before: ","bad"]}]}, + {_,[{L1,_,{undefined_function,{yeccpars2_2_,1}}}, + {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], + [{_,[{16,_,{unused_function,{foo,0}}}]}]} = + compile:file(Parserfile1, [basic_validation, return]), + ?line L1 = 34 + SzYeccPre, + ?line L2 = 41 + SzYeccPre + end(), + + YeccPre = filename:join(Dir, "yeccpre.hrl"), + ?line ok = file:write_file(YeccPre, + [<<"-export([parse/1, parse_and_scan/1, format_error/1]).\n">>, + yeccpre_v1_2()]), + Inc = [{includefile,YeccPre}], + ?line {ok, _, []} = yecc:file(Filename, ParserFile3 ++ Inc ++ Ret), + fun() -> + SzYeccPre = yeccpre_size(YeccPre), + ?line {error, + [{_,[{5,_,["syntax error before: ","bad"]}]}, + {_,[{L1,_,{undefined_function,{yeccpars2_2_,1}}}, + {L2,_,{bad_inline,{yeccpars2_2_,1}}}]}], + [{_,[{16,_,{unused_function,{foo,0}}}]}]} = + compile:file(Parserfile1, [basic_validation, return]), + ?line L1 = 33 + SzYeccPre, + ?line L2 = 40 + SzYeccPre + end(), + + file:delete(YeccPre), + file:delete(Parserfile1 ++ ".erl"), + file:delete(Filename), + + ok. + +yeccpre_v1_2() -> + <<" +parse(Tokens) -> + yeccpars0(Tokens, false). + +parse_and_scan({F, A}) -> % Fun or {M, F} + yeccpars0([], {F, A}); +parse_and_scan({M, F, A}) -> + yeccpars0([], {{M, F}, A}). + +format_error(Message) -> + case io_lib:deep_char_list(Message) of + true -> + Message; + _ -> + io_lib:write(Message) + end. + +-define(CODE_VERSION, \"1.2\"). + +yeccpars0(Tokens, MFA) -> + try yeccpars1(Tokens, MFA, 0, [], []) + catch + error: Error -> + Stacktrace = erlang:get_stacktrace(), + try yecc_error_type(Error, Stacktrace) of + {syntax_error, Token} -> + yeccerror(Token); + {missing_in_goto_table=Tag, State} -> + Desc = {State, Tag}, + erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, + Stacktrace); + {missing_in_goto_table=Tag, Symbol, State} -> + Desc = {Symbol, State, Tag}, + erlang:raise(error, {yecc_bug, ?CODE_VERSION, Desc}, + Stacktrace) + catch _:_ -> erlang:raise(error, Error, Stacktrace) + end; + throw: {error, {_Line, ?MODULE, _M}} = Error -> + Error % probably from return_error/2 + end. + +yecc_error_type(function_clause, [{?MODULE,F,[_,_,_,_,Token,_,_]} | _]) -> + \"yeccpars2\" ++ _ = atom_to_list(F), + {syntax_error, Token}; +yecc_error_type({case_clause,{State}}, [{?MODULE,yeccpars2,_}|_]) -> + %% Inlined goto-function + {missing_in_goto_table, State}; +yecc_error_type(function_clause, [{?MODULE,F,[State]}|_]) -> + \"yeccgoto_\" ++ SymbolL = atom_to_list(F), + {ok,[{atom,_,Symbol}]} = erl_scan:string(SymbolL), + {missing_in_goto_table, Symbol, State}. + +yeccpars1([Token | Tokens], Tokenizer, State, States, Vstack) -> + yeccpars2(State, element(1, Token), States, Vstack, Token, Tokens, + Tokenizer); +yeccpars1([], {F, A}, State, States, Vstack) -> + case apply(F, A) of + {ok, Tokens, _Endline} -> + yeccpars1(Tokens, {F, A}, State, States, Vstack); + {eof, _Endline} -> + yeccpars1([], false, State, States, Vstack); + {error, Descriptor, _Endline} -> + {error, Descriptor} + end; +yeccpars1([], false, State, States, Vstack) -> + yeccpars2(State, '$end', States, Vstack, {'$end', 999999}, [], false). + +yeccpars1(State1, State, States, Vstack, Stack1, [Token | Tokens], + Tokenizer) -> + yeccpars2(State, element(1, Token), [State1 | States], + [Stack1 | Vstack], Token, Tokens, Tokenizer); +yeccpars1(State1, State, States, Vstack, Stack1, [], {F, A}) -> + case apply(F, A) of + {ok, Tokens, _Endline} -> + yeccpars1(State1, State, States, Vstack, Stack1, Tokens, {F, A}); + {eof, _Endline} -> + yeccpars1(State1, State, States, Vstack, Stack1, [], false); + {error, Descriptor, _Endline} -> + {error, Descriptor} + end; +yeccpars1(State1, State, States, Vstack, Stack1, [], false) -> + yeccpars2(State, '$end', [State1 | States], [Stack1 | Vstack], + {'$end', 999999}, [], false). + +yeccerror(Token) -> + {error, + {element(2, Token), ?MODULE, + [\"syntax error before: \", yecctoken2string(Token)]}}. + +yecctoken2string({atom, _, A}) -> io_lib:write(A); +yecctoken2string({integer,_,N}) -> io_lib:write(N); +yecctoken2string({float,_,F}) -> io_lib:write(F); +yecctoken2string({char,_,C}) -> io_lib:write_char(C); +yecctoken2string({var,_,V}) -> io_lib:format('~s', [V]); +yecctoken2string({string,_,S}) -> io_lib:write_string(S); +yecctoken2string({reserved_symbol, _, A}) -> io_lib:format('~w', [A]); +yecctoken2string({_Cat, _, Val}) -> io_lib:format('~w', [Val]); +yecctoken2string({'dot', _}) -> io_lib:format('~w', ['.']); +yecctoken2string({'$end', _}) -> + []; +yecctoken2string({Other, _}) when is_atom(Other) -> + io_lib:format('~w', [Other]); +yecctoken2string(Other) -> + io_lib:write(Other). +">>. + +run(Config, Tests) -> + F = fun({N,P,Pre,E}) -> + case catch run_test(Config, P, Pre) of + E -> + ok; + Bad -> + ?t:format("~nTest ~p failed. Expected~n ~p~n" + "but got~n ~p~n", [N, E, Bad]), + fail() + end + end, + lists:foreach(F, Tests). + +otp_7969(doc) -> + "OTP-7969. Interface to the I/O protocol.."; +otp_7969(suite) -> []; +otp_7969(Config) when is_list(Config) -> + ?line {ok,Ts1,_} = + erl_scan:string("'foo\nbar'", {1,1}, [text]), + ?line {error,{2,_,["syntax error before: ",[]]}} = erl_parse:parse(Ts1), + + ?line {ok,Ts1_1,_} = erl_scan:string("'foo\nbar'", 1, [text]), + ?line {error,{2,_,["syntax error before: ",[]]}} = erl_parse:parse(Ts1_1), + + ?line {ok,Ts2,_EndLocation} = + erl_scan:string("'foo\nbar'", {1,1}, []), + %% Can't do better than report possibly wrong line: + ?line {error,{1,_,["syntax error before: ",[]]}} = erl_parse:parse(Ts2), + + ?line {ok, Ts11, _}=R1 = erl_scan:string("f() -> a."), + ?line F1 = fun() -> {ok,Ts11 ++ [{'$end',2}],2} end, + ?line{ok,{function,1,f,0,[{clause,1,[],[],[{atom,1,a}]}]}} = + erl_parse:parse_and_scan({F1, []}), + ?line F2 = fun() -> erl_scan:string("f() -> ,") end, + ?line {error,{1,erl_parse,_}} = erl_parse:parse_and_scan({F2, []}), + ?line F3 = fun() -> case erase(foo) of + bar -> + {ok,[{'$end',2}],3}; + undefined -> + put(foo,bar), R1 + end + end, + ?line {ok,{function,1,f,0,[{clause,1,[],[],[{atom,1,a}]}]}} = + erl_parse:parse_and_scan({F3,[]}), + F4 = fun() -> {error, {1, ?MODULE, bad}, 2} end, + ?line {error, {1,?MODULE,bad}} = erl_parse:parse_and_scan({F4, []}), + F5 = fun() -> {eof, 3} end, + ?line {error,{3,erl_parse,_}} = erl_parse:parse_and_scan({F5, []}), + ?line {error,{999999,erl_parse,_}} = erl_parse:parse([]), + ?line {ok, Ts21, EL} = erl_scan:string("f() -> a; g() -> b. ", {1,1}), + ?line F6 = fun() -> {ok, Ts21, EL} end, + ?line {error,{{1,11},erl_parse,_}} = erl_parse:parse_and_scan({F6, []}), + ok. + +yeccpre_size() -> + yeccpre_size(default_yeccpre()). + +yeccpre_size(File) -> + n_lines(File). + +default_yeccpre() -> + filename:join([code:lib_dir(parsetools),"include","yeccpre.hrl"]). + +n_lines(File) -> + {ok, Bin} = file:read_file(File), + length([C || C=$\n <- binary_to_list(Bin)]). + +run_test(Config, Def, Pre) -> + %% io:format("testing ~s~n", [binary_to_list(Def)]), + DefFile = 'yecc_test.yrl', + Filename = 'yecc_test.erl', + DataDir = ?privdir, + YrlFile = filename:join(DataDir, DefFile), + ErlFile = filename:join(DataDir, Filename), + Opts = [return, warn_unused_vars,{outdir,DataDir}], + ok = file:write_file(YrlFile, Def), + YOpts = [return, {report, false} | + case Pre of + default -> + []; + _ -> + [{includefile,Pre}] + end], + P0 = pps(), + YRet = yecc:file(YrlFile, [verbose | YOpts]), + case {pps(), YRet} of + {P0, {ok, _Outfile, _YWs}} -> + case compile:file(ErlFile, Opts) of + {ok, _M, _Ws} -> + AbsFile = filename:rootname(ErlFile, ".erl"), + Mod = yecc_test, + code:purge(Mod), + code:load_abs(AbsFile, Mod), + Mod:t(); + %% warnings(ErlFile, Ws); + {error, [{ErlFile,Es}], []} -> {error, Es, []}; + {error, [{ErlFile,Es}], [{ErlFile,Ws}]} -> {error, Es, Ws}; + Error -> Error + end; + {P0, {error, [{YrlFile,YEs}], []}} -> {error, YEs, []}; + {P0, {error, [{YrlFile,YEs}], [{YrlFile,YWs}]}} -> {error, YEs, YWs}; + {P0, YError} -> YError; + {P, _} -> + io:format("failure, got ~p~n, expected ~p\n", [P, P0]), + fail() + end. + +extract(File, {error, Es, Ws}) -> + {errors, extract(File, Es), extract(File, Ws)}; +extract(File, Ts) -> + lists:append([T || {F, T} <- Ts, F =:= File]). + +pps() -> + {port_list(), process_list()}. + +port_list() -> + [{P,safe_second_element(erlang:port_info(P, name))} || P <- erlang:ports()]. + +process_list() -> + [{P,process_info(P, registered_name), + safe_second_element(process_info(P, initial_call))} || + P <- processes(), is_process_alive(P)]. + +safe_second_element({_,Info}) -> Info; +safe_second_element(Other) -> Other. + +fail() -> + ?t:fail(). diff --git a/lib/parsetools/vsn.mk b/lib/parsetools/vsn.mk index e3db40c193..b1354e89d8 100644 --- a/lib/parsetools/vsn.mk +++ b/lib/parsetools/vsn.mk @@ -1 +1 @@ -PARSETOOLS_VSN = 2.0.1 +PARSETOOLS_VSN = 2.0.2 |