From 921925be3c10d86f27597f1aebdbe0cde893a06a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Mon, 1 Feb 2010 14:53:39 +0000 Subject: Add test suite for parsetools --- lib/parsetools/test/leex_SUITE.erl | 909 +++++++++++++++++++++++++++++++++++++ 1 file changed, 909 insertions(+) create mode 100644 lib/parsetools/test/leex_SUITE.erl (limited to 'lib/parsetools/test/leex_SUITE.erl') diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl new file mode 100644 index 0000000000..069f780b5e --- /dev/null +++ b/lib/parsetools/test/leex_SUITE.erl @@ -0,0 +1,909 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(leex_SUITE). + +%-define(debug, true). + +-include_lib("stdlib/include/erl_compile.hrl"). +-include_lib("kernel/include/file.hrl"). + +-ifdef(debug). +-define(line, put(line, ?LINE), ). +-define(config(X,Y), foo). +-define(datadir, "leex_SUITE_data"). +-define(privdir, "leex_SUITE_priv"). +-define(t, test_server). +-else. +-include("test_server.hrl"). +-define(datadir, ?config(data_dir, Config)). +-define(privdir, ?config(priv_dir, Config)). +-endif. + +-export([all/1, init_per_testcase/2, fin_per_testcase/2]). + +-export([checks/1, + file/1, compile/1, syntax/1, + examples/1, + pt/1, man/1, ex/1, ex2/1, not_yet/1]). + +% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +init_per_testcase(_Case, Config) -> + ?line Dog = ?t:timetrap(?default_timeout), + [{watchdog, Dog} | Config]. + +fin_per_testcase(_Case, Config) -> + Dog = ?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +all(suite) -> [checks, examples]. + +checks(suite) -> + [file, compile, syntax]. + +file(doc) -> + "Bad files and options."; +file(suite) -> []; +file(Config) when is_list(Config) -> + Dir = ?privdir, + Ret = [return, {report, false}], + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file("not_a_file", Ret), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file("not_a_file", [{return,true}]), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file("not_a_file", [{report,false},return_errors]), + ?line error = leex:file("not_a_file"), + ?line error = leex:file("not_a_file", [{return,false},report]), + ?line error = leex:file("not_a_file", [return_warnings,{report,false}]), + + Filename = filename:join(Dir, "file.xrl"), + file:delete(Filename), + + ?line {'EXIT', {badarg, _}} = (catch leex:file({foo})), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {parserfile,{foo}})), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {includefile,{foo}})), + + ?line {'EXIT', {badarg, _}} = (catch leex:file(Filename, no_option)), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, [return | report])), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, {return,foo})), + ?line {'EXIT', {badarg, _}} = + (catch leex:file(Filename, includefile)), + + Mini = <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" + "Erlang code.\n">>, + ?line ok = file:write_file(Filename, Mini), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [{scannerfile,"//"} | Ret]), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [{includefile,"//"} | Ret]), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [{includefile,"/ /"} | Ret]), + + LeexPre = filename:join(Dir, "leexinc.hrl"), + ?line ok = file:write_file(LeexPre, <<"syntax error.\n">>), + PreErrors = run_test(Config, Mini, LeexPre), + ?line {errors, + [{1,_,["syntax error before: ","error"]}, + {3,_,undefined_module}], + []} = + extract(LeexPre, PreErrors), + file:delete(LeexPre), + + Ret2 = [return, report_errors, report_warnings, verbose], + Scannerfile = filename:join(Dir, "file.erl"), + ?line ok = file:write_file(Scannerfile, <<"nothing">>), + ?line unwritable(Scannerfile), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, Ret2), + ?line writable(Scannerfile), + file:delete(Scannerfile), + + Dotfile = filename:join(Dir, "file.dot"), + ?line ok = file:write_file(Dotfile, <<"nothing">>), + ?line unwritable(Dotfile), + ?line {error,[{_,[{none,leex,{file_error,_}}]}],[]} = + leex:file(Filename, [dfa_graph | Ret2]), + ?line writable(Dotfile), + file:delete(Dotfile), + + file:delete(Filename), + ok. + +compile(doc) -> + "Check of compile/3."; +compile(suite) -> []; +compile(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + Scannerfile = filename:join(Dir, "file.erl"), + Mini = <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" + "Erlang code.\n">>, + ?line ok = file:write_file(Filename, Mini), + ?line error = leex:compile(Filename, "//", #options{}), + ?line ok = leex:compile(Filename, Scannerfile, #options{}), + file:delete(Scannerfile), + file:delete(Filename), + ok. + +syntax(doc) -> + "Syntax checks."; +syntax(suite) -> []; +syntax(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + Ret = [return, {report, true}], + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "%% comment\n" + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n + ">>), + ?line {error,[{_,[{7,leex,missing_code}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : \n">>), + ?line {error,[{_,[{5,leex,missing_code}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[] :">>), + ?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : .\n" + "[] : ">>), + ?line {error,[{_,[{5,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[] : .\n">>), + ?line {error,[{_,[{4,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ ">>), + ?line {error,[{_,[{5,leex,bad_rule}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ ; ">>), + ?line {error,[{_,[{4,leex,bad_rule}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[] : '99\n">>), + ?line {error,[{_,[{4,erl_scan,_}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n">>), + ?line {error,[{_,[{3,leex,empty_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "Erlang code.\n">>), + ?line {error,[{_,[{4,leex,empty_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n">>), + ?line {error,[{_,[{2,leex,missing_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Erlang code.\n">>), + ?line {error,[{_,[{3,leex,missing_rules}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"">>), + %% This is a weird line: + ?line {error,[{_,[{0,leex,missing_defs}]}],[]} = leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Rules.\n">>), + ?line {error,[{_,[{1,leex,missing_defs}]}],[]} = leex:file(Filename, Ret), + + %% Check that correct line number is used in messages. + ErlFile = filename:join(Dir, "file.erl"), + Ret1 = [{scannerfile,ErlFile}|Ret], + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,\n" + " {word,TokenLine,TokenChars,\n" + " DDDD}}.\n" % unbound + "Erlang code.\n" + "an error.\n">>), % syntax error + ?line {ok, _, []} = leex:file(Filename, Ret1), + ?line {error, + [{_,[{8,_,["syntax error before: ","error"]}]}, + {_,[{6,_,{unbound_var,'DDDD'}}]}], + []} = + compile:file(ErlFile, [basic_validation, return]), + + %% Ignored characters + ?line ok = file:write_file(Filename, + <<"Definitions. D = [0-9]\n" + "Rules. [a-z] : .\n" + "1 : skip_token.\n" + "Erlang code. f() -> a.\n">>), + ?line {ok,_,[{_, + [{1,leex,ignored_characters}, + {2,leex,ignored_characters}, + {4,leex,ignored_characters}]}]} = + leex:file(Filename, Ret), + + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+\\ : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\"}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+\\x : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{illegal_char,"\\x"}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+\\x{ : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"\\x{"}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[^ab : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"["}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "(a : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{unterminated,"("}}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "[b-a] : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp,{char_class,"b-a"}}}]}],[]} = + leex:file(Filename, Ret), + + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "\\x{333333333333333333333333} : token.\n">>), + ?line {error,[{_,[{4,leex,{regexp, + {illegal_char, + "\\x{333333333333333333333333}"}}}]}],[]} = + leex:file(Filename, Ret), + ok. + +examples(suite) -> + [pt,man,ex,ex2,not_yet]. + +pt(doc) -> + "Pushing back characters."; +pt(suite) -> []; +pt(Config) when is_list(Config) -> + %% Needs more testing... + Ts = [{pt_1, + <<"Definitions.\n" + "D = [0-9]\n" + "L = [a-z]\n" + + "Rules.\n" + "{L}+ : {token,{word,TokenLine,TokenChars}}.\n" + "abc{D}+ : {skip_token,\"sture\" ++ string:substr(TokenChars, 4)}.\n" + "{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + "\\s : .\n" + "\\r\\n : {end_token,{crlf,TokenLine}}.\n" + + "Erlang code.\n" + "-export([t/0]).\n" + "t() -> + {ok,[{word,1,\"sture\"},{integer,1,123}],1} = + string(\"abc123\"), ok. ">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +man(doc) -> + "Examples from the manpage."; +man(suite) -> []; +man(Config) when is_list(Config) -> + Ts = [{man_1, + <<"Definitions.\n" + "Rules.\n" + "[a-z][0-9a-zA-Z_]* :\n" + " {token,{atom,TokenLine,list_to_atom(TokenChars)}}.\n" + "[A-Z_][0-9a-zA-Z_]* :\n" + " {token,{var,TokenLine,list_to_atom(TokenChars)}}.\n" + "(\\+|-)?[0-9]+\\.[0-9]+((E|e)(\\+|-)?[0-9]+)? : \n" + " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" + "\\s : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{float,1,3.14},{atom,1,atom},{var,1,'V314'}],1} =\n" + " string(\"3.14atom V314\"),\n" + " ok.\n">>, + default, + ok}, + + {man_2, + <<"Definitions.\n" + "D = [0-9]\n" + "Rules.\n" + "{D}+ :\n" + " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + "{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? :\n" + " {token,{float,TokenLine,list_to_float(TokenChars)}}.\n" + "\\s : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{float,1,3.14},{integer,1,314}],1} = \n" + " string(\"3.14 314\"),\n" + " ok.\n">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +ex(doc) -> + "Examples."; +ex(suite) -> []; +ex(Config) when is_list(Config) -> + Ts = [{ex_1, + <<"Definitions.\n" + "D = [0-543-705-982]\n" + "Rules.\n" + "{D}+ :\n" + " {token,{integer,TokenLine,list_to_integer(TokenChars)}}.\n" + "[^235]+ :\n" + " {token,{list_to_atom(TokenChars),TokenLine}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{integer,1,12},{' c\\na',1},{integer,2,34},{b789a,2}],2} =\n" + " string(\"12 c\\na34b789a\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_2, + <<"Definitions.\n" + "L = [a-z]\n" + "D = [0-9]\n" + "Rules.\n" + "{L}+ : {token,chars}.\n" + "zyx{D}+ : {token,zyx}.\n" + "\\s : skip_token.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[chars,zyx],1} = string(\"abcdef zyx123\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_3, + <<"Definitions.\n" + "NL = [\\n]\n" + "Rules.\n" + "{NL}* : {token,newlines}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[],1} = string(\"\"), ok.\n">>, % string("a") would loop... + default, + ok}, + + {ex_4, + <<"Definitions.\n" + "SP1 = [\\n-\\s]\n" + "SP0 = [\\000-\\n]\n" + "Rules.\n" + "{SP0}+ : {token,{small,TokenChars}}.\n" + "{SP1}+ : {token,{big,TokenChars}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " string(\"\\x00\\n\\s\\n\\n\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_5, + <<"Definitions.\n" + "L = [a-z]\n" + "W = [\\s\\b\\n\\r\\t\\e\\v\\d\\f]\n" + "Rules.\n" + "\\[{L}+(,{L}+)*\\] : {token,{list,TokenChars}}.\n" + "\"{L}+\" : {token,{string,TokenChars}}.\n" + "\\$. : {token,{char,TokenChars}}.\n" + "{W}+ : {token,{white,TokenChars}}.\n" + "ff\\f+ : {token,{form,TokenChars}}.\n" + "\\$\\^+\\\\+ : {token,{other,TokenChars}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{white,\"\\b\\f\"}],1} = string(\"\\b\\f\"),\n" + " {ok,[{form,\"ff\\f\"}],1} = string(\"ff\\f\"),\n" + " {ok,[{string,\"\\\"foo\\\"\"}],1} = string(\"\\\"foo\\\"\"),\n" + " {ok,[{char,\"$.\"}],1} = string(\"$\\.\"),\n" + " {ok,[{list,\"[a,b,c]\"}],1} = string(\"[a,b,c]\"),\n" + " {ok,[{other,\"$^\\\\\"}],1} = string(\"$^\\\\\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_6, + <<"Definitions.\n" + "L = [a-z]\n" + "Rules.\n" + "L}+ : {token,{TokenChars,#r.f}}.\n" + "Erlang code.\n" + "-record(r, {f}).\n" + "-export([t/0]).\n" + "t() ->\n" + " string(\"abc\"),\n" + " ok.\n">>, + default, + ok}, + + {ex_7, %% Assumes regexp can handle \x + <<"Definitions.\n" + "H1 = \\x11\\x{ab}\n" + "H2 = [\\x{30}\\x{ac}]\n" + "Rules.\n" + "{H1}{H2}+ : {token,{hex,TokenChars}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{hex,[17,171,48,172]}],1} =\n" + " string(\"\\x{11}\\xab0\\xac\"),\n" + " ok.\n">>, + default, + ok}], + + ?line run(Config, Ts), + ok. + +ex2(doc) -> + "More examples."; +ex2(suite) -> []; +ex2(Config) when is_list(Config) -> + Xrl = + <<" +%%% File : erlang_scan.xrl +%%% Author : Robert Virding +%%% Purpose : Tkoen definitions for Erlang. + +Definitions. +O = [0-7] +D = [0-9] +H = [0-9a-fA-F] +U = [A-Z] +L = [a-z] +A = ({U}|{L}|{D}|_|@) +WS = ([\\000-\\s]|%.*) + +Rules. +{D}+\\.{D}+((E|e)(\\+|\\-)?{D}+)? : + {token,{float,TokenLine,list_to_float(TokenChars)}}. +{D}+#{H}+ : base(TokenLine, TokenChars). +{D}+ : {token,{integer,TokenLine,list_to_integer(TokenChars)}}. +{L}{A}* : Atom = list_to_atom(TokenChars), + {token,case reserved_word(Atom) of + true -> {Atom,TokenLine}; + false -> {atom,TokenLine,Atom} + end}. +'(\\\\\\^.|\\\\.|[^'])*' : + %% Strip quotes. + S = lists:sublist(TokenChars, 2, TokenLen - 2), + case catch list_to_atom(string_gen(S)) of + {'EXIT',_} -> {error,\"illegal atom \" ++ TokenChars}; + Atom -> {token,{atom,TokenLine,Atom}} + end. +({U}|_){A}* : {token,{var,TokenLine,list_to_atom(TokenChars)}}. +\"(\\\\\\^.|\\\\.|[^\"])*\" : + %% Strip quotes. + S = lists:sublist(TokenChars, 2, TokenLen - 2), + {token,{string,TokenLine,string_gen(S)}}. +\\$(\\\\{O}{O}{O}|\\\\\\^.|\\\\.|.) : + {token,{char,TokenLine,cc_convert(TokenChars)}}. +-> : {token,{'->',TokenLine}}. +:- : {token,{':-',TokenLine}}. +\\|\\| : {token,{'||',TokenLine}}. +<- : {token,{'<-',TokenLine}}. +\\+\\+ : {token,{'++',TokenLine}}. +-- : {token,{'--',TokenLine}}. +=/= : {token,{'=/=',TokenLine}}. +== : {token,{'==',TokenLine}}. +=:= : {token,{'=:=',TokenLine}}. +/= : {token,{'/=',TokenLine}}. +>= : {token,{'>=',TokenLine}}. +=< : {token,{'=<',TokenLine}}. +<= : {token,{'<=',TokenLine}}. +<< : {token,{'<<',TokenLine}}. +>> : {token,{'>>',TokenLine}}. +:: : {token,{'::',TokenLine}}. +[]()[}{|!?/;:,.*+#<>=-] : + {token,{list_to_atom(TokenChars),TokenLine}}. +\\.{WS} : {end_token,{dot,TokenLine}}. +{WS}+ : skip_token. + +Erlang code. + +-export([reserved_word/1]). + +%% reserved_word(Atom) -> Bool +%% return 'true' if Atom is an Erlang reserved word, else 'false'. + +reserved_word('after') -> true; +reserved_word('begin') -> true; +reserved_word('case') -> true; +reserved_word('try') -> true; +reserved_word('cond') -> true; +reserved_word('catch') -> true; +reserved_word('andalso') -> true; +reserved_word('orelse') -> true; +reserved_word('end') -> true; +reserved_word('fun') -> true; +reserved_word('if') -> true; +reserved_word('let') -> true; +reserved_word('of') -> true; +reserved_word('query') -> true; +reserved_word('receive') -> true; +reserved_word('when') -> true; +reserved_word('bnot') -> true; +reserved_word('not') -> true; +reserved_word('div') -> true; +reserved_word('rem') -> true; +reserved_word('band') -> true; +reserved_word('and') -> true; +reserved_word('bor') -> true; +reserved_word('bxor') -> true; +reserved_word('bsl') -> true; +reserved_word('bsr') -> true; +reserved_word('or') -> true; +reserved_word('xor') -> true; +reserved_word('spec') -> true; +reserved_word(_) -> false. + +base(L, Cs) -> + H = string:chr(Cs, $#), + case list_to_integer(string:substr(Cs, 1, H-1)) of + B when B > 16 -> {error,\"illegal base\"}; + B -> + case base(string:substr(Cs, H+1), B, 0) of + error -> {error,\"illegal based number\"}; + N -> {token,{integer,L,N}} + end + end. + +base([C|Cs], Base, SoFar) when C >= $0, C =< $9, C < Base + $0 -> + Next = SoFar * Base + (C - $0), + base(Cs, Base, Next); +base([C|Cs], Base, SoFar) when C >= $a, C =< $f, C < Base + $a - 10 -> + Next = SoFar * Base + (C - $a + 10), + base(Cs, Base, Next); +base([C|Cs], Base, SoFar) when C >= $A, C =< $F, C < Base + $A - 10 -> + Next = SoFar * Base + (C - $A + 10), + base(Cs, Base, Next); +base([_|_], _, _) -> error; %Unknown character +base([], _, N) -> N. + +cc_convert([$$,$\\\\|Cs]) -> + hd(string_escape(Cs)); +cc_convert([$$,C]) -> C. + +string_gen([$\\\\|Cs]) -> + string_escape(Cs); +string_gen([C|Cs]) -> + [C|string_gen(Cs)]; +string_gen([]) -> []. + +string_escape([O1,O2,O3|S]) when + O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> + [(O1*8 + O2)*8 + O3 - 73*$0|string_gen(S)]; +string_escape([$^,C|Cs]) -> + [C band 31|string_gen(Cs)]; +string_escape([C|Cs]) when C >= $\\000, C =< $\\s -> + string_gen(Cs); +string_escape([C|Cs]) -> + [escape_char(C)|string_gen(Cs)]. + +escape_char($n) -> $\\n; %\\n = LF +escape_char($r) -> $\\r; %\\r = CR +escape_char($t) -> $\\t; %\\t = TAB +escape_char($v) -> $\\v; %\\v = VT +escape_char($b) -> $\\b; %\\b = BS +escape_char($f) -> $\\f; %\\f = FF +escape_char($e) -> $\\e; %\\e = ESC +escape_char($s) -> $\\s; %\\s = SPC +escape_char($d) -> $\\d; %\\d = DEL +escape_char(C) -> C. + ">>, + Dir = ?privdir, + XrlFile = filename:join(Dir, "erlang_scan.xrl"), + ?line ok = file:write_file(XrlFile, Xrl), + ErlFile = filename:join(Dir, "erlang_scan.erl"), + ?line {ok, _} = leex:file(XrlFile, []), + ?line {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), + code:purge(erlang_scan), + AbsFile = filename:rootname(ErlFile, ".erl"), + code:load_abs(AbsFile, erlang_scan), + + F = fun(Cont, Chars, Location) -> + erlang_scan:tokens(Cont, Chars, Location) + end, + F1 = fun(Cont, Chars, Location) -> + erlang_scan:token(Cont, Chars, Location) + end, + fun() -> + S = "ab cd. ", + {ok, Ts, 1} = scan_tokens_1(S, F, 1), + {ok, Ts, 1} = scan_token_1(S, F1, 1), + {ok, Ts, 1} = scan_tokens(S, F, 1), + {ok, Ts, 1} = erlang_scan:string(S, 1) + end(), + fun() -> + S = "'ab\n cd'. ", + {ok, Ts, 2} = scan_tokens_1(S, F, 1), + {ok, Ts, 2} = scan_token_1(S, F1, 1), + {ok, Ts, 2} = scan_tokens(S, F, 1), + {ok, Ts, 2} = erlang_scan:string(S, 1) + end(), + fun() -> + S = "99. ", + {ok, Ts, 1} = scan_tokens_1(S, F, 1), + {ok, Ts, 1} = scan_token_1(S, F1, 1), + {ok, Ts, 1} = scan_tokens(S, F, 1), + {ok, Ts, 1} = erlang_scan:string(S, 1) + end(), + {ok,[{integer,1,99},{dot,1}],1} = erlang_scan:string("99. "), + fun() -> + Atom = "'" ++ lists:duplicate(1000,$a) ++ "'", + S = Atom ++ ". ", + Reason = "illegal atom " ++ Atom, + Err = {error,{1,erlang_scan,{user,Reason}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[]} = scan_token_1(S, F1, 1), + {done,Err,[]} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa}. ", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,[]} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa} + 1. 34", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa} \x{bbb}. 34", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa} 18#34. 34", + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,[]} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,"34"} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + fun() -> + S = "\x{aaa}"++eof, + Err = {error,{1,erlang_scan,{illegal,[2730]}},1}, + {done,Err,eof} = scan_tokens_1(S, F, 1), + {done,Err,[_]} = scan_token_1(S, F1, 1), % Note: Rest non-empty + {done,Err,eof} = scan_tokens(S, F, 1), + Err = erlang_scan:string(S, 1) + end(), + ok. + +scan_tokens(String, Fun, Location) -> + scan_tokens(String, Fun, Location, []). + +scan_tokens(String, Fun, Location, Rs) -> + case Fun([], String, Location) of + {done, {error,_,_}, _} = Error -> + Error; + {done, {ok,Ts,End}, ""} -> + {ok, lists:append(lists:reverse([Ts|Rs])), End}; + {done, {ok,Ts,End}, Rest} -> + scan_tokens(Rest, Fun, End, [Ts|Rs]) + end. + +scan_tokens_1(String, Fun, Location) -> + scan_tokens_1({more, []}, String, Fun, Location, []). + +scan_tokens_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) -> + Error; +scan_tokens_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) -> + {ok,lists:append(lists:reverse([Ts|Rs])),End}; +scan_tokens_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) -> + scan_tokens_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]); +scan_tokens_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> + R = Fun(Cont, [C], Loc), + scan_tokens_1(R, Cs, Fun, Loc, Rs); +scan_tokens_1({more, Cont}, eof, Fun, Loc, Rs) -> + R = Fun(Cont, eof, Loc), + scan_tokens_1(R, eof, Fun, Loc, Rs). + +scan_token_1(String, Fun, Location) -> + scan_token_1({more, []}, String, Fun, Location, []). + +scan_token_1({done, {error, _, _}, _}=Error, _Cs, _Fun, _Location, _Rs) -> + Error; +scan_token_1({done, {ok,Ts,End}, ""}, "", _Fun, _Location, Rs) -> + {ok,lists:reverse([Ts|Rs]),End}; +scan_token_1({done, {ok,Ts,End}, Rest}, Cs, Fun, _Location, Rs) -> + scan_token_1({more,[]}, Rest++Cs, Fun, End, [Ts|Rs]); +scan_token_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> + R = Fun(Cont, [C], Loc), + scan_token_1(R, Cs, Fun, Loc, Rs). + +%% End of ex2 + +not_yet(doc) -> + "Not yet implemented."; +not_yet(suite) -> []; +not_yet(Config) when is_list(Config) -> + Dir = ?privdir, + Filename = filename:join(Dir, "file.xrl"), + Ret = [return, {report, true}], + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "Rules.\n" + "$ : .\n" + "Erlang code.\n">>), + ?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + ?line ok = file:write_file(Filename, + <<"Definitions.\n" + "Rules.\n" + "^ : .\n" + "Erlang code.\n">>), + ?line {error,[{_,[{3,leex,{regexp,_}}]}],[]} = + leex:file(Filename, Ret), + + ok. + +unwritable(Fname) -> + {ok, Info} = file:read_file_info(Fname), + Mode = Info#file_info.mode - 8#00200, + ok = file:write_file_info(Fname, Info#file_info{mode = Mode}). + +writable(Fname) -> + {ok, Info} = file:read_file_info(Fname), + Mode = Info#file_info.mode bor 8#00200, + ok = file:write_file_info(Fname, Info#file_info{mode = Mode}). + +run(Config, Tests) -> + F = fun({N,P,Pre,E}) -> + case catch run_test(Config, P, Pre) of + E -> + ok; + Bad -> + ?t:format("~nTest ~p failed. Expected~n ~p~n" + "but got~n ~p~n", [N, E, Bad]), + fail() + end + end, + lists:foreach(F, Tests). + +run_test(Config, Def, Pre) -> + %% io:format("testing ~s~n", [binary_to_list(Def)]), + DefFile = 'leex_test.xrl', + Filename = 'leex_test.erl', + DataDir = ?privdir, + XrlFile = filename:join(DataDir, DefFile), + ErlFile = filename:join(DataDir, Filename), + Opts = [return, warn_unused_vars,{outdir,DataDir}], + ok = file:write_file(XrlFile, Def), + LOpts = [return, {report, false} | + case Pre of + default -> + []; + _ -> + [{includefile,Pre}] + end], + XOpts = [verbose, dfa_graph], % just to get some code coverage... + LRet = leex:file(XrlFile, XOpts ++ LOpts), + case LRet of + {ok, _Outfile, _LWs} -> + CRet = compile:file(ErlFile, Opts), + case CRet of + {ok, _M, _Ws} -> + AbsFile = filename:rootname(ErlFile, ".erl"), + Mod = leex_test, + code:purge(Mod), + code:load_abs(AbsFile, Mod), + Mod:t(); + %% warnings(ErlFile, Ws); + {error, [{ErlFile,Es}], []} -> {error, Es, []}; + {error, [{ErlFile,Es}], [{ErlFile,Ws}]} -> {error, Es, Ws}; + Error -> Error + end; + {error, [{XrlFile,LEs}], []} -> {error, LEs, []}; + {error, [{XrlFile,LEs}], [{XrlFile,LWs}]} -> {error, LEs, LWs}; + LError -> LError + end. + +extract(File, {error, Es, Ws}) -> + {errors, extract(File, Es), extract(File, Ws)}; +extract(File, Ts) -> + lists:append([T || {F, T} <- Ts, F =:= File]). + +fail() -> + ?t:fail(). -- cgit v1.2.3