From ae622fb6655bf3c490ff5ce272c0e474f4d9df0f Mon Sep 17 00:00:00 2001 From: archimed Date: Tue, 15 Jul 2014 09:25:44 +0400 Subject: Fix the line counting in string function The line counter becomes invalid, when the rules with linewrap are used. This issue appears, because the parsing FSM does not rollback the line counter after attempting such rule. --- lib/parsetools/include/leexinc.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/parsetools/include/leexinc.hrl b/lib/parsetools/include/leexinc.hrl index dbbb688d2d..7a0d65e28e 100644 --- a/lib/parsetools/include/leexinc.hrl +++ b/lib/parsetools/include/leexinc.hrl @@ -36,8 +36,8 @@ string(Ics0, L0, Tcs, Ts) -> string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L0), Ts); {reject,_Alen,Tlen,_Ics1,L1,_S1} -> % After a non-accepting state {error,{L0,?MODULE,{illegal,yypre(Tcs, Tlen+1)}},L1}; - {A,Alen,_Tlen,_Ics1,L1,_S1} -> - string_cont(yysuf(Tcs, Alen), L1, yyaction(A, Alen, Tcs, L0), Ts) + {A,Alen,_Tlen,_Ics1,_L1,_S1} -> + string_cont(yysuf(Tcs, Alen), L0, yyaction(A, Alen, Tcs, L0), Ts) end. %% string_cont(RestChars, Line, Token, Tokens) -- cgit v1.2.3 From 10414ad49c1dce62e1022c80c0ed6edd45abc20d Mon Sep 17 00:00:00 2001 From: archimed Date: Tue, 15 Jul 2014 12:34:23 +0400 Subject: Add unit test for string fun in leex test suite --- lib/parsetools/test/leex_SUITE.erl | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index eb15bebf63..44a60aff88 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -43,8 +43,8 @@ file/1, compile/1, syntax/1, pt/1, man/1, ex/1, ex2/1, not_yet/1, - - otp_10302/1, otp_11286/1, unicode/1]). + line_wrap/1, + otp_10302/1, otp_11286/1, unicode/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(1)). @@ -61,12 +61,13 @@ end_per_testcase(_Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [{group, checks}, {group, examples}]. + [{group, checks}, {group, examples}, {group, bugs}]. groups() -> [{checks, [], [file, compile, syntax]}, {examples, [], [pt, man, ex, ex2, not_yet, unicode]}, - {tickets, [], [otp_10302, otp_11286]}]. + {tickets, [], [otp_10302, otp_11286]}, + {bugs, [], [line_wrap]}]. init_per_suite(Config) -> Config. @@ -871,6 +872,35 @@ scan_token_1({more, Cont}, [C | Cs], Fun, Loc, Rs) -> %% End of ex2 +line_wrap(doc) -> "Much more examples."; +line_wrap(suite) -> []; +line_wrap(Config) when is_list(Config) -> + Xrl = + <<" +Definitions. +Rules. +[a]+[\\n]*= : {token, {first, TokenLine}}. +[a]+ : {token, {second, TokenLine}}. +[\\s\\r\\n\\t]+ : skip_token. +Erlang code. + ">>, + Dir = ?privdir, + XrlFile = filename:join(Dir, "test_line_wrap.xrl"), + ?line ok = file:write_file(XrlFile, Xrl), + ErlFile = filename:join(Dir, "test_line_wrap.erl"), + ?line {ok, _} = leex:file(XrlFile, []), + ?line {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), + code:purge(test_line_wrap), + AbsFile = filename:rootname(ErlFile, ".erl"), + code:load_abs(AbsFile, test_line_wrap), + fun() -> + S = "aaa\naaa", + {ok,[{second,1},{second,2}],2} = test_line_wrap:string(S) + end(), + ok. + +%% End of line_wrap + not_yet(doc) -> "Not yet implemented."; not_yet(suite) -> []; -- cgit v1.2.3 From c9bc5c944bf064727385d071e9331f521a2b1447 Mon Sep 17 00:00:00 2001 From: archimed Date: Wed, 23 Jul 2014 09:57:13 +0400 Subject: Fix line counting in token and tokens functions The line counter becomes invalid, when the rules with linewrap are used. This issue appears, because the parsing FSM does not rollback the line counter after attempting such rule. Unit tests for 'token' and 'tokens' are also added. --- lib/parsetools/include/leexinc.hrl | 8 ++++---- lib/parsetools/test/leex_SUITE.erl | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/parsetools/include/leexinc.hrl b/lib/parsetools/include/leexinc.hrl index 7a0d65e28e..938aef58f9 100644 --- a/lib/parsetools/include/leexinc.hrl +++ b/lib/parsetools/include/leexinc.hrl @@ -105,8 +105,8 @@ token(S0, Ics0, L0, Tcs, Tlen0, Tline, A0, Alen0) -> {reject,_Alen1,Tlen1,Ics1,L1,_S1} -> % No token match Error = {Tline,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, {done,{error,Error,L1},Ics1}; - {A1,Alen1,_Tlen1,_Ics1,L1,_S1} -> % Use last accept match - token_cont(yysuf(Tcs, Alen1), L1, yyaction(A1, Alen1, Tcs, Tline)) + {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} -> % Use last accept match + token_cont(yysuf(Tcs, Alen1), L0, yyaction(A1, Alen1, Tcs, Tline)) end. %% token_cont(RestChars, Line, Token) @@ -177,9 +177,9 @@ tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Ts, A0, Alen0) -> %% Skip rest of tokens. Error = {L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error); - {A1,Alen1,_Tlen1,_Ics1,L1,_S1} -> + {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} -> Token = yyaction(A1, Alen1, Tcs, Tline), - tokens_cont(yysuf(Tcs, Alen1), L1, Token, Ts) + tokens_cont(yysuf(Tcs, Alen1), L0, Token, Ts) end. %% tokens_cont(RestChars, Line, Token, Tokens) diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index 44a60aff88..6d2afe061e 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -888,8 +888,8 @@ Erlang code. XrlFile = filename:join(Dir, "test_line_wrap.xrl"), ?line ok = file:write_file(XrlFile, Xrl), ErlFile = filename:join(Dir, "test_line_wrap.erl"), - ?line {ok, _} = leex:file(XrlFile, []), - ?line {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), + {ok, _} = leex:file(XrlFile, []), + {ok, _} = compile:file(ErlFile, [{outdir,Dir}]), code:purge(test_line_wrap), AbsFile = filename:rootname(ErlFile, ".erl"), code:load_abs(AbsFile, test_line_wrap), @@ -897,6 +897,19 @@ Erlang code. S = "aaa\naaa", {ok,[{second,1},{second,2}],2} = test_line_wrap:string(S) end(), + fun() -> + S = "aaa\naaa", + {ok,[{second,3},{second,4}],4} = test_line_wrap:string(S, 3) + end(), + fun() -> + {done,{ok,{second,1},1},"\na"} = test_line_wrap:token([], "a\na"), + {more,Cont1} = test_line_wrap:token([], "\na"), + {done,{ok,{second,2},2},eof} = test_line_wrap:token(Cont1, eof) + end(), + fun() -> + {more,Cont1} = test_line_wrap:tokens([], "a\na"), + {done,{ok,[{second,1},{second,2}],2},eof} = test_line_wrap:tokens(Cont1, eof) + end(), ok. %% End of line_wrap -- cgit v1.2.3