From 2990325d5ae7f7fd119aea540a2edae4f97e76d9 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Tue, 27 Sep 2016 12:50:21 +0200 Subject: parsetools: Correct counting of newlines See https://bugs.erlang.org/browse/ERL-263 The fix in commit c9bc5c94 of PR-431 (https://github.com/erlang/otp/pull/431) introduced new problems. --- lib/parsetools/include/leexinc.hrl | 35 +++++++++++++++++++++++------- lib/parsetools/test/leex_SUITE.erl | 44 ++++++++++++++++++++++++++++++++++---- lib/parsetools/test/yecc_SUITE.erl | 2 +- 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/lib/parsetools/include/leexinc.hrl b/lib/parsetools/include/leexinc.hrl index 2657fdcfaa..b4449607cb 100644 --- a/lib/parsetools/include/leexinc.hrl +++ b/lib/parsetools/include/leexinc.hrl @@ -36,8 +36,10 @@ string(Ics0, L0, Tcs, Ts) -> string_cont(Ics1, L1, yyaction(A, Alen, Tcs, L0), Ts); {reject,_Alen,Tlen,_Ics1,L1,_S1} -> % After a non-accepting state {error,{L0,?MODULE,{illegal,yypre(Tcs, Tlen+1)}},L1}; - {A,Alen,_Tlen,_Ics1,_L1,_S1} -> - string_cont(yysuf(Tcs, Alen), L0, yyaction(A, Alen, Tcs, L0), Ts) + {A,Alen,Tlen,_Ics1,L1,_S1} -> + Tcs1 = yysuf(Tcs, Alen), + L2 = adjust_line(Tlen, Alen, Tcs1, L1), + string_cont(Tcs1, L2, yyaction(A, Alen, Tcs, L0), Ts) end. %% string_cont(RestChars, Line, Token, Tokens) @@ -107,8 +109,10 @@ token(S0, Ics0, L0, Tcs, Tlen0, Tline, A0, Alen0) -> {reject,_Alen1,Tlen1,Ics1,L1,_S1} -> % No token match Error = {Tline,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, {done,{error,Error,L1},Ics1}; - {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} -> % Use last accept match - token_cont(yysuf(Tcs, Alen1), L0, yyaction(A1, Alen1, Tcs, Tline)) + {A1,Alen1,Tlen1,_Ics1,L1,_S1} -> % Use last accept match + Tcs1 = yysuf(Tcs, Alen1), + L2 = adjust_line(Tlen1, Alen1, Tcs1, L1), + token_cont(Tcs1, L2, yyaction(A1, Alen1, Tcs, Tline)) end. %% token_cont(RestChars, Line, Token) @@ -181,9 +185,11 @@ tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Ts, A0, Alen0) -> %% Skip rest of tokens. Error = {L1,?MODULE,{illegal,yypre(Tcs, Tlen1+1)}}, skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error); - {A1,Alen1,_Tlen1,_Ics1,_L1,_S1} -> + {A1,Alen1,Tlen1,_Ics1,L1,_S1} -> Token = yyaction(A1, Alen1, Tcs, Tline), - tokens_cont(yysuf(Tcs, Alen1), L0, Token, Ts) + Tcs1 = yysuf(Tcs, Alen1), + L2 = adjust_line(Tlen1, Alen1, Tcs1, L1), + tokens_cont(Tcs1, L2, Token, Ts) end. %% tokens_cont(RestChars, Line, Token, Tokens) @@ -235,9 +241,11 @@ skip_tokens(S0, Ics0, L0, Tcs, Tlen0, Tline, Error, A0, Alen0) -> {done,{error,Error,L1},eof}; {reject,_Alen1,Tlen1,_Ics1,L1,_S1} -> skip_tokens(yysuf(Tcs, Tlen1+1), L1, Error); - {A1,Alen1,_Tlen1,_Ics1,L1,_S1} -> + {A1,Alen1,Tlen1,_Ics1,L1,_S1} -> Token = yyaction(A1, Alen1, Tcs, Tline), - skip_cont(yysuf(Tcs, Alen1), L1, Token, Error) + Tcs1 = yysuf(Tcs, Alen1), + L2 = adjust_line(Tlen1, Alen1, Tcs1, L1), + skip_cont(Tcs1, L2, Token, Error) end. %% skip_cont(RestChars, Line, Token, Error) @@ -269,6 +277,17 @@ yyrev(List, Tail) -> lists:reverse(List, Tail). yypre(List, N) -> lists:sublist(List, N). yysuf(List, N) -> lists:nthtail(N, List). +%% adjust_line(TokenLength, AcceptLength, Chars, Line) -> NewLine +%% Make sure that newlines in Chars are not counted twice. +%% Line has been updated with respect to newlines in the prefix of +%% Chars consisting of (TokenLength - AcceptLength) characters. + +adjust_line(N, N, _Cs, L) -> L; +adjust_line(T, A, [$\n|Cs], L) -> + adjust_line(T-1, A, Cs, L-1); +adjust_line(T, A, [_|Cs], L) -> + adjust_line(T-1, A, Cs, L). + %% yystate() -> InitialState. %% yystate(State, InChars, Line, CurrTokLen, AcceptAction, AcceptLen) -> %% {Action, AcceptLen, RestChars, Line} | diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl index 84f9c996ac..54602848ec 100644 --- a/lib/parsetools/test/leex_SUITE.erl +++ b/lib/parsetools/test/leex_SUITE.erl @@ -45,7 +45,7 @@ pt/1, man/1, ex/1, ex2/1, not_yet/1, line_wrap/1, - otp_10302/1, otp_11286/1, unicode/1]). + otp_10302/1, otp_11286/1, unicode/1, otp_13916/1]). % Default timetrap timeout (set in init_per_testcase). -define(default_timeout, ?t:minutes(1)). @@ -62,12 +62,12 @@ end_per_testcase(_Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [{group, checks}, {group, examples}, {group, bugs}]. + [{group, checks}, {group, examples}, {group, tickets}, {group, bugs}]. groups() -> [{checks, [], [file, compile, syntax]}, {examples, [], [pt, man, ex, ex2, not_yet, unicode]}, - {tickets, [], [otp_10302, otp_11286]}, + {tickets, [], [otp_10302, otp_11286, otp_13916]}, {bugs, [], [line_wrap]}]. init_per_suite(Config) -> @@ -1052,7 +1052,7 @@ otp_11286(Config) when is_list(Config) -> Dir = ?privdir, UName = [1024] ++ "u", UDir = filename:join(Dir, UName), - ok = rpc:call(Node, file, make_dir, [UDir]), + _ = rpc:call(Node, file, make_dir, [UDir]), %% Note: Cannot use UName as filename since the filename is used %% as module name. To be fixed in R18. @@ -1095,6 +1095,42 @@ otp_11286(Config) when is_list(Config) -> true = test_server:stop_node(Node), ok. +otp_13916(doc) -> + "OTP-13916. Leex rules with newlines result in bad line numbers"; +otp_13916(suite) -> []; +otp_13916(Config) when is_list(Config) -> + Ts = [{otp_13916_1, + <<"Definitions.\n" + "W = [a-zA-Z0-9]\n" + "S = [\\s\\t]\n" + "B = [\\n\\r]\n" + "Rules.\n" + "%% mark line break(s) and empty lines by token 'break'\n" + "%% in order to use as delimiters\n" + "{B}({S}*{B})+ : {token, {break, TokenLine}}.\n" + "{B} : {token, {break, TokenLine}}.\n" + "{S}+ : {token, {blank, TokenLine, TokenChars}}.\n" + "{W}+ : {token, {word, TokenLine, TokenChars}}.\n" + "Erlang code.\n" + "-export([t/0]).\n" + "t() ->\n" + " {ok,[{break,1},{blank,4,\" \"},{word,4,\"breaks\"}],4} =\n" + " string(\"\\n\\n \\n breaks\"),\n" + " {ok,[{break,1},{word,4,\"works\"}],4} =\n" + " string(\"\\n\\n \\nworks\"),\n" + " {ok,[{break,1},{word,4,\"L4\"},{break,4},\n" + " {word,5,\"L5\"},{break,5},{word,7,\"L7\"}], 7} =\n" + " string(\"\\n\\n \\nL4\\nL5\\n\\nL7\"),\n" + " {ok,[{break,1},{blank,4,\" \"},{word,4,\"L4\"},\n" + " {break,4},{blank,5,\" \"},{word,5,\"L5\"},\n" + " {break,5},{blank,7,\" \"},{word,7,\"L7\"}], 7} =\n" + " string(\"\\n\\n \\n L4\\n L5\\n\\n L7\"),\n" + " ok.\n">>, + default, + ok}], + ?line run(Config, Ts), + ok. + start_node(Name, Args) -> [_,Host] = string:tokens(atom_to_list(node()), "@"), ct:log("Trying to start ~w@~s~n", [Name,Host]), diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl index e91ddb11d1..5bd71d5d19 100644 --- a/lib/parsetools/test/yecc_SUITE.erl +++ b/lib/parsetools/test/yecc_SUITE.erl @@ -2009,7 +2009,7 @@ otp_11286(Config) when is_list(Config) -> Dir = ?privdir, UName = [1024] ++ "u", UDir = filename:join(Dir, UName), - ok = rpc:call(Node, file, make_dir, [UDir]), + _ = rpc:call(Node, file, make_dir, [UDir]), %% Note: Cannot use UName as filename since the filename is used %% as module name. To be fixed in R18. -- cgit v1.2.3