diff options
Diffstat (limited to 'lib/stdlib/test/erl_scan_SUITE.erl')
| -rw-r--r-- | lib/stdlib/test/erl_scan_SUITE.erl | 1282 |
1 files changed, 652 insertions, 630 deletions
diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 361abbb771..4ae734eb65 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -1,28 +1,29 @@ -%% -*- coding: utf-8 -*- %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% -module(erl_scan_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2, init_per_group/2,end_per_group/2]). --export([ error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1, - otp_10990/1, otp_10992/1]). +-export([error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1, + otp_10990/1, otp_10992/1, otp_11807/1]). -import(lists, [nth/2,flatten/1]). -import(io_lib, [print/1]). @@ -30,7 +31,7 @@ %% %% Define to run outside of test server %% -%-define(STANDALONE,1). +%%-define(STANDALONE,1). -ifdef(STANDALONE). -compile(export_all). @@ -42,26 +43,22 @@ %% config(data_dir, _) -> %% ".". -else. --include_lib("test_server/include/test_server.hrl"). --export([init_per_testcase/2, end_per_testcase/2]). +-include_lib("common_test/include/ct.hrl"). +-endif. -init_per_testcase(_Case, Config) when is_list(Config) -> - ?line Dog=test_server:timetrap(test_server:seconds(1200)), - [{watchdog, Dog}|Config]. +init_per_testcase(_Case, Config) -> + Config. -end_per_testcase(_Case, Config) -> - Dog=?config(watchdog, Config), - test_server:timetrap_cancel(Dog), +end_per_testcase(_Case, _Config) -> ok. --endif. - -% Default timetrap timeout (set in init_per_testcase). --define(default_timeout, ?t:minutes(1)). -suite() -> [{ct_hooks,[ts_install_cth]}]. +suite() -> + [{ct_hooks,[ts_install_cth]}, + {timetrap,{minutes,20}}]. all() -> - [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992]. + [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992, + otp_11807]. groups() -> [{error, [], [error_1, error_2]}]. @@ -80,20 +77,14 @@ end_per_group(_GroupName, Config) -> -error_1(doc) -> - ["(OTP-2347)"]; -error_1(suite) -> - []; +%% (OTP-2347) error_1(Config) when is_list(Config) -> - ?line {error, _, _} = erl_scan:string("'a"), + {error, _, _} = erl_scan:string("'a"), ok. -error_2(doc) -> - ["Checks that format_error works on the error cases."]; -error_2(suite) -> - []; +%% Checks that format_error works on the error cases. error_2(Config) when is_list(Config) -> - ?line lists:foreach(fun check/1, error_cases()), + lists:foreach(fun check/1, error_cases()), ok. error_cases() -> @@ -106,7 +97,7 @@ error_cases() -> "2.3e", "2.3e-", "91#9" -]. + ]. assert_type(N, integer) when is_integer(N) -> ok; @@ -127,71 +118,66 @@ check_error({error, Info, EndLine}, Module0) -> String = lists:flatten(Module0:format_error(Desc)), true = io_lib:printable_list(String). -iso88591(doc) -> ["Tests the support for ISO-8859-1 i.e Latin-1"]; -iso88591(suite) -> []; +%% Tests the support for ISO-8859-1 i.e Latin-1. iso88591(Config) when is_list(Config) -> - ?line ok = - case catch begin - %% Some atom and variable names - V1s = [$Á,$á,$é,$ë], - V2s = [$N,$ä,$r], - A1s = [$h,$ä,$r], - A2s = [$ö,$r,$e], - %% Test parsing atom and variable characters. - {ok,Ts1,_} = erl_scan:string(V1s ++ " " ++ V2s ++ - "\327" ++ - A1s ++ " " ++ A2s), - V1s = atom_to_list(element(3, nth(1, Ts1))), - V2s = atom_to_list(element(3, nth(2, Ts1))), - A1s = atom_to_list(element(3, nth(4, Ts1))), - A2s = atom_to_list(element(3, nth(5, Ts1))), - %% Test printing atoms - A1s = flatten(print(element(3, nth(4, Ts1)))), - A2s = flatten(print(element(3, nth(5, Ts1)))), - %% Test parsing and printing strings. - S1 = V1s ++ "\327" ++ A1s ++ "\250" ++ A2s, - S1s = "\"" ++ S1 ++ "\"", - {ok,Ts2,_} = erl_scan:string(S1s), - S1 = element(3, nth(1, Ts2)), - S1s = flatten(print(element(3, nth(1, Ts2)))), - ok %It all worked - end of - {'EXIT',R} -> %Something went wrong! - {error,R}; - ok -> ok %Aok - end. - -otp_7810(doc) -> - ["OTP-7810. White spaces, comments, and more.."]; -otp_7810(suite) -> - []; + ok = + case catch begin + %% Some atom and variable names + V1s = [$Á,$á,$é,$ë], + V2s = [$N,$ä,$r], + A1s = [$h,$ä,$r], + A2s = [$ö,$r,$e], + %% Test parsing atom and variable characters. + {ok,Ts1,_} = erl_scan_string(V1s ++ " " ++ V2s ++ + "\327" ++ + A1s ++ " " ++ A2s), + V1s = atom_to_list(element(3, nth(1, Ts1))), + V2s = atom_to_list(element(3, nth(2, Ts1))), + A1s = atom_to_list(element(3, nth(4, Ts1))), + A2s = atom_to_list(element(3, nth(5, Ts1))), + %% Test printing atoms + A1s = flatten(print(element(3, nth(4, Ts1)))), + A2s = flatten(print(element(3, nth(5, Ts1)))), + %% Test parsing and printing strings. + S1 = V1s ++ "\327" ++ A1s ++ "\250" ++ A2s, + S1s = "\"" ++ S1 ++ "\"", + {ok,Ts2,_} = erl_scan_string(S1s), + S1 = element(3, nth(1, Ts2)), + S1s = flatten(print(element(3, nth(1, Ts2)))), + ok %It all worked + end of + {'EXIT',R} -> %Something went wrong! + {error,R}; + ok -> ok %Aok + end. + +%% OTP-7810. White spaces, comments, and more... otp_7810(Config) when is_list(Config) -> - ?line ok = reserved_words(), - ?line ok = atoms(), - ?line ok = punctuations(), - ?line ok = comments(), - ?line ok = errors(), - ?line ok = integers(), - ?line ok = base_integers(), - ?line ok = floats(), - ?line ok = dots(), - ?line ok = chars(), - ?line ok = variables(), - ?line ok = eof(), - ?line ok = illegal(), - ?line ok = crashes(), - - ?line ok = options(), - ?line ok = token_info(), - ?line ok = column_errors(), - ?line ok = white_spaces(), - - ?line ok = unicode(), - - ?line ok = more_chars(), - ?line ok = more_options(), - ?line ok = attributes_info(), - ?line ok = set_attribute(), + ok = reserved_words(), + ok = atoms(), + ok = punctuations(), + ok = comments(), + ok = errors(), + ok = integers(), + ok = base_integers(), + ok = floats(), + ok = dots(), + ok = chars(), + ok = variables(), + ok = eof(), + ok = illegal(), + ok = crashes(), + + ok = options(), + ok = token_info(), + ok = column_errors(), + ok = white_spaces(), + + ok = unicode(), + + ok = more_chars(), + ok = more_options(), + ok = anno_info(), ok. @@ -202,36 +188,36 @@ reserved_words() -> 'rem', 'band', 'and', 'bor', 'bxor', 'bsl', 'bsr', 'or', 'xor'], [begin - ?line {RW, true} = {RW, erl_scan:reserved_word(RW)}, + {RW, true} = {RW, erl_scan:reserved_word(RW)}, S = atom_to_list(RW), - Ts = [{RW,1}], - ?line test_string(S, Ts) + Ts = [{RW,{1,1}}], + test_string(S, Ts) end || RW <- L], ok. atoms() -> - ?line test_string("a - b", [{atom,1,a},{atom,2,b}]), - ?line test_string("'a b'", [{atom,1,'a b'}]), - ?line test_string("a", [{atom,1,a}]), - ?line test_string("a@2", [{atom,1,a@2}]), - ?line test_string([39,65,200,39], [{atom,1,'AÈ'}]), - ?line test_string("ärlig östen", [{atom,1,ärlig},{atom,1,östen}]), - ?line {ok,[{atom,_,'$a'}],{1,6}} = - erl_scan:string("'$\\a'", {1,1}), - ?line test("'$\\a'"), - ok. + test_string("a + b", [{atom,{1,1},a},{atom,{2,18},b}]), + test_string("'a b'", [{atom,{1,1},'a b'}]), + test_string("a", [{atom,{1,1},a}]), + test_string("a@2", [{atom,{1,1},a@2}]), + test_string([39,65,200,39], [{atom,{1,1},'AÈ'}]), + test_string("ärlig östen", [{atom,{1,1},ärlig},{atom,{1,7},östen}]), + {ok,[{atom,_,'$a'}],{1,6}} = + erl_scan_string("'$\\a'", {1,1}), + test("'$\\a'"), + ok. punctuations() -> L = ["<<", "<-", "<=", "<", ">>", ">=", ">", "->", "--", - "-", "++", "+", "=:=", "=/=", "=<", "==", "=", "/=", - "/", "||", "|", ":-", "::", ":"], + "-", "++", "+", "=:=", "=/=", "=<", "=>", "==", "=", "/=", + "/", "||", "|", ":=", "::", ":"], %% One token at a time: [begin W = list_to_atom(S), - Ts = [{W,1}], - ?line test_string(S, Ts) + Ts = [{W,{1,1}}], + test_string(S, Ts) end || S <- L], Three = ["/=:=", "<=:=", "==:=", ">=:="], % three tokens... No = Three ++ L, @@ -246,66 +232,73 @@ punctuations() -> [begin W1 = list_to_atom(S1), W2 = list_to_atom(S2), - Ts = [{W1,1},{W2,1}], - ?line test_string(S, Ts) - end || {S,[{_,S1,S2}|_]} <- SL], - - PTs1 = [{'!',1},{'(',1},{')',1},{',',1},{';',1},{'=',1},{'[',1}, - {']',1},{'{',1},{'|',1},{'}',1}], - ?line test_string("!(),;=[]{|}", PTs1), - - PTs2 = [{'#',1},{'&',1},{'*',1},{'+',1},{'/',1}, - {':',1},{'<',1},{'>',1},{'?',1},{'@',1}, - {'\\',1},{'^',1},{'`',1},{'~',1}], - ?line test_string("#&*+/:<>?@\\^`~", PTs2), - - ?line test_string(".. ", [{'..',1}]), - ?line test("1 .. 2"), - ?line test_string("...", [{'...',1}]), + Ts = [{W1,{1,1}},{W2,{1,-L2+1}}], + test_string(S, Ts) + end || {S,[{L2,S1,S2}|_]} <- SL], + + PTs1 = [{'!',{1,1}},{'(',{1,2}},{')',{1,3}},{',',{1,4}},{';',{1,5}}, + {'=',{1,6}},{'[',{1,7}},{']',{1,8}},{'{',{1,9}},{'|',{1,10}}, + {'}',{1,11}}], + test_string("!(),;=[]{|}", PTs1), + + PTs2 = [{'#',{1,1}},{'&',{1,2}},{'*',{1,3}},{'+',{1,4}},{'/',{1,5}}, + {':',{1,6}},{'<',{1,7}},{'>',{1,8}},{'?',{1,9}},{'@',{1,10}}, + {'\\',{1,11}},{'^',{1,12}},{'`',{1,13}},{'~',{1,14}}], + test_string("#&*+/:<>?@\\^`~", PTs2), + + test_string(".. ", [{'..',{1,1}}]), + test_string("1 .. 2", + [{integer,{1,1},1},{'..',{1,3}},{integer,{1,6},2}]), + test_string("...", [{'...',{1,1}}]), ok. comments() -> - ?line test("a %%\n b"), - ?line {ok,[],1} = erl_scan:string("%"), - ?line test("a %%\n b"), - ?line {ok,[{atom,_,a},{atom,_,b}],{2,3}} = - erl_scan:string("a %%\n b",{1,1}), - ?line {ok,[{atom,_,a},{comment,_,"%%"},{atom,_,b}],{2,3}} = - erl_scan:string("a %%\n b",{1,1}, [return_comments]), - ?line {ok,[{atom,_,a}, - {white_space,_," "}, - {white_space,_,"\n "}, - {atom,_,b}], - {2,3}} = - erl_scan:string("a %%\n b",{1,1},[return_white_spaces]), - ?line {ok,[{atom,_,a}, - {white_space,_," "}, - {comment,_,"%%"}, - {white_space,_,"\n "}, - {atom,_,b}], - {2,3}} = erl_scan:string("a %%\n b",{1,1},[return]), + test("a %%\n b"), + {ok,[],1} = erl_scan_string("%"), + test("a %%\n b"), + {ok,[{atom,{1,1},a},{atom,{2,2},b}],{2,3}} = + erl_scan_string("a %%\n b", {1,1}), + {ok,[{atom,{1,1},a},{comment,{1,3},"%%"},{atom,{2,2},b}],{2,3}} = + erl_scan_string("a %%\n b",{1,1}, [return_comments]), + {ok,[{atom,{1,1},a}, + {white_space,{1,2}," "}, + {white_space,{1,5},"\n "}, + {atom,{2,2},b}], + {2,3}} = + erl_scan_string("a %%\n b",{1,1},[return_white_spaces]), + {ok,[{atom,{1,1},a}, + {white_space,{1,2}," "}, + {comment,{1,3},"%%"}, + {white_space,{1,5},"\n "}, + {atom,{2,2},b}], + {2,3}} = erl_scan_string("a %%\n b",{1,1},[return]), ok. errors() -> - ?line {error,{1,erl_scan,{string,$',"qa"}},1} = erl_scan:string("'qa"), %' - ?line {error,{1,erl_scan,{string,$","str"}},1} = %" + {error,{1,erl_scan,{string,$',"qa"}},1} = erl_scan:string("'qa"), %' + {error,{{1,1},erl_scan,{string,$',"qa"}},{1,4}} = %' + erl_scan:string("'qa", {1,1}, []), %' + {error,{1,erl_scan,{string,$","str"}},1} = %" erl_scan:string("\"str"), %" - ?line {error,{1,erl_scan,char},1} = erl_scan:string("$"), - ?line test_string([34,65,200,34], [{string,1,"AÈ"}]), - ?line test_string("\\", [{'\\',1}]), - ?line {'EXIT',_} = + {error,{{1,1},erl_scan,{string,$","str"}},{1,5}} = %" + erl_scan:string("\"str", {1,1}, []), %" + {error,{1,erl_scan,char},1} = erl_scan:string("$"), + {error,{{1,1},erl_scan,char},{1,2}} = erl_scan:string("$", {1,1}, []), + test_string([34,65,200,34], [{string,{1,1},"AÈ"}]), + test_string("\\", [{'\\',{1,1}}]), + {'EXIT',_} = (catch {foo, erl_scan:string('$\\a', {1,1})}), % type error - ?line {'EXIT',_} = + {'EXIT',_} = (catch {foo, erl_scan:tokens([], '$\\a', {1,1})}), % type error - ?line "{a,tuple}" = erl_scan:format_error({a,tuple}), + "{a,tuple}" = erl_scan:format_error({a,tuple}), ok. integers() -> [begin I = list_to_integer(S), - Ts = [{integer,1,I}], - ?line test_string(S, Ts) + Ts = [{integer,{1,1},I}], + test_string(S, Ts) end || S <- [[N] || N <- lists:seq($0, $9)] ++ ["2323","000"] ], ok. @@ -313,122 +306,132 @@ base_integers() -> [begin B = list_to_integer(BS), I = erlang:list_to_integer(S, B), - Ts = [{integer,1,I}], - ?line test_string(BS++"#"++S, Ts) + Ts = [{integer,{1,1},I}], + test_string(BS++"#"++S, Ts) end || {BS,S} <- [{"2","11"}, {"5","23234"}, {"12","05a"}, {"16","abcdef"}, {"16","ABCDEF"}] ], - ?line {error,{1,erl_scan,{base,1}},1} = erl_scan:string("1#000"), + {error,{1,erl_scan,{base,1}},1} = erl_scan:string("1#000"), + {error,{{1,1},erl_scan,{base,1}},{1,2}} = + erl_scan:string("1#000", {1,1}, []), - ?line test_string("12#bc", [{integer,1,11},{atom,1,c}]), + test_string("12#bc", [{integer,{1,1},11},{atom,{1,5},c}]), [begin Str = BS ++ "#" ++ S, - ?line {error,{1,erl_scan,{illegal,integer}},1} = + {error,{1,erl_scan,{illegal,integer}},1} = erl_scan:string(Str) end || {BS,S} <- [{"3","3"},{"15","f"}, {"12","c"}] ], - ?line {ok,[{integer,1,239},{'@',1}],1} = erl_scan:string("16#ef@"), - ?line {ok,[{integer,1,14},{atom,1,g@}],1} = erl_scan:string("16#eg@"), + {ok,[{integer,1,239},{'@',1}],1} = erl_scan_string("16#ef@"), + {ok,[{integer,{1,1},239},{'@',{1,6}}],{1,7}} = + erl_scan_string("16#ef@", {1,1}, []), + {ok,[{integer,{1,1},14},{atom,{1,5},g@}],{1,7}} = + erl_scan_string("16#eg@", {1,1}, []), ok. floats() -> [begin F = list_to_float(FS), - Ts = [{float,1,F}], - ?line test_string(FS, Ts) + Ts = [{float,{1,1},F}], + test_string(FS, Ts) end || FS <- ["1.0","001.17","3.31200","1.0e0","1.0E17", "34.21E-18", "17.0E+14"]], - ?line test_string("1.e2", [{integer,1,1},{'.',1},{atom,1,e2}]), + test_string("1.e2", [{integer,{1,1},1},{'.',{1,2}},{atom,{1,3},e2}]), - ?line {error,{1,erl_scan,{illegal,float}},1} = + {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string("1.0e400"), + {error,{{1,1},erl_scan,{illegal,float}},{1,8}} = + erl_scan:string("1.0e400", {1,1}, []), [begin - ?line {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(S) + {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(S), + {error,{{1,1},erl_scan,{illegal,float}},{1,_}} = + erl_scan:string(S, {1,1}, []) end || S <- ["1.14Ea"]], ok. dots() -> - Dot = [{".", {ok,[{dot,1}],1}}, - {". ", {ok,[{dot,1}],1}}, - {".\n", {ok,[{dot,1}],2}}, - {".%", {ok,[{dot,1}],1}}, - {".\210",{ok,[{dot,1}],1}}, - {".% öh",{ok,[{dot,1}],1}}, - {".%\n", {ok,[{dot,1}],2}}, - {".$", {error,{1,erl_scan,char},1}}, - {".$\\", {error,{1,erl_scan,char},1}}, - {".a", {ok,[{'.',1},{atom,1,a}],1}} + Dot = [{".", {ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,2}}}, + {". ", {ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,3}}}, + {".\n", {ok,[{dot,1}],2}, {ok,[{dot,{1,1}}],{2,1}}}, + {".%", {ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,3}}}, + {".\210",{ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,3}}}, + {".% öh",{ok,[{dot,1}],1}, {ok,[{dot,{1,1}}],{1,6}}}, + {".%\n", {ok,[{dot,1}],2}, {ok,[{dot,{1,1}}],{2,1}}}, + {".$", {error,{1,erl_scan,char},1}, + {error,{{1,2},erl_scan,char},{1,3}}}, + {".$\\", {error,{1,erl_scan,char},1}, + {error,{{1,2},erl_scan,char},{1,4}}}, + {".a", {ok,[{'.',1},{atom,1,a}],1}, + {ok,[{'.',{1,1}},{atom,{1,2},a}],{1,3}}} ], - ?line [R = erl_scan:string(S) || {S, R} <- Dot], - - ?line {ok,[{dot,_}=T1],{1,2}} = erl_scan:string(".", {1,1}, text), - ?line [{column,1},{length,1},{line,1},{text,"."}] = - erl_scan:token_info(T1, [column, length, line, text]), - ?line {ok,[{dot,_}=T2],{1,3}} = erl_scan:string(".%", {1,1}, text), - ?line [{column,1},{length,1},{line,1},{text,"."}] = - erl_scan:token_info(T2, [column, length, line, text]), - ?line {ok,[{dot,_}=T3],{1,6}} = + [begin + R = erl_scan_string(S), + R2 = erl_scan_string(S, {1,1}, []) + end || {S, R, R2} <- Dot], + + {ok,[{dot,_}=T1],{1,2}} = erl_scan:string(".", {1,1}, text), + [1, 1, "."] = token_info(T1), + {ok,[{dot,_}=T2],{1,3}} = erl_scan:string(".%", {1,1}, text), + [1, 1, "."] = token_info(T2), + {ok,[{dot,_}=T3],{1,6}} = erl_scan:string(".% öh", {1,1}, text), - ?line [{column,1},{length,1},{line,1},{text,"."}] = - erl_scan:token_info(T3, [column, length, line, text]), - ?line {error,{{1,2},erl_scan,char},{1,3}} = - erl_scan:string(".$", {1,1}), - ?line {error,{{1,2},erl_scan,char},{1,4}} = - erl_scan:string(".$\\", {1,1}), - - ?line test(". "), - ?line test(". "), - ?line test(".\n"), - ?line test(".\n\n"), - ?line test(".\n\r"), - ?line test(".\n\n\n"), - ?line test(".\210"), - ?line test(".%\n"), - ?line test(".a"), - - ?line test("%. \n. "), - ?line {more,C} = erl_scan:tokens([], "%. ",{1,1}, return), - ?line {done,{ok,[{comment,_,"%. "}, - {white_space,_,"\n"}, - {dot,_}], - {2,3}}, ""} = - erl_scan:tokens(C, "\n. ", {1,1}, return), % any loc, any options - - ?line [test_string(S, R) || - {S, R} <- [{".$\n", [{'.',1},{char,1,$\n}]}, - {"$\\\n", [{char,1,$\n}]}, - {"'\\\n'", [{atom,1,'\n'}]}, - {"$\n", [{char,1,$\n}]}] ], + [1, 1, "."] = token_info(T3), + {error,{{1,2},erl_scan,char},{1,3}} = erl_scan:string(".$", {1,1}), + {error,{{1,2},erl_scan,char},{1,4}} = erl_scan:string(".$\\", {1,1}), + + test_string(". ", [{dot,{1,1}}]), + test_string(". ", [{dot,{1,1}}]), + test_string(".\n", [{dot,{1,1}}]), + test_string(".\n\n", [{dot,{1,1}}]), + test_string(".\n\r", [{dot,{1,1}}]), + test_string(".\n\n\n", [{dot,{1,1}}]), + test_string(".\210", [{dot,{1,1}}]), + test_string(".%\n", [{dot,{1,1}}]), + test_string(".a", [{'.',{1,1}},{atom,{1,2},a}]), + + test_string("%. \n. ", [{dot,{2,1}}]), + {more,C} = erl_scan:tokens([], "%. ",{1,1}, return), + {done,{ok,[{comment,{1,1},"%. "}, + {white_space,{1,4},"\n"}, + {dot,{2,1}}], + {2,3}}, ""} = + erl_scan_tokens(C, "\n. ", {1,1}, return), % any loc, any options + + [test_string(S, R) || + {S, R} <- [{".$\n", [{'.',{1,1}},{char,{1,2},$\n}]}, + {"$\\\n", [{char,{1,1},$\n}]}, + {"'\\\n'", [{atom,{1,1},'\n'}]}, + {"$\n", [{char,{1,1},$\n}]}] ], ok. chars() -> [begin L = lists:flatten(io_lib:format("$\\~.8b", [C])), - Ts = [{char,1,C}], - ?line test_string(L, Ts) + Ts = [{char,{1,1},C}], + test_string(L, Ts) end || C <- lists:seq(0, 255)], %% Leading zeroes... [begin L = lists:flatten(io_lib:format("$\\~3.8.0b", [C])), - Ts = [{char,1,C}], - ?line test_string(L, Ts) + Ts = [{char,{1,1},C}], + test_string(L, Ts) end || C <- lists:seq(0, 255)], %% $\^\n now increments the line... [begin L = "$\\^" ++ [C], - Ts = [{char,1,C band 2#11111}], - ?line test_string(L, Ts) + Ts = [{char,{1,1},C band 2#11111}], + test_string(L, Ts) end || C <- lists:seq(0, 255)], [begin L = "$\\" ++ [C], - Ts = [{char,1,V}], - ?line test_string(L, Ts) + Ts = [{char,{1,1},V}], + test_string(L, Ts) end || {C,V} <- [{$n,$\n}, {$r,$\r}, {$t,$\t}, {$v,$\v}, {$b,$\b}, {$f,$\f}, {$e,$\e}, {$s,$\s}, {$d,$\d}]], @@ -440,177 +443,191 @@ chars() -> No = EC ++ Ds ++ X ++ New, [begin L = "$\\" ++ [C], - Ts = [{char,1,C}], - ?line test_string(L, Ts) + Ts = [{char,{1,1},C}], + test_string(L, Ts) end || C <- lists:seq(0, 255) -- No], [begin L = "'$\\" ++ [C] ++ "'", - Ts = [{atom,1,list_to_atom("$"++[C])}], - ?line test_string(L, Ts) + Ts = [{atom,{1,1},list_to_atom("$"++[C])}], + test_string(L, Ts) end || C <- lists:seq(0, 255) -- No], - ?line test_string("\"\\013a\\\n\"", [{string,1,"\va\n"}]), + test_string("\"\\013a\\\n\"", [{string,{1,1},"\va\n"}]), - ?line test_string("'\n'", [{atom,1,'\n'}]), - ?line test_string("\"\n\a\"", [{string,1,"\na"}]), + test_string("'\n'", [{atom,{1,1},'\n'}]), + test_string("\"\n\a\"", [{string,{1,1},"\na"}]), %% No escape [begin L = "$" ++ [C], - Ts = [{char,1,C}], - ?line test_string(L, Ts) + Ts = [{char,{1,1},C}], + test_string(L, Ts) end || C <- lists:seq(0, 255) -- (No ++ [$\\])], - ?line test_string("$\n", [{char,1,$\n}]), + test_string("$\n", [{char,{1,1},$\n}]), - ?line {error,{{1,1},erl_scan,char},{1,4}} = + {error,{{1,1},erl_scan,char},{1,4}} = erl_scan:string("$\\^",{1,1}), - ?line test_string("$\\\n", [{char,1,$\n}]), + test_string("$\\\n", [{char,{1,1},$\n}]), %% Robert's scanner returns line 1: - ?line test_string("$\\\n", [{char,1,$\n}]), - ?line test_string("$\n\n", [{char,1,$\n}]), - ?line test("$\n\n"), + test_string("$\\\n", [{char,{1,1},$\n}]), + test_string("$\n\n", [{char,{1,1},$\n}]), + test("$\n\n"), ok. variables() -> - ?line test_string(" \237_Aouåeiyäö", [{var,1,'_Aouåeiyäö'}]), - ?line test_string("A_b_c@", [{var,1,'A_b_c@'}]), - ?line test_string("V@2", [{var,1,'V@2'}]), - ?line test_string("ABDÀ", [{var,1,'ABDÀ'}]), - ?line test_string("Ärlig Östen", [{var,1,'Ärlig'},{var,1,'Östen'}]), + test_string(" \237_Aouåeiyäö", [{var,{1,7},'_Aouåeiyäö'}]), + test_string("A_b_c@", [{var,{1,1},'A_b_c@'}]), + test_string("V@2", [{var,{1,1},'V@2'}]), + test_string("ABDÀ", [{var,{1,1},'ABDÀ'}]), + test_string("Ärlig Östen", [{var,{1,1},'Ärlig'},{var,{1,7},'Östen'}]), ok. eof() -> - ?line {done,{eof,1},eof} = erl_scan:tokens([], eof, 1), + {done,{eof,1},eof} = erl_scan:tokens([], eof, 1), {more, C1} = erl_scan:tokens([]," \n", 1), - ?line {done,{eof,2},eof} = erl_scan:tokens(C1, eof, 1), + {done,{eof,2},eof} = erl_scan:tokens(C1, eof, 1), {more, C2} = erl_scan:tokens([], "abra", 1), %% An error before R13A. - %% ?line {done,Err={error,{1,erl_scan,scan},1},eof} = - ?line {done,{ok,[{atom,1,abra}],1},eof} = - erl_scan:tokens(C2, eof, 1), + %% {done,Err={error,{1,erl_scan,scan},1},eof} = + {done,{ok,[{atom,1,abra}],1},eof} = + erl_scan_tokens(C2, eof, 1), %% With column. - ?line {more, C3} = erl_scan:tokens([]," \n",{1,1}), - ?line {done,{eof,{2,1}},eof} = erl_scan:tokens(C3, eof, 1), + {more, C3} = erl_scan:tokens([]," \n",{1,1}), + {done,{eof,{2,1}},eof} = erl_scan:tokens(C3, eof, 1), {more, C4} = erl_scan:tokens([], "abra", {1,1}), %% An error before R13A. - %% ?line {done,{error,{{1,1},erl_scan,scan},{1,5}},eof} = - ?line {done,{ok,[{atom,_,abra}],{1,5}},eof} = - erl_scan:tokens(C4, eof, 1), + %% {done,{error,{{1,1},erl_scan,scan},{1,5}},eof} = + {done,{ok,[{atom,_,abra}],{1,5}},eof} = + erl_scan_tokens(C4, eof, 1), %% Robert's scanner returns "" as LeftoverChars; %% the R12B scanner returns eof as LeftoverChars: (eof is correct) - ?line {more, C5} = erl_scan:tokens([], "a", 1), + {more, C5} = erl_scan:tokens([], "a", 1), + %% An error before R13A. + %% {done,{error,{1,erl_scan,scan},1},eof} = + {done,{ok,[{atom,1,a}],1},eof} = + erl_scan_tokens(C5,eof,1), + + %% With column. + {more, C6} = erl_scan:tokens([], "a", {1,1}), %% An error before R13A. - %% ?line {done,{error,{1,erl_scan,scan},1},eof} = - ?line {done,{ok,[{atom,1,a}],1},eof} = - erl_scan:tokens(C5,eof,1), + %% {done,{error,{1,erl_scan,scan},1},eof} = + {done,{ok,[{atom,{1,1},a}],{1,2}},eof} = + erl_scan_tokens(C6,eof,1), %% A dot followed by eof is special: - ?line {more, C} = erl_scan:tokens([], "a.", 1), - ?line {done,{ok,[{atom,1,a},{dot,1}],1},eof} = erl_scan:tokens(C,eof,1), - ?line {ok,[{atom,1,foo},{dot,1}],1} = erl_scan:string("foo."), + {more, C} = erl_scan:tokens([], "a.", 1), + {done,{ok,[{atom,1,a},{dot,1}],1},eof} = erl_scan_tokens(C,eof,1), + {ok,[{atom,1,foo},{dot,1}],1} = erl_scan_string("foo."), + + %% With column. + {more, CCol} = erl_scan:tokens([], "a.", {1,1}), + {done,{ok,[{atom,{1,1},a},{dot,{1,2}}],{1,3}},eof} = + erl_scan_tokens(CCol,eof,1), + {ok,[{atom,{1,1},foo},{dot,{1,4}}],{1,5}} = + erl_scan_string("foo.", {1,1}, []), ok. illegal() -> Atom = lists:duplicate(1000, $a), - ?line {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(Atom), - ?line {done,{error,{1,erl_scan,{illegal,atom}},1},". "} = + {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(Atom), + {done,{error,{1,erl_scan,{illegal,atom}},1},". "} = erl_scan:tokens([], Atom++". ", 1), QAtom = "'" ++ Atom ++ "'", - ?line {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(QAtom), - ?line {done,{error,{1,erl_scan,{illegal,atom}},1},". "} = + {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(QAtom), + {done,{error,{1,erl_scan,{illegal,atom}},1},". "} = erl_scan:tokens([], QAtom++". ", 1), Var = lists:duplicate(1000, $A), - ?line {error,{1,erl_scan,{illegal,var}},1} = erl_scan:string(Var), - ?line {done,{error,{1,erl_scan,{illegal,var}},1},". "} = + {error,{1,erl_scan,{illegal,var}},1} = erl_scan:string(Var), + {done,{error,{1,erl_scan,{illegal,var}},1},". "} = erl_scan:tokens([], Var++". ", 1), Float = "1" ++ lists:duplicate(400, $0) ++ ".0", - ?line {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(Float), - ?line {done,{error,{1,erl_scan,{illegal,float}},1},". "} = + {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(Float), + {done,{error,{1,erl_scan,{illegal,float}},1},". "} = erl_scan:tokens([], Float++". ", 1), String = "\"43\\x{aaaaaa}34\"", - ?line {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string(String), - ?line {done,{error,{1,erl_scan,{illegal,character}},1},"34\". "} = + {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string(String), + {done,{error,{1,erl_scan,{illegal,character}},1},"34\". "} = %% Would be nice if `34\"' were skipped... %% Maybe, but then the LeftOverChars would not be the characters %% immediately following the end location of the error. erl_scan:tokens([], String++". ", 1), - ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,1001}} = + {error,{{1,1},erl_scan,{illegal,atom}},{1,1001}} = erl_scan:string(Atom, {1,1}), - ?line {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1005}},". "} = + {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1005}},". "} = erl_scan:tokens([], "foo "++Atom++". ", {1,1}), - ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,1003}} = + {error,{{1,1},erl_scan,{illegal,atom}},{1,1003}} = erl_scan:string(QAtom, {1,1}), - ?line {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1007}},". "} = + {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1007}},". "} = erl_scan:tokens([], "foo "++QAtom++". ", {1,1}), - ?line {error,{{1,1},erl_scan,{illegal,var}},{1,1001}} = + {error,{{1,1},erl_scan,{illegal,var}},{1,1001}} = erl_scan:string(Var, {1,1}), - ?line {done,{error,{{1,5},erl_scan,{illegal,var}},{1,1005}},". "} = + {done,{error,{{1,5},erl_scan,{illegal,var}},{1,1005}},". "} = erl_scan:tokens([], "foo "++Var++". ", {1,1}), - ?line {error,{{1,1},erl_scan,{illegal,float}},{1,404}} = + {error,{{1,1},erl_scan,{illegal,float}},{1,404}} = erl_scan:string(Float, {1,1}), - ?line {done,{error,{{1,5},erl_scan,{illegal,float}},{1,408}},". "} = + {done,{error,{{1,5},erl_scan,{illegal,float}},{1,408}},". "} = erl_scan:tokens([], "foo "++Float++". ", {1,1}), - ?line {error,{{1,4},erl_scan,{illegal,character}},{1,14}} = + {error,{{1,4},erl_scan,{illegal,character}},{1,14}} = erl_scan:string(String, {1,1}), - ?line {done,{error,{{1,4},erl_scan,{illegal,character}},{1,14}},"34\". "} = + {done,{error,{{1,4},erl_scan,{illegal,character}},{1,14}},"34\". "} = erl_scan:tokens([], String++". ", {1,1}), ok. crashes() -> - ?line {'EXIT',_} = (catch {foo, erl_scan:string([-1])}), % type error - ?line {'EXIT',_} = (catch {foo, erl_scan:string("$"++[-1])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[-1])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[-1])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"],{1,1})}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[-1,$"])}), %$" - ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[-1])}), - ?line {'EXIT',_} = + {'EXIT',_} = (catch {foo, erl_scan:string([-1])}), % type error + {'EXIT',_} = (catch {foo, erl_scan:string("$"++[-1])}), + {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[-1])}), + {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[-1])}), + {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"],{1,1})}), + {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[-1,$"])}), %$" + {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"])}), + {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[-1])}), + {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[-1],{1,1})}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string([a])}), % type error - ?line {'EXIT',_} = (catch {foo, erl_scan:string("$"++[a])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[a])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[a])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"],{1,1})}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[a,$"])}), %$" - ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"])}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[a])}), - ?line {'EXIT',_} = + {'EXIT',_} = (catch {foo, erl_scan:string([a])}), % type error + {'EXIT',_} = (catch {foo, erl_scan:string("$"++[a])}), + {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[a])}), + {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[a])}), + {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"],{1,1})}), + {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[a,$"])}), %$" + {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"])}), + {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[a])}), + {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[a],{1,1})}), - ?line {'EXIT',_} = (catch {foo, erl_scan:string([3.0])}), % type error + {'EXIT',_} = (catch {foo, erl_scan:string([3.0])}), % type error ok. options() -> %% line and column are not options, but tested here - ?line {ok,[{atom,1,foo},{white_space,1," "},{comment,1,"% bar"}], 1} = - erl_scan:string("foo % bar", 1, return), - ?line {ok,[{atom,1,foo},{white_space,1," "}],1} = - erl_scan:string("foo % bar", 1, return_white_spaces), - ?line {ok,[{atom,1,foo},{comment,1,"% bar"}],1} = - erl_scan:string("foo % bar", 1, return_comments), - ?line {ok,[{atom,17,foo}],17} = - erl_scan:string("foo % bar", 17), - ?line {'EXIT',{function_clause,_}} = + {ok,[{atom,1,foo},{white_space,1," "},{comment,1,"% bar"}], 1} = + erl_scan_string("foo % bar", 1, return), + {ok,[{atom,1,foo},{white_space,1," "}],1} = + erl_scan_string("foo % bar", 1, return_white_spaces), + {ok,[{atom,1,foo},{comment,1,"% bar"}],1} = + erl_scan_string("foo % bar", 1, return_comments), + {ok,[{atom,17,foo}],17} = + erl_scan_string("foo % bar", 17), + {'EXIT',{function_clause,_}} = (catch {foo, erl_scan:string("foo % bar", {a,1}, [])}), % type error - ?line {ok,[{atom,_,foo}],{17,18}} = - erl_scan:string("foo % bar", {17,9}, []), - ?line {'EXIT',{function_clause,_}} = + {ok,[{atom,_,foo}],{17,18}} = + erl_scan_string("foo % bar", {17,9}, []), + {'EXIT',{function_clause,_}} = (catch {foo, erl_scan:string("foo % bar", {1,0}, [])}), % type error - ?line {ok,[{foo,1}],1} = - erl_scan:string("foo % bar",1, [{reserved_word_fun, + {ok,[{foo,1}],1} = + erl_scan_string("foo % bar",1, [{reserved_word_fun, fun(W) -> W =:= foo end}]), - ?line {'EXIT',{badarg,_}} = + {'EXIT',{badarg,_}} = (catch {foo, erl_scan:string("foo % bar",1, % type error [{reserved_word_fun, @@ -618,168 +635,107 @@ options() -> ok. more_options() -> - ?line {ok,[{atom,A1,foo}],{19,20}} = + {ok,[{atom,_,foo}=T1],{19,20}} = erl_scan:string("foo", {19,17},[]), - ?line [{column,17},{line,19}] = erl_scan:attributes_info(A1), - ?line {done,{ok,[{atom,A2,foo},{dot,_}],{19,22}},[]} = + {19,17} = erl_scan:location(T1), + {done,{ok,[{atom,_,foo}=T2,{dot,_}],{19,22}},[]} = erl_scan:tokens([], "foo. ", {19,17}, [bad_opt]), % type error - ?line [{column,17},{line,19}] = erl_scan:attributes_info(A2), - ?line {ok,[{atom,A3,foo}],{19,20}} = + {19,17} = erl_scan:location(T2), + {ok,[{atom,_,foo}=T3],{19,20}} = erl_scan:string("foo", {19,17},[text]), - ?line [{column,17},{length,3},{line,19},{text,"foo"}] = - erl_scan:attributes_info(A3), + {19,17} = erl_scan:location(T3), + "foo" = erl_scan:text(T3), - ?line {ok,[{atom,A4,foo}],1} = erl_scan:string("foo", 1, [text]), - ?line [{length,3},{line,1},{text,"foo"}] = erl_scan:attributes_info(A4), + {ok,[{atom,_,foo}=T4],1} = erl_scan:string("foo", 1, [text]), + 1 = erl_scan:line(T4), + 1 = erl_scan:location(T4), + "foo" = erl_scan:text(T4), ok. token_info() -> - ?line {ok,[T1],_} = erl_scan:string("foo", {1,18}, [text]), + {ok,[T1],_} = erl_scan:string("foo", {1,18}, [text]), {'EXIT',{badarg,_}} = - (catch {foo, erl_scan:token_info(T1, foo)}), % type error - ?line {line,1} = erl_scan:token_info(T1, line), - ?line {column,18} = erl_scan:token_info(T1, column), - ?line {length,3} = erl_scan:token_info(T1, length), - ?line {text,"foo"} = erl_scan:token_info(T1, text), - ?line [{category,atom},{column,18},{length,3},{line,1}, - {symbol,foo},{text,"foo"}] = - erl_scan:token_info(T1), - ?line [{length,3},{column,18}] = - erl_scan:token_info(T1, [length, column]), - ?line [{location,{1,18}}] = - erl_scan:token_info(T1, [location]), - ?line {category,atom} = erl_scan:token_info(T1, category), - ?line [{symbol,foo}] = erl_scan:token_info(T1, [symbol]), - - ?line {ok,[T2],_} = erl_scan:string("foo", 1, []), - ?line {line,1} = erl_scan:token_info(T2, line), - ?line undefined = erl_scan:token_info(T2, column), - ?line undefined = erl_scan:token_info(T2, length), - ?line undefined = erl_scan:token_info(T2, text), - ?line {location,1} = erl_scan:token_info(T2, location), - ?line [{category,atom},{line,1},{symbol,foo}] = erl_scan:token_info(T2), - ?line [{line,1}] = erl_scan:token_info(T2, [length, line]), - - ?line {ok,[T3],_} = erl_scan:string("=", 1, []), - ?line [{line,1}] = erl_scan:token_info(T3, [column, line]), - ?line {category,'='} = erl_scan:token_info(T3, category), - ?line [{symbol,'='}] = erl_scan:token_info(T3, [symbol]), + (catch {foo, erl_scan:category(foo)}), % type error + {'EXIT',{badarg,_}} = + (catch {foo, erl_scan:symbol(foo)}), % type error + atom = erl_scan:category(T1), + foo = erl_scan:symbol(T1), + + {ok,[T2],_} = erl_scan:string("foo", 1, []), + 1 = erl_scan:line(T2), + undefined = erl_scan:column(T2), + undefined = erl_scan:text(T2), + 1 = erl_scan:location(T2), + + {ok,[T3],_} = erl_scan:string("=", 1, []), + '=' = erl_scan:category(T3), + '=' = erl_scan:symbol(T3), ok. -attributes_info() -> - ?line {'EXIT',_} = - (catch {foo,erl_scan:attributes_info(foo)}), % type error - ?line [{line,18}] = erl_scan:attributes_info(18), - ?line {location,19} = erl_scan:attributes_info(19, location), - ?line {ok,[{atom,A0,foo}],_} = erl_scan:string("foo", 19, [text]), - ?line {location,19} = erl_scan:attributes_info(A0, location), - - ?line {ok,[{atom,A3,foo}],_} = erl_scan:string("foo", {1,3}, [text]), - ?line {line,1} = erl_scan:attributes_info(A3, line), - ?line {column,3} = erl_scan:attributes_info(A3, column), - ?line {location,{1,3}} = erl_scan:attributes_info(A3, location), - ?line {text,"foo"} = erl_scan:attributes_info(A3, text), - - ?line {ok,[{atom,A4,foo}],_} = erl_scan:string("foo", 2, [text]), - ?line {line,2} = erl_scan:attributes_info(A4, line), - ?line undefined = erl_scan:attributes_info(A4, column), - ?line {location,2} = erl_scan:attributes_info(A4, location), - ?line {text,"foo"} = erl_scan:attributes_info(A4, text), - - ?line {ok,[{atom,A5,foo}],_} = erl_scan:string("foo", {1,3}, []), - ?line {line,1} = erl_scan:attributes_info(A5, line), - ?line {column,3} = erl_scan:attributes_info(A5, column), - ?line {location,{1,3}} = erl_scan:attributes_info(A5, location), - ?line undefined = erl_scan:attributes_info(A5, text), - - ?line undefined = erl_scan:attributes_info([], line), % type error - - ok. +anno_info() -> + {'EXIT',_} = + (catch {foo,erl_scan:line(foo)}), % type error + {ok,[{atom,_,foo}=T0],_} = erl_scan:string("foo", 19, [text]), + 19 = erl_scan:location(T0), + 19 = erl_scan:end_location(T0), + + {ok,[{atom,_,foo}=T3],_} = erl_scan:string("foo", {1,3}, [text]), + 1 = erl_scan:line(T3), + 3 = erl_scan:column(T3), + {1,3} = erl_scan:location(T3), + {1,6} = erl_scan:end_location(T3), + "foo" = erl_scan:text(T3), + + {ok,[{atom,_,foo}=T4],_} = erl_scan:string("foo", 2, [text]), + 2 = erl_scan:line(T4), + undefined = erl_scan:column(T4), + 2 = erl_scan:location(T4), + "foo" = erl_scan:text(T4), + + {ok,[{atom,_,foo}=T5],_} = erl_scan:string("foo", {1,3}, []), + 1 = erl_scan:line(T5), + 3 = erl_scan:column(T5), + {1,3} = erl_scan:location(T5), + undefined = erl_scan:text(T5), -set_attribute() -> - F = fun(Line) -> -Line end, - ?line -2 = erl_scan:set_attribute(line, 2, F), - ?line {ok,[{atom,A1,foo}],_} = erl_scan:string("foo", {9,17}), - ?line A2 = erl_scan:set_attribute(line, A1, F), - ?line {line,-9} = erl_scan:attributes_info(A2, line), - ?line {location,{-9,17}} = erl_scan:attributes_info(A2, location), - ?line [{line,-9},{column,17}] = - erl_scan:attributes_info(A2, [line,column,text]), - - F2 = fun(Line) -> {17,Line} end, - ?line Attr1 = erl_scan:set_attribute(line, 2, F2), - ?line {line,{17,2}} = erl_scan:attributes_info(Attr1, line), - ?line undefined = erl_scan:attributes_info(Attr1, column), - ?line {location,{17,2}} = % a bit mixed up - erl_scan:attributes_info(Attr1, location), - - ?line A3 = erl_scan:set_attribute(line, A1, F2), - ?line {line,{17,9}} = erl_scan:attributes_info(A3, line), - ?line {location,{{17,9},17}} = erl_scan:attributes_info(A3, location), - ?line [{line,{17,9}},{column,17}] = - erl_scan:attributes_info(A3, [line,column,text]), - - ?line {ok,[{atom,A4,foo}],_} = erl_scan:string("foo", {9,17}, [text]), - ?line A5 = erl_scan:set_attribute(line, A4, F), - ?line {line,-9} = erl_scan:attributes_info(A5, line), - ?line {location,{-9,17}} = erl_scan:attributes_info(A5, location), - ?line [{line,-9},{column,17},{text,"foo"}] = - erl_scan:attributes_info(A5, [line,column,text]), - - ?line {ok,[{atom,A6,foo}],_} = erl_scan:string("foo", 11, [text]), - ?line A7 = erl_scan:set_attribute(line, A6, F2), - ?line {line,{17,11}} = erl_scan:attributes_info(A7, line), - ?line {location,{17,11}} = % mixed up - erl_scan:attributes_info(A7, location), - ?line [{line,{17,11}},{text,"foo"}] = - erl_scan:attributes_info(A7, [line,column,text]), - - ?line {'EXIT',_} = - (catch {foo, erl_scan:set_attribute(line, [], F2)}), % type error - ?line {'EXIT',{badarg,_}} = - (catch {foo, erl_scan:set_attribute(column, [], F2)}), % type error - - %% OTP-9412 - ?line 8 = erl_scan:set_attribute(line, [{line,{nos,'X',8}}], - fun({nos,_V,VL}) -> VL end), ok. column_errors() -> - ?line {error,{{1,1},erl_scan,{string,$',""}},{1,3}} = % $' + {error,{{1,1},erl_scan,{string,$',""}},{1,3}} = % $' erl_scan:string("'\\",{1,1}), - ?line {error,{{1,1},erl_scan,{string,$",""}},{1,3}} = % $" + {error,{{1,1},erl_scan,{string,$",""}},{1,3}} = % $" erl_scan:string("\"\\",{1,1}), - ?line {error,{{1,1},erl_scan,{string,$',""}},{1,2}} = % $' + {error,{{1,1},erl_scan,{string,$',""}},{1,2}} = % $' erl_scan:string("'",{1,1}), - ?line {error,{{1,1},erl_scan,{string,$",""}},{1,2}} = % $" + {error,{{1,1},erl_scan,{string,$",""}},{1,2}} = % $" erl_scan:string("\"",{1,1}), - ?line {error,{{1,1},erl_scan,char},{1,2}} = + {error,{{1,1},erl_scan,char},{1,2}} = erl_scan:string("$",{1,1}), - ?line {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{1,20}} = %' + {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{1,20}} = %' erl_scan:string(" '12345678901234567", {1,1}), - ?line {error,{{1,2},erl_scan,{string,$',"123456789012345 "}}, {1,20}} = %' + {error,{{1,2},erl_scan,{string,$',"123456789012345 "}}, {1,20}} = %' erl_scan:string(" '123456789012345\\s", {1,1}), - ?line {error,{{1,2},erl_scan,{string,$","1234567890123456"}},{1,20}} = %" + {error,{{1,2},erl_scan,{string,$","1234567890123456"}},{1,20}} = %" erl_scan:string(" \"12345678901234567", {1,1}), - ?line {error,{{1,2},erl_scan,{string,$","123456789012345 "}}, {1,20}} = %" + {error,{{1,2},erl_scan,{string,$","123456789012345 "}}, {1,20}} = %" erl_scan:string(" \"123456789012345\\s", {1,1}), - ?line {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{2,1}} = %' + {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{2,1}} = %' erl_scan:string(" '12345678901234567\n", {1,1}), ok. white_spaces() -> - ?line {ok,[{white_space,_,"\r"}, + {ok,[{white_space,_,"\r"}, {white_space,_," "}, {atom,_,a}, {white_space,_,"\n"}], - _} = erl_scan:string("\r a\n", {1,1}, return), - ?line test("\r a\n"), + _} = erl_scan_string("\r a\n", {1,1}, return), + test("\r a\n"), L = "{\"a\nb\", \"a\\nb\",\nabc\r,def}.\n\n", - ?line {ok,[{'{',_}, + {ok,[{'{',_}, {string,_,"a\nb"}, {',',_}, {white_space,_," "}, @@ -793,92 +749,91 @@ white_spaces() -> {'}',_}, {dot,_}, {white_space,_,"\n"}], - _} = erl_scan:string(L, {1,1}, return), - ?line test(L), - ?line test("\"\n\"\n"), - ?line test("\n\r\n"), - ?line test("\n\r"), - ?line test("\r\n"), - ?line test("\n\f"), - ?line [test(lists:duplicate(N, $\t)) || N <- lists:seq(1, 20)], - ?line [test([$\n|lists:duplicate(N, $\t)]) || N <- lists:seq(1, 20)], - ?line [test(lists:duplicate(N, $\s)) || N <- lists:seq(1, 20)], - ?line [test([$\n|lists:duplicate(N, $\s)]) || N <- lists:seq(1, 20)], - ?line test("\v\f\n\v "), - ?line test("\n\e\n\b\f\n\da\n"), + _} = erl_scan_string(L, {1,1}, return), + test(L), + test("\"\n\"\n"), + test("\n\r\n"), + test("\n\r"), + test("\r\n"), + test("\n\f"), + [test(lists:duplicate(N, $\t)) || N <- lists:seq(1, 20)], + [test([$\n|lists:duplicate(N, $\t)]) || N <- lists:seq(1, 20)], + [test(lists:duplicate(N, $\s)) || N <- lists:seq(1, 20)], + [test([$\n|lists:duplicate(N, $\s)]) || N <- lists:seq(1, 20)], + test("\v\f\n\v "), + test("\n\e\n\b\f\n\da\n"), ok. unicode() -> - ?line {ok,[{char,1,83},{integer,1,45}],1} = - erl_scan:string("$\\12345"), % not unicode + {ok,[{char,1,83},{integer,1,45}],1} = + erl_scan_string("$\\12345"), % not unicode - ?line {error,{1,erl_scan,{illegal,character}},1} = + {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string([1089]), - ?line {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = + {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = erl_scan:string([1089], {1,1}), - ?line {error,{1,erl_scan,{illegal,atom}},1} = + {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string("'a"++[1089]++"b'", 1), - ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,6}} = + {error,{{1,1},erl_scan,{illegal,atom}},{1,6}} = erl_scan:string("'a"++[1089]++"b'", {1,1}), - ?line test("\"a"++[1089]++"b\""), - ?line {ok,[{char,1,1}],1} = - erl_scan:string([$$,$\\,$^,1089], 1), + test("\"a"++[1089]++"b\""), + {ok,[{char,1,1}],1} = + erl_scan_string([$$,$\\,$^,1089], 1), - ?line {error,{1,erl_scan,Error},1} = + {error,{1,erl_scan,Error},1} = erl_scan:string("\"qa\x{aaa}", 1), - ?line "unterminated string starting with \"qa"++[2730]++"\"" = + "unterminated string starting with \"qa"++[2730]++"\"" = erl_scan:format_error(Error), - ?line {error,{{1,1},erl_scan,_},{1,11}} = + {error,{{1,1},erl_scan,_},{1,11}} = erl_scan:string("\"qa\\x{aaa}",{1,1}), - ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} = + {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} = erl_scan:string("'qa\\x{aaa}'",{1,1}), - ?line {ok,[{char,1,1089}],1} = - erl_scan:string([$$,1089], 1), - ?line {ok,[{char,1,1089}],1} = - erl_scan:string([$$,$\\,1089], 1), + {ok,[{char,1,1089}],1} = + erl_scan_string([$$,1089], 1), + {ok,[{char,1,1089}],1} = + erl_scan_string([$$,$\\,1089], 1), Qs = "$\\x{aaa}", - ?line {ok,[{char,1,$\x{aaa}}],1} = - erl_scan:string(Qs, 1), - ?line {ok,[Q2],{1,9}} = + {ok,[{char,1,$\x{aaa}}],1} = + erl_scan_string(Qs, 1), + {ok,[Q2],{1,9}} = erl_scan:string("$\\x{aaa}", {1,1}, [text]), - ?line [{category,char},{column,1},{length,8}, - {line,1},{symbol,16#aaa},{text,Qs}] = - erl_scan:token_info(Q2), + [{category,char},{column,1},{line,1},{symbol,16#aaa},{text,Qs}] = + token_info_long(Q2), U1 = "\"\\x{aaa}\"", - {ok, - [{string,[{line,1},{column,1},{text,"\"\\x{aaa}\""}],[2730]}], - {1,10}} = erl_scan:string(U1, {1,1}, [text]), - {ok,[{string,1,[2730]}],1} = erl_scan:string(U1, 1), + {ok,[{string,_,[2730]}=T1],{1,10}} = erl_scan:string(U1, {1,1}, [text]), + {1,1} = erl_scan:location(T1), + "\"\\x{aaa}\"" = erl_scan:text(T1), + {ok,[{string,1,[2730]}],1} = erl_scan_string(U1, 1), U2 = "\"\\x41\\x{fff}\\x42\"", - {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan:string(U2, 1), + {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan_string(U2, 1), U3 = "\"a\n\\x{fff}\n\"", - {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan:string(U3, 1), + {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan_string(U3, 1), U4 = "\"\\^\n\\x{aaa}\\^\n\"", - {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan:string(U4, 1), + {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan_string(U4, 1), %% Keep these tests: - ?line test(Qs), - ?line test(U1), - ?line test(U2), - ?line test(U3), - ?line test(U4), + test(Qs), + test(U1), + test(U2), + test(U3), + test(U4), Str1 = "\"ab" ++ [1089] ++ "cd\"", - {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan:string(Str1, 1), + {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan_string(Str1, 1), {ok,[{string,{1,1},[$a,$b,1089,$c,$d]}],{1,8}} = - erl_scan:string(Str1, {1,1}), - ?line test(Str1), + erl_scan_string(Str1, {1,1}), + test(Str1), Comment = "%% "++[1089], {ok,[{comment,1,[$%,$%,$\s,1089]}],1} = - erl_scan:string(Comment, 1, [return]), + erl_scan_string(Comment, 1, [return]), {ok,[{comment,{1,1},[$%,$%,$\s,1089]}],{1,5}} = - erl_scan:string(Comment, {1,1}, [return]), + erl_scan_string(Comment, {1,1}, [return]), ok. more_chars() -> @@ -886,70 +841,67 @@ more_chars() -> %% $\x{...}, $\xHH %% All kinds of tests... - ?line {ok,[{char,_,123}],{1,4}} = - erl_scan:string("$\\{",{1,1}), - ?line {more, C1} = erl_scan:tokens([], "$\\{", {1,1}), - ?line {done,{ok,[{char,_,123}],{1,4}},eof} = - erl_scan:tokens(C1, eof, 1), - ?line {ok,[{char,1,123},{atom,1,a},{'}',1}],1} = - erl_scan:string("$\\{a}"), - - ?line {error,{{1,1},erl_scan,char},{1,4}} = + {ok,[{char,_,123}],{1,4}} = + erl_scan_string("$\\{",{1,1}), + {more, C1} = erl_scan:tokens([], "$\\{", {1,1}), + {done,{ok,[{char,_,123}],{1,4}},eof} = + erl_scan_tokens(C1, eof, 1), + {ok,[{char,1,123},{atom,1,a},{'}',1}],1} = + erl_scan_string("$\\{a}"), + + {error,{{1,1},erl_scan,char},{1,4}} = erl_scan:string("$\\x", {1,1}), - ?line {error,{{1,1},erl_scan,char},{1,5}} = + {error,{{1,1},erl_scan,char},{1,5}} = erl_scan:string("$\\x{",{1,1}), - ?line {more, C3} = erl_scan:tokens([], "$\\x", {1,1}), - ?line {done,{error,{{1,1},erl_scan,char},{1,4}},eof} = + {more, C3} = erl_scan:tokens([], "$\\x", {1,1}), + {done,{error,{{1,1},erl_scan,char},{1,4}},eof} = erl_scan:tokens(C3, eof, 1), - ?line {error,{{1,1},erl_scan,char},{1,5}} = + {error,{{1,1},erl_scan,char},{1,5}} = erl_scan:string("$\\x{",{1,1}), - ?line {more, C2} = erl_scan:tokens([], "$\\x{", {1,1}), - ?line {done,{error,{{1,1},erl_scan,char},{1,5}},eof} = + {more, C2} = erl_scan:tokens([], "$\\x{", {1,1}), + {done,{error,{{1,1},erl_scan,char},{1,5}},eof} = erl_scan:tokens(C2, eof, 1), - ?line {error,{1,erl_scan,{illegal,character}},1} = + {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string("$\\x{g}"), - ?line {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = + {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = erl_scan:string("$\\x{g}", {1,1}), - ?line {error,{{1,1},erl_scan,{illegal,character}},{1,6}} = + {error,{{1,1},erl_scan,{illegal,character}},{1,6}} = erl_scan:string("$\\x{}",{1,1}), - ?line test("\"\\{0}\""), - ?line test("\"\\x{0}\""), - ?line test("\'\\{0}\'"), - ?line test("\'\\x{0}\'"), + test("\"\\{0}\""), + test("\"\\x{0}\""), + test("\'\\{0}\'"), + test("\'\\x{0}\'"), - ?line {error,{{2,3},erl_scan,{illegal,character}},{2,6}} = + {error,{{2,3},erl_scan,{illegal,character}},{2,6}} = erl_scan:string("\"ab \n $\\x{g}\"",{1,1}), - ?line {error,{{2,3},erl_scan,{illegal,character}},{2,6}} = + {error,{{2,3},erl_scan,{illegal,character}},{2,6}} = erl_scan:string("\'ab \n $\\x{g}\'",{1,1}), - ?line test("$\\{34}"), - ?line test("$\\x{34}"), - ?line test("$\\{377}"), - ?line test("$\\x{FF}"), - ?line test("$\\{400}"), - ?line test("$\\x{100}"), - ?line test("$\\x{10FFFF}"), - ?line test("$\\x{10ffff}"), - ?line test("\"$\n \\{1}\""), - ?line {error,{1,erl_scan,{illegal,character}},1} = + test("$\\{34}"), + test("$\\x{34}"), + test("$\\{377}"), + test("$\\x{FF}"), + test("$\\{400}"), + test("$\\x{100}"), + test("$\\x{10FFFF}"), + test("$\\x{10ffff}"), + test("\"$\n \\{1}\""), + {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string("$\\x{110000}"), - ?line {error,{{1,1},erl_scan,{illegal,character}},{1,12}} = + {error,{{1,1},erl_scan,{illegal,character}},{1,12}} = erl_scan:string("$\\x{110000}", {1,1}), - ?line {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = + {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = erl_scan:string("$\\xfg", {1,1}), - ?line test("$\\xffg"), + test("$\\xffg"), - ?line {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = + {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = erl_scan:string("$\\xg", {1,1}), ok. -otp_10302(doc) -> - "OTP-10302. Unicode characters scanner/parser."; -otp_10302(suite) -> - []; +%% OTP-10302. Unicode characters scanner/parser. otp_10302(Config) when is_list(Config) -> %% From unicode(): {error,{1,erl_scan,{illegal,atom}},1} = @@ -957,37 +909,34 @@ otp_10302(Config) when is_list(Config) -> {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} = erl_scan:string("'qa\\x{aaa}'",{1,1}), - {ok,[{char,1,1089}],1} = erl_scan:string([$$,1089], 1), - {ok,[{char,1,1089}],1} = erl_scan:string([$$,$\\,1089],1), + {ok,[{char,1,1089}],1} = erl_scan_string([$$,1089], 1), + {ok,[{char,1,1089}],1} = erl_scan_string([$$,$\\,1089],1), Qs = "$\\x{aaa}", - {ok,[{char,1,2730}],1} = erl_scan:string(Qs,1), + {ok,[{char,1,2730}],1} = erl_scan_string(Qs, 1), {ok,[Q2],{1,9}} = erl_scan:string(Qs,{1,1},[text]), - [{category,char},{column,1},{length,8}, - {line,1},{symbol,16#aaa},{text,Qs}] = - erl_scan:token_info(Q2), - - Tags = [category, column, length, line, symbol, text], + [{category,char},{column,1},{line,1},{symbol,16#aaa},{text,Qs}] = + token_info_long(Q2), U1 = "\"\\x{aaa}\"", {ok,[T1],{1,10}} = erl_scan:string(U1, {1,1}, [text]), - [{category,string},{column,1},{length,9},{line,1}, - {symbol,[16#aaa]},{text,U1}] = erl_scan:token_info(T1, Tags), + [{category,string},{column,1},{line,1},{symbol,[16#aaa]},{text,U1}] = + token_info_long(T1), U2 = "\"\\x41\\x{fff}\\x42\"", - {ok,[{string,1,[65,4095,66]}],1} = erl_scan:string(U2, 1), + {ok,[{string,1,[65,4095,66]}],1} = erl_scan_string(U2, 1), U3 = "\"a\n\\x{fff}\n\"", - {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan:string(U3, 1), + {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan_string(U3, 1), U4 = "\"\\^\n\\x{aaa}\\^\n\"", - {ok,[{string,1,[10,2730,10]}],3} = erl_scan:string(U4, 1,[]), + {ok,[{string,1,[10,2730,10]}],3} = erl_scan_string(U4, 1,[]), Str1 = "\"ab" ++ [1089] ++ "cd\"", {ok,[{string,1,[97,98,1089,99,100]}],1} = - erl_scan:string(Str1,1), + erl_scan_string(Str1,1), {ok,[{string,{1,1},[97,98,1089,99,100]}],{1,8}} = - erl_scan:string(Str1, {1,1}), + erl_scan_string(Str1, {1,1}), OK1 = 16#D800-1, OK2 = 16#DFFF+1, @@ -1002,19 +951,19 @@ otp_10302(Config) when is_list(Config) -> IllegalL = [Illegal1,Illegal2,Illegal3,Illegal4], [{ok,[{comment,1,[$%,$%,$\s,OK]}],1} = - erl_scan:string("%% "++[OK], 1, [return]) || + erl_scan_string("%% "++[OK], 1, [return]) || OK <- OKL], {ok,[{comment,_,[$%,$%,$\s,OK1]}],{1,5}} = - erl_scan:string("%% "++[OK1], {1,1}, [return]), + erl_scan_string("%% "++[OK1], {1,1}, [return]), [{error,{1,erl_scan,{illegal,character}},1} = erl_scan:string("%% "++[Illegal], 1, [return]) || Illegal <- IllegalL], {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = erl_scan:string("%% "++[Illegal1], {1,1}, [return]), - [{ok,[],1} = erl_scan:string("%% "++[OK], 1, []) || + [{ok,[],1} = erl_scan_string("%% "++[OK], 1, []) || OK <- OKL], - {ok,[],{1,5}} = erl_scan:string("%% "++[OK1], {1,1}, []), + {ok,[],{1,5}} = erl_scan_string("%% "++[OK1], {1,1}, []), [{error,{1,erl_scan,{illegal,character}},1} = erl_scan:string("%% "++[Illegal], 1, []) || Illegal <- IllegalL], @@ -1022,7 +971,7 @@ otp_10302(Config) when is_list(Config) -> erl_scan:string("%% "++[Illegal1], {1,1}, []), [{ok,[{string,{1,1},[OK]}],{1,4}} = - erl_scan:string("\""++[OK]++"\"",{1,1}) || + erl_scan_string("\""++[OK]++"\"",{1,1}) || OK <- OKL], [{error,{{1,2},erl_scan,{illegal,character}},{1,3}} = erl_scan:string("\""++[OK]++"\"",{1,1}) || @@ -1033,122 +982,181 @@ otp_10302(Config) when is_list(Config) -> Illegal <- IllegalL], {ok,[{char,{1,1},OK1}],{1,3}} = - erl_scan:string([$$,OK1],{1,1}), + erl_scan_string([$$,OK1],{1,1}), {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = erl_scan:string([$$,Illegal1],{1,1}), {ok,[{char,{1,1},OK1}],{1,4}} = - erl_scan:string([$$,$\\,OK1],{1,1}), + erl_scan_string([$$,$\\,OK1],{1,1}), {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = erl_scan:string([$$,$\\,Illegal1],{1,1}), {ok,[{string,{1,1},[55295]}],{1,5}} = - erl_scan:string("\"\\"++[OK1]++"\"",{1,1}), + erl_scan_string("\"\\"++[OK1]++"\"",{1,1}), {error,{{1,2},erl_scan,{illegal,character}},{1,4}} = erl_scan:string("\"\\"++[Illegal1]++"\"",{1,1}), {ok,[{char,{1,1},OK1}],{1,10}} = - erl_scan:string("$\\x{D7FF}",{1,1}), + erl_scan_string("$\\x{D7FF}",{1,1}), {error,{{1,1},erl_scan,{illegal,character}},{1,10}} = erl_scan:string("$\\x{D800}",{1,1}), %% Not erl_scan, but erl_parse. - {integer,0,1} = erl_parse:abstract(1), - Float = 3.14, {float,0,Float} = erl_parse:abstract(Float), - {nil,0} = erl_parse:abstract([]), + {integer,0,1} = erl_parse_abstract(1), + Float = 3.14, {float,0,Float} = erl_parse_abstract(Float), + {nil,0} = erl_parse_abstract([]), {bin,0, [{bin_element,0,{integer,0,1},default,default}, {bin_element,0,{integer,0,2},default,default}]} = - erl_parse:abstract(<<1,2>>), + erl_parse_abstract(<<1,2>>), {cons,0,{tuple,0,[{atom,0,a}]},{atom,0,b}} = - erl_parse:abstract([{a} | b]), - {string,0,"str"} = erl_parse:abstract("str"), + erl_parse_abstract([{a} | b]), + {string,0,"str"} = erl_parse_abstract("str"), {cons,0, {integer,0,$a}, - {cons,0,{integer,0,1024},{string,0,"c"}}} = - erl_parse:abstract("a"++[1024]++"c"), + {cons,0,{integer,0,55296},{string,0,"c"}}} = + erl_parse_abstract("a"++[55296]++"c"), Line = 17, - {integer,Line,1} = erl_parse:abstract(1, Line), - Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Line), - {nil,Line} = erl_parse:abstract([], Line), + {integer,Line,1} = erl_parse_abstract(1, Line), + Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Line), + {nil,Line} = erl_parse_abstract([], Line), {bin,Line, [{bin_element,Line,{integer,Line,1},default,default}, {bin_element,Line,{integer,Line,2},default,default}]} = - erl_parse:abstract(<<1,2>>, Line), + erl_parse_abstract(<<1,2>>, Line), {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = - erl_parse:abstract([{a} | b], Line), - {string,Line,"str"} = erl_parse:abstract("str", Line), + erl_parse_abstract([{a} | b], Line), + {string,Line,"str"} = erl_parse_abstract("str", Line), {cons,Line, {integer,Line,$a}, - {cons,Line,{integer,Line,1024},{string,Line,"c"}}} = - erl_parse:abstract("a"++[1024]++"c", Line), + {cons,Line,{integer,Line,55296},{string,Line,"c"}}} = + erl_parse_abstract("a"++[55296]++"c", Line), Opts1 = [{line,17}], - {integer,Line,1} = erl_parse:abstract(1, Opts1), - Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts1), - {nil,Line} = erl_parse:abstract([], Opts1), + {integer,Line,1} = erl_parse_abstract(1, Opts1), + Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Opts1), + {nil,Line} = erl_parse_abstract([], Opts1), {bin,Line, [{bin_element,Line,{integer,Line,1},default,default}, {bin_element,Line,{integer,Line,2},default,default}]} = - erl_parse:abstract(<<1,2>>, Opts1), + erl_parse_abstract(<<1,2>>, Opts1), {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = - erl_parse:abstract([{a} | b], Opts1), - {string,Line,"str"} = erl_parse:abstract("str", Opts1), + erl_parse_abstract([{a} | b], Opts1), + {string,Line,"str"} = erl_parse_abstract("str", Opts1), {cons,Line, {integer,Line,$a}, - {cons,Line,{integer,Line,1024},{string,Line,"c"}}} = - erl_parse:abstract("a"++[1024]++"c", Opts1), + {cons,Line,{integer,Line,55296},{string,Line,"c"}}} = + erl_parse_abstract("a"++[55296]++"c", Opts1), [begin - {integer,Line,1} = erl_parse:abstract(1, Opts2), - Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts2), - {nil,Line} = erl_parse:abstract([], Opts2), + {integer,Line,1} = erl_parse_abstract(1, Opts2), + Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Opts2), + {nil,Line} = erl_parse_abstract([], Opts2), {bin,Line, [{bin_element,Line,{integer,Line,1},default,default}, {bin_element,Line,{integer,Line,2},default,default}]} = - erl_parse:abstract(<<1,2>>, Opts2), + erl_parse_abstract(<<1,2>>, Opts2), {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = - erl_parse:abstract([{a} | b], Opts2), - {string,Line,"str"} = erl_parse:abstract("str", Opts2), + erl_parse_abstract([{a} | b], Opts2), + {string,Line,"str"} = erl_parse_abstract("str", Opts2), {string,Line,[97,1024,99]} = - erl_parse:abstract("a"++[1024]++"c", Opts2) + erl_parse_abstract("a"++[1024]++"c", Opts2) end || Opts2 <- [[{encoding,unicode},{line,Line}], [{encoding,utf8},{line,Line}]]], {cons,0, {integer,0,97}, {cons,0,{integer,0,1024},{string,0,"c"}}} = - erl_parse:abstract("a"++[1024]++"c", [{encoding,latin1}]), + erl_parse_abstract("a"++[1024]++"c", [{encoding,latin1}]), ok. -otp_10990(doc) -> - "OTP-10990. Floating point number in input string."; -otp_10990(suite) -> - []; +%% OTP-10990. Floating point number in input string. otp_10990(Config) when is_list(Config) -> {'EXIT',_} = (catch {foo, erl_scan:string([$",42.0,$"],1)}), ok. -otp_10992(doc) -> - "OTP-10992. List of floats to abstract format."; -otp_10992(suite) -> - []; +%% OTP-10992. List of floats to abstract format. otp_10992(Config) when is_list(Config) -> {cons,0,{float,0,42.0},{nil,0}} = - erl_parse:abstract([42.0], [{encoding,unicode}]), + erl_parse_abstract([42.0], [{encoding,unicode}]), {cons,0,{float,0,42.0},{nil,0}} = - erl_parse:abstract([42.0], [{encoding,utf8}]), + erl_parse_abstract([42.0], [{encoding,utf8}]), {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} = - erl_parse:abstract([$A,42.0], [{encoding,unicode}]), + erl_parse_abstract([$A,42.0], [{encoding,unicode}]), {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} = - erl_parse:abstract([$A,42.0], [{encoding,utf8}]), + erl_parse_abstract([$A,42.0], [{encoding,utf8}]), ok. -test_string(String, Expected) -> - {ok, Expected, _End} = erl_scan:string(String), +%% OTP-11807. Generalize erl_parse:abstract/2. +otp_11807(Config) when is_list(Config) -> + {cons,0,{integer,0,97},{cons,0,{integer,0,98},{nil,0}}} = + erl_parse_abstract("ab", [{encoding,none}]), + {cons,0,{integer,0,-1},{nil,0}} = + erl_parse_abstract([-1], [{encoding,latin1}]), + ASCII = fun(I) -> I >= 0 andalso I < 128 end, + {string,0,"xyz"} = erl_parse_abstract("xyz", [{encoding,ASCII}]), + {cons,0,{integer,0,228},{nil,0}} = + erl_parse_abstract([228], [{encoding,ASCII}]), + {cons,0,{integer,0,97},{atom,0,a}} = + erl_parse_abstract("a"++a, [{encoding,latin1}]), + {'EXIT', {{badarg,bad},_}} = % minor backward incompatibility + (catch erl_parse:abstract("string", [{encoding,bad}])), + ok. + +test_string(String, ExpectedWithCol) -> + {ok, ExpectedWithCol, _EndWithCol} = erl_scan_string(String, {1, 1}, []), + Expected = [ begin + {L,_C} = element(2, T), + setelement(2, T, L) + end + || T <- ExpectedWithCol ], + {ok, Expected, _End} = erl_scan_string(String), test(String). +erl_scan_string(String) -> + erl_scan_string(String, 1, []). + +erl_scan_string(String, StartLocation) -> + erl_scan_string(String, StartLocation, []). + +erl_scan_string(String, StartLocation, Options) -> + case erl_scan:string(String, StartLocation, Options) of + {ok, Tokens, EndLocation} -> + {ok, unopaque_tokens(Tokens), EndLocation}; + Else -> + Else + end. + +erl_scan_tokens(C, S, L) -> + erl_scan_tokens(C, S, L, []). + +erl_scan_tokens(C, S, L, O) -> + case erl_scan:tokens(C, S, L, O) of + {done, {ok, Ts, End}, R} -> + {done, {ok, unopaque_tokens(Ts), End}, R}; + Else -> + Else + end. + +unopaque_tokens([]) -> + []; +unopaque_tokens([Token|Tokens]) -> + Attrs = element(2, Token), + Term = erl_anno:to_term(Attrs), + T = setelement(2, Token, Term), + [T | unopaque_tokens(Tokens)]. + +erl_parse_abstract(Term) -> + erl_parse_abstract(Term, []). + +erl_parse_abstract(Term, Options) -> + Abstr = erl_parse:abstract(Term, Options), + unopaque_abstract(Abstr). + +unopaque_abstract(Abstr) -> + erl_parse:anno_to_term(Abstr). + %% test_string(String, Expected, StartLocation, Options) -> %% {ok, Expected, _End} = erl_scan:string(String, StartLocation, Options), %% test(String). @@ -1236,9 +1244,7 @@ test_wsc([], []) -> ok; test_wsc([Token|Tokens], [Token2|Tokens2]) -> [Text, Text2] = [Text || - {text, Text} <- - [erl_scan:token_info(T, text) || - T <- [Token, Token2]]], + Text <- [erl_scan:text(T) || T <- [Token, Token2]]], Sz = erts_debug:size(Text), Sz2 = erts_debug:size({Text, Text2}), IsCompacted = Sz2 < 2*Sz+erts_debug:size({a,a}), @@ -1277,7 +1283,7 @@ all_same(L, Char) -> newlines_first([]) -> ok; newlines_first([Token|Tokens]) -> - {text,Text} = erl_scan:token_info(Token, text), + Text = erl_scan:text(Token), Nnls = length([C || C <- Text, C =:= $\n]), OK = case Text of [$\n|_] -> @@ -1297,8 +1303,8 @@ select_tokens(Tokens, Tags) -> lists:filter(fun(T) -> lists:member(element(1, T), Tags) end, Tokens). simplify([Token|Tokens]) -> - {line,Line} = erl_scan:token_info(Token, line), - [setelement(2, Token, Line) | simplify(Tokens)]; + Line = erl_scan:line(Token), + [setelement(2, Token, erl_anno:new(Line)) | simplify(Tokens)]; simplify([]) -> []. @@ -1306,17 +1312,31 @@ get_text(Tokens) -> lists:flatten( [T || Token <- Tokens, - ({text,T} = erl_scan:token_info(Token, text)) =/= []]). + (T = erl_scan:text(Token)) =/= []]). test_decorated_tokens(String, Tokens) -> ToksAttrs = token_attrs(Tokens), test_strings(ToksAttrs, String, 1, 1). token_attrs(Tokens) -> - [{L,C,Len,T} || + [{L,C,length(T),T} || Token <- Tokens, - ([{line,L},{column,C},{length,Len},{text,T}] = - erl_scan:token_info(Token, [line,column,length,text])) =/= []]. + ([C,L,T] = token_info(Token)) =/= []]. + +token_info(T) -> + Column = erl_scan:column(T), + Line = erl_scan:line(T), + Text = erl_scan:text(T), + [Column, Line, Text]. + +token_info_long(T) -> + Column = erl_scan:column(T), + Line = erl_scan:line(T), + Text = erl_scan:text(T), + Category = erl_scan:category(T), + Symbol = erl_scan:symbol(T), + [{category,Category},{column,Column},{line,Line}, + {symbol,Symbol},{text,Text}]. test_strings([], _S, Line, Column) -> {Line,Column}; @@ -1397,8 +1417,7 @@ consistent_attributes([Ts | TsL]) -> L = [T || T <- Ts, is_integer(element(2, T))], case L of [] -> - TagsL = [[Tag || {Tag,_} <- - erl_scan:attributes_info(element(2, T))] || + TagsL = [[Tag || {Tag,_} <- defined(token_info_long(T))] || T <- Ts], case lists:usort(TagsL) of [_] -> @@ -1414,6 +1433,9 @@ consistent_attributes([Ts | TsL]) -> Ts end. +defined(L) -> + [{T,V} || {T,V} <- L, V =/= undefined]. + family_list(L) -> sofs:to_external(family(L)). |