aboutsummaryrefslogtreecommitdiffstats
path: root/lib/stdlib/test/erl_scan_SUITE.erl
diff options
context:
space:
mode:
Diffstat (limited to 'lib/stdlib/test/erl_scan_SUITE.erl')
-rw-r--r--lib/stdlib/test/erl_scan_SUITE.erl306
1 files changed, 181 insertions, 125 deletions
diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl
index 9be9f641c8..12ea3d128c 100644
--- a/lib/stdlib/test/erl_scan_SUITE.erl
+++ b/lib/stdlib/test/erl_scan_SUITE.erl
@@ -1,18 +1,19 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1998-2014. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2015. All Rights Reserved.
%%
-%% The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved online at http://www.erlang.org/.
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
%%
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
%%
%% %CopyrightEnd%
@@ -138,7 +139,7 @@ iso88591(Config) when is_list(Config) ->
A1s = [$h,$ä,$r],
A2s = [$ö,$r,$e],
%% Test parsing atom and variable characters.
- {ok,Ts1,_} = erl_scan:string(V1s ++ " " ++ V2s ++
+ {ok,Ts1,_} = erl_scan_string(V1s ++ " " ++ V2s ++
"\327" ++
A1s ++ " " ++ A2s),
V1s = atom_to_list(element(3, nth(1, Ts1))),
@@ -151,7 +152,7 @@ iso88591(Config) when is_list(Config) ->
%% Test parsing and printing strings.
S1 = V1s ++ "\327" ++ A1s ++ "\250" ++ A2s,
S1s = "\"" ++ S1 ++ "\"",
- {ok,Ts2,_} = erl_scan:string(S1s),
+ {ok,Ts2,_} = erl_scan_string(S1s),
S1 = element(3, nth(1, Ts2)),
S1s = flatten(print(element(3, nth(1, Ts2)))),
ok %It all worked
@@ -219,14 +220,14 @@ atoms() ->
test_string([39,65,200,39], [{atom,{1,1},'AÈ'}]),
test_string("ärlig östen", [{atom,{1,1},ärlig},{atom,{1,7},östen}]),
?line {ok,[{atom,_,'$a'}],{1,6}} =
- erl_scan:string("'$\\a'", {1,1}),
+ erl_scan_string("'$\\a'", {1,1}),
?line test("'$\\a'"),
ok.
punctuations() ->
L = ["<<", "<-", "<=", "<", ">>", ">=", ">", "->", "--",
"-", "++", "+", "=:=", "=/=", "=<", "=>", "==", "=", "/=",
- "/", "||", "|", ":=", ":-", "::", ":"],
+ "/", "||", "|", ":=", "::", ":"],
%% One token at a time:
[begin
W = list_to_atom(S),
@@ -268,24 +269,24 @@ punctuations() ->
comments() ->
?line test("a %%\n b"),
- ?line {ok,[],1} = erl_scan:string("%"),
+ {ok,[],1} = erl_scan_string("%"),
?line test("a %%\n b"),
{ok,[{atom,{1,1},a},{atom,{2,2},b}],{2,3}} =
- erl_scan:string("a %%\n b",{1,1}),
+ erl_scan_string("a %%\n b", {1,1}),
{ok,[{atom,{1,1},a},{comment,{1,3},"%%"},{atom,{2,2},b}],{2,3}} =
- erl_scan:string("a %%\n b",{1,1}, [return_comments]),
+ erl_scan_string("a %%\n b",{1,1}, [return_comments]),
{ok,[{atom,{1,1},a},
{white_space,{1,2}," "},
{white_space,{1,5},"\n "},
{atom,{2,2},b}],
{2,3}} =
- erl_scan:string("a %%\n b",{1,1},[return_white_spaces]),
+ erl_scan_string("a %%\n b",{1,1},[return_white_spaces]),
{ok,[{atom,{1,1},a},
{white_space,{1,2}," "},
{comment,{1,3},"%%"},
{white_space,{1,5},"\n "},
{atom,{2,2},b}],
- {2,3}} = erl_scan:string("a %%\n b",{1,1},[return]),
+ {2,3}} = erl_scan_string("a %%\n b",{1,1},[return]),
ok.
errors() ->
@@ -337,11 +338,11 @@ base_integers() ->
erl_scan:string(Str)
end || {BS,S} <- [{"3","3"},{"15","f"}, {"12","c"}] ],
- ?line {ok,[{integer,1,239},{'@',1}],1} = erl_scan:string("16#ef@"),
+ {ok,[{integer,1,239},{'@',1}],1} = erl_scan_string("16#ef@"),
{ok,[{integer,{1,1},239},{'@',{1,6}}],{1,7}} =
- erl_scan:string("16#ef@", {1,1}, []),
+ erl_scan_string("16#ef@", {1,1}, []),
{ok,[{integer,{1,1},14},{atom,{1,5},g@}],{1,7}} =
- erl_scan:string("16#eg@", {1,1}, []),
+ erl_scan_string("16#eg@", {1,1}, []),
ok.
@@ -382,8 +383,8 @@ dots() ->
{ok,[{'.',{1,1}},{atom,{1,2},a}],{1,3}}}
],
[begin
- R = erl_scan:string(S),
- R2 = erl_scan:string(S, {1,1}, [])
+ R = erl_scan_string(S),
+ R2 = erl_scan_string(S, {1,1}, [])
end || {S, R, R2} <- Dot],
?line {ok,[{dot,_}=T1],{1,2}} = erl_scan:string(".", {1,1}, text),
@@ -417,7 +418,7 @@ dots() ->
{white_space,{1,4},"\n"},
{dot,{2,1}}],
{2,3}}, ""} =
- erl_scan:tokens(C, "\n. ", {1,1}, return), % any loc, any options
+ erl_scan_tokens(C, "\n. ", {1,1}, return), % any loc, any options
?line [test_string(S, R) ||
{S, R} <- [{".$\n", [{'.',{1,1}},{char,{1,2},$\n}]},
@@ -511,7 +512,7 @@ eof() ->
%% An error before R13A.
%% ?line {done,Err={error,{1,erl_scan,scan},1},eof} =
?line {done,{ok,[{atom,1,abra}],1},eof} =
- erl_scan:tokens(C2, eof, 1),
+ erl_scan_tokens(C2, eof, 1),
%% With column.
?line {more, C3} = erl_scan:tokens([]," \n",{1,1}),
@@ -520,7 +521,7 @@ eof() ->
%% An error before R13A.
%% ?line {done,{error,{{1,1},erl_scan,scan},{1,5}},eof} =
?line {done,{ok,[{atom,_,abra}],{1,5}},eof} =
- erl_scan:tokens(C4, eof, 1),
+ erl_scan_tokens(C4, eof, 1),
%% Robert's scanner returns "" as LeftoverChars;
%% the R12B scanner returns eof as LeftoverChars: (eof is correct)
@@ -528,26 +529,26 @@ eof() ->
%% An error before R13A.
%% ?line {done,{error,{1,erl_scan,scan},1},eof} =
?line {done,{ok,[{atom,1,a}],1},eof} =
- erl_scan:tokens(C5,eof,1),
+ erl_scan_tokens(C5,eof,1),
%% With column.
{more, C6} = erl_scan:tokens([], "a", {1,1}),
%% An error before R13A.
%% {done,{error,{1,erl_scan,scan},1},eof} =
{done,{ok,[{atom,{1,1},a}],{1,2}},eof} =
- erl_scan:tokens(C6,eof,1),
+ erl_scan_tokens(C6,eof,1),
%% A dot followed by eof is special:
?line {more, C} = erl_scan:tokens([], "a.", 1),
- ?line {done,{ok,[{atom,1,a},{dot,1}],1},eof} = erl_scan:tokens(C,eof,1),
- ?line {ok,[{atom,1,foo},{dot,1}],1} = erl_scan:string("foo."),
+ {done,{ok,[{atom,1,a},{dot,1}],1},eof} = erl_scan_tokens(C,eof,1),
+ {ok,[{atom,1,foo},{dot,1}],1} = erl_scan_string("foo."),
%% With column.
{more, CCol} = erl_scan:tokens([], "a.", {1,1}),
{done,{ok,[{atom,{1,1},a},{dot,{1,2}}],{1,3}},eof} =
- erl_scan:tokens(CCol,eof,1),
+ erl_scan_tokens(CCol,eof,1),
{ok,[{atom,{1,1},foo},{dot,{1,4}}],{1,5}} =
- erl_scan:string("foo.", {1,1}, []),
+ erl_scan_string("foo.", {1,1}, []),
ok.
@@ -628,23 +629,23 @@ crashes() ->
options() ->
%% line and column are not options, but tested here
?line {ok,[{atom,1,foo},{white_space,1," "},{comment,1,"% bar"}], 1} =
- erl_scan:string("foo % bar", 1, return),
+ erl_scan_string("foo % bar", 1, return),
?line {ok,[{atom,1,foo},{white_space,1," "}],1} =
- erl_scan:string("foo % bar", 1, return_white_spaces),
+ erl_scan_string("foo % bar", 1, return_white_spaces),
?line {ok,[{atom,1,foo},{comment,1,"% bar"}],1} =
- erl_scan:string("foo % bar", 1, return_comments),
+ erl_scan_string("foo % bar", 1, return_comments),
?line {ok,[{atom,17,foo}],17} =
- erl_scan:string("foo % bar", 17),
+ erl_scan_string("foo % bar", 17),
?line {'EXIT',{function_clause,_}} =
(catch {foo,
erl_scan:string("foo % bar", {a,1}, [])}), % type error
?line {ok,[{atom,_,foo}],{17,18}} =
- erl_scan:string("foo % bar", {17,9}, []),
+ erl_scan_string("foo % bar", {17,9}, []),
?line {'EXIT',{function_clause,_}} =
(catch {foo,
erl_scan:string("foo % bar", {1,0}, [])}), % type error
?line {ok,[{foo,1}],1} =
- erl_scan:string("foo % bar",1, [{reserved_word_fun,
+ erl_scan_string("foo % bar",1, [{reserved_word_fun,
fun(W) -> W =:= foo end}]),
?line {'EXIT',{badarg,_}} =
(catch {foo,
@@ -706,8 +707,9 @@ token_info() ->
attributes_info() ->
?line {'EXIT',_} =
(catch {foo,erl_scan:attributes_info(foo)}), % type error
- ?line [{line,18}] = erl_scan:attributes_info(18),
- ?line {location,19} = erl_scan:attributes_info(19, location),
+ [{line,18}] = erl_scan:attributes_info(erl_anno:new(18)),
+ {location,19} =
+ erl_scan:attributes_info(erl_anno:new(19), location),
?line {ok,[{atom,A0,foo}],_} = erl_scan:string("foo", 19, [text]),
?line {location,19} = erl_scan:attributes_info(A0, location),
@@ -735,7 +737,9 @@ attributes_info() ->
set_attribute() ->
F = fun(Line) -> -Line end,
- ?line -2 = erl_scan:set_attribute(line, 2, F),
+ Anno2 = erl_anno:new(2),
+ A0 = erl_scan:set_attribute(line, Anno2, F),
+ {line, -2} = erl_scan:attributes_info(A0, line),
?line {ok,[{atom,A1,foo}],_} = erl_scan:string("foo", {9,17}),
?line A2 = erl_scan:set_attribute(line, A1, F),
?line {line,-9} = erl_scan:attributes_info(A2, line),
@@ -765,10 +769,15 @@ set_attribute() ->
?line {ok,[{atom,A6,foo}],_} = erl_scan:string("foo", 11, [text]),
?line A7 = erl_scan:set_attribute(line, A6, F2),
- ?line {line,{17,11}} = erl_scan:attributes_info(A7, line),
+ %% Incompatible with pre 18:
+ %% {line,{17,11}} = erl_scan:attributes_info(A7, line),
+ {line,17} = erl_scan:attributes_info(A7, line),
?line {location,{17,11}} = % mixed up
erl_scan:attributes_info(A7, location),
- ?line [{line,{17,11}},{text,"foo"}] =
+ %% Incompatible with pre 18:
+ %% [{line,{17,11}},{text,"foo"}] =
+ %% erl_scan:attributes_info(A7, [line,column,text]),
+ [{line,17},{column,11},{text,"foo"}] =
erl_scan:attributes_info(A7, [line,column,text]),
?line {'EXIT',_} =
@@ -776,9 +785,13 @@ set_attribute() ->
?line {'EXIT',{badarg,_}} =
(catch {foo, erl_scan:set_attribute(column, [], F2)}), % type error
+ Attr10 = erl_anno:new(8),
+ Attr20 = erl_scan:set_attribute(line, Attr10,
+ fun(L) -> {nos,'X',L} end),
%% OTP-9412
- ?line 8 = erl_scan:set_attribute(line, [{line,{nos,'X',8}}],
- fun({nos,_V,VL}) -> VL end),
+ Attr30 = erl_scan:set_attribute(line, Attr20,
+ fun({nos,_V,VL}) -> VL end),
+ 8 = erl_anno:to_term(Attr30),
ok.
column_errors() ->
@@ -812,7 +825,7 @@ white_spaces() ->
{white_space,_," "},
{atom,_,a},
{white_space,_,"\n"}],
- _} = erl_scan:string("\r a\n", {1,1}, return),
+ _} = erl_scan_string("\r a\n", {1,1}, return),
?line test("\r a\n"),
L = "{\"a\nb\", \"a\\nb\",\nabc\r,def}.\n\n",
?line {ok,[{'{',_},
@@ -829,7 +842,7 @@ white_spaces() ->
{'}',_},
{dot,_},
{white_space,_,"\n"}],
- _} = erl_scan:string(L, {1,1}, return),
+ _} = erl_scan_string(L, {1,1}, return),
?line test(L),
?line test("\"\n\"\n"),
?line test("\n\r\n"),
@@ -846,7 +859,7 @@ white_spaces() ->
unicode() ->
?line {ok,[{char,1,83},{integer,1,45}],1} =
- erl_scan:string("$\\12345"), % not unicode
+ erl_scan_string("$\\12345"), % not unicode
?line {error,{1,erl_scan,{illegal,character}},1} =
erl_scan:string([1089]),
@@ -858,7 +871,7 @@ unicode() ->
erl_scan:string("'a"++[1089]++"b'", {1,1}),
?line test("\"a"++[1089]++"b\""),
{ok,[{char,1,1}],1} =
- erl_scan:string([$$,$\\,$^,1089], 1),
+ erl_scan_string([$$,$\\,$^,1089], 1),
{error,{1,erl_scan,Error},1} =
erl_scan:string("\"qa\x{aaa}", 1),
@@ -870,13 +883,13 @@ unicode() ->
erl_scan:string("'qa\\x{aaa}'",{1,1}),
{ok,[{char,1,1089}],1} =
- erl_scan:string([$$,1089], 1),
+ erl_scan_string([$$,1089], 1),
{ok,[{char,1,1089}],1} =
- erl_scan:string([$$,$\\,1089], 1),
+ erl_scan_string([$$,$\\,1089], 1),
Qs = "$\\x{aaa}",
{ok,[{char,1,$\x{aaa}}],1} =
- erl_scan:string(Qs, 1),
+ erl_scan_string(Qs, 1),
{ok,[Q2],{1,9}} =
erl_scan:string("$\\x{aaa}", {1,1}, [text]),
[{category,char},{column,1},{length,8},
@@ -884,19 +897,19 @@ unicode() ->
erl_scan:token_info(Q2),
U1 = "\"\\x{aaa}\"",
- {ok,
- [{string,[{line,1},{column,1},{text,"\"\\x{aaa}\""}],[2730]}],
- {1,10}} = erl_scan:string(U1, {1,1}, [text]),
- {ok,[{string,1,[2730]}],1} = erl_scan:string(U1, 1),
+ {ok,[{string,A1,[2730]}],{1,10}} = erl_scan:string(U1, {1,1}, [text]),
+ [{line,1},{column,1},{text,"\"\\x{aaa}\""}] =
+ erl_scan:attributes_info(A1, [line, column, text]),
+ {ok,[{string,1,[2730]}],1} = erl_scan_string(U1, 1),
U2 = "\"\\x41\\x{fff}\\x42\"",
- {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan:string(U2, 1),
+ {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan_string(U2, 1),
U3 = "\"a\n\\x{fff}\n\"",
- {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan:string(U3, 1),
+ {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan_string(U3, 1),
U4 = "\"\\^\n\\x{aaa}\\^\n\"",
- {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan:string(U4, 1),
+ {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan_string(U4, 1),
%% Keep these tests:
?line test(Qs),
@@ -906,15 +919,15 @@ unicode() ->
?line test(U4),
Str1 = "\"ab" ++ [1089] ++ "cd\"",
- {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan:string(Str1, 1),
+ {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan_string(Str1, 1),
{ok,[{string,{1,1},[$a,$b,1089,$c,$d]}],{1,8}} =
- erl_scan:string(Str1, {1,1}),
+ erl_scan_string(Str1, {1,1}),
?line test(Str1),
Comment = "%% "++[1089],
{ok,[{comment,1,[$%,$%,$\s,1089]}],1} =
- erl_scan:string(Comment, 1, [return]),
+ erl_scan_string(Comment, 1, [return]),
{ok,[{comment,{1,1},[$%,$%,$\s,1089]}],{1,5}} =
- erl_scan:string(Comment, {1,1}, [return]),
+ erl_scan_string(Comment, {1,1}, [return]),
ok.
more_chars() ->
@@ -923,12 +936,12 @@ more_chars() ->
%% All kinds of tests...
?line {ok,[{char,_,123}],{1,4}} =
- erl_scan:string("$\\{",{1,1}),
+ erl_scan_string("$\\{",{1,1}),
?line {more, C1} = erl_scan:tokens([], "$\\{", {1,1}),
?line {done,{ok,[{char,_,123}],{1,4}},eof} =
- erl_scan:tokens(C1, eof, 1),
+ erl_scan_tokens(C1, eof, 1),
?line {ok,[{char,1,123},{atom,1,a},{'}',1}],1} =
- erl_scan:string("$\\{a}"),
+ erl_scan_string("$\\{a}"),
?line {error,{{1,1},erl_scan,char},{1,4}} =
erl_scan:string("$\\x", {1,1}),
@@ -993,11 +1006,11 @@ otp_10302(Config) when is_list(Config) ->
{error,{{1,1},erl_scan,{illegal,atom}},{1,12}} =
erl_scan:string("'qa\\x{aaa}'",{1,1}),
- {ok,[{char,1,1089}],1} = erl_scan:string([$$,1089], 1),
- {ok,[{char,1,1089}],1} = erl_scan:string([$$,$\\,1089],1),
+ {ok,[{char,1,1089}],1} = erl_scan_string([$$,1089], 1),
+ {ok,[{char,1,1089}],1} = erl_scan_string([$$,$\\,1089],1),
Qs = "$\\x{aaa}",
- {ok,[{char,1,2730}],1} = erl_scan:string(Qs,1),
+ {ok,[{char,1,2730}],1} = erl_scan_string(Qs, 1),
{ok,[Q2],{1,9}} = erl_scan:string(Qs,{1,1},[text]),
[{category,char},{column,1},{length,8},
{line,1},{symbol,16#aaa},{text,Qs}] =
@@ -1011,19 +1024,19 @@ otp_10302(Config) when is_list(Config) ->
{symbol,[16#aaa]},{text,U1}] = erl_scan:token_info(T1, Tags),
U2 = "\"\\x41\\x{fff}\\x42\"",
- {ok,[{string,1,[65,4095,66]}],1} = erl_scan:string(U2, 1),
+ {ok,[{string,1,[65,4095,66]}],1} = erl_scan_string(U2, 1),
U3 = "\"a\n\\x{fff}\n\"",
- {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan:string(U3, 1),
+ {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan_string(U3, 1),
U4 = "\"\\^\n\\x{aaa}\\^\n\"",
- {ok,[{string,1,[10,2730,10]}],3} = erl_scan:string(U4, 1,[]),
+ {ok,[{string,1,[10,2730,10]}],3} = erl_scan_string(U4, 1,[]),
Str1 = "\"ab" ++ [1089] ++ "cd\"",
{ok,[{string,1,[97,98,1089,99,100]}],1} =
- erl_scan:string(Str1,1),
+ erl_scan_string(Str1,1),
{ok,[{string,{1,1},[97,98,1089,99,100]}],{1,8}} =
- erl_scan:string(Str1, {1,1}),
+ erl_scan_string(Str1, {1,1}),
OK1 = 16#D800-1,
OK2 = 16#DFFF+1,
@@ -1038,19 +1051,19 @@ otp_10302(Config) when is_list(Config) ->
IllegalL = [Illegal1,Illegal2,Illegal3,Illegal4],
[{ok,[{comment,1,[$%,$%,$\s,OK]}],1} =
- erl_scan:string("%% "++[OK], 1, [return]) ||
+ erl_scan_string("%% "++[OK], 1, [return]) ||
OK <- OKL],
{ok,[{comment,_,[$%,$%,$\s,OK1]}],{1,5}} =
- erl_scan:string("%% "++[OK1], {1,1}, [return]),
+ erl_scan_string("%% "++[OK1], {1,1}, [return]),
[{error,{1,erl_scan,{illegal,character}},1} =
erl_scan:string("%% "++[Illegal], 1, [return]) ||
Illegal <- IllegalL],
{error,{{1,1},erl_scan,{illegal,character}},{1,5}} =
erl_scan:string("%% "++[Illegal1], {1,1}, [return]),
- [{ok,[],1} = erl_scan:string("%% "++[OK], 1, []) ||
+ [{ok,[],1} = erl_scan_string("%% "++[OK], 1, []) ||
OK <- OKL],
- {ok,[],{1,5}} = erl_scan:string("%% "++[OK1], {1,1}, []),
+ {ok,[],{1,5}} = erl_scan_string("%% "++[OK1], {1,1}, []),
[{error,{1,erl_scan,{illegal,character}},1} =
erl_scan:string("%% "++[Illegal], 1, []) ||
Illegal <- IllegalL],
@@ -1058,7 +1071,7 @@ otp_10302(Config) when is_list(Config) ->
erl_scan:string("%% "++[Illegal1], {1,1}, []),
[{ok,[{string,{1,1},[OK]}],{1,4}} =
- erl_scan:string("\""++[OK]++"\"",{1,1}) ||
+ erl_scan_string("\""++[OK]++"\"",{1,1}) ||
OK <- OKL],
[{error,{{1,2},erl_scan,{illegal,character}},{1,3}} =
erl_scan:string("\""++[OK]++"\"",{1,1}) ||
@@ -1069,93 +1082,93 @@ otp_10302(Config) when is_list(Config) ->
Illegal <- IllegalL],
{ok,[{char,{1,1},OK1}],{1,3}} =
- erl_scan:string([$$,OK1],{1,1}),
+ erl_scan_string([$$,OK1],{1,1}),
{error,{{1,1},erl_scan,{illegal,character}},{1,2}} =
erl_scan:string([$$,Illegal1],{1,1}),
{ok,[{char,{1,1},OK1}],{1,4}} =
- erl_scan:string([$$,$\\,OK1],{1,1}),
+ erl_scan_string([$$,$\\,OK1],{1,1}),
{error,{{1,1},erl_scan,{illegal,character}},{1,4}} =
erl_scan:string([$$,$\\,Illegal1],{1,1}),
{ok,[{string,{1,1},[55295]}],{1,5}} =
- erl_scan:string("\"\\"++[OK1]++"\"",{1,1}),
+ erl_scan_string("\"\\"++[OK1]++"\"",{1,1}),
{error,{{1,2},erl_scan,{illegal,character}},{1,4}} =
erl_scan:string("\"\\"++[Illegal1]++"\"",{1,1}),
{ok,[{char,{1,1},OK1}],{1,10}} =
- erl_scan:string("$\\x{D7FF}",{1,1}),
+ erl_scan_string("$\\x{D7FF}",{1,1}),
{error,{{1,1},erl_scan,{illegal,character}},{1,10}} =
erl_scan:string("$\\x{D800}",{1,1}),
%% Not erl_scan, but erl_parse.
- {integer,0,1} = erl_parse:abstract(1),
- Float = 3.14, {float,0,Float} = erl_parse:abstract(Float),
- {nil,0} = erl_parse:abstract([]),
+ {integer,0,1} = erl_parse_abstract(1),
+ Float = 3.14, {float,0,Float} = erl_parse_abstract(Float),
+ {nil,0} = erl_parse_abstract([]),
{bin,0,
[{bin_element,0,{integer,0,1},default,default},
{bin_element,0,{integer,0,2},default,default}]} =
- erl_parse:abstract(<<1,2>>),
+ erl_parse_abstract(<<1,2>>),
{cons,0,{tuple,0,[{atom,0,a}]},{atom,0,b}} =
- erl_parse:abstract([{a} | b]),
- {string,0,"str"} = erl_parse:abstract("str"),
+ erl_parse_abstract([{a} | b]),
+ {string,0,"str"} = erl_parse_abstract("str"),
{cons,0,
{integer,0,$a},
{cons,0,{integer,0,55296},{string,0,"c"}}} =
- erl_parse:abstract("a"++[55296]++"c"),
+ erl_parse_abstract("a"++[55296]++"c"),
Line = 17,
- {integer,Line,1} = erl_parse:abstract(1, Line),
- Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Line),
- {nil,Line} = erl_parse:abstract([], Line),
+ {integer,Line,1} = erl_parse_abstract(1, Line),
+ Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Line),
+ {nil,Line} = erl_parse_abstract([], Line),
{bin,Line,
[{bin_element,Line,{integer,Line,1},default,default},
{bin_element,Line,{integer,Line,2},default,default}]} =
- erl_parse:abstract(<<1,2>>, Line),
+ erl_parse_abstract(<<1,2>>, Line),
{cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
- erl_parse:abstract([{a} | b], Line),
- {string,Line,"str"} = erl_parse:abstract("str", Line),
+ erl_parse_abstract([{a} | b], Line),
+ {string,Line,"str"} = erl_parse_abstract("str", Line),
{cons,Line,
{integer,Line,$a},
{cons,Line,{integer,Line,55296},{string,Line,"c"}}} =
- erl_parse:abstract("a"++[55296]++"c", Line),
+ erl_parse_abstract("a"++[55296]++"c", Line),
Opts1 = [{line,17}],
- {integer,Line,1} = erl_parse:abstract(1, Opts1),
- Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts1),
- {nil,Line} = erl_parse:abstract([], Opts1),
+ {integer,Line,1} = erl_parse_abstract(1, Opts1),
+ Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Opts1),
+ {nil,Line} = erl_parse_abstract([], Opts1),
{bin,Line,
[{bin_element,Line,{integer,Line,1},default,default},
{bin_element,Line,{integer,Line,2},default,default}]} =
- erl_parse:abstract(<<1,2>>, Opts1),
+ erl_parse_abstract(<<1,2>>, Opts1),
{cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
- erl_parse:abstract([{a} | b], Opts1),
- {string,Line,"str"} = erl_parse:abstract("str", Opts1),
+ erl_parse_abstract([{a} | b], Opts1),
+ {string,Line,"str"} = erl_parse_abstract("str", Opts1),
{cons,Line,
{integer,Line,$a},
{cons,Line,{integer,Line,55296},{string,Line,"c"}}} =
- erl_parse:abstract("a"++[55296]++"c", Opts1),
+ erl_parse_abstract("a"++[55296]++"c", Opts1),
[begin
- {integer,Line,1} = erl_parse:abstract(1, Opts2),
- Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts2),
- {nil,Line} = erl_parse:abstract([], Opts2),
+ {integer,Line,1} = erl_parse_abstract(1, Opts2),
+ Float = 3.14, {float,Line,Float} = erl_parse_abstract(Float, Opts2),
+ {nil,Line} = erl_parse_abstract([], Opts2),
{bin,Line,
[{bin_element,Line,{integer,Line,1},default,default},
{bin_element,Line,{integer,Line,2},default,default}]} =
- erl_parse:abstract(<<1,2>>, Opts2),
+ erl_parse_abstract(<<1,2>>, Opts2),
{cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} =
- erl_parse:abstract([{a} | b], Opts2),
- {string,Line,"str"} = erl_parse:abstract("str", Opts2),
+ erl_parse_abstract([{a} | b], Opts2),
+ {string,Line,"str"} = erl_parse_abstract("str", Opts2),
{string,Line,[97,1024,99]} =
- erl_parse:abstract("a"++[1024]++"c", Opts2)
+ erl_parse_abstract("a"++[1024]++"c", Opts2)
end || Opts2 <- [[{encoding,unicode},{line,Line}],
[{encoding,utf8},{line,Line}]]],
{cons,0,
{integer,0,97},
{cons,0,{integer,0,1024},{string,0,"c"}}} =
- erl_parse:abstract("a"++[1024]++"c", [{encoding,latin1}]),
+ erl_parse_abstract("a"++[1024]++"c", [{encoding,latin1}]),
ok.
otp_10990(doc) ->
@@ -1172,13 +1185,13 @@ otp_10992(suite) ->
[];
otp_10992(Config) when is_list(Config) ->
{cons,0,{float,0,42.0},{nil,0}} =
- erl_parse:abstract([42.0], [{encoding,unicode}]),
+ erl_parse_abstract([42.0], [{encoding,unicode}]),
{cons,0,{float,0,42.0},{nil,0}} =
- erl_parse:abstract([42.0], [{encoding,utf8}]),
+ erl_parse_abstract([42.0], [{encoding,utf8}]),
{cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} =
- erl_parse:abstract([$A,42.0], [{encoding,unicode}]),
+ erl_parse_abstract([$A,42.0], [{encoding,unicode}]),
{cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} =
- erl_parse:abstract([$A,42.0], [{encoding,utf8}]),
+ erl_parse_abstract([$A,42.0], [{encoding,utf8}]),
ok.
otp_11807(doc) ->
@@ -1187,29 +1200,72 @@ otp_11807(suite) ->
[];
otp_11807(Config) when is_list(Config) ->
{cons,0,{integer,0,97},{cons,0,{integer,0,98},{nil,0}}} =
- erl_parse:abstract("ab", [{encoding,none}]),
+ erl_parse_abstract("ab", [{encoding,none}]),
{cons,0,{integer,0,-1},{nil,0}} =
- erl_parse:abstract([-1], [{encoding,latin1}]),
+ erl_parse_abstract([-1], [{encoding,latin1}]),
ASCII = fun(I) -> I >= 0 andalso I < 128 end,
- {string,0,"xyz"} = erl_parse:abstract("xyz", [{encoding,ASCII}]),
+ {string,0,"xyz"} = erl_parse_abstract("xyz", [{encoding,ASCII}]),
{cons,0,{integer,0,228},{nil,0}} =
- erl_parse:abstract([228], [{encoding,ASCII}]),
+ erl_parse_abstract([228], [{encoding,ASCII}]),
{cons,0,{integer,0,97},{atom,0,a}} =
- erl_parse:abstract("a"++a, [{encoding,latin1}]),
+ erl_parse_abstract("a"++a, [{encoding,latin1}]),
{'EXIT', {{badarg,bad},_}} = % minor backward incompatibility
(catch erl_parse:abstract("string", [{encoding,bad}])),
ok.
test_string(String, ExpectedWithCol) ->
- {ok, ExpectedWithCol, _EndWithCol} = erl_scan:string(String, {1, 1}, []),
+ {ok, ExpectedWithCol, _EndWithCol} = erl_scan_string(String, {1, 1}, []),
Expected = [ begin
{L,_C} = element(2, T),
setelement(2, T, L)
end
|| T <- ExpectedWithCol ],
- {ok, Expected, _End} = erl_scan:string(String),
+ {ok, Expected, _End} = erl_scan_string(String),
test(String).
+erl_scan_string(String) ->
+ erl_scan_string(String, 1, []).
+
+erl_scan_string(String, StartLocation) ->
+ erl_scan_string(String, StartLocation, []).
+
+erl_scan_string(String, StartLocation, Options) ->
+ case erl_scan:string(String, StartLocation, Options) of
+ {ok, Tokens, EndLocation} ->
+ {ok, unopaque_tokens(Tokens), EndLocation};
+ Else ->
+ Else
+ end.
+
+erl_scan_tokens(C, S, L) ->
+ erl_scan_tokens(C, S, L, []).
+
+erl_scan_tokens(C, S, L, O) ->
+ case erl_scan:tokens(C, S, L, O) of
+ {done, {ok, Ts, End}, R} ->
+ {done, {ok, unopaque_tokens(Ts), End}, R};
+ Else ->
+ Else
+ end.
+
+unopaque_tokens([]) ->
+ [];
+unopaque_tokens([Token|Tokens]) ->
+ Attrs = element(2, Token),
+ Term = erl_anno:to_term(Attrs),
+ T = setelement(2, Token, Term),
+ [T | unopaque_tokens(Tokens)].
+
+erl_parse_abstract(Term) ->
+ erl_parse_abstract(Term, []).
+
+erl_parse_abstract(Term, Options) ->
+ Abstr = erl_parse:abstract(Term, Options),
+ unopaque_abstract(Abstr).
+
+unopaque_abstract(Abstr) ->
+ erl_parse:anno_to_term(Abstr).
+
%% test_string(String, Expected, StartLocation, Options) ->
%% {ok, Expected, _End} = erl_scan:string(String, StartLocation, Options),
%% test(String).
@@ -1359,7 +1415,7 @@ select_tokens(Tokens, Tags) ->
simplify([Token|Tokens]) ->
{line,Line} = erl_scan:token_info(Token, line),
- [setelement(2, Token, Line) | simplify(Tokens)];
+ [setelement(2, Token, erl_anno:new(Line)) | simplify(Tokens)];
simplify([]) ->
[].