diff options
Diffstat (limited to 'lib/stdlib/test/erl_scan_SUITE.erl')
| -rw-r--r-- | lib/stdlib/test/erl_scan_SUITE.erl | 301 |
1 files changed, 232 insertions, 69 deletions
diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index 4298b2c701..ecd181e87c 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -1,7 +1,8 @@ +%% -*- coding: utf-8 -*- %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2011. All Rights Reserved. +%% Copyright Ericsson AB 1998-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,7 +21,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). --export([ error_1/1, error_2/1, iso88591/1, otp_7810/1]). +-export([ error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1]). -import(lists, [nth/2,flatten/1]). -import(io_lib, [print/1]). @@ -59,7 +60,7 @@ end_per_testcase(_Case, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [{group, error}, iso88591, otp_7810]. + [{group, error}, iso88591, otp_7810, otp_10302]. groups() -> [{error, [], [error_1, error_2]}]. @@ -117,13 +118,13 @@ check(String) -> %%% (This should be useful for all format_error functions.) check_error({error, Info, EndLine}, Module0) -> - ?line {ErrorLine, Module, Desc} = Info, - ?line true = (Module == Module0), - ?line assert_type(EndLine, integer), - ?line assert_type(ErrorLine, integer), - ?line true = (ErrorLine =< EndLine), - ?line String = lists:flatten(Module0:format_error(Desc)), - ?line true = io_lib:printable_list(String). + {ErrorLine, Module, Desc} = Info, + true = (Module == Module0), + assert_type(EndLine, integer), + assert_type(ErrorLine, integer), + true = (ErrorLine =< EndLine), + String = lists:flatten(Module0:format_error(Desc)), + true = io_lib:printable_list(String). iso88591(doc) -> ["Tests the support for ISO-8859-1 i.e Latin-1"]; iso88591(suite) -> []; @@ -131,10 +132,10 @@ iso88591(Config) when is_list(Config) -> ?line ok = case catch begin %% Some atom and variable names - V1s = [$�,$�,$�,$�], - V2s = [$N,$�,$r], - A1s = [$h,$�,$r], - A2s = [$�,$r,$e], + V1s = [$Á,$á,$é,$ë], + V2s = [$N,$ä,$r], + A1s = [$h,$ä,$r], + A2s = [$ö,$r,$e], %% Test parsing atom and variable characters. {ok,Ts1,_} = erl_scan:string(V1s ++ " " ++ V2s ++ "\327" ++ @@ -196,7 +197,7 @@ otp_7810(Config) when is_list(Config) -> reserved_words() -> L = ['after', 'begin', 'case', 'try', 'cond', 'catch', 'andalso', 'orelse', 'end', 'fun', 'if', 'let', 'of', - 'query', 'receive', 'when', 'bnot', 'not', 'div', + 'receive', 'when', 'bnot', 'not', 'div', 'rem', 'band', 'and', 'bor', 'bxor', 'bsl', 'bsr', 'or', 'xor'], [begin @@ -214,8 +215,8 @@ atoms() -> ?line test_string("'a b'", [{atom,1,'a b'}]), ?line test_string("a", [{atom,1,a}]), ?line test_string("a@2", [{atom,1,a@2}]), - ?line test_string([39,65,200,39], [{atom,1,'A�'}]), - ?line test_string("�rlig �sten", [{atom,1,�rlig},{atom,1,�sten}]), + ?line test_string([39,65,200,39], [{atom,1,'AÈ'}]), + ?line test_string("ärlig östen", [{atom,1,ärlig},{atom,1,östen}]), ?line {ok,[{atom,_,'$a'}],{1,6}} = erl_scan:string("'$\\a'", {1,1}), ?line test("'$\\a'"), @@ -289,7 +290,7 @@ errors() -> ?line {error,{1,erl_scan,{string,$","str"}},1} = %" erl_scan:string("\"str"), %" ?line {error,{1,erl_scan,char},1} = erl_scan:string("$"), - ?line test_string([34,65,200,34], [{string,1,"A�"}]), + ?line test_string([34,65,200,34], [{string,1,"AÈ"}]), ?line test_string("\\", [{'\\',1}]), ?line {'EXIT',_} = (catch {foo, erl_scan:string('$\\a', {1,1})}), % type error @@ -354,7 +355,7 @@ dots() -> {".\n", {ok,[{dot,1}],2}}, {".%", {ok,[{dot,1}],1}}, {".\210",{ok,[{dot,1}],1}}, - {".% �h",{ok,[{dot,1}],1}}, + {".% öh",{ok,[{dot,1}],1}}, {".%\n", {ok,[{dot,1}],2}}, {".$", {error,{1,erl_scan,char},1}}, {".$\\", {error,{1,erl_scan,char},1}}, @@ -369,7 +370,7 @@ dots() -> ?line [{column,1},{length,1},{line,1},{text,"."}] = erl_scan:token_info(T2, [column, length, line, text]), ?line {ok,[{dot,_}=T3],{1,6}} = - erl_scan:string(".% �h", {1,1}, text), + erl_scan:string(".% öh", {1,1}, text), ?line [{column,1},{length,1},{line,1},{text,"."}] = erl_scan:token_info(T3, [column, length, line, text]), ?line {error,{{1,2},erl_scan,char},{1,3}} = @@ -472,11 +473,11 @@ chars() -> variables() -> - ?line test_string(" \237_Aou�eiy��", [{var,1,'_Aou�eiy��'}]), + ?line test_string(" \237_Aouåeiyäö", [{var,1,'_Aouåeiyäö'}]), ?line test_string("A_b_c@", [{var,1,'A_b_c@'}]), ?line test_string("V@2", [{var,1,'V@2'}]), - ?line test_string("ABD�", [{var,1,'ABD�'}]), - ?line test_string("�rlig �sten", [{var,1,'�rlig'},{var,1,'�sten'}]), + ?line test_string("ABDÀ", [{var,1,'ABDÀ'}]), + ?line test_string("Ärlig Östen", [{var,1,'Ärlig'},{var,1,'Östen'}]), ok. eof() -> @@ -814,62 +815,51 @@ unicode() -> erl_scan:string([1089]), ?line {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = erl_scan:string([1089], {1,1}), - ?line {error,{1,erl_scan,{illegal,character}},1} = - %% ?line {error,{1,erl_scan,{illegal,atom}},1} = - erl_scan:string("'a"++[1089]++"b'"), - ?line {error,{{1,3},erl_scan,{illegal,character}},{1,4}} = + ?line {error,{1,erl_scan,{illegal,atom}},1} = + erl_scan:string("'a"++[1089]++"b'", 1), + ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,6}} = erl_scan:string("'a"++[1089]++"b'", {1,1}), ?line test("\"a"++[1089]++"b\""), - ?line {ok,[{char,1,1}],1} = erl_scan:string([$$,$\\,$^,1089]), + ?line {ok,[{char,1,1}],1} = + erl_scan:string([$$,$\\,$^,1089], 1), - ?line {error,{1,erl_scan,Error},1} = erl_scan:string("\"qa\x{aaa}"), - ?line "unterminated string starting with \"qa\\x{AAA}\"" = + ?line {error,{1,erl_scan,Error},1} = + erl_scan:string("\"qa\x{aaa}", 1), + ?line "unterminated string starting with \"qa"++[2730]++"\"" = erl_scan:format_error(Error), ?line {error,{{1,1},erl_scan,_},{1,11}} = erl_scan:string("\"qa\\x{aaa}",{1,1}), - ?line {error,{{1,4},erl_scan,{illegal,character}},{1,11}} = + ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} = erl_scan:string("'qa\\x{aaa}'",{1,1}), - Tags = [category, column, length, line, symbol, text], - - %% Workaround. No character codes greater than 255! To be changed. - %% Note: don't remove these tests, just modify them! - - ?line {ok,[{integer,1,1089}],1} = erl_scan:string([$$,1089]), - ?line {ok,[{integer,1,1089}],1} = erl_scan:string([$$,$\\,1089]), + ?line {ok,[{char,1,1089}],1} = + erl_scan:string([$$,1089], 1), + ?line {ok,[{char,1,1089}],1} = + erl_scan:string([$$,$\\,1089], 1), Qs = "$\\x{aaa}", - ?line {ok,[{integer,1,16#aaa}],1} = erl_scan:string(Qs), - ?line {ok,[Q2],{1,9}} = erl_scan:string("$\\x{aaa}", {1,1}, text), - ?line [{category,integer},{column,1},{length,8}, + ?line {ok,[{char,1,$\x{aaa}}],1} = + erl_scan:string(Qs, 1), + ?line {ok,[Q2],{1,9}} = + erl_scan:string("$\\x{aaa}", {1,1}, [text]), + ?line [{category,char},{column,1},{length,8}, {line,1},{symbol,16#aaa},{text,Qs}] = erl_scan:token_info(Q2), U1 = "\"\\x{aaa}\"", - ?line {ok,[T1,T2,T3],{1,10}} = erl_scan:string(U1, {1,1}, text), - ?line [{category,'['},{column,1},{length,1},{line,1}, - {symbol,'['},{text,"\""}] = erl_scan:token_info(T1, Tags), - ?line [{category,integer},{column,2},{length,7}, - {line,1},{symbol,16#aaa},{text,"\\x{aaa}"}] = - erl_scan:token_info(T2, Tags), - ?line [{category,']'},{column,9},{length,1},{line,1}, - {symbol,']'},{text,"\""}] = erl_scan:token_info(T3, Tags), - ?line {ok,[{'[',1},{integer,1,16#aaa},{']',1}],1} = - erl_scan:string(U1, 1), + {ok, + [{string,[{line,1},{column,1},{text,"\"\\x{aaa}\""}],[2730]}], + {1,10}} = erl_scan:string(U1, {1,1}, [text]), + {ok,[{string,1,[2730]}],1} = erl_scan:string(U1, 1), U2 = "\"\\x41\\x{fff}\\x42\"", - ?line {ok,[{'[',1},{char,1,16#41},{',',1},{integer,1,16#fff}, - {',',1},{char,1,16#42},{']',1}],1} = erl_scan:string(U2, 1), + {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan:string(U2, 1), U3 = "\"a\n\\x{fff}\n\"", - ?line {ok,[{'[',1},{char,1,$a},{',',1},{char,1,$\n}, - {',',2},{integer,2,16#fff},{',',2},{char,2,$\n}, - {']',3}],3} = - erl_scan:string(U3, 1), + {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan:string(U3, 1), U4 = "\"\\^\n\\x{aaa}\\^\n\"", - ?line {ok,[{'[',1},{char,1,$\n},{',',2},{integer,2,16#aaa}, - {',',2},{char,2,$\n},{']',3}],3} = erl_scan:string(U4, 1), + {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan:string(U4, 1), %% Keep these tests: ?line test(Qs), @@ -879,18 +869,15 @@ unicode() -> ?line test(U4), Str1 = "\"ab" ++ [1089] ++ "cd\"", - ?line {ok,[{'[',1},{char,1,$a},{',',1},{char,1,$b},{',',1}, - {integer,1,1089},{',',1},{char,1,$c},{',',1}, - {char,1,$d},{']',1}],1} = erl_scan:string(Str1), - ?line {ok,[{'[',_},{char,_,$a},{',',_},{char,_,$b},{',',_}, - {integer,_,1089},{',',_},{char,_,$c},{',',_}, - {char,_,$d},{']',_}],{1,8}} = erl_scan:string(Str1, {1,1}), + {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan:string(Str1, 1), + {ok,[{string,{1,1},[$a,$b,1089,$c,$d]}],{1,8}} = + erl_scan:string(Str1, {1,1}), ?line test(Str1), Comment = "%% "++[1089], - ?line {ok,[{comment,1,[$%,$%,$\s,1089]}],1} = - erl_scan:string(Comment, 1, return), - ?line {ok,[{comment,_,[$%,$%,$\s,1089]}],{1,5}} = - erl_scan:string(Comment, {1,1}, return), + {ok,[{comment,1,[$%,$%,$\s,1089]}],1} = + erl_scan:string(Comment, 1, [return]), + {ok,[{comment,{1,1},[$%,$%,$\s,1089]}],{1,5}} = + erl_scan:string(Comment, {1,1}, [return]), ok. more_chars() -> @@ -958,6 +945,182 @@ more_chars() -> erl_scan:string("$\\xg", {1,1}), ok. +otp_10302(doc) -> + "OTP-10302. Unicode characters scanner/parser."; +otp_10302(suite) -> + []; +otp_10302(Config) when is_list(Config) -> + %% From unicode(): + {error,{1,erl_scan,{illegal,atom}},1} = + erl_scan:string("'a"++[1089]++"b'", 1), + {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} = + erl_scan:string("'qa\\x{aaa}'",{1,1}), + + {ok,[{char,1,1089}],1} = erl_scan:string([$$,1089], 1), + {ok,[{char,1,1089}],1} = erl_scan:string([$$,$\\,1089],1), + + Qs = "$\\x{aaa}", + {ok,[{char,1,2730}],1} = erl_scan:string(Qs,1), + {ok,[Q2],{1,9}} = erl_scan:string(Qs,{1,1},[text]), + [{category,char},{column,1},{length,8}, + {line,1},{symbol,16#aaa},{text,Qs}] = + erl_scan:token_info(Q2), + + Tags = [category, column, length, line, symbol, text], + + U1 = "\"\\x{aaa}\"", + {ok,[T1],{1,10}} = erl_scan:string(U1, {1,1}, [text]), + [{category,string},{column,1},{length,9},{line,1}, + {symbol,[16#aaa]},{text,U1}] = erl_scan:token_info(T1, Tags), + + U2 = "\"\\x41\\x{fff}\\x42\"", + {ok,[{string,1,[65,4095,66]}],1} = erl_scan:string(U2, 1), + + U3 = "\"a\n\\x{fff}\n\"", + {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan:string(U3, 1), + + U4 = "\"\\^\n\\x{aaa}\\^\n\"", + {ok,[{string,1,[10,2730,10]}],3} = erl_scan:string(U4, 1,[]), + + Str1 = "\"ab" ++ [1089] ++ "cd\"", + {ok,[{string,1,[97,98,1089,99,100]}],1} = + erl_scan:string(Str1,1), + {ok,[{string,{1,1},[97,98,1089,99,100]}],{1,8}} = + erl_scan:string(Str1, {1,1}), + + OK1 = 16#D800-1, + OK2 = 16#DFFF+1, + OK3 = 16#FFFE-1, + OK4 = 16#FFFF+1, + OKL = [OK1,OK2,OK3,OK4], + + Illegal1 = 16#D800, + Illegal2 = 16#DFFF, + Illegal3 = 16#FFFE, + Illegal4 = 16#FFFF, + IllegalL = [Illegal1,Illegal2,Illegal3,Illegal4], + + [{ok,[{comment,1,[$%,$%,$\s,OK]}],1} = + erl_scan:string("%% "++[OK], 1, [return]) || + OK <- OKL], + {ok,[{comment,_,[$%,$%,$\s,OK1]}],{1,5}} = + erl_scan:string("%% "++[OK1], {1,1}, [return]), + [{error,{1,erl_scan,{illegal,character}},1} = + erl_scan:string("%% "++[Illegal], 1, [return]) || + Illegal <- IllegalL], + {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = + erl_scan:string("%% "++[Illegal1], {1,1}, [return]), + + [{ok,[],1} = erl_scan:string("%% "++[OK], 1, []) || + OK <- OKL], + {ok,[],{1,5}} = erl_scan:string("%% "++[OK1], {1,1}, []), + [{error,{1,erl_scan,{illegal,character}},1} = + erl_scan:string("%% "++[Illegal], 1, []) || + Illegal <- IllegalL], + {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = + erl_scan:string("%% "++[Illegal1], {1,1}, []), + + [{ok,[{string,{1,1},[OK]}],{1,4}} = + erl_scan:string("\""++[OK]++"\"",{1,1}) || + OK <- OKL], + [{error,{{1,2},erl_scan,{illegal,character}},{1,3}} = + erl_scan:string("\""++[OK]++"\"",{1,1}) || + OK <- IllegalL], + + [{error,{{1,1},erl_scan,{illegal,character}},{1,2}} = + erl_scan:string([Illegal],{1,1}) || + Illegal <- IllegalL], + + {ok,[{char,{1,1},OK1}],{1,3}} = + erl_scan:string([$$,OK1],{1,1}), + {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = + erl_scan:string([$$,Illegal1],{1,1}), + + {ok,[{char,{1,1},OK1}],{1,4}} = + erl_scan:string([$$,$\\,OK1],{1,1}), + {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = + erl_scan:string([$$,$\\,Illegal1],{1,1}), + + {ok,[{string,{1,1},[55295]}],{1,5}} = + erl_scan:string("\"\\"++[OK1]++"\"",{1,1}), + {error,{{1,2},erl_scan,{illegal,character}},{1,4}} = + erl_scan:string("\"\\"++[Illegal1]++"\"",{1,1}), + + {ok,[{char,{1,1},OK1}],{1,10}} = + erl_scan:string("$\\x{D7FF}",{1,1}), + {error,{{1,1},erl_scan,{illegal,character}},{1,10}} = + erl_scan:string("$\\x{D800}",{1,1}), + + %% Not erl_scan, but erl_parse. + {integer,0,1} = erl_parse:abstract(1), + Float = 3.14, {float,0,Float} = erl_parse:abstract(Float), + {nil,0} = erl_parse:abstract([]), + {bin,0, + [{bin_element,0,{integer,0,1},default,default}, + {bin_element,0,{integer,0,2},default,default}]} = + erl_parse:abstract(<<1,2>>), + {cons,0,{tuple,0,[{atom,0,a}]},{atom,0,b}} = + erl_parse:abstract([{a} | b]), + {string,0,"str"} = erl_parse:abstract("str"), + {cons,0, + {integer,0,$a}, + {cons,0,{integer,0,1024},{string,0,"c"}}} = + erl_parse:abstract("a"++[1024]++"c"), + + Line = 17, + {integer,Line,1} = erl_parse:abstract(1, Line), + Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Line), + {nil,Line} = erl_parse:abstract([], Line), + {bin,Line, + [{bin_element,Line,{integer,Line,1},default,default}, + {bin_element,Line,{integer,Line,2},default,default}]} = + erl_parse:abstract(<<1,2>>, Line), + {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = + erl_parse:abstract([{a} | b], Line), + {string,Line,"str"} = erl_parse:abstract("str", Line), + {cons,Line, + {integer,Line,$a}, + {cons,Line,{integer,Line,1024},{string,Line,"c"}}} = + erl_parse:abstract("a"++[1024]++"c", Line), + + Opts1 = [{line,17}], + {integer,Line,1} = erl_parse:abstract(1, Opts1), + Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts1), + {nil,Line} = erl_parse:abstract([], Opts1), + {bin,Line, + [{bin_element,Line,{integer,Line,1},default,default}, + {bin_element,Line,{integer,Line,2},default,default}]} = + erl_parse:abstract(<<1,2>>, Opts1), + {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = + erl_parse:abstract([{a} | b], Opts1), + {string,Line,"str"} = erl_parse:abstract("str", Opts1), + {cons,Line, + {integer,Line,$a}, + {cons,Line,{integer,Line,1024},{string,Line,"c"}}} = + erl_parse:abstract("a"++[1024]++"c", Opts1), + + [begin + {integer,Line,1} = erl_parse:abstract(1, Opts2), + Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts2), + {nil,Line} = erl_parse:abstract([], Opts2), + {bin,Line, + [{bin_element,Line,{integer,Line,1},default,default}, + {bin_element,Line,{integer,Line,2},default,default}]} = + erl_parse:abstract(<<1,2>>, Opts2), + {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = + erl_parse:abstract([{a} | b], Opts2), + {string,Line,"str"} = erl_parse:abstract("str", Opts2), + {string,Line,[97,1024,99]} = + erl_parse:abstract("a"++[1024]++"c", Opts2) + end || Opts2 <- [[{encoding,unicode},{line,Line}], + [{encoding,utf8},{line,Line}]]], + + {cons,0, + {integer,0,97}, + {cons,0,{integer,0,1024},{string,0,"c"}}} = + erl_parse:abstract("a"++[1024]++"c", [{encoding,latin1}]), + ok. + test_string(String, Expected) -> {ok, Expected, _End} = erl_scan:string(String), test(String). |