aboutsummaryrefslogtreecommitdiffstats
path: root/lib/parsetools/test
diff options
context:
space:
mode:
Diffstat (limited to 'lib/parsetools/test')
-rw-r--r--lib/parsetools/test/leex_SUITE.erl116
-rw-r--r--lib/parsetools/test/yecc_SUITE.erl154
2 files changed, 263 insertions, 7 deletions
diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl
index 1e50aedf07..9d591c0d05 100644
--- a/lib/parsetools/test/leex_SUITE.erl
+++ b/lib/parsetools/test/leex_SUITE.erl
@@ -1,7 +1,8 @@
+%% -*- coding: latin-1 -*-
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2013. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +43,9 @@
-export([
file/1, compile/1, syntax/1,
- pt/1, man/1, ex/1, ex2/1, not_yet/1]).
+ pt/1, man/1, ex/1, ex2/1, not_yet/1,
+
+ otp_10302/1]).
% Default timetrap timeout (set in init_per_testcase).
-define(default_timeout, ?t:minutes(1)).
@@ -63,7 +66,8 @@ all() ->
groups() ->
[{checks, [], [file, compile, syntax]},
- {examples, [], [pt, man, ex, ex2, not_yet]}].
+ {examples, [], [pt, man, ex, ex2, not_yet]},
+ {tickets, [], [otp_10302]}].
init_per_suite(Config) ->
Config.
@@ -645,7 +649,6 @@ reserved_word('fun') -> true;
reserved_word('if') -> true;
reserved_word('let') -> true;
reserved_word('of') -> true;
-reserved_word('query') -> true;
reserved_word('receive') -> true;
reserved_word('when') -> true;
reserved_word('bnot') -> true;
@@ -875,6 +878,111 @@ not_yet(Config) when is_list(Config) ->
ok.
+otp_10302(doc) ->
+ "OTP-10302. Unicode characters scanner/parser.";
+otp_10302(suite) -> [];
+otp_10302(Config) when is_list(Config) ->
+ Dir = ?privdir,
+ Filename = filename:join(Dir, "file.xrl"),
+ Ret = [return, {report, true}],
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "�"
+ >>),
+ {error,[{_,[{2,leex,cannot_parse}]}],[]} =
+ leex:file(Filename, Ret),
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "Definitions.\n"
+ "�"
+ >>),
+ {error,[{_,[{3,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars),H�pp}}.\n"
+ >>),
+ {error,[{_,[{7,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars)}}.\n"
+ "Erlang code.\n"
+ "-export([t/0]).\n"
+ "t() ->\n"
+ " H�pp\n"
+ >>),
+ {error,[{_,[{11,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+ Mini = <<"Definitions.\n"
+ "D = [0-9]\n"
+ "Rules.\n"
+ "{L}+ : {token,{word,TokenLine,TokenChars}}.\n"
+ "Erlang code.\n">>,
+ LeexPre = filename:join(Dir, "leexinc.hrl"),
+ ?line ok = file:write_file(LeexPre, <<"%% coding: UTF-8\n �">>),
+ PreErrors = run_test(Config, Mini, LeexPre),
+ {error,[{IncludeFile,[{2,leex,cannot_parse}]}],[]} = PreErrors,
+ "leexinc.hrl" = filename:basename(IncludeFile),
+
+ Ts = [{uni_1,
+ <<"%% coding: UTF-8\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars),\n"
+ "begin Häpp = foo, Häpp end,"
+ " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+ "Erlang code.\n"
+ "-export([t/0]).\n"
+ "t() ->\n"
+ " %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+ " {ok, [R], 1} = string(\"tip\"),\n"
+ " {tip,foo,'Häpp',[1024,66],[246,114,110,95,1024]} = R,\n"
+ " Häpp = foo,\n"
+ " {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+ " ok.\n">>,
+ default,
+ ok},
+ {uni_2,
+ <<"%% coding: Latin-1\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars),\n"
+ "begin H�pp = foo, H�pp end,"
+ " 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+ "Erlang code.\n"
+ "-export([t/0]).\n"
+ "t() ->\n"
+ " %% H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+ " {ok, [R], 1} = string(\"tip\"),\n"
+ " {tip,foo,'H�pp',[1024,66],[195,182,114,110,95,208,128]} = R,\n"
+ " H�pp = foo,\n"
+ " {tip, H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+ " ok.\n">>,
+ default,
+ ok}],
+ run(Config, Ts),
+
+ ok.
+
unwritable(Fname) ->
{ok, Info} = file:read_file_info(Fname),
Mode = Info#file_info.mode - 8#00200,
diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl
index 3d26adf1be..c306dbe833 100644
--- a/lib/parsetools/test/yecc_SUITE.erl
+++ b/lib/parsetools/test/yecc_SUITE.erl
@@ -1,7 +1,8 @@
+%% -*- coding: latin-1 -*-
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2012. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -48,7 +49,7 @@
otp_5369/1, otp_6362/1, otp_7945/1, otp_8483/1, otp_8486/1,
- otp_7292/1, otp_7969/1, otp_8919/1]).
+ otp_7292/1, otp_7969/1, otp_8919/1, otp_10302/1]).
% Default timetrap timeout (set in init_per_testcase).
-define(default_timeout, ?t:minutes(1)).
@@ -75,7 +76,7 @@ groups() ->
[empty, prec, yeccpre, lalr, old_yecc, other_examples]},
{bugs, [],
[otp_5369, otp_6362, otp_7945, otp_8483, otp_8486]},
- {improvements, [], [otp_7292, otp_7969, otp_8919]}].
+ {improvements, [], [otp_7292, otp_7969, otp_8919, otp_10302]}].
init_per_suite(Config) ->
Config.
@@ -1815,6 +1816,153 @@ otp_8919(Config) when is_list(Config) ->
"syntax error before: \"hello\"" = lists:flatten(Mod:format_error(Mess)),
ok.
+otp_10302(doc) ->
+ "OTP-10302. Unicode characters scanner/parser.";
+otp_10302(suite) -> [];
+otp_10302(Config) when is_list(Config) ->
+ Dir = ?privdir,
+ Filename = filename:join(Dir, "OTP-10302.yrl"),
+ Ret = [return, {report, true}],
+ Mini1 = <<"%% coding: utf-8
+ Nonterminals H�pp.
+ nt -> t.">>,
+ ok = file:write_file(Filename, Mini1),
+ %% This could (and should) be refined:
+ {error,[{Filename,[{2,Mod1,Err1}]}],[]} =
+ yecc:file(Filename, Ret),
+ "cannot translate from UTF-8" = Mod1:format_error(Err1),
+
+ Mini2 = <<"%% coding: Utf-8
+ Nonterminals Hopp.
+ Terminals t.
+ Rootsymbol Hopp.
+
+ Hopp -> t.
+
+ Erlang code.
+
+ t() ->
+ H�pp.">>,
+ ok = file:write_file(Filename, Mini2),
+ {error,[{Filename,[{11,Mod2,Err2}]}],[]} =
+ yecc:file(Filename, Ret),
+ "cannot parse; possibly encoding mismatch" = Mod2:format_error(Err2),
+
+ Mini3 = <<"%% coding: latin-1
+ Nonterminals Hopp.
+ Terminals t.
+ Rootsymbol Hopp.
+
+ Hopp -> t.
+
+ Erlang code.
+
+ t() ->
+ H�pp.">>,
+ ok = file:write_file(Filename, Mini3),
+ YeccPre = filename:join(Dir, "yeccpre.hrl"),
+ ok = file:write_file(YeccPre, [<<"%% coding: UTF-8\n �.\n">>]),
+ Inc = [{includefile,YeccPre}],
+ {error,[{_,[{2,yecc,cannot_parse}]}],[]} =
+ yecc:file(Filename, Inc ++ Ret),
+
+ ok = file:write_file(Filename,
+ <<"%% coding: UTF-8
+ Nonterminals Hopp.
+ Terminals t.
+ Rootsymbol \"örn_Ѐ\".
+ Hopp -> t : '$1'.">>),
+ {error,[{Filename,[{4,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+ yecc:file(Filename, Ret),
+
+ ok = file:write_file(Filename,
+ <<"%% coding: UTF-8
+ Nonterminals Hopp.
+ Terminals t.
+ Rootsymbol Hopp.
+ Endsymbol \"örn_Ѐ\".
+ Hopp -> t : '$1'.">>),
+ {error,[{Filename,[{5,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+ yecc:file(Filename, Ret),
+
+ ok = file:write_file(Filename,
+ <<"%% coding: UTF-8
+ Nonterminals Hopp.
+ Terminals t.
+ Rootsymbol Hopp.
+ Expect \"örn_Ѐ\".
+ Hopp -> t : '$1'.">>),
+ {error,[{Filename,[{5,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+ yecc:file(Filename, Ret),
+
+ ok = file:write_file(Filename,
+ <<"%% coding: UTF-8
+ Nonterminals Hopp.
+ Terminals t.
+ Rootsymbol Hopp.
+ States \"örn_Ѐ\".
+ Hopp -> t : '$1'.">>),
+ {error,[{Filename,[{5,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+ yecc:file(Filename, Ret),
+
+ Ts = [{otp_10302_1,<<"
+ %% coding: UTF-8
+ Header \"%% örn_Ѐ\" \"%% \\x{400}B\".
+ Nonterminals Häpp list.
+ Terminals element.
+ Rootsymbol Häpp.
+
+ Häpp -> list : '$1'.
+
+ list -> element : '$1'.
+ list -> list element :
+ begin
+ Häpp = foo,
+ {Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}
+ end.
+
+ Erlang code.
+
+ -export([t/0]).
+
+ t() ->
+ L = [{element, 1}, {element,2}],
+ {ok, R} = parse(L),
+ Häpp = foo,
+ {_,_,[1024,66],[246,114,110,95,1024]} = R,
+ {Häpp,'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,
+ ok.
+ ">>,default,ok},
+ {otp_10302_2,<<"
+ %% coding: Latin-1
+ Nonterminals H�pp list.
+ Terminals element.
+ Rootsymbol H�pp.
+
+ H�pp -> list : '$1'.
+
+ list -> element : '$1'.
+ list -> list element :
+ begin
+ H�pp = foo,
+ {H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"}
+ end.
+
+ Erlang code.
+
+ -export([t/0]).
+
+ t() ->
+ L = [{element, 1}, {element,2}],
+ {ok, R} = parse(L),
+ H�pp = foo,
+ {_,_,[1024,66],[195,182,114,110,95,208,128]} = R,
+ {H�pp,'H�pp',\"\\x{400}B\",\"örn_Ѐ\"} = R,
+ ok.
+ ">>,default,ok}],
+ run(Config, Ts),
+ ok.
+
yeccpre_size() ->
yeccpre_size(default_yeccpre()).
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-30 18:20:03 +0000