aboutsummaryrefslogtreecommitdiffstats
path: root/lib/parsetools/test/leex_SUITE.erl
diff options
context:
space:
mode:
authorHans Bolinder <[email protected]>2012-10-04 15:58:26 +0200
committerHans Bolinder <[email protected]>2013-01-02 10:15:17 +0100
commit300c5466a7c9cfe3ed22bba2a88ba21058406402 (patch)
treeb8c30800b17d5ae98255de2fd2818d8b5d4d6eba /lib/parsetools/test/leex_SUITE.erl
parent7a884a31cfcaaf23f7920ba1a006aa2855529030 (diff)
downloadotp-300c5466a7c9cfe3ed22bba2a88ba21058406402.tar.gz
otp-300c5466a7c9cfe3ed22bba2a88ba21058406402.tar.bz2
otp-300c5466a7c9cfe3ed22bba2a88ba21058406402.zip
[stdlib, kernel] Introduce Unicode support for Erlang source files
Expect modifications, additions and corrections. There is a kludge in file_io_server and erl_scan:continuation_location() that's not so pleasing.
Diffstat (limited to 'lib/parsetools/test/leex_SUITE.erl')
-rw-r--r--lib/parsetools/test/leex_SUITE.erl115
1 files changed, 112 insertions, 3 deletions
diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl
index 1e50aedf07..a0d4fd7c48 100644
--- a/lib/parsetools/test/leex_SUITE.erl
+++ b/lib/parsetools/test/leex_SUITE.erl
@@ -1,7 +1,8 @@
+%% -*= coding: latin-1 -*-
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +43,9 @@
-export([
file/1, compile/1, syntax/1,
- pt/1, man/1, ex/1, ex2/1, not_yet/1]).
+ pt/1, man/1, ex/1, ex2/1, not_yet/1,
+
+ otp_10302/1]).
% Default timetrap timeout (set in init_per_testcase).
-define(default_timeout, ?t:minutes(1)).
@@ -63,7 +66,8 @@ all() ->
groups() ->
[{checks, [], [file, compile, syntax]},
- {examples, [], [pt, man, ex, ex2, not_yet]}].
+ {examples, [], [pt, man, ex, ex2, not_yet]},
+ {tickets, [], [otp_10302]}].
init_per_suite(Config) ->
Config.
@@ -875,6 +879,111 @@ not_yet(Config) when is_list(Config) ->
ok.
+otp_10302(doc) ->
+ "OTP-10302. Unicode characters scanner/parser.";
+otp_10302(suite) -> [];
+otp_10302(Config) when is_list(Config) ->
+ Dir = ?privdir,
+ Filename = filename:join(Dir, "file.xrl"),
+ Ret = [return, {report, true}],
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "�"
+ >>),
+ {error,[{_,[{2,leex,cannot_parse}]}],[]} =
+ leex:file(Filename, Ret),
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "Definitions.\n"
+ "�"
+ >>),
+ {error,[{_,[{3,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars),H�pp}}.\n"
+ >>),
+ {error,[{_,[{7,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+ ok = file:write_file(Filename,<<
+ "%% coding: UTF-8\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars)}}.\n"
+ "Erlang code.\n"
+ "-export([t/0]).\n"
+ "t() ->\n"
+ " H�pp\n"
+ >>),
+ {error,[{_,[{11,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+ Mini = <<"Definitions.\n"
+ "D = [0-9]\n"
+ "Rules.\n"
+ "{L}+ : {token,{word,TokenLine,TokenChars}}.\n"
+ "Erlang code.\n">>,
+ LeexPre = filename:join(Dir, "leexinc.hrl"),
+ ?line ok = file:write_file(LeexPre, <<"%% coding: UTF-8\n �">>),
+ PreErrors = run_test(Config, Mini, LeexPre),
+ {error,[{IncludeFile,[{2,leex,cannot_parse}]}],[]} = PreErrors,
+ "leexinc.hrl" = filename:basename(IncludeFile),
+
+ Ts = [{uni_1,
+ <<"%% coding: UTF-8\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars),\n"
+ "begin Häpp = foo, Häpp end,"
+ " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+ "Erlang code.\n"
+ "-export([t/0]).\n"
+ "t() ->\n"
+ " %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+ " {ok, [R], 1} = string(\"tip\"),\n"
+ " {tip,foo,'Häpp',[1024,66],[246,114,110,95,1024]} = R,\n"
+ " Häpp = foo,\n"
+ " {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+ " ok.\n">>,
+ default,
+ ok},
+ {uni_2,
+ <<"%% coding: Latin-1\n"
+ "Definitions.\n"
+ "A = a\n"
+ "L = [{A}-{Z}]\n"
+ "Z = z\n"
+ "Rules.\n"
+ "{L}+ : {token,{list_to_atom(TokenChars),\n"
+ "begin H�pp = foo, H�pp end,"
+ " 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+ "Erlang code.\n"
+ "-export([t/0]).\n"
+ "t() ->\n"
+ " %% H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+ " {ok, [R], 1} = string(\"tip\"),\n"
+ " {tip,foo,'H�pp',[1024,66],[195,182,114,110,95,208,128]} = R,\n"
+ " H�pp = foo,\n"
+ " {tip, H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+ " ok.\n">>,
+ default,
+ ok}],
+ run(Config, Ts),
+
+ ok.
+
unwritable(Fname) ->
{ok, Info} = file:read_file_info(Fname),
Mode = Info#file_info.mode - 8#00200,