[stdlib, kernel] Introduce Unicode support for Erlang source files

Expect modifications, additions and corrections.

There is a kludge in file_io_server and
erl_scan:continuation_location() that's not so pleasing.

2 files changed, 263 insertions, 6 deletions

diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl
index 1e50aedf07..a0d4fd7c48 100644
--- a/lib/parsetools/test/leex_SUITE.erl
+++ b/lib/parsetools/test/leex_SUITE.erl
@@ -1,7 +1,8 @@
+%% -*= coding: latin-1 -*-
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +43,9 @@
 -export([
 	 file/1, compile/1, syntax/1,
 	 
-	 pt/1, man/1, ex/1, ex2/1, not_yet/1]).
+	 pt/1, man/1, ex/1, ex2/1, not_yet/1,
+
+         otp_10302/1]).
 
 % Default timetrap timeout (set in init_per_testcase).
 -define(default_timeout, ?t:minutes(1)).
@@ -63,7 +66,8 @@ all() ->
 
 groups() -> 
     [{checks, [], [file, compile, syntax]},
-     {examples, [], [pt, man, ex, ex2, not_yet]}].
+     {examples, [], [pt, man, ex, ex2, not_yet]},
+     {tickets, [], [otp_10302]}].
 
 init_per_suite(Config) ->
     Config.
@@ -875,6 +879,111 @@ not_yet(Config) when is_list(Config) ->
 
     ok.
 
+otp_10302(doc) ->
+    "OTP-10302. Unicode characters scanner/parser.";
+otp_10302(suite) -> [];
+otp_10302(Config) when is_list(Config) ->
+    Dir = ?privdir,
+    Filename = filename:join(Dir, "file.xrl"),
+    Ret = [return, {report, true}],
+
+    ok = file:write_file(Filename,<<
+         "%% coding: UTF-8\n"
+         "�"
+     >>),
+    {error,[{_,[{2,leex,cannot_parse}]}],[]} =
+        leex:file(Filename, Ret),
+
+    ok = file:write_file(Filename,<<
+         "%% coding: UTF-8\n"
+         "Definitions.\n"
+         "�"
+     >>),
+    {error,[{_,[{3,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+    ok = file:write_file(Filename,<<
+         "%% coding: UTF-8\n"
+         "Definitions.\n"
+         "A = a\n"
+         "L = [{A}-{Z}]\n"
+         "Z = z\n"
+         "Rules.\n"
+         "{L}+ : {token,{list_to_atom(TokenChars),H�pp}}.\n"
+     >>),
+    {error,[{_,[{7,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+    ok = file:write_file(Filename,<<
+         "%% coding: UTF-8\n"
+         "Definitions.\n"
+         "A = a\n"
+         "L = [{A}-{Z}]\n"
+         "Z = z\n"
+         "Rules.\n"
+         "{L}+ : {token,{list_to_atom(TokenChars)}}.\n"
+         "Erlang code.\n"
+         "-export([t/0]).\n"
+         "t() ->\n"
+         "    H�pp\n"
+      >>),
+    {error,[{_,[{11,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
+
+    Mini = <<"Definitions.\n"
+             "D  = [0-9]\n"
+             "Rules.\n"
+             "{L}+  : {token,{word,TokenLine,TokenChars}}.\n"
+             "Erlang code.\n">>,
+    LeexPre = filename:join(Dir, "leexinc.hrl"),
+    ?line ok = file:write_file(LeexPre, <<"%% coding: UTF-8\n �">>),
+    PreErrors = run_test(Config, Mini, LeexPre),
+    {error,[{IncludeFile,[{2,leex,cannot_parse}]}],[]} = PreErrors,
+    "leexinc.hrl" = filename:basename(IncludeFile),
+
+    Ts = [{uni_1,
+       <<"%% coding: UTF-8\n"
+         "Definitions.\n"
+         "A = a\n"
+         "L = [{A}-{Z}]\n"
+         "Z = z\n"
+         "Rules.\n"
+         "{L}+ : {token,{list_to_atom(TokenChars),\n"
+         "begin Häpp = foo, Häpp end,"
+         " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+         "Erlang code.\n"
+         "-export([t/0]).\n"
+         "t() ->\n"
+         "    %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+         "    {ok, [R], 1} = string(\"tip\"),\n"
+         "    {tip,foo,'Häpp',[1024,66],[246,114,110,95,1024]} = R,\n"
+         "    Häpp = foo,\n"
+         "    {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+         "    ok.\n">>,
+          default,
+          ok},
+      {uni_2,
+       <<"%% coding: Latin-1\n"
+         "Definitions.\n"
+         "A = a\n"
+         "L = [{A}-{Z}]\n"
+         "Z = z\n"
+         "Rules.\n"
+         "{L}+ : {token,{list_to_atom(TokenChars),\n"
+         "begin H�pp = foo, H�pp end,"
+         " 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+         "Erlang code.\n"
+         "-export([t/0]).\n"
+         "t() ->\n"
+         "    %% H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+         "    {ok, [R], 1} = string(\"tip\"),\n"
+         "    {tip,foo,'H�pp',[1024,66],[195,182,114,110,95,208,128]} = R,\n"
+         "    H�pp = foo,\n"
+         "    {tip, H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+         "    ok.\n">>,
+          default,
+          ok}],
+    run(Config, Ts),
+
+    ok.
+
 unwritable(Fname) ->
     {ok, Info} = file:read_file_info(Fname),
     Mode = Info#file_info.mode - 8#00200,
diff --git a/lib/parsetools/test/yecc_SUITE.erl b/lib/parsetools/test/yecc_SUITE.erl
index 3d26adf1be..c306dbe833 100644
--- a/lib/parsetools/test/yecc_SUITE.erl
+++ b/lib/parsetools/test/yecc_SUITE.erl
@@ -1,7 +1,8 @@
+%% -*- coding: latin-1 -*-
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -48,7 +49,7 @@
 	 
 	 otp_5369/1, otp_6362/1, otp_7945/1, otp_8483/1, otp_8486/1,
 	 
-	 otp_7292/1, otp_7969/1, otp_8919/1]).
+	 otp_7292/1, otp_7969/1, otp_8919/1, otp_10302/1]).
 
 % Default timetrap timeout (set in init_per_testcase).
 -define(default_timeout, ?t:minutes(1)).
@@ -75,7 +76,7 @@ groups() ->
       [empty, prec, yeccpre, lalr, old_yecc, other_examples]},
      {bugs, [],
       [otp_5369, otp_6362, otp_7945, otp_8483, otp_8486]},
-     {improvements, [], [otp_7292, otp_7969, otp_8919]}].
+     {improvements, [], [otp_7292, otp_7969, otp_8919, otp_10302]}].
 
 init_per_suite(Config) ->
     Config.
@@ -1815,6 +1816,153 @@ otp_8919(Config) when is_list(Config) ->
     "syntax error before: \"hello\"" = lists:flatten(Mod:format_error(Mess)),
     ok.
 
+otp_10302(doc) ->
+    "OTP-10302. Unicode characters scanner/parser.";
+otp_10302(suite) -> [];
+otp_10302(Config) when is_list(Config) ->
+    Dir = ?privdir,
+    Filename = filename:join(Dir, "OTP-10302.yrl"),
+    Ret = [return, {report, true}],
+    Mini1 = <<"%% coding: utf-8
+               Nonterminals H�pp.
+               nt -> t.">>,
+    ok = file:write_file(Filename, Mini1),
+    %% This could (and should) be refined:
+    {error,[{Filename,[{2,Mod1,Err1}]}],[]} =
+        yecc:file(Filename, Ret),
+    "cannot translate from UTF-8" = Mod1:format_error(Err1),
+
+    Mini2 = <<"%% coding: Utf-8
+               Nonterminals Hopp.
+               Terminals t.
+               Rootsymbol Hopp.
+
+               Hopp -> t.
+
+               Erlang code.
+
+               t() ->
+                   H�pp.">>,
+    ok = file:write_file(Filename, Mini2),
+    {error,[{Filename,[{11,Mod2,Err2}]}],[]} =
+        yecc:file(Filename, Ret),
+    "cannot parse; possibly encoding mismatch" = Mod2:format_error(Err2),
+
+    Mini3 = <<"%% coding: latin-1
+               Nonterminals Hopp.
+               Terminals t.
+               Rootsymbol Hopp.
+
+               Hopp -> t.
+
+               Erlang code.
+
+               t() ->
+                   H�pp.">>,
+    ok = file:write_file(Filename, Mini3),
+    YeccPre = filename:join(Dir, "yeccpre.hrl"),
+    ok = file:write_file(YeccPre, [<<"%% coding: UTF-8\n �.\n">>]),
+    Inc = [{includefile,YeccPre}],
+    {error,[{_,[{2,yecc,cannot_parse}]}],[]} =
+        yecc:file(Filename, Inc ++ Ret),
+
+    ok = file:write_file(Filename,
+     <<"%% coding: UTF-8
+        Nonterminals Hopp.
+        Terminals t.
+        Rootsymbol \"örn_Ѐ\".
+        Hopp -> t : '$1'.">>),
+    {error,[{Filename,[{4,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+        yecc:file(Filename, Ret),
+
+    ok = file:write_file(Filename,
+     <<"%% coding: UTF-8
+        Nonterminals Hopp.
+        Terminals t.
+        Rootsymbol Hopp.
+        Endsymbol \"örn_Ѐ\".
+        Hopp -> t : '$1'.">>),
+    {error,[{Filename,[{5,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+        yecc:file(Filename, Ret),
+
+    ok = file:write_file(Filename,
+     <<"%% coding: UTF-8
+        Nonterminals Hopp.
+        Terminals t.
+        Rootsymbol Hopp.
+        Expect \"örn_Ѐ\".
+        Hopp -> t : '$1'.">>),
+    {error,[{Filename,[{5,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+        yecc:file(Filename, Ret),
+
+    ok = file:write_file(Filename,
+     <<"%% coding: UTF-8
+        Nonterminals Hopp.
+        Terminals t.
+        Rootsymbol Hopp.
+        States \"örn_Ѐ\".
+        Hopp -> t : '$1'.">>),
+    {error,[{Filename,[{5,yecc,{bad_symbol,"�rn_"++[1024]}}]}],[]} =
+        yecc:file(Filename, Ret),
+
+    Ts = [{otp_10302_1,<<"
+           %% coding: UTF-8
+           Header \"%% örn_Ѐ\" \"%% \\x{400}B\".
+           Nonterminals Häpp list.
+           Terminals element.
+           Rootsymbol Häpp.
+
+           Häpp -> list : '$1'.
+
+           list -> element : '$1'.
+           list -> list element :
+                       begin
+                           Häpp = foo,
+                           {Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}
+                       end.
+
+           Erlang code.
+
+           -export([t/0]).
+
+           t() ->
+               L = [{element, 1}, {element,2}],
+               {ok, R} = parse(L),
+               Häpp = foo,
+               {_,_,[1024,66],[246,114,110,95,1024]} = R,
+               {Häpp,'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,
+               ok.
+          ">>,default,ok},
+          {otp_10302_2,<<"
+           %% coding: Latin-1
+           Nonterminals H�pp list.
+           Terminals element.
+           Rootsymbol H�pp.
+
+           H�pp -> list : '$1'.
+
+           list -> element : '$1'.
+           list -> list element :
+                       begin
+                           H�pp = foo,
+                           {H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"}
+                       end.
+
+           Erlang code.
+
+           -export([t/0]).
+
+           t() ->
+               L = [{element, 1}, {element,2}],
+               {ok, R} = parse(L),
+               H�pp = foo,
+               {_,_,[1024,66],[195,182,114,110,95,208,128]} = R,
+               {H�pp,'H�pp',\"\\x{400}B\",\"örn_Ѐ\"} = R,
+               ok.
+          ">>,default,ok}],
+    run(Config, Ts),
+    ok.
+
 yeccpre_size() ->
     yeccpre_size(default_yeccpre()).