[stdlib, kernel] Introduce Unicode support for Erlang source files

Expect modifications, additions and corrections.

There is a kludge in file_io_server and
erl_scan:continuation_location() that's not so pleasing.

3 files changed, 41 insertions, 0 deletions

diff --git a/lib/stdlib/test/escript_SUITE_data/unicode1 b/lib/stdlib/test/escript_SUITE_data/unicode1
new file mode 100755
index 0000000000..a77574625e
--- /dev/null
+++ b/lib/stdlib/test/escript_SUITE_data/unicode1
@@ -0,0 +1,14 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+-export([main/1]).
+
+main(_) ->
+    ok = io:setopts([{encoding,unicode}]),
+    _D = erlang:system_flag(backtrace_depth, 0),
+    A = <<"\x{aa}">>,
+    S = lists:flatten(io_lib:format("~p/~p.", [A, A])),
+    {ok, Ts, _} = erl_scan:string(S, 1, [unicode]),
+    {ok, Es} = erl_parse:parse_exprs(Ts),
+    B = erl_eval:new_bindings(),
+    erl_eval:exprs(Es, B).
diff --git a/lib/stdlib/test/escript_SUITE_data/unicode2 b/lib/stdlib/test/escript_SUITE_data/unicode2
new file mode 100755
index 0000000000..495188f6f0
--- /dev/null
+++ b/lib/stdlib/test/escript_SUITE_data/unicode2
@@ -0,0 +1,14 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+-export([main/1]).
+
+main(_) ->
+    ok = io:setopts([{encoding,latin1}]),
+    _D = erlang:system_flag(backtrace_depth, 0),
+    A = <<"\x{aa}">>,
+    S = lists:flatten(io_lib:format("~p/~p.", [A, A])),
+    {ok, Ts, _} = erl_scan:string(S, 1, [unicode]),
+    {ok, Es} = erl_parse:parse_exprs(Ts),
+    B = erl_eval:new_bindings(),
+    erl_eval:exprs(Es, B).
diff --git a/lib/stdlib/test/escript_SUITE_data/unicode3 b/lib/stdlib/test/escript_SUITE_data/unicode3
new file mode 100755
index 0000000000..944487dcae
--- /dev/null
+++ b/lib/stdlib/test/escript_SUITE_data/unicode3
@@ -0,0 +1,13 @@
+#!/usr/bin/env escript
+%% -*- erlang; coding: utf-8 -*-
+
+-export([main/1]).
+
+main(_) ->
+    ok = io:setopts([{encoding,unicode}]),
+    Bin1 = <<"örn_Ѐ שלום-שלום+של 日本語">>,
+
+    L = [246,114,110,95,1024,32,1513,1500,1493,1501,45,1513,1500,1493,
+         1501,43,1513,1500,32,26085,26412,35486],
+    L = unicode:characters_to_list(Bin1, utf8),
+    ok.