Merge branch 'hb/unicode/OTP-10302'

OTP-10742
OTP-10745
OTP-10749

* hb/unicode/OTP-10302:
  [stdlib] Remove documentation of ~tp
  Remove one use of iolist_size/1 in io_lib_pretty.erl
  Add a new function proc_lib:format/2 which takes encoding
  Export the type erl_scan:token()
  Make adjustments for Unicode
  [stdlib] Change default of erl_scan's unicode option
  Update preloaded
  Correct recently introduced Unicode related type errors
  [stdlib] Introduce new functions epp:read_encoding_from_binary/1,2
  Extend char() to Unicode characters
  [kernel] Correct bugs in the old shell (user.erl)
  [parsetools] Change the encoding of test suites to UTF-8
  [stdlib] Fix a contract bug
  [stdlib] Update the Unicode examples in STDLIB User's Guide
  [stdlib] Remove documentation of Unicode functions in io_lib

1 files changed, 18 insertions, 18 deletions

diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl
index 9d591c0d05..afedd79a4e 100644
--- a/lib/parsetools/test/leex_SUITE.erl
+++ b/lib/parsetools/test/leex_SUITE.erl
@@ -1,4 +1,4 @@
-%% -*- coding: latin-1 -*-
+%% -*- coding: utf-8 -*-
 %%
 %% %CopyrightBegin%
 %% 
@@ -888,7 +888,7 @@ otp_10302(Config) when is_list(Config) ->
 
     ok = file:write_file(Filename,<<
          "%% coding: UTF-8\n"
-         "�"
+         "ä"
      >>),
     {error,[{_,[{2,leex,cannot_parse}]}],[]} =
         leex:file(Filename, Ret),
@@ -896,7 +896,7 @@ otp_10302(Config) when is_list(Config) ->
     ok = file:write_file(Filename,<<
          "%% coding: UTF-8\n"
          "Definitions.\n"
-         "�"
+         "ä"
      >>),
     {error,[{_,[{3,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
 
@@ -907,7 +907,7 @@ otp_10302(Config) when is_list(Config) ->
          "L = [{A}-{Z}]\n"
          "Z = z\n"
          "Rules.\n"
-         "{L}+ : {token,{list_to_atom(TokenChars),H�pp}}.\n"
+         "{L}+ : {token,{list_to_atom(TokenChars),Häpp}}.\n"
      >>),
     {error,[{_,[{7,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
 
@@ -922,7 +922,7 @@ otp_10302(Config) when is_list(Config) ->
          "Erlang code.\n"
          "-export([t/0]).\n"
          "t() ->\n"
-         "    H�pp\n"
+         "    Häpp\n"
       >>),
     {error,[{_,[{11,leex,cannot_parse}]}],[]} = leex:file(Filename, Ret),
 
@@ -932,7 +932,7 @@ otp_10302(Config) when is_list(Config) ->
              "{L}+  : {token,{word,TokenLine,TokenChars}}.\n"
              "Erlang code.\n">>,
     LeexPre = filename:join(Dir, "leexinc.hrl"),
-    ?line ok = file:write_file(LeexPre, <<"%% coding: UTF-8\n �">>),
+    ?line ok = file:write_file(LeexPre, <<"%% coding: UTF-8\n ä">>),
     PreErrors = run_test(Config, Mini, LeexPre),
     {error,[{IncludeFile,[{2,leex,cannot_parse}]}],[]} = PreErrors,
     "leexinc.hrl" = filename:basename(IncludeFile),
@@ -945,16 +945,16 @@ otp_10302(Config) when is_list(Config) ->
          "Z = z\n"
          "Rules.\n"
          "{L}+ : {token,{list_to_atom(TokenChars),\n"
-         "begin Häpp = foo, Häpp end,"
-         " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+         "begin Häpp = foo, Häpp end,"
+         " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
          "Erlang code.\n"
          "-export([t/0]).\n"
          "t() ->\n"
-         "    %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+         "    %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n"
          "    {ok, [R], 1} = string(\"tip\"),\n"
-         "    {tip,foo,'Häpp',[1024,66],[246,114,110,95,1024]} = R,\n"
-         "    Häpp = foo,\n"
-         "    {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+         "    {tip,foo,'Häpp',[1024,66],[246,114,110,95,1024]} = R,\n"
+         "    Häpp = foo,\n"
+         "    {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
          "    ok.\n">>,
           default,
           ok},
@@ -966,16 +966,16 @@ otp_10302(Config) when is_list(Config) ->
          "Z = z\n"
          "Rules.\n"
          "{L}+ : {token,{list_to_atom(TokenChars),\n"
-         "begin H�pp = foo, H�pp end,"
-         " 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
+         "begin Häpp = foo, Häpp end,"
+         " 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"}}.\n"
          "Erlang code.\n"
          "-export([t/0]).\n"
          "t() ->\n"
-         "    %% H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"\n"
+         "    %% Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"\n"
          "    {ok, [R], 1} = string(\"tip\"),\n"
-         "    {tip,foo,'H�pp',[1024,66],[195,182,114,110,95,208,128]} = R,\n"
-         "    H�pp = foo,\n"
-         "    {tip, H�pp, 'H�pp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
+         "    {tip,foo,'Häpp',[1024,66],[195,182,114,110,95,208,128]} = R,\n"
+         "    Häpp = foo,\n"
+         "    {tip, Häpp, 'Häpp',\"\\x{400}B\",\"örn_Ѐ\"} = R,\n"
          "    ok.\n">>,
           default,
           ok}],