diff options
author | Hans Bolinder <[email protected]> | 2013-01-25 13:04:01 +0100 |
---|---|---|
committer | Hans Bolinder <[email protected]> | 2013-01-25 13:04:10 +0100 |
commit | 4adc1523254d5e326e2d5cf942b5eeee84fdb8ea (patch) | |
tree | 527d936f6295f3d8f86ceecf44ff2310ab5508b2 /lib/stdlib/src/erl_scan.erl | |
parent | 9afbb879f0397a650a7c403911a8cc30daa6dbbe (diff) | |
parent | 5b68f914f08479759d36557282aee29f44683d97 (diff) | |
download | otp-4adc1523254d5e326e2d5cf942b5eeee84fdb8ea.tar.gz otp-4adc1523254d5e326e2d5cf942b5eeee84fdb8ea.tar.bz2 otp-4adc1523254d5e326e2d5cf942b5eeee84fdb8ea.zip |
Merge branch 'hb/unicode/OTP-10302'
OTP-10742
OTP-10745
OTP-10749
* hb/unicode/OTP-10302:
[stdlib] Remove documentation of ~tp
Remove one use of iolist_size/1 in io_lib_pretty.erl
Add a new function proc_lib:format/2 which takes encoding
Export the type erl_scan:token()
Make adjustments for Unicode
[stdlib] Change default of erl_scan's unicode option
Update preloaded
Correct recently introduced Unicode related type errors
[stdlib] Introduce new functions epp:read_encoding_from_binary/1,2
Extend char() to Unicode characters
[kernel] Correct bugs in the old shell (user.erl)
[parsetools] Change the encoding of test suites to UTF-8
[stdlib] Fix a contract bug
[stdlib] Update the Unicode examples in STDLIB User's Guide
[stdlib] Remove documentation of Unicode functions in io_lib
Diffstat (limited to 'lib/stdlib/src/erl_scan.erl')
-rw-r--r-- | lib/stdlib/src/erl_scan.erl | 20 |
1 files changed, 12 insertions, 8 deletions
diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl index bc0eaf015d..26d5747ee7 100644 --- a/lib/stdlib/src/erl_scan.erl +++ b/lib/stdlib/src/erl_scan.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -64,6 +64,7 @@ location/0, options/0, return_cont/0, + token/0, tokens_result/0]). %%% @@ -106,7 +107,7 @@ ws = false :: boolean(), comment = false :: boolean(), text = false :: boolean(), - unicode = false :: boolean()}). + unicode = true :: boolean()}). %%---------------------------------------------------------------------------- @@ -115,7 +116,7 @@ format_error({string,Quote,Head}) -> lists:flatten(["unterminated " ++ string_thing(Quote) ++ " starting with " ++ - io_lib:write_unicode_string(Head, Quote)]); + io_lib:write_string(Head, Quote)]); format_error({illegal,Type}) -> lists:flatten(io_lib:fwrite("illegal ~w", [Type])); format_error(char) -> "unterminated character"; @@ -349,14 +350,14 @@ string_thing(_) -> "string". %% erl_scan:string("[98,2730,99]."). This is to protect the caller %% from character codes greater than 255. Search for UNI to find code %% implementing this "feature". The 'unicode' option is undocumented -%% and will probably be removed later. +%% and will be removed later. -define(NO_UNICODE, 0). -define(UNI255(C), (C =< 16#ff)). options(Opts0) when is_list(Opts0) -> Opts = lists:foldr(fun expand_opt/2, [], Opts0), - [RW_fun] = - case opts(Opts, [reserved_word_fun], []) of + [RW_fun, Unicode] = + case opts(Opts, [reserved_word_fun, unicode], []) of badarg -> erlang:error(badarg, [Opts0]); R -> @@ -365,7 +366,6 @@ options(Opts0) when is_list(Opts0) -> Comment = proplists:get_bool(return_comments, Opts), WS = proplists:get_bool(return_white_spaces, Opts), Txt = proplists:get_bool(text, Opts), - Unicode = proplists:get_bool(unicode, Opts), #erl_scan{resword_fun = RW_fun, comment = Comment, ws = WS, @@ -378,6 +378,8 @@ opts(Options, [Key|Keys], L) -> V = case lists:keyfind(Key, 1, Options) of {reserved_word_fun,F} when ?RESWORDFUN(F) -> {ok,F}; + {unicode, Bool} when is_boolean(Bool) -> + {ok,Bool}; {Key,_} -> badarg; false -> @@ -393,7 +395,9 @@ opts(_Options, [], L) -> lists:reverse(L). default_option(reserved_word_fun) -> - fun reserved_word/1. + fun reserved_word/1; +default_option(unicode) -> + true. expand_opt(return, Os) -> [return_comments,return_white_spaces|Os]; |