aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/asn1/src/asn1ct_check.erl6
-rw-r--r--lib/asn1/src/asn1ct_gen.erl4
-rw-r--r--lib/asn1/src/asn1rtt_real_common.erl4
-rw-r--r--lib/common_test/src/ct.erl2
-rw-r--r--lib/common_test/src/ct_config.erl4
-rw-r--r--lib/common_test/src/ct_config_plain.erl2
-rw-r--r--lib/common_test/src/ct_conn_log_h.erl4
-rw-r--r--lib/common_test/src/ct_framework.erl7
-rw-r--r--lib/common_test/src/ct_ftp.erl2
-rw-r--r--lib/common_test/src/ct_logs.erl83
-rw-r--r--lib/common_test/src/ct_master.erl2
-rw-r--r--lib/common_test/src/ct_master_logs.erl18
-rw-r--r--lib/common_test/src/ct_netconfc.erl2
-rw-r--r--lib/common_test/src/ct_release_test.erl2
-rw-r--r--lib/common_test/src/ct_run.erl8
-rw-r--r--lib/common_test/src/ct_slave.erl55
-rw-r--r--lib/common_test/src/ct_ssh.erl3
-rw-r--r--lib/common_test/src/ct_util.erl18
-rw-r--r--lib/common_test/src/cth_log_redirect.erl4
-rw-r--r--lib/common_test/src/cth_surefire.erl11
-rw-r--r--lib/common_test/src/test_server.erl23
-rw-r--r--lib/common_test/src/test_server_ctrl.erl29
-rw-r--r--lib/common_test/src/test_server_node.erl10
-rw-r--r--lib/common_test/src/unix_telnet.erl3
-rw-r--r--lib/common_test/test/ct_config_SUITE.erl4
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_log_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_pre_post_test_io_SUITE.erl4
-rw-r--r--lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl4
-rw-r--r--lib/common_test/test/ct_test_support.erl12
-rw-r--r--lib/common_test/test/ct_unicode_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_userconfig_callback.erl2
-rw-r--r--lib/common_test/test/erl2html2_SUITE.erl4
-rw-r--r--lib/common_test/test/telnet_server.erl20
-rw-r--r--lib/common_test/test/test_server_SUITE.erl6
-rw-r--r--lib/common_test/test/test_server_test_lib.erl2
-rw-r--r--lib/common_test/test_server/ts.erl2
-rw-r--r--lib/common_test/test_server/ts_autoconf_win32.erl12
-rw-r--r--lib/common_test/test_server/ts_erl_config.erl12
-rw-r--r--lib/common_test/test_server/ts_install.erl34
-rw-r--r--lib/common_test/test_server/ts_lib.erl8
-rw-r--r--lib/common_test/test_server/ts_run.erl8
-rw-r--r--lib/compiler/doc/src/compile.xml11
-rw-r--r--lib/compiler/internal_doc/cerl-notes.md75
-rw-r--r--lib/compiler/src/Makefile13
-rw-r--r--lib/compiler/src/beam_asm.erl38
-rw-r--r--lib/compiler/src/beam_clean.erl58
-rw-r--r--lib/compiler/src/beam_jump.erl223
-rw-r--r--lib/compiler/src/beam_listing.erl67
-rw-r--r--lib/compiler/src/beam_peep.erl28
-rw-r--r--lib/compiler/src/beam_receive.erl2
-rw-r--r--lib/compiler/src/beam_utils.erl93
-rw-r--r--lib/compiler/src/cerl_clauses.erl2
-rw-r--r--lib/compiler/src/compile.erl34
-rw-r--r--lib/compiler/src/compiler.app.src4
-rw-r--r--lib/compiler/src/core_pp.erl4
-rw-r--r--lib/compiler/src/core_scan.erl4
-rw-r--r--lib/compiler/src/sys_core_alias.erl308
-rw-r--r--lib/compiler/src/sys_core_fold.erl32
-rw-r--r--lib/compiler/src/v3_codegen.erl1105
-rw-r--r--lib/compiler/src/v3_core.erl45
-rw-r--r--lib/compiler/src/v3_kernel.erl168
-rw-r--r--lib/compiler/src/v3_kernel.hrl2
-rw-r--r--lib/compiler/src/v3_kernel_pp.erl4
-rw-r--r--lib/compiler/src/v3_life.erl468
-rw-r--r--lib/compiler/test/Makefile1
-rw-r--r--lib/compiler/test/bs_match_SUITE.erl19
-rw-r--r--lib/compiler/test/compilation_SUITE_data/opt_crash.erl6
-rw-r--r--lib/compiler/test/compile_SUITE.erl58
-rw-r--r--lib/compiler/test/core_alias_SUITE.erl195
-rw-r--r--lib/compiler/test/core_fold_SUITE.erl28
-rw-r--r--lib/compiler/test/guard_SUITE.erl4
-rw-r--r--lib/compiler/test/match_SUITE.erl64
-rw-r--r--lib/compiler/test/misc_SUITE.erl11
-rw-r--r--lib/compiler/test/trycatch_SUITE.erl6
-rw-r--r--lib/crypto/doc/src/crypto.xml125
-rw-r--r--lib/crypto/doc/src/crypto_app.xml18
-rw-r--r--lib/crypto/src/crypto.app.src2
-rw-r--r--lib/crypto/src/crypto.erl47
-rw-r--r--lib/debugger/src/i.erl2
-rw-r--r--lib/dialyzer/src/dialyzer_cl.erl2
-rw-r--r--lib/dialyzer/src/dialyzer_gui_wx.erl4
-rw-r--r--lib/dialyzer/src/dialyzer_races.erl66
-rw-r--r--lib/dialyzer/src/dialyzer_utils.erl86
-rw-r--r--lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args2
-rw-r--r--lib/dialyzer/test/dialyzer_common.erl10
-rw-r--r--lib/edoc/src/edoc_layout.erl6
-rw-r--r--lib/edoc/src/edoc_specs.erl2
-rw-r--r--lib/eldap/src/eldap.erl4
-rw-r--r--lib/erl_docgen/priv/dtd/erlref.dtd1
-rw-r--r--lib/erl_docgen/priv/xsl/db_html.xsl6
-rw-r--r--lib/erl_docgen/src/docgen_edoc_xml_cb.erl4
-rw-r--r--lib/erl_docgen/src/docgen_otp_specs.erl6
-rw-r--r--lib/et/src/et_collector.erl2
-rw-r--r--lib/eunit/src/eunit_lib.erl2
-rw-r--r--lib/hipe/cerl/erl_types.erl45
-rw-r--r--lib/hipe/icode/hipe_beam_to_icode.erl6
-rw-r--r--lib/hipe/llvm/hipe_llvm.erl11
-rw-r--r--lib/hipe/llvm/hipe_llvm_main.erl2
-rw-r--r--lib/hipe/llvm/hipe_rtl_to_llvm.erl4
-rw-r--r--lib/hipe/main/hipe.erl6
-rw-r--r--lib/hipe/test/basic_SUITE_data/basic_exceptions.erl118
-rw-r--r--lib/hipe/test/hipe_testsuite_driver.erl10
-rw-r--r--lib/hipe/test/opt_verify_SUITE.erl6
-rw-r--r--lib/kernel/doc/src/file.xml43
-rw-r--r--lib/kernel/doc/src/inet_res.xml2
-rw-r--r--lib/kernel/doc/src/os.xml109
-rw-r--r--lib/kernel/examples/Makefile2
-rw-r--r--lib/kernel/examples/gen_tcp_dist/Makefile20
-rw-r--r--lib/kernel/examples/gen_tcp_dist/ebin/.gitignore0
-rw-r--r--lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl781
-rw-r--r--lib/kernel/include/dist.hrl30
-rw-r--r--lib/kernel/include/dist_util.hrl12
-rw-r--r--lib/kernel/src/dist_util.erl252
-rw-r--r--lib/kernel/src/erl_boot_server.erl6
-rw-r--r--lib/kernel/src/erl_epmd.erl8
-rw-r--r--lib/kernel/src/erl_reply.erl4
-rw-r--r--lib/kernel/src/erts_debug.erl19
-rw-r--r--lib/kernel/src/group.erl91
-rw-r--r--lib/kernel/src/inet_config.erl4
-rw-r--r--lib/kernel/src/inet_dns.erl6
-rw-r--r--lib/kernel/src/inet_parse.erl4
-rw-r--r--lib/kernel/src/inet_res.erl12
-rw-r--r--lib/kernel/src/kernel.app.src2
-rw-r--r--lib/kernel/src/kernel.appup.src4
-rw-r--r--lib/kernel/src/net_kernel.erl4
-rw-r--r--lib/kernel/src/os.erl65
-rw-r--r--lib/kernel/test/file_name_SUITE.erl2
-rw-r--r--lib/kernel/test/gen_tcp_misc_SUITE.erl62
-rw-r--r--lib/kernel/test/gen_udp_SUITE.erl120
-rw-r--r--lib/kernel/test/os_SUITE.erl14
-rw-r--r--lib/kernel/test/zlib_SUITE.erl47
-rw-r--r--lib/mnesia/doc/src/Mnesia_chap5.xmlsrc6
-rw-r--r--lib/mnesia/test/mnesia_test_lib.erl2
-rw-r--r--lib/observer/src/cdv_dist_cb.erl2
-rw-r--r--lib/observer/src/cdv_mem_cb.erl4
-rw-r--r--lib/observer/src/cdv_wx.erl6
-rw-r--r--lib/observer/src/crashdump_viewer.erl22
-rw-r--r--lib/observer/src/observer_html_lib.erl10
-rw-r--r--lib/observer/src/observer_lib.erl2
-rw-r--r--lib/observer/src/observer_procinfo.erl3
-rw-r--r--lib/observer/src/observer_traceoptions_wx.erl2
-rw-r--r--lib/observer/src/observer_wx.erl2
-rw-r--r--lib/observer/src/ttb.erl6
-rw-r--r--lib/observer/test/ttb_SUITE.erl6
-rw-r--r--lib/os_mon/src/cpu_sup.erl2
-rw-r--r--lib/os_mon/src/disksup.erl98
-rw-r--r--lib/os_mon/src/memsup.erl2
-rw-r--r--lib/os_mon/test/disksup_SUITE.erl38
-rw-r--r--lib/parsetools/src/leex.erl34
-rw-r--r--lib/parsetools/src/yecc.erl12
-rw-r--r--lib/reltool/src/reltool.hrl4
-rw-r--r--lib/reltool/src/reltool_mod_win.erl4
-rw-r--r--lib/reltool/src/reltool_target.erl10
-rw-r--r--lib/reltool/src/reltool_utils.erl9
-rw-r--r--lib/reltool/test/reltool_server_SUITE.erl23
-rw-r--r--lib/runtime_tools/src/erts_alloc_config.erl28
-rw-r--r--lib/runtime_tools/src/msacc.erl3
-rw-r--r--lib/runtime_tools/src/system_information.erl4
-rw-r--r--lib/runtime_tools/test/dbg_SUITE.erl97
-rw-r--r--lib/runtime_tools/test/dyntrace_SUITE.erl6
-rw-r--r--lib/sasl/src/Makefile2
-rw-r--r--lib/sasl/src/erlsrv.erl25
-rw-r--r--lib/sasl/src/format_lib_supp.erl6
-rw-r--r--lib/sasl/src/rb.erl6
-rw-r--r--lib/sasl/src/rb_format_supp.erl2
-rw-r--r--lib/sasl/src/release_handler.erl18
-rw-r--r--lib/sasl/src/sasl.app.src2
-rw-r--r--lib/sasl/src/sasl.appup.src8
-rw-r--r--lib/sasl/src/si.erl169
-rw-r--r--lib/sasl/src/si_sasl_supp.erl380
-rw-r--r--lib/sasl/test/release_handler_SUITE.erl40
-rw-r--r--lib/sasl/test/test_lib.hrl4
-rw-r--r--lib/ssl/src/Makefile5
-rw-r--r--lib/ssl/src/dtls_connection.erl2
-rw-r--r--lib/ssl/src/dtls_udp_listener.erl22
-rw-r--r--lib/ssl/src/inet6_tls_dist.erl7
-rw-r--r--lib/ssl/src/inet_tls_dist.erl664
-rw-r--r--lib/ssl/src/ssl.app.src7
-rw-r--r--lib/ssl/src/ssl_cipher.erl91
-rw-r--r--lib/ssl/src/ssl_cipher.hrl51
-rw-r--r--lib/ssl/src/ssl_connection.erl321
-rw-r--r--lib/ssl/src/ssl_dist_sup.erl14
-rw-r--r--lib/ssl/src/ssl_handshake.erl85
-rw-r--r--lib/ssl/src/ssl_handshake.hrl11
-rw-r--r--lib/ssl/src/ssl_tls_dist_proxy.erl479
-rw-r--r--lib/ssl/src/tls_connection.erl11
-rw-r--r--lib/ssl/test/ssl_test_lib.erl4
-rw-r--r--lib/stdlib/doc/src/Makefile1
-rw-r--r--lib/stdlib/doc/src/c.xml9
-rw-r--r--lib/stdlib/doc/src/filelib.xml29
-rw-r--r--lib/stdlib/doc/src/filename.xml30
-rw-r--r--lib/stdlib/doc/src/gen_server.xml52
-rw-r--r--lib/stdlib/doc/src/gen_statem.xml12
-rw-r--r--lib/stdlib/doc/src/rand.xml118
-rw-r--r--lib/stdlib/doc/src/ref_man.xml1
-rw-r--r--lib/stdlib/doc/src/specs.xml1
-rw-r--r--lib/stdlib/doc/src/string.xml396
-rw-r--r--lib/stdlib/doc/src/unicode_usage.xml8
-rw-r--r--lib/stdlib/doc/src/uri_string.xml230
-rw-r--r--lib/stdlib/src/Makefile1
-rw-r--r--lib/stdlib/src/base64.erl108
-rw-r--r--lib/stdlib/src/c.erl18
-rw-r--r--lib/stdlib/src/epp.erl5
-rw-r--r--lib/stdlib/src/erl_lint.erl7
-rw-r--r--lib/stdlib/src/erl_pp.erl40
-rw-r--r--lib/stdlib/src/erl_scan.erl6
-rw-r--r--lib/stdlib/src/escript.erl12
-rw-r--r--lib/stdlib/src/ets.erl4
-rw-r--r--lib/stdlib/src/filelib.erl42
-rw-r--r--lib/stdlib/src/filename.erl106
-rw-r--r--lib/stdlib/src/gen_server.erl31
-rw-r--r--lib/stdlib/src/gen_statem.erl4
-rw-r--r--lib/stdlib/src/io_lib_format.erl10
-rw-r--r--lib/stdlib/src/lib.erl2
-rw-r--r--lib/stdlib/src/otp_internal.erl46
-rw-r--r--lib/stdlib/src/pool.erl6
-rw-r--r--lib/stdlib/src/rand.erl261
-rw-r--r--lib/stdlib/src/slave.erl2
-rw-r--r--lib/stdlib/src/stdlib.app.src3
-rw-r--r--lib/stdlib/src/stdlib.appup.src6
-rw-r--r--lib/stdlib/src/string.erl10
-rw-r--r--lib/stdlib/src/uri_string.erl1842
-rw-r--r--lib/stdlib/test/Makefile4
-rw-r--r--lib/stdlib/test/erl_internal_SUITE.erl4
-rw-r--r--lib/stdlib/test/ets_SUITE.erl34
-rw-r--r--lib/stdlib/test/filelib_SUITE.erl33
-rw-r--r--lib/stdlib/test/gen_server_SUITE.erl111
-rw-r--r--lib/stdlib/test/gen_server_SUITE_data/oc_server.erl12
-rw-r--r--lib/stdlib/test/property_test/README12
-rw-r--r--lib/stdlib/test/property_test/uri_string_recompose.erl361
-rw-r--r--lib/stdlib/test/rand_SUITE.erl568
-rw-r--r--lib/stdlib/test/stdlib.spec1
-rw-r--r--lib/stdlib/test/stdlib_bench.spec7
-rw-r--r--lib/stdlib/test/stdlib_bench_SUITE.erl107
-rw-r--r--lib/stdlib/test/string_SUITE.erl14
-rw-r--r--lib/stdlib/test/unicode_util_SUITE.erl16
-rw-r--r--lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt175
-rw-r--r--lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt23
-rw-r--r--lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt30
-rw-r--r--lib/stdlib/test/uri_string_SUITE.erl844
-rw-r--r--lib/stdlib/test/uri_string_property_test_SUITE.erl (renamed from lib/compiler/src/v3_life.hrl)32
-rw-r--r--lib/stdlib/uc_spec/CaseFolding.txt8
-rw-r--r--lib/stdlib/uc_spec/CompositionExclusions.txt6
-rw-r--r--lib/stdlib/uc_spec/GraphemeBreakProperty.txt78
-rw-r--r--lib/stdlib/uc_spec/PropList.txt83
-rw-r--r--lib/stdlib/uc_spec/SpecialCasing.txt8
-rw-r--r--lib/stdlib/uc_spec/UnicodeData.txt1028
-rwxr-xr-xlib/stdlib/uc_spec/gen_unicode_mod.escript26
-rw-r--r--lib/syntax_tools/src/Makefile2
-rw-r--r--lib/syntax_tools/src/erl_comment_scan.erl2
-rw-r--r--lib/syntax_tools/src/merl_transform.erl2
-rw-r--r--lib/tools/emacs/erlang.el5
-rw-r--r--lib/tools/src/cover.erl14
-rw-r--r--lib/tools/src/xref_utils.erl9
-rw-r--r--lib/tools/test/xref_SUITE.erl22
-rw-r--r--lib/wx/examples/demo/demo.erl2
257 files changed, 11658 insertions, 4323 deletions
diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl
index 83d12600b7..321980e5e4 100644
--- a/lib/asn1/src/asn1ct_check.erl
+++ b/lib/asn1/src/asn1ct_check.erl
@@ -1576,13 +1576,15 @@ printable_string_1(#'Externalvaluereference'{value=Type}) ->
printable_string_1({Atom,Line}) when is_atom(Atom), is_integer(Line) ->
q(Atom);
printable_string_1({object,definedsyntax,L}) ->
- q(string:join([printable_string_1(Item) || Item <- L], " "));
+ Str = lists:join($\s, [printable_string_1(Item) || Item <- L]),
+ q(lists:flatten(Str));
printable_string_1([_|_]=Def) ->
case lists:all(fun is_integer/1, Def) of
true ->
lists:flatten(io_lib:format("~p", [Def]));
false ->
- q(string:join([printable_string_1(Item) || Item <- Def], " "))
+ Str = lists:join($\s, [printable_string_1(Item) || Item <- Def]),
+ q(lists:flatten(Str))
end;
printable_string_1(Def) ->
lists:flatten(io_lib:format("~p", [Def])).
diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl
index 806f8420ec..84c7f39d55 100644
--- a/lib/asn1/src/asn1ct_gen.erl
+++ b/lib/asn1/src/asn1ct_gen.erl
@@ -796,7 +796,7 @@ result_line(NoOkWrapper, Items) ->
result_line_1([SingleItem]) ->
SingleItem;
result_line_1(Items) ->
- ["{",string:join(Items, ","),"}"].
+ ["{",lists:join(",",Items),"}"].
try_catch() ->
[" catch",nl,
@@ -943,7 +943,7 @@ open_hrl(OutFile,Module) ->
hrl_protector(OutFile) ->
BaseName = filename:basename(OutFile),
- P = "_" ++ string:to_upper(BaseName) ++ "_HRL_",
+ P = "_" ++ string:uppercase(BaseName) ++ "_HRL_",
[if
$A =< C, C =< $Z -> C;
$a =< C, C =< $a -> C;
diff --git a/lib/asn1/src/asn1rtt_real_common.erl b/lib/asn1/src/asn1rtt_real_common.erl
index 3a79209015..81c1f54d74 100644
--- a/lib/asn1/src/asn1rtt_real_common.erl
+++ b/lib/asn1/src/asn1rtt_real_common.erl
@@ -125,7 +125,7 @@ encode_real(_C, {_,Base,_}) ->
encode_real(C, Real) when is_list(Real) ->
%% The Real string may come in as a NR1, NR2 or NR3 string.
{Mantissa, Exponent} =
- case string:tokens(Real,"Ee") of
+ case string:lexemes(Real,"Ee") of
[NR2] ->
{NR2,0};
[NR3MB,NR3E] ->
@@ -144,7 +144,7 @@ encode_real(C, Real) when is_list(Real) ->
NewMan = remove_trailing_zeros(Dec),
{NewMan,length(ZeroDecimal(NewMan))};
_ ->
- case string:tokens(Mantissa,",.") of
+ case string:lexemes(Mantissa,",.") of
[Num] -> %% No decimal-mark
{integer_to_list(list_to_integer(Num)),0};
[Num,Dec] ->
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index a12c0c9101..875301a8b2 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -1030,7 +1030,7 @@ make_and_load(Dir, Suite) ->
EnvInclude =
case os:getenv("CT_INCLUDE_PATH") of
false -> [];
- CtInclPath -> string:tokens(CtInclPath, [$:,$ ,$,])
+ CtInclPath -> string:lexemes(CtInclPath, [$:,$ ,$,])
end,
StartInclude =
case init:get_argument(include) of
diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl
index d48ae830bb..b3f983dd46 100644
--- a/lib/common_test/src/ct_config.erl
+++ b/lib/common_test/src/ct_config.erl
@@ -659,7 +659,7 @@ decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) ->
get_crypt_key_from_file(File) ->
case file:read_file(File) of
{ok,Bin} ->
- case catch string:tokens(binary_to_list(Bin), [$\n,$\r]) of
+ case catch string:lexemes(binary_to_list(Bin), [$\n,$\r]) of
[Key] ->
Key;
_ ->
@@ -693,7 +693,7 @@ get_crypt_key_from_file() ->
noent ->
Result;
_ ->
- case catch string:tokens(binary_to_list(Result), [$\n,$\r]) of
+ case catch string:lexemes(binary_to_list(Result), [$\n,$\r]) of
[Key] ->
io:format("~nCrypt key file: ~ts~n", [FullName]),
Key;
diff --git a/lib/common_test/src/ct_config_plain.erl b/lib/common_test/src/ct_config_plain.erl
index e72b55971b..e77381d7cf 100644
--- a/lib/common_test/src/ct_config_plain.erl
+++ b/lib/common_test/src/ct_config_plain.erl
@@ -106,7 +106,7 @@ read_config_terms1({done,{eof,EL},_}, L, _, _) ->
read_config_terms1({done,{error,Info,EL},_}, L, _, _) ->
{error,{Info,{L,EL}}};
read_config_terms1({more,_}, L, Terms, Rest) ->
- case string:tokens(Rest, [$\n,$\r,$\t]) of
+ case string:lexemes(Rest, [$\n,$\r,$\t]) of
[] ->
lists:reverse(Terms);
_ ->
diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl
index cf0a228e1b..3e83020f45 100644
--- a/lib/common_test/src/ct_conn_log_h.erl
+++ b/lib/common_test/src/ct_conn_log_h.erl
@@ -224,7 +224,7 @@ now_to_time({_,_,MicroS}=Now) ->
{calendar:now_to_local_time(Now),MicroS}.
pretty_head({{{Y,Mo,D},{H,Mi,S}},MicroS},ConnMod,Text0) ->
- Text = string:to_upper(atom_to_list(ConnMod) ++ Text0),
+ Text = string:uppercase(atom_to_list(ConnMod) ++ Text0),
io_lib:format("= ~s ==== ~s-~s-~w::~s:~s:~s,~s ",
[Text,t(D),month(Mo),Y,t(H),t(Mi),t(S),
micro2milli(MicroS)]).
@@ -275,7 +275,7 @@ pad0(N,Str) ->
lists:duplicate(N-M,$0) ++ Str.
pad_char_end(N,Str,Char) ->
- case length(lists:flatten(Str)) of
+ case string:length(Str) of
M when M<N -> Str ++ lists:duplicate(N-M,Char);
_ -> Str
end.
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 6066470233..7f7e9ae6f9 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -921,9 +921,10 @@ error_notification(Mod,Func,_Args,{Error,Loc}) ->
end,
ErrorStr = case ErrorSpec of
{badmatch,Descr} ->
- Descr1 = lists:flatten(io_lib:format("~tP",[Descr,10])),
- if length(Descr1) > 50 ->
- Descr2 = string:substr(Descr1,1,50),
+ Descr1 = io_lib:format("~tP",[Descr,10]),
+ DescrLength = string:length(Descr1),
+ if DescrLength > 50 ->
+ Descr2 = string:slice(Descr1,0,50),
io_lib:format("{badmatch,~ts...}",[Descr2]);
true ->
io_lib:format("{badmatch,~ts}",[Descr1])
diff --git a/lib/common_test/src/ct_ftp.erl b/lib/common_test/src/ct_ftp.erl
index 8effb06e7e..ee4a6a6c45 100644
--- a/lib/common_test/src/ct_ftp.erl
+++ b/lib/common_test/src/ct_ftp.erl
@@ -285,7 +285,7 @@ init(KeyOrName,{IP,Port},{Username,Password}) ->
{ok,FtpPid} ->
log(heading(init,KeyOrName),
"Opened ftp connection:\nIP: ~tp\nUsername: ~tp\nPassword: ~p\n",
- [IP,Username,lists:duplicate(length(Password),$*)]),
+ [IP,Username,lists:duplicate(string:length(Password),$*)]),
{ok,FtpPid,#state{ftp_pid=FtpPid,target_name=KeyOrName}};
Error ->
Error
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index ba7660fe6a..028c265420 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -1188,26 +1188,23 @@ print_style(Fd, IoFormat, StyleSheet) ->
case file:read_file(StyleSheet) of
{ok,Bin} ->
Str = b2s(Bin,encoding(StyleSheet)),
- Pos0 = case string:str(Str,"<style>") of
- 0 -> string:str(Str,"<STYLE>");
- N0 -> N0
- end,
- Pos1 = case string:str(Str,"</style>") of
- 0 -> string:str(Str,"</STYLE>");
- N1 -> N1
- end,
- if (Pos0 == 0) and (Pos1 /= 0) ->
- print_style_error(Fd, IoFormat,
- StyleSheet, missing_style_start_tag);
- (Pos0 /= 0) and (Pos1 == 0) ->
- print_style_error(Fd, IoFormat,
- StyleSheet,missing_style_end_tag);
- Pos0 /= 0 ->
- Style = string:sub_string(Str,Pos0,Pos1+7),
- IoFormat(Fd,"~ts\n",[Style]);
- Pos0 == 0 ->
- IoFormat(Fd,"<style>\n~ts</style>\n",[Str])
- end;
+ case re:run(Str,"<style>.*</style>",
+ [dotall,caseless,{capture,all,list}]) of
+ nomatch ->
+ case re:run(Str,"</?style>",[caseless,{capture,all,list}]) of
+ nomatch ->
+ IoFormat(Fd,"<style>\n~ts</style>\n",[Str]);
+ {match,["</"++_]} ->
+ print_style_error(Fd, IoFormat,
+ StyleSheet,
+ missing_style_start_tag);
+ {match,[_]} ->
+ print_style_error(Fd, IoFormat,
+ StyleSheet,missing_style_end_tag)
+ end;
+ {match,[Style]} ->
+ IoFormat(Fd,"~ts\n",[Style])
+ end;
{error,Reason} ->
print_style_error(Fd,IoFormat,StyleSheet,Reason)
end.
@@ -1414,9 +1411,9 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip,
{Lbl,Timestamp,Node,AllInfo} =
case All of
{true,OldRuns} ->
- [_Prefix,NodeOrDate|_] = string:tokens(Link,"."),
- Node1 = case string:chr(NodeOrDate,$@) of
- 0 -> "-";
+ [_Prefix,NodeOrDate|_] = string:lexemes(Link,"."),
+ Node1 = case string:find(NodeOrDate,[$@]) of
+ nomatch -> "-";
_ -> NodeOrDate
end,
@@ -1523,7 +1520,7 @@ not_built(BaseName,_LogDir,_All,Missing) ->
%% Top.ObjDir | Top.ObjDir.suites | Top.ObjDir.Suite |
%% Top.ObjDir.Suite.cases | Top.ObjDir.Suite.Case
Failed =
- case string:tokens(BaseName,".") of
+ case string:lexemes(BaseName,".") of
[T,O] when is_list(T) -> % all under Top.ObjDir
locate_info({T,O},all,Missing);
[T,O,"suites"] ->
@@ -2051,9 +2048,9 @@ sort_all_runs(Dirs) ->
%% "YYYY-MM-DD_HH.MM.SS"
lists:sort(fun(Dir1,Dir2) ->
[SS1,MM1,HH1,Date1|_] =
- lists:reverse(string:tokens(Dir1,[$.,$_])),
+ lists:reverse(string:lexemes(Dir1,[$.,$_])),
[SS2,MM2,HH2,Date2|_] =
- lists:reverse(string:tokens(Dir2,[$.,$_])),
+ lists:reverse(string:lexemes(Dir2,[$.,$_])),
{Date1,HH1,MM1,SS1} > {Date2,HH2,MM2,SS2}
end, Dirs).
@@ -2063,9 +2060,9 @@ sort_ct_runs(Dirs) ->
lists:sort(
fun(Dir1,Dir2) ->
[SS1,MM1,DateHH1 | _] =
- lists:reverse(string:tokens(filename:dirname(Dir1),[$.])),
+ lists:reverse(string:lexemes(filename:dirname(Dir1),[$.])),
[SS2,MM2,DateHH2 | _] =
- lists:reverse(string:tokens(filename:dirname(Dir2),[$.])),
+ lists:reverse(string:lexemes(filename:dirname(Dir2),[$.])),
{DateHH1,MM1,SS1} =< {DateHH2,MM2,SS2}
end, Dirs).
@@ -2211,27 +2208,15 @@ runentry(Dir, Totals={Node,Label,Logs,
0 -> "-";
N -> integer_to_list(N)
end,
- StripExt =
- fun(File) ->
- string:sub_string(File,1,
- length(File)-
- length(?logdir_ext)) ++ ", "
- end,
- Polish = fun(S) -> case lists:reverse(S) of
- [32,$,|Rev] -> lists:reverse(Rev);
- [$,|Rev] -> lists:reverse(Rev);
- _ -> S
- end
- end,
- TestNames = Polish(lists:flatten(lists:map(StripExt,Logs))),
+
+ RootNames = lists:map(fun(F) -> filename:rootname(F,?logdir_ext) end, Logs),
+ TestNames = lists:flatten(lists:join(", ", RootNames)),
TestNamesTrunc =
- if TestNames=="" ->
- "";
- length(TestNames) < ?testname_width ->
+ if length(TestNames) < ?testname_width ->
TestNames;
true ->
- Trunc = Polish(string:substr(TestNames,1,
- ?testname_width-3)),
+ Trunc = string:trim(string:slice(TestNames,0,?testname_width-3),
+ trailing,",\s"),
lists:flatten(io_lib:format("~ts...",[Trunc]))
end,
TotMissingStr =
@@ -2374,7 +2359,7 @@ force_rename(From,To,Number) ->
timestamp(Dir) ->
- TsR = lists:reverse(string:tokens(Dir,".-_")),
+ TsR = lists:reverse(string:lexemes(Dir,".-_")),
[S,Min,H,D,M,Y] = [list_to_integer(N) || N <- lists:sublist(TsR,6)],
format_time({{Y,M,D},{H,Min,S}}).
@@ -2923,7 +2908,7 @@ cache_vsn() ->
VSNfile = filename:join([EbinDir,"..","vsn.mk"]),
case file:read_file(VSNfile) of
{ok,Bin} ->
- [_,VSN] = string:tokens(binary_to_list(Bin),[$=,$\n,$ ]),
+ [_,VSN] = string:lexemes(binary_to_list(Bin),[$=,$\n,$ ]),
VSN;
_ ->
undefined
@@ -3320,7 +3305,7 @@ insert_javascript({tablesorter,TableName,
end, [{"CTDateSorter",DateCols},
{"CTTextSorter",TextCols},
{"CTValSorter",ValCols}]))),
- Headers1 = string:substr(Headers, 1, length(Headers)-2),
+ Headers1 = string:trim(Headers, trailing, ",\n"),
["<script type=\"text/javascript\">\n",
"// Parser for date format, e.g: Wed Jul 4 2012 11:24:15\n",
diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl
index 6e6d1879c2..44d3fb8f64 100644
--- a/lib/common_test/src/ct_master.erl
+++ b/lib/common_test/src/ct_master.erl
@@ -807,7 +807,7 @@ filter_accessible(InitOptions, Inaccessible)->
start_nodes(InitOptions)->
lists:foreach(fun({NodeName, Options})->
- [NodeS,HostS]=string:tokens(atom_to_list(NodeName), "@"),
+ [NodeS,HostS]=string:lexemes(atom_to_list(NodeName), "@"),
Node=list_to_atom(NodeS),
Host=list_to_atom(HostS),
HasNodeStart = lists:keymember(node_start, 1, Options),
diff --git a/lib/common_test/src/ct_master_logs.erl b/lib/common_test/src/ct_master_logs.erl
index d8ecd641ed..fd92f73f63 100644
--- a/lib/common_test/src/ct_master_logs.erl
+++ b/lib/common_test/src/ct_master_logs.erl
@@ -297,7 +297,7 @@ sort_all_runs(Dirs) ->
%% "YYYY-MM-DD_HH.MM.SS"
KeyList =
lists:map(fun(Dir) ->
- case lists:reverse(string:tokens(Dir,[$.,$_])) of
+ case lists:reverse(string:lexemes(Dir,[$.,$_])) of
[SS,MM,HH,Date|_] ->
{{Date,HH,MM,SS},Dir};
_Other ->
@@ -312,18 +312,8 @@ runentry(Dir) ->
{MasterStr,NodesStr} =
case read_details_file(Dir) of
{Master,Nodes} when is_list(Nodes) ->
- [_,Host] = string:tokens(atom_to_list(Master),"@"),
- NodesList =
- lists:reverse(lists:map(fun(N) ->
- atom_to_list(N) ++ ", "
- end,Nodes)),
- case NodesList of
- [N|NListR] ->
- N1 = string:sub_string(N,1,length(N)-2),
- {Host,lists:flatten(lists:reverse([N1|NListR]))};
- [] ->
- {Host,""}
- end;
+ [_,Host] = string:lexemes(atom_to_list(Master),"@"),
+ {Host,lists:concat(lists:join(", ",Nodes))};
_Error ->
{"unknown",""}
end,
@@ -348,7 +338,7 @@ all_runs_header() ->
xhtml("", "</tr></thead>\n<tbody>\n")]].
timestamp(Dir) ->
- [S,Min,H,D,M,Y|_] = lists:reverse(string:tokens(Dir,".-_")),
+ [S,Min,H,D,M,Y|_] = lists:reverse(string:lexemes(Dir,".-_")),
[S1,Min1,H1,D1,M1,Y1] = [list_to_integer(N) || N <- [S,Min,H,D,M,Y]],
format_time({{Y1,M1,D1},{H1,Min1,S1}}).
diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl
index 2c4b97df20..29188a648e 100644
--- a/lib/common_test/src/ct_netconfc.erl
+++ b/lib/common_test/src/ct_netconfc.erl
@@ -1467,7 +1467,7 @@ decode_data(Other) ->
{error,{unexpected_rpc_reply,Other}}.
get_qualified_name(Tag) ->
- case string:tokens(atom_to_list(Tag),":") of
+ case string:lexemes(atom_to_list(Tag),":") of
[TagStr] -> {[],TagStr};
[PrefixStr,TagStr] -> {PrefixStr,TagStr}
end.
diff --git a/lib/common_test/src/ct_release_test.erl b/lib/common_test/src/ct_release_test.erl
index 551a7e06d7..4af9afb45b 100644
--- a/lib/common_test/src/ct_release_test.erl
+++ b/lib/common_test/src/ct_release_test.erl
@@ -817,7 +817,7 @@ get_runtime_deps([App|Apps],StartApps,Acc,Visited) ->
RuntimeDeps =
lists:flatmap(
fun(Str) ->
- [RuntimeAppStr,_] = string:tokens(Str,"-"),
+ [RuntimeAppStr,_] = string:lexemes(Str,"-"),
RuntimeApp = list_to_atom(RuntimeAppStr),
case {lists:keymember(RuntimeApp,1,Acc),
lists:member(RuntimeApp,StartApps)} of
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 14f28f9ca3..9436236719 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -147,7 +147,7 @@ script_start(Args) ->
CTVsn =
case filename:basename(code:lib_dir(common_test)) of
CTBase when is_list(CTBase) ->
- case string:tokens(CTBase, "-") of
+ case string:lexemes(CTBase, "-") of
["common_test",Vsn] -> " v"++Vsn;
_ -> ""
end
@@ -315,7 +315,7 @@ script_start1(Parent, Args) ->
{undefined,InclDirs};
CtInclPath ->
AllInclDirs =
- string:tokens(CtInclPath,[$:,$ ,$,]) ++ InclDirs,
+ string:lexemes(CtInclPath,[$:,$ ,$,]) ++ InclDirs,
application:set_env(common_test, include, AllInclDirs),
{undefined,AllInclDirs}
end;
@@ -1096,7 +1096,7 @@ run_test2(StartOpts) ->
application:set_env(common_test, include, InclDirs),
{undefined,InclDirs};
CtInclPath ->
- InclDirs1 = string:tokens(CtInclPath, [$:,$ ,$,]),
+ InclDirs1 = string:lexemes(CtInclPath, [$:,$ ,$,]),
AllInclDirs = InclDirs1++InclDirs,
application:set_env(common_test, include, AllInclDirs),
{undefined,AllInclDirs}
@@ -1482,7 +1482,7 @@ run_testspec2(TestSpec) ->
false ->
Opts#opts.include;
CtInclPath ->
- EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]),
+ EnvInclude = string:lexemes(CtInclPath, [$:,$ ,$,]),
EnvInclude++Opts#opts.include
end,
application:set_env(common_test, include, AllInclude),
diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl
index 4188bd7c3b..61e6446df8 100644
--- a/lib/common_test/src/ct_slave.erl
+++ b/lib/common_test/src/ct_slave.erl
@@ -38,7 +38,8 @@
-record(options, {username, password, boot_timeout, init_timeout,
startup_timeout, startup_functions, monitor_master,
- kill_if_fail, erl_flags, env, ssh_port, ssh_opts}).
+ kill_if_fail, erl_flags, env, ssh_port, ssh_opts,
+ stop_timeout}).
%%%-----------------------------------------------------------------
%%% @spec start(Node) -> Result
@@ -198,6 +199,7 @@ start(Host, Node, Opts) ->
end
end.
+%%%-----------------------------------------------------------------
%%% @spec stop(Node) -> Result
%%% Node = atom()
%%% Result = {ok, NodeName} |
@@ -205,16 +207,41 @@ start(Host, Node, Opts) ->
%%% Reason = not_started |
%%% not_connected |
%%% stop_timeout
-
%%% NodeName = atom()
%%% @doc Stops the running Erlang node with name <code>Node</code> on
%%% the localhost.
stop(Node) ->
stop(gethostname(), Node).
-%%% @spec stop(Host, Node) -> Result
+%%%-----------------------------------------------------------------
+%%% @spec stop(HostOrNode, NodeOrOpts) -> Result
+%%% HostOrNode = atom()
+%%% NodeOrOpts = atom() | list()
+%%% Result = {ok, NodeName} |
+%%% {error, Reason, NodeName}
+%%% Reason = not_started |
+%%% not_connected |
+%%% stop_timeout
+%%% NodeName = atom()
+%%% @doc Stops the running Erlang node with default options on a specified
+%%% host, or on the local host with specified options. That is,
+%%% the call is interpreted as <code>stop(Host, Node)</code> when the
+%%% second argument is atom-valued and <code>stop(Node, Opts)</code>
+%%% when it's list-valued.
+%%% @see stop/3
+stop(_HostOrNode = Node, _NodeOrOpts = Opts) %% match to satiate edoc
+ when is_list(Opts) ->
+ stop(gethostname(), Node, Opts);
+
+stop(Host, Node) ->
+ stop(Host, Node, []).
+
+%%% @spec stop(Host, Node, Opts) -> Result
%%% Host = atom()
%%% Node = atom()
+%%% Opts = [OptTuples]
+%%% OptTuples = {stop_timeout, StopTimeout}
+%%% StopTimeout = integer()
%%% Result = {ok, NodeName} |
%%% {error, Reason, NodeName}
%%% Reason = not_started |
@@ -222,12 +249,19 @@ stop(Node) ->
%%% stop_timeout
%%% NodeName = atom()
%%% @doc Stops the running Erlang node with name <code>Node</code> on
-%%% host <code>Host</code>.
-stop(Host, Node) ->
+%%% host <code>Host</code> as specified by options <code>Opts</code>.
+%%%
+%%% <p>Option <code>stop_timeout</code> specifies, in seconds,
+%%% the time to wait until the node is disconnected.
+%%% Defaults to 5 seconds. If this timeout occurs,
+%%% the result <code>{error, stop_timeout, NodeName}</code> is returned.
+%%%
+stop(Host, Node, Opts) ->
ENode = enodename(Host, Node),
case is_started(ENode) of
{true, connected}->
- do_stop(ENode);
+ OptionsRec = fetch_options(Opts),
+ do_stop(ENode, OptionsRec);
{true, not_connected}->
{error, not_connected, ENode};
false->
@@ -257,11 +291,13 @@ fetch_options(Options) ->
EnvVars = get_option_value(env, Options, []),
SSHPort = get_option_value(ssh_port, Options, []),
SSHOpts = get_option_value(ssh_opts, Options, []),
+ StopTimeout = get_option_value(stop_timeout, Options, 5),
#options{username=UserName, password=Password,
boot_timeout=BootTimeout, init_timeout=InitTimeout,
startup_timeout=StartupTimeout, startup_functions=StartupFunctions,
monitor_master=Monitor, kill_if_fail=KillIfFail,
- erl_flags=ErlFlags, env=EnvVars, ssh_port=SSHPort, ssh_opts=SSHOpts}.
+ erl_flags=ErlFlags, env=EnvVars, ssh_port=SSHPort, ssh_opts=SSHOpts,
+ stop_timeout=StopTimeout}.
% send a message when slave node is started
% @hidden
@@ -461,6 +497,8 @@ wait_for_node_alive(Node, N) ->
% call init:stop on a remote node
do_stop(ENode) ->
+ do_stop(ENode, fetch_options([])).
+do_stop(ENode, Options) ->
{Cover,MainCoverNode} =
case test_server:is_cover() of
true ->
@@ -471,7 +509,8 @@ do_stop(ENode) ->
{false,undefined}
end,
spawn(ENode, init, stop, []),
- case wait_for_node_dead(ENode, 5) of
+ StopTimeout = Options#options.stop_timeout,
+ case wait_for_node_dead(ENode, StopTimeout) of
{ok,ENode} ->
if Cover ->
%% To avoid that cover is started again if a node
diff --git a/lib/common_test/src/ct_ssh.erl b/lib/common_test/src/ct_ssh.erl
index ca62357e1c..491d56dfc1 100644
--- a/lib/common_test/src/ct_ssh.erl
+++ b/lib/common_test/src/ct_ssh.erl
@@ -996,7 +996,8 @@ init(KeyOrName, {ConnType,Addr,Port}, AllOpts) ->
try_log(heading(init,KeyOrName),
"Opened ~w connection:\n"
"Host: ~tp (~p)\nUser: ~tp\nPassword: ~p\n",
- [ConnType,Addr,Port,User,lists:duplicate(length(Password),$*)]),
+ [ConnType,Addr,Port,User,
+ lists:duplicate(string:length(Password),$*)]),
{ok,SSHRef,#state{ssh_ref=SSHRef, conn_type=ConnType,
target=KeyOrName}}
end.
diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl
index abf131f4df..3c0fead5b2 100644
--- a/lib/common_test/src/ct_util.erl
+++ b/lib/common_test/src/ct_util.erl
@@ -795,25 +795,25 @@ parse_table(Data) ->
{Heading,Lines}.
get_headings(["|" ++ Headings | Rest]) ->
- {remove_space(string:tokens(Headings, "|"),[]), Rest};
+ {remove_space(string:lexemes(Headings, "|"),[]), Rest};
get_headings([_ | Rest]) ->
get_headings(Rest);
get_headings([]) ->
{{},[]}.
parse_row(["|" ++ _ = Row | T], Rows, NumCols) when NumCols > 1 ->
- case string:tokens(Row, "|") of
+ case string:lexemes(Row, "|") of
Values when length(Values) =:= NumCols ->
parse_row(T,[remove_space(Values,[])|Rows], NumCols);
Values when length(Values) < NumCols ->
parse_row([Row ++"\n"++ hd(T) | tl(T)], Rows, NumCols)
end;
-parse_row(["|" ++ _ = Row | T], Rows, 1 = NumCols) ->
- case string:rchr(Row, $|) of
- 1 ->
+parse_row(["|" ++ X = Row | T], Rows, 1 = NumCols) ->
+ case string:find(X, [$|]) of
+ nomatch ->
parse_row([Row ++"\n"++hd(T) | tl(T)], Rows, NumCols);
_Else ->
- parse_row(T, [remove_space(string:tokens(Row,"|"),[])|Rows],
+ parse_row(T, [remove_space(string:lexemes(Row,"|"),[])|Rows],
NumCols)
end;
parse_row([_Skip | T], Rows, NumCols) ->
@@ -822,7 +822,7 @@ parse_row([], Rows, _NumCols) ->
lists:reverse(Rows).
remove_space([Str|Rest],Acc) ->
- remove_space(Rest,[string:strip(string:strip(Str),both,$')|Acc]);
+ remove_space(Rest,[string:trim(string:trim(Str,both,[$\s]),both,[$'])|Acc]);
remove_space([],Acc) ->
list_to_tuple(lists:reverse(Acc)).
@@ -832,7 +832,7 @@ remove_space([],Acc) ->
%%%
%%% @doc
is_test_dir(Dir) ->
- lists:last(string:tokens(filename:basename(Dir), "_")) == "test".
+ lists:last(string:lexemes(filename:basename(Dir), "_")) == "test".
%%%-----------------------------------------------------------------
%%% @spec
@@ -1077,7 +1077,7 @@ open_url(iexplore, Args, URL) ->
_ = case win32reg:values(R) of
{ok, Paths} ->
Path = proplists:get_value(default, Paths),
- [Cmd | _] = string:tokens(Path, "%"),
+ [Cmd | _] = string:lexemes(Path, "%"),
Cmd1 = Cmd ++ " " ++ Args ++ " " ++ URL,
io:format(?def_gl, "~nOpening ~ts with command:~n ~ts~n", [URL,Cmd1]),
open_port({spawn,Cmd1}, []);
diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl
index 8b29d0f96d..1c55e17686 100644
--- a/lib/common_test/src/cth_log_redirect.erl
+++ b/lib/common_test/src/cth_log_redirect.erl
@@ -121,8 +121,8 @@ handle_event({_Type,GL,_Msg}, #eh_state{handle_remote_events = false} = State)
when node(GL) /= node() ->
{ok, State};
handle_event(Event, #eh_state{log_func = LogFunc} = State) ->
- case lists:keyfind(sasl, 1, application:which_applications()) of
- false ->
+ case whereis(sasl_sup) of
+ undefined ->
sasl_not_started;
_Else ->
{ok, ErrLogType} = application:get_env(sasl, errlog_type),
diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl
index da68bd105e..4407ff56c1 100644
--- a/lib/common_test/src/cth_surefire.erl
+++ b/lib/common_test/src/cth_surefire.erl
@@ -184,15 +184,14 @@ end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite,
Log =
case Log0 of
"" ->
- LowerSuiteName = string:to_lower(atom_to_list(Suite)),
+ LowerSuiteName = string:lowercase(atom_to_list(Suite)),
filename:join(CurrLogDir,LowerSuiteName++"."++Name++".html");
_ ->
Log0
end,
Url = make_url(UrlBase,Log),
ClassName = atom_to_list(Suite),
- PGroup = string:join([ atom_to_list(Group)||
- Group <- lists:reverse(Groups)],"."),
+ PGroup = lists:concat(lists:join(".",lists:reverse(Groups))),
TimeTakes = io_lib:format("~f",[timer:now_diff(?now,TS) / 1000000]),
State#state{ test_cases = [#testcase{ log = Log,
url = Url,
@@ -317,9 +316,9 @@ make_url(undefined,_) ->
make_url(_,[]) ->
undefined;
make_url(UrlBase0,Log) ->
- UrlBase = string:strip(UrlBase0,right,$/),
+ UrlBase = string:trim(UrlBase0,trailing,[$/]),
RelativeLog = get_relative_log_url(Log),
- string:join([UrlBase,RelativeLog],"/").
+ lists:flatten(lists:join($/,[UrlBase,RelativeLog])).
get_test_root(Log) ->
LogParts = filename:split(Log),
@@ -329,7 +328,7 @@ get_relative_log_url(Log) ->
LogParts = filename:split(Log),
Start = length(LogParts)-?log_depth,
Length = ?log_depth+1,
- string:join(lists:sublist(LogParts,Start,Length),"/").
+ lists:flatten(lists:join($/,lists:sublist(LogParts,Start,Length))).
count_tcs([#testcase{name=ConfCase}|TCs],Ok,F,S)
when ConfCase=="init_per_suite";
diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl
index dc6b7a536c..35a73e6d2e 100644
--- a/lib/common_test/src/test_server.erl
+++ b/lib/common_test/src/test_server.erl
@@ -462,11 +462,12 @@ run_test_case_msgloop(#st{ref=Ref,pid=Pid,end_conf_pid=EndConfPid0}=St0) ->
%% it as a comment, potentially replacing user data
Error = lists:flatten(io_lib:format("Aborted: ~tp",
[Reason])),
- Error1 = lists:flatten([string:strip(S,left) ||
- S <- string:tokens(Error,
- [$\n])]),
- Comment = if length(Error1) > 63 ->
- string:substr(Error1,1,60) ++ "...";
+ Error1 = lists:flatten([string:trim(S,leading,"\s") ||
+ S <- string:lexemes(Error,
+ [$\n])]),
+ ErrorLength = string:length(Error1),
+ Comment = if ErrorLength > 63 ->
+ string:slice(Error1,0,60) ++ "...";
true ->
Error1
end,
@@ -2697,9 +2698,9 @@ is_cover() ->
is_debug() ->
case catch erlang:system_info(debug_compiled) of
{'EXIT', _} ->
- case string:str(erlang:system_info(system_version), "debug") of
- Int when is_integer(Int), Int > 0 -> true;
- _ -> false
+ case string:find(erlang:system_info(system_version), "debug") of
+ nomatch -> false;
+ _ -> true
end;
Res ->
Res
@@ -2735,9 +2736,9 @@ has_superfluous_schedulers() ->
%% We might want to do more tests on a commercial platform, for instance
%% ensuring that all applications have documentation).
is_commercial() ->
- case string:str(erlang:system_info(system_version), "source") of
- Int when is_integer(Int), Int > 0 -> false;
- _ -> true
+ case string:find(erlang:system_info(system_version), "source") of
+ nomatch -> true;
+ _ -> false
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index 71978c7267..c70ea4ef9d 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -1464,13 +1464,14 @@ get_suites([], Mods) ->
lists:reverse(Mods).
add_mod(Mod, Mods) ->
- case string:rstr(atom_to_list(Mod), "_SUITE") of
- 0 -> false;
- _ -> % test suite
+ case lists:reverse(atom_to_list(Mod)) of
+ "ETIUS_" ++ _ -> % test suite
case lists:member(Mod, Mods) of
true -> false;
false -> true
- end
+ end;
+ _ ->
+ false
end.
@@ -2611,7 +2612,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0,
NumStr ->
%% Ex: "123 456 789" or "123,456,789" -> {123,456,789}
list_to_tuple([list_to_integer(NS) ||
- NS <- string:tokens(NumStr, [$ ,$:,$,])])
+ NS <- string:lexemes(NumStr, [$ ,$:,$,])])
end,
{shuffle_cases(Ref, Cs0, UseSeed),{shuffle,UseSeed}}
end;
@@ -3979,11 +3980,12 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
true -> "~w"
end, [Time]),
ReasonStr = escape_chars(reason_to_string(Reason1)),
- ReasonStr1 = lists:flatten([string:strip(S,left) ||
- S <- string:tokens(ReasonStr,[$\n])]),
+ ReasonStr1 = lists:flatten([string:trim(S,leading,"\s") ||
+ S <- string:lexemes(ReasonStr,[$\n])]),
+ ReasonLength = string:length(ReasonStr1),
ReasonStr2 =
- if length(ReasonStr1) > 80 ->
- string:substr(ReasonStr1, 1, 77) ++ "...";
+ if ReasonLength > 80 ->
+ string:slice(ReasonStr1, 0, 77) ++ "...";
true ->
ReasonStr1
end,
@@ -4067,11 +4069,12 @@ progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time,
true -> "~w"
end, [Time]),
ErrorReason = escape_chars(lists:flatten(io_lib:format("~tp", [Reason]))),
- ErrorReason1 = lists:flatten([string:strip(S,left) ||
- S <- string:tokens(ErrorReason,[$\n])]),
+ ErrorReason1 = lists:flatten([string:trim(S,leading,"\s") ||
+ S <- string:lexemes(ErrorReason,[$\n])]),
+ ErrorReasonLength = string:length(ErrorReason1),
ErrorReason2 =
- if length(ErrorReason1) > 63 ->
- string:substr(ErrorReason1, 1, 60) ++ "...";
+ if ErrorReasonLength > 63 ->
+ string:slice(ErrorReason1, 0, 60) ++ "...";
true ->
ErrorReason1
end,
diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl
index c0d7e12721..f0f9cea6e0 100644
--- a/lib/common_test/src/test_server_node.erl
+++ b/lib/common_test/src/test_server_node.erl
@@ -315,9 +315,11 @@ start_node_peer(SlaveName, OptList, From, TI) ->
Prog0 = start_node_get_option_value(erl, OptList, default),
Prog = quote_progname(pick_erl_program(Prog0)),
Args =
- case string:str(SuppliedArgs,"-setcookie") of
- 0 -> "-setcookie " ++ TI#target_info.cookie ++ " " ++ SuppliedArgs;
- _ -> SuppliedArgs
+ case string:find(SuppliedArgs,"-setcookie") of
+ nomatch ->
+ "-setcookie " ++ TI#target_info.cookie ++ " " ++ SuppliedArgs;
+ _ ->
+ SuppliedArgs
end,
Cmd = lists:concat([Prog,
" -detached ",
@@ -612,7 +614,7 @@ pick_erl_program(L) ->
%% emulator and flags as the test node. The return from lib:progname()
%% could then typically be '/<full_path_to>/cerl -gcov').
quote_progname(Progname) ->
- do_quote_progname(string:tokens(Progname," ")).
+ do_quote_progname(string:lexemes(Progname," ")).
do_quote_progname([Prog]) ->
"\""++Prog++"\"";
diff --git a/lib/common_test/src/unix_telnet.erl b/lib/common_test/src/unix_telnet.erl
index 8ac467014c..5992f26e6d 100644
--- a/lib/common_test/src/unix_telnet.erl
+++ b/lib/common_test/src/unix_telnet.erl
@@ -121,7 +121,8 @@ connect1(Name,Ip,Port,Timeout,KeepAlive,TCPNoDelay,Username,Password) ->
prompt,?prx,[]) of
{ok,{prompt,?password},_} ->
ok = ct_telnet_client:send_data(Pid,Password),
- Stars = lists:duplicate(length(Password),$*),
+ Stars =
+ lists:duplicate(string:length(Password),$*),
log(Name,send,"Password: ~s",[Stars]),
% ok = ct_telnet_client:send_data(Pid,""),
case ct_telnet:silent_teln_expect(Name,Pid,[],
diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl
index 250700741c..5ffc735d6a 100644
--- a/lib/common_test/test/ct_config_SUITE.erl
+++ b/lib/common_test/test/ct_config_SUITE.erl
@@ -213,8 +213,8 @@ reformat_events(Events, EH) ->
skip_dynamic() ->
case os:getenv("TS_EXTRA_PLATFORM_LABEL") of
TSExtraPlatformLabel when is_list(TSExtraPlatformLabel) ->
- case string:str(TSExtraPlatformLabel,"TimeWarpingOS") of
- 0 -> false;
+ case string:find(TSExtraPlatformLabel,"TimeWarpingOS") of
+ nomatch -> false;
_ -> true
end;
_ ->
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 8ba14e63bc..3c1e887f65 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -257,7 +257,7 @@ cth_log(Config) when is_list(Config) ->
lists:foreach(
fun(UnexpIoLog) ->
{ok,Bin} = file:read_file(UnexpIoLog),
- Ts = string:tokens(binary_to_list(Bin),[$\n]),
+ Ts = string:lexemes(binary_to_list(Bin),[$\n]),
Matches = lists:foldl(fun([$=,$E,$R,$R,$O,$R|_], N) ->
N+1;
([$L,$o,$g,$g,$e,$r|_], N) ->
diff --git a/lib/common_test/test/ct_log_SUITE.erl b/lib/common_test/test/ct_log_SUITE.erl
index 93affda398..9655b6f09a 100644
--- a/lib/common_test/test/ct_log_SUITE.erl
+++ b/lib/common_test/test/ct_log_SUITE.erl
@@ -174,7 +174,7 @@ verify(Config) ->
TcLogFile = proplists:get_value(the_logfile, SavedCfg),
Pid = proplists:get_value(the_pid, SavedCfg),
StrPid = lists:flatten(io_lib:format("~p",[Pid])),
- EscPid = "&lt;" ++ string:substr(StrPid, 2, length(StrPid)-2) ++ "&gt;",
+ EscPid = "&lt;" ++ string:slice(StrPid, 1, length(StrPid)-2) ++ "&gt;",
String = proplists:get_value(the_string, SavedCfg),
ct:log("Read from prev testcase: ~p & ~p", [TcLogFile,Pid]),
{ok,Dev} = file:open(TcLogFile, [read]),
diff --git a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl
index 7ffe6f045b..0b85392009 100644
--- a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl
+++ b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl
@@ -164,7 +164,7 @@ pre_post_io(Config) ->
fun(PrePostIoFile) ->
ct:log("Reading Pre/Post Test IO Log file: ~ts", [PrePostIoFile]),
{ok,Bin} = file:read_file(PrePostIoFile),
- Ts = string:tokens(binary_to_list(Bin),[$\n]),
+ Ts = string:lexemes(binary_to_list(Bin),[$\n]),
PrePostIOEntries =
lists:foldl(fun([$L,$o,$g,$g,$e,$r|_],
{pre,PreLogN,PreErrN,0,0}) ->
@@ -203,7 +203,7 @@ pre_post_io(Config) ->
fun(UnexpIoFile) ->
ct:log("Reading Unexpected IO Log file: ~ts", [UnexpIoFile]),
{ok,Bin} = file:read_file(UnexpIoFile),
- Ts = string:tokens(binary_to_list(Bin),[$\n]),
+ Ts = string:lexemes(binary_to_list(Bin),[$\n]),
UnexpIOEntries =
lists:foldl(fun([$L,$o,$g,$g,$e,$r|_], [LogN,ErrN]) ->
[LogN+1,ErrN];
diff --git a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
index 1b171801a3..b1d191873d 100644
--- a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
+++ b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
@@ -126,12 +126,12 @@ default(Config) ->
auto_per_tc(Config) ->
PrivDir = proplists:get_value(priv_dir, Config),
- ["log_private",_] = string:tokens(filename:basename(PrivDir), "."),
+ ["log_private",_] = string:lexemes(filename:basename(PrivDir), "."),
{ok,_} = file:list_dir(PrivDir).
manual_per_tc(Config) ->
PrivDir = proplists:get_value(priv_dir, Config),
- ["log_private",_] = string:tokens(filename:basename(PrivDir), "."),
+ ["log_private",_] = string:lexemes(filename:basename(PrivDir), "."),
{error,_} = file:list_dir(PrivDir),
ok = ct:make_priv_dir(),
{ok,_} = file:list_dir(PrivDir).
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 44c27e54c2..2ba7c7c13f 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -88,7 +88,7 @@ start_slave(Config, Level) ->
start_slave(ct, Config, Level).
start_slave(NodeName, Config, Level) ->
- [_,Host] = string:tokens(atom_to_list(node()), "@"),
+ [_,Host] = string:lexemes(atom_to_list(node()), "@"),
test_server:format(0, "Trying to start ~s~n",
[atom_to_list(NodeName)++"@"++Host]),
PR = proplists:get_value(printable_range,Config,io:printable_range()),
@@ -1088,8 +1088,8 @@ locate({TEH,Name,{'DEF','RUNDIR'}}, Node, [Ev|Evs], Config) ->
{TEH,#event{name=Name, node=Node, data=EvData}} ->
{_,{_,LogDir}} = lists:keysearch(logdir, 1, get_opts(Config)),
D = filename:join(LogDir, "ct_run." ++ atom_to_list(Node)),
- case string:str(EvData, D) of
- 0 -> exit({badmatch,EvData});
+ case string:find(EvData, D) of
+ nomatch -> exit({badmatch,EvData});
_ -> ok
end,
{Config,Evs};
@@ -1104,8 +1104,8 @@ locate({TEH,Name,{'DEF',{'START_TIME','LOGDIR'}}}, Node, [Ev|Evs], Config) ->
{DT={{_,_,_},{_,_,_}},Dir} when is_list(Dir) ->
{_,{_,LogDir}} = lists:keysearch(logdir, 1, get_opts(Config)),
D = filename:join(LogDir, "ct_run." ++ atom_to_list(Node)),
- case string:str(Dir, D) of
- 0 -> exit({badmatch,Dir});
+ case string:find(Dir, D) of
+ nomatch -> exit({badmatch,Dir});
_ -> ok
end,
{[{start_time,DT}|Config],Evs};
@@ -1373,7 +1373,7 @@ delete_dirs(LogDir) ->
Dirs2Del =
lists:foldl(fun(Dir, Del) ->
[S,Mi,H,D,Mo,Y|_] =
- lists:reverse(string:tokens(Dir, [$.,$-,$_])),
+ lists:reverse(string:lexemes(Dir, [$.,$-,$_])),
S2I = fun(Str) -> list_to_integer(Str) end,
DT = {{S2I(Y),S2I(Mo),S2I(D)}, {S2I(H),S2I(Mi),S2I(S)}},
Then = calendar:datetime_to_gregorian_seconds(DT),
diff --git a/lib/common_test/test/ct_unicode_SUITE.erl b/lib/common_test/test/ct_unicode_SUITE.erl
index 355503a5dc..6f6ec97ceb 100644
--- a/lib/common_test/test/ct_unicode_SUITE.erl
+++ b/lib/common_test/test/ct_unicode_SUITE.erl
@@ -191,7 +191,7 @@ check_logs(Dirs) ->
[] ->
ok;
Match ->
- MatchStr = string:join(Match,"\n"),
+ MatchStr = lists:join("\n",Match),
ct:log("ERROR: Escaped unicode characters found in:~n~ts",[MatchStr]),
ct:fail(escaped_unicode_characters_found)
end.
diff --git a/lib/common_test/test/ct_userconfig_callback.erl b/lib/common_test/test/ct_userconfig_callback.erl
index c723f4ca1c..14e3d9a688 100644
--- a/lib/common_test/test/ct_userconfig_callback.erl
+++ b/lib/common_test/test/ct_userconfig_callback.erl
@@ -21,7 +21,7 @@
-export([check_parameter/1, read_config/1]).
read_config(Str) ->
- KeyVals = string:tokens(Str, " "),
+ KeyVals = string:lexemes(Str, " "),
{ok,read_config1(KeyVals)}.
read_config1([Key,Val | KeyVals]) ->
diff --git a/lib/common_test/test/erl2html2_SUITE.erl b/lib/common_test/test/erl2html2_SUITE.erl
index 53a63578b2..b2336ff0bc 100644
--- a/lib/common_test/test/erl2html2_SUITE.erl
+++ b/lib/common_test/test/erl2html2_SUITE.erl
@@ -214,10 +214,10 @@ check_line_number(Last,Line,OrigLine) ->
[$>|Rest] = lists:dropwhile(fun($>) -> false; (_) -> true end,Line),
check_line_number(Last,Rest,OrigLine);
_ ->
- [N |_] = string:tokens(Line,":"),
+ [N |_] = string:lexemes(Line,":"),
% erlang:display(N),
Num =
- try list_to_integer(string:strip(N))
+ try list_to_integer(string:trim(N,both,"\s"))
catch _:_ -> ct:fail({no_line_number_after,Last,OrigLine})
end,
if Num == Last+1 ->
diff --git a/lib/common_test/test/telnet_server.erl b/lib/common_test/test/telnet_server.erl
index 65300b0bdf..cef7784333 100644
--- a/lib/common_test/test/telnet_server.erl
+++ b/lib/common_test/test/telnet_server.erl
@@ -249,7 +249,7 @@ do_handle_data("echo " ++ Data,State) ->
send(Data++"\r\n> ",State),
{ok,State};
do_handle_data("echo_sep " ++ Data,State) ->
- Msgs = string:tokens(Data," "),
+ Msgs = string:lexemes(Data," "),
lists:foreach(fun(Msg) ->
send(Msg,State),
timer:sleep(10)
@@ -260,28 +260,28 @@ do_handle_data("echo_no_prompt " ++ Data,State) ->
send(Data,State),
{ok,State};
do_handle_data("echo_ml " ++ Data,State) ->
- Lines = string:tokens(Data," "),
- ReturnData = string:join(Lines,"\n"),
+ Lines = string:lexemes(Data," "),
+ ReturnData = lists:flatten(lists:join("\n",Lines)),
send(ReturnData++"\r\n> ",State),
{ok,State};
do_handle_data("echo_ml_no_prompt " ++ Data,State) ->
- Lines = string:tokens(Data," "),
- ReturnData = string:join(Lines,"\n"),
+ Lines = string:lexemes(Data," "),
+ ReturnData = lists:flatten(lists:join("\n",Lines)),
send(ReturnData,State),
{ok,State};
do_handle_data("echo_loop " ++ Data,State) ->
- [TStr|Lines] = string:tokens(Data," "),
- ReturnData = string:join(Lines,"\n"),
+ [TStr|Lines] = string:lexemes(Data," "),
+ ReturnData = lists:flatten(lists:join("\n",Lines)),
send_loop(list_to_integer(TStr),ReturnData,State),
{ok,State};
do_handle_data("echo_delayed_prompt "++Data,State) ->
- [MsStr|EchoData] = string:tokens(Data, " "),
- send(string:join(EchoData,"\n"),State),
+ [MsStr|EchoData] = string:lexemes(Data, " "),
+ send(lists:flatten(lists:join("\n",EchoData)),State),
timer:sleep(list_to_integer(MsStr)),
send("\r\n> ",State),
{ok,State};
do_handle_data("disconnect_after " ++WaitStr,State) ->
- Wait = list_to_integer(string:strip(WaitStr,right,$\n)),
+ Wait = list_to_integer(string:trim(WaitStr,trailing,"\n")),
dbg("Server will close connection in ~w ms...", [Wait]),
erlang:send_after(Wait,self(),disconnect),
{ok,State};
diff --git a/lib/common_test/test/test_server_SUITE.erl b/lib/common_test/test/test_server_SUITE.erl
index 50d8bdd1ac..edfaea1d08 100644
--- a/lib/common_test/test/test_server_SUITE.erl
+++ b/lib/common_test/test/test_server_SUITE.erl
@@ -71,7 +71,7 @@ init_per_testcase(_TestCase, Config) ->
%% @spec end_per_testcase(TestCase, Config0) ->
%% void() | {save_config,Config1} | {fail,Reason}
end_per_testcase(test_server_unicode, _Config) ->
- [_,Host] = string:tokens(atom_to_list(node()), "@"),
+ [_,Host] = string:lexemes(atom_to_list(node()), "@"),
N1 = list_to_atom("test_server_tester_latin1" ++ "@" ++ Host),
N2 = list_to_atom("test_server_tester_utf8" ++ "@" ++ Host),
test_server:stop_node(N1),
@@ -347,7 +347,7 @@ generate_and_run_unicode_test(Config0,Encoding) ->
RunDir = get_latest_run_dir(LogDir),
true = filelib:is_dir(RunDir),
- LowerModStr = string:to_lower(ModStr),
+ LowerModStr = string:lowercase(ModStr),
SuiteHtml = translate_filename(LowerModStr++".src.html",Encoding),
true = filelib:is_regular(filename:join(RunDir,SuiteHtml)),
@@ -362,7 +362,7 @@ generate_and_run_unicode_test(Config0,Encoding) ->
%% remote file system on master - i.e. they will use same file name
%% mode as the master.
start_node(Config,Name,Args) ->
- [_,Host] = string:tokens(atom_to_list(node()), "@"),
+ [_,Host] = string:lexemes(atom_to_list(node()), "@"),
ct:log("Trying to start ~w@~s~n",[Name,Host]),
case test_server:start_node(Name, peer, [{args,Args}]) of
{error,Reason} ->
diff --git a/lib/common_test/test/test_server_test_lib.erl b/lib/common_test/test/test_server_test_lib.erl
index c18b89b178..e3d987a2ea 100644
--- a/lib/common_test/test/test_server_test_lib.erl
+++ b/lib/common_test/test/test_server_test_lib.erl
@@ -43,7 +43,7 @@ pre_init_per_testcase(_TC,Config,State) ->
{start_slave(Config, 50),State}.
start_slave(Config,_Level) ->
- [_,Host] = string:tokens(atom_to_list(node()), "@"),
+ [_,Host] = string:lexemes(atom_to_list(node()), "@"),
ct:log("Trying to start ~s~n",
["test_server_tester@"++Host]),
diff --git a/lib/common_test/test_server/ts.erl b/lib/common_test/test_server/ts.erl
index 5bfea9f4de..330652e73f 100644
--- a/lib/common_test/test_server/ts.erl
+++ b/lib/common_test/test_server/ts.erl
@@ -583,7 +583,7 @@ is_list_of_suites(List) ->
S = if is_atom(Suite) -> atom_to_list(Suite);
true -> Suite
end,
- try lists:last(string:tokens(S,"_")) of
+ try lists:last(string:lexemes(S,"_")) of
"SUITE" -> true;
"suite" -> true;
_ -> false
diff --git a/lib/common_test/test_server/ts_autoconf_win32.erl b/lib/common_test/test_server/ts_autoconf_win32.erl
index 52e5ac8e69..6f6caaeb70 100644
--- a/lib/common_test/test_server/ts_autoconf_win32.erl
+++ b/lib/common_test/test_server/ts_autoconf_win32.erl
@@ -228,7 +228,7 @@ make(Vars) ->
end.
find_make(MakeCmd, Vars) ->
- [Make|_] = string:tokens(MakeCmd, " \t"),
+ [Make|_] = string:lexemes(MakeCmd, " \t"),
case os:find_executable(Make) of
false ->
{no, Vars};
@@ -248,9 +248,9 @@ javac(Vars) ->
end.
is_debug_build() ->
- case catch string:str(erlang:system_info(system_version), "debug") of
- Int when is_integer(Int), Int > 0 ->
- true;
- _ ->
- false
+ case catch string:find(erlang:system_info(system_version), "debug") of
+ nomatch ->
+ false;
+ _Else ->
+ true
end.
diff --git a/lib/common_test/test_server/ts_erl_config.erl b/lib/common_test/test_server/ts_erl_config.erl
index 032593bdda..c7fe4ccf83 100644
--- a/lib/common_test/test_server/ts_erl_config.erl
+++ b/lib/common_test/test_server/ts_erl_config.erl
@@ -311,7 +311,7 @@ lib_dir(Vars, Lib) ->
end,
CLibDir = filename:join(CLibDirList),
Cmd = "ls -d " ++ CLibDir ++ "*",
- XLibDir = lists:last(string:tokens(os:cmd(Cmd),"\n")),
+ XLibDir = lists:last(string:lexemes(os:cmd(Cmd),"\n")),
case file:list_dir(XLibDir) of
{error, enoent} ->
[];
@@ -361,15 +361,11 @@ emu_vars(Vars) ->
{erl_name, atom_to_list(lib:progname())}|Vars].
is_source_build() ->
- string:str(erlang:system_info(system_version), "[source]") > 0.
+ string:find(erlang:system_info(system_version), "source") =/= nomatch.
is_debug_build() ->
- case catch string:str(erlang:system_info(system_version), "debug") of
- Int when is_integer(Int), Int > 0 ->
- true;
- _ ->
- false
- end.
+ string:find(erlang:system_info(system_version), "debug") =/= nomatch.
+
%%
%% ssl_libdir
%%
diff --git a/lib/common_test/test_server/ts_install.erl b/lib/common_test/test_server/ts_install.erl
index c4e0223ac7..048e5493d2 100644
--- a/lib/common_test/test_server/ts_install.erl
+++ b/lib/common_test/test_server/ts_install.erl
@@ -115,7 +115,7 @@ get_vars(_, _, _, _) ->
config_flags() ->
case os:getenv("CONFIG_FLAGS") of
false -> [];
- CF -> string:tokens(CF, " \t\n")
+ CF -> string:lexemes(CF, " \t\n")
end.
unix_autoconf(XConf) ->
@@ -127,7 +127,7 @@ unix_autoconf(XConf) ->
Threads = [" --enable-shlib-thread-safety" ||
erlang:system_info(threads) /= false],
Debug = [" --enable-debug-mode" ||
- string:str(erlang:system_info(system_version),"debug") > 0],
+ string:find(erlang:system_info(system_version),"debug") =/= nomatch],
MXX_Build = [Y || Y <- config_flags(),
Y == "--enable-m64-build"
orelse Y == "--enable-m32-build"],
@@ -159,10 +159,8 @@ assign_vars([]) ->
assign_vars([{VAR,FlagsStr} | VARs]) ->
[{VAR,assign_vars(FlagsStr)} | assign_vars(VARs)];
assign_vars(FlagsStr) ->
- Flags = [assign_all_vars(Str,[]) || Str <- string:tokens(FlagsStr, [$ ])],
- string:strip(lists:flatten(lists:map(fun(Flag) ->
- Flag ++ " "
- end, Flags)), right).
+ Flags = [assign_all_vars(Str,[]) || Str <- string:lexemes(FlagsStr, [$\s])],
+ lists:flatten(lists:join(" ", Flags)).
assign_all_vars([$$ | Rest], FlagSoFar) ->
{VarName,Rest1} = get_var_name(Rest, []),
@@ -292,7 +290,7 @@ add_vars(Vars0, Opts0) ->
get_testcase_callback() ->
case os:getenv("TS_TESTCASE_CALLBACK") of
ModFunc when is_list(ModFunc), ModFunc /= "" ->
- case string:tokens(ModFunc, " ") of
+ case string:lexemes(ModFunc, " ") of
[_Mod,_Func] -> ModFunc;
_ -> ""
end;
@@ -408,17 +406,13 @@ off_heap_msgq() ->
end.
schedulers() ->
- case catch erlang:system_info(smp_support) of
- true ->
- case {erlang:system_info(schedulers),
- erlang:system_info(schedulers_online)} of
- {S,S} ->
- "/S"++integer_to_list(S);
- {S,O} ->
- "/S"++integer_to_list(S) ++ ":" ++
- integer_to_list(O)
- end;
- _ -> ""
+ case {erlang:system_info(schedulers),
+ erlang:system_info(schedulers_online)} of
+ {S,S} ->
+ "/S"++integer_to_list(S);
+ {S,O} ->
+ "/S"++integer_to_list(S) ++ ":" ++
+ integer_to_list(O)
end.
bind_type() ->
@@ -434,8 +428,8 @@ bind_type() ->
debug() ->
- case string:str(erlang:system_info(system_version), "debug") of
- 0 -> "";
+ case string:find(erlang:system_info(system_version), "debug") of
+ nomatch -> "";
_ -> "/Debug"
end.
diff --git a/lib/common_test/test_server/ts_lib.erl b/lib/common_test/test_server/ts_lib.erl
index ea039a2c2b..da8d676b18 100644
--- a/lib/common_test/test_server/ts_lib.erl
+++ b/lib/common_test/test_server/ts_lib.erl
@@ -99,7 +99,7 @@ specialized_specs(Dir,PostFix) ->
sort_tests([begin
DirPart = filename:dirname(Name),
AppTest = hd(lists:reverse(filename:split(DirPart))),
- list_to_atom(string:substr(AppTest, 1, length(AppTest)-5))
+ list_to_atom(string:slice(AppTest, 0, string:length(AppTest)-5))
end || Name <- Specs]).
specs(Dir) ->
@@ -111,9 +111,9 @@ specs(Dir) ->
[Spec,TestDir|_] =
lists:reverse(filename:split(FullName)),
[_TestSuffix|TDParts] =
- lists:reverse(string:tokens(TestDir,[$_,$.])),
+ lists:reverse(string:lexemes(TestDir,[$_,$.])),
[_SpecSuffix|SParts] =
- lists:reverse(string:tokens(Spec,[$_,$.])),
+ lists:reverse(string:lexemes(Spec,[$_,$.])),
if TDParts == SParts ->
[filename_to_atom(FullName)];
true ->
@@ -273,7 +273,7 @@ do_test(Rest, Vars, Test) ->
get_arg([$(|Rest], Vars, Stop, _) ->
get_arg(Rest, Vars, Stop, []);
get_arg([Stop|Rest], Vars, Stop, Acc) ->
- Arg = string:strip(lists:reverse(Acc)),
+ Arg = string:trim(lists:reverse(Acc),both,[$\s]),
Subst = subst(Arg, Vars),
{Subst,Rest};
get_arg([C|Rest], Vars, Stop, Acc) ->
diff --git a/lib/common_test/test_server/ts_run.erl b/lib/common_test/test_server/ts_run.erl
index e22fa8d196..3f594236bc 100644
--- a/lib/common_test/test_server/ts_run.erl
+++ b/lib/common_test/test_server/ts_run.erl
@@ -207,11 +207,7 @@ make_command(Vars, Spec, State) ->
_ ->
ok
end,
- "cerl -valgrind" ++
- case erlang:system_info(smp_support) of
- true -> " -smp";
- false -> ""
- end
+ "cerl -valgrind"
end,
Naming =
case ts_lib:var(longnames, Vars) of
@@ -468,4 +464,4 @@ split_one(Path) ->
filename:split(Path).
split_path(Path) ->
- string:tokens(Path,";").
+ string:lexemes(Path,";").
diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml
index 10164890f2..b398871ddf 100644
--- a/lib/compiler/doc/src/compile.xml
+++ b/lib/compiler/doc/src/compile.xml
@@ -123,6 +123,17 @@
in the Efficiency Guide.</p>
</item>
+ <tag><c>{compile_info, [{atom(), term()}]}</c></tag>
+ <item>
+ <p>Allows compilers built on top of <c>compile</c> to attach
+ extra compilation metadata to the <c>compile_info</c> chunk
+ in the generated beam file.</p>
+
+ <p>It is advised for compilers to remove all non-deterministic
+ information if the <c>deterministic</c> option is supported and
+ it was supplied by the user.</p>
+ </item>
+
<tag><c>compressed</c></tag>
<item>
<p>The compiler will compress the generated object code,
diff --git a/lib/compiler/internal_doc/cerl-notes.md b/lib/compiler/internal_doc/cerl-notes.md
new file mode 100644
index 0000000000..705fe8f42d
--- /dev/null
+++ b/lib/compiler/internal_doc/cerl-notes.md
@@ -0,0 +1,75 @@
+Some notes on the cerl modules
+==============================
+
+Maps in cerl_clauses:match/3
+----------------------------
+
+Not much optimization is done for maps in `cerl_clauses:match/3`.
+
+The reason is that the inliner (`cerl_inline`) was not designed for
+data types that depend on variables bound in the enclosing environment
+(for example, the keys for maps). If we attempt to extend the
+optimizations for maps similar to the optimizations for the other data
+types, the inliner will crash for certain code. Here is an example of
+code that would crash the inliner:
+
+ t() ->
+ f(key).
+
+ f(K) ->
+ case #{K => value} of
+ #{key := V} -> V
+ end.
+
+The reason for the crash is that the inliner works in several
+passes and calls `cerl_clauses:match/3` twice. The inliner will
+assume that the same result will be returned both times, but
+for this example different bindings will be returned.
+
+Here is the rough outline of what happens:
+
+* The first time `cerl_clauses:match/3` will be asked to match the
+pattern `#{K := V}` against `#{key => value}`. It cannot say more
+than that the pattern *may* match.
+
+* Now the inliner will add the bindings to body of the clause (which
+is simply `V`). In this case, the bindings are still empty, so
+nothing is added.
+
+* The inliner will now do some substitutions and renaming. The
+variable `K` will be replaced with `key`.
+
+* The next time `cerl_clauses:match/3` is called, it will be asked to
+match the pattern `#{key := V}` against `#{key => value#}`. In this
+case, there will be a match and the bindings can be extended with
+`{V,value}`.
+
+* The inliner will see that the entire case can be removed. Only
+the body for the clause needs to be kept.
+
+Thus, after inlining the function `t/0` will look like this:
+
+ t() ->
+ V.
+
+The problem here is that the inliner assumed that the bindings from
+the first and second call to `cer_clauses:match/3` would be the same.
+It used the empty bindings from the first call for the body.
+
+The correct way would be to add the bindings from the second call:
+
+ t() ->
+ let V = value in V.
+
+### How to fix this ###
+
+The inliner will need to call `cerl_clauses:match/3` after doing
+all substitutions and renaming. It is not clear to me how difficult
+that would be. I assume that the inliner is written the way it is
+for a good reason. That means that switching the order things are
+done would lead to correctness and/or performance problems.
+
+### What must also be done to fix this ###
+
+`cerl_inline:make_template/3` must be extended to create a template
+for maps. That is relatively straightforward.
diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile
index ef6db66ff6..9e96147787 100644
--- a/lib/compiler/src/Makefile
+++ b/lib/compiler/src/Makefile
@@ -83,6 +83,7 @@ MODULES = \
core_scan \
erl_bifs \
rec_env \
+ sys_core_alias \
sys_core_bsm \
sys_core_dsetel \
sys_core_fold \
@@ -92,16 +93,14 @@ MODULES = \
v3_codegen \
v3_core \
v3_kernel \
- v3_kernel_pp \
- v3_life
+ v3_kernel_pp
BEAM_H = $(wildcard ../priv/beam_h/*.h)
HRL_FILES= \
beam_disasm.hrl \
core_parse.hrl \
- v3_kernel.hrl \
- v3_life.hrl
+ v3_kernel.hrl
YRL_FILE = core_parse.yrl
@@ -186,7 +185,7 @@ release_docs_spec:
# ----------------------------------------------------
$(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl
-$(EBIN)/beam_listing.beam: v3_life.hrl
+$(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl
$(EBIN)/beam_validator.beam: beam_disasm.hrl
$(EBIN)/cerl.beam: core_parse.hrl
$(EBIN)/compile.beam: core_parse.hrl ../../stdlib/include/erl_compile.hrl
@@ -194,12 +193,12 @@ $(EBIN)/core_lib.beam: core_parse.hrl
$(EBIN)/core_lint.beam: core_parse.hrl
$(EBIN)/core_parse.beam: core_parse.hrl $(EGEN)/core_parse.erl
$(EBIN)/core_pp.beam: core_parse.hrl
+$(EBIN)/sys_core_alias.beam: core_parse.hrl
$(EBIN)/sys_core_dsetel.beam: core_parse.hrl
$(EBIN)/sys_core_fold.beam: core_parse.hrl
$(EBIN)/sys_core_fold_lists.beam: core_parse.hrl
$(EBIN)/sys_core_inline.beam: core_parse.hrl
-$(EBIN)/v3_codegen.beam: v3_life.hrl
+$(EBIN)/v3_codegen.beam: v3_kernel.hrl
$(EBIN)/v3_core.beam: core_parse.hrl
$(EBIN)/v3_kernel.beam: core_parse.hrl v3_kernel.hrl
$(EBIN)/v3_kernel_pp.beam: v3_kernel.hrl
-$(EBIN)/v3_life.beam: v3_kernel.hrl v3_life.hrl
diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl
index c35efdfc9d..9ecbb7884c 100644
--- a/lib/compiler/src/beam_asm.erl
+++ b/lib/compiler/src/beam_asm.erl
@@ -21,7 +21,7 @@
-module(beam_asm).
--export([module/5]).
+-export([module/4]).
-export([encode/2]).
-export_type([fail/0,label/0,reg/0,src/0,module_code/0,function_name/0]).
@@ -55,20 +55,20 @@
-type module_code() ::
{module(),[_],[_],[asm_function()],pos_integer()}.
--spec module(module_code(), [{binary(), binary()}], [_], [compile:option()], [compile:option()]) ->
+-spec module(module_code(), [{binary(), binary()}], [{atom(),term()}], [compile:option()]) ->
{'ok',binary()}.
-module(Code, ExtraChunks, SourceFile, Opts, CompilerOpts) ->
- {ok,assemble(Code, ExtraChunks, SourceFile, Opts, CompilerOpts)}.
+module(Code, ExtraChunks, CompileInfo, CompilerOpts) ->
+ {ok,assemble(Code, ExtraChunks, CompileInfo, CompilerOpts)}.
-assemble({Mod,Exp0,Attr0,Asm0,NumLabels}, ExtraChunks, SourceFile, Opts, CompilerOpts) ->
+assemble({Mod,Exp0,Attr0,Asm0,NumLabels}, ExtraChunks, CompileInfo, CompilerOpts) ->
{1,Dict0} = beam_dict:atom(Mod, beam_dict:new()),
{0,Dict1} = beam_dict:fname(atom_to_list(Mod) ++ ".erl", Dict0),
NumFuncs = length(Asm0),
{Asm,Attr} = on_load(Asm0, Attr0),
Exp = cerl_sets:from_list(Exp0),
{Code,Dict2} = assemble_1(Asm, Exp, Dict1, []),
- build_file(Code, Attr, Dict2, NumLabels, NumFuncs, ExtraChunks, SourceFile, Opts, CompilerOpts).
+ build_file(Code, Attr, Dict2, NumLabels, NumFuncs, ExtraChunks, CompileInfo, CompilerOpts).
on_load(Fs0, Attr0) ->
case proplists:get_value(on_load, Attr0) of
@@ -111,7 +111,7 @@ assemble_function([H|T], Acc, Dict0) ->
assemble_function([], Code, Dict) ->
{Code, Dict}.
-build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks, SourceFile, Opts, CompilerOpts) ->
+build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks, CompileInfo, CompilerOpts) ->
%% Create the code chunk.
CodeChunk = chunk(<<"Code">>,
@@ -182,7 +182,7 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks, SourceFile, Opts,
Essentials1 = [iolist_to_binary(C) || C <- Essentials0],
MD5 = module_md5(Essentials1),
Essentials = finalize_fun_table(Essentials1, MD5),
- {Attributes,Compile} = build_attributes(Opts, SourceFile, Attr, MD5),
+ {Attributes,Compile} = build_attributes(Attr, CompileInfo, MD5),
AttrChunk = chunk(<<"Attr">>, Attributes),
CompileChunk = chunk(<<"CInf">>, Compile),
@@ -192,7 +192,7 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks, SourceFile, Opts,
%% Create IFF chunk.
- Chunks = case member(slim, Opts) of
+ Chunks = case member(slim, CompilerOpts) of
true ->
[Essentials,AttrChunk];
false ->
@@ -264,22 +264,10 @@ flatten_exports(Exps) ->
flatten_imports(Imps) ->
list_to_binary(map(fun({M,F,A}) -> <<M:32,F:32,A:32>> end, Imps)).
-build_attributes(Opts, SourceFile, Attr, MD5) ->
- Misc0 = case SourceFile of
- [] -> [];
- [_|_] -> [{source,SourceFile}]
- end,
- Misc = case member(slim, Opts) of
- false -> Misc0;
- true -> []
- end,
- Compile = case member(deterministic, Opts) of
- false ->
- [{options,Opts},{version,?COMPILER_VSN}|Misc];
- true ->
- [{version,?COMPILER_VSN}]
- end,
- {term_to_binary(set_vsn_attribute(Attr, MD5)),term_to_binary(Compile)}.
+build_attributes(Attr, Compile, MD5) ->
+ AttrBinary = term_to_binary(set_vsn_attribute(Attr, MD5)),
+ CompileBinary = term_to_binary([{version,?COMPILER_VSN}|Compile]),
+ {AttrBinary,CompileBinary}.
build_line_table(Dict) ->
{NumLineInstrs,NumFnames0,Fnames0,NumLines,Lines0} =
diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl
index b736d39f9c..e094c2c320 100644
--- a/lib/compiler/src/beam_clean.erl
+++ b/lib/compiler/src/beam_clean.erl
@@ -24,7 +24,7 @@
-export([module/2]).
-export([bs_clean_saves/1]).
-export([clean_labels/1]).
--import(lists, [map/2,foldl/3,reverse/1,filter/2]).
+-import(lists, [foldl/3,reverse/1,filter/2]).
-spec module(beam_utils:module_code(), [compile:option()]) ->
{'ok',beam_utils:module_code()}.
@@ -118,7 +118,7 @@ add_to_work_list(F, {Fs,Used}=Sets) ->
clean_labels(Fs0) ->
St0 = #st{lmap=[],entry=1,lc=1},
{Fs1,#st{lmap=Lmap0,lc=Lc}} = function_renumber(Fs0, St0, []),
- Lmap = gb_trees:from_orddict(ordsets:from_list(Lmap0)),
+ Lmap = maps:from_list(Lmap0),
Fs = function_replace(Fs1, Lmap, []),
{Fs,Lc}.
@@ -187,7 +187,8 @@ is_record_tuple(_, _, _) -> no.
function_replace([{function,Name,Arity,Entry,Asm0}|Fs], Dict, Acc) ->
Asm = try
- replace(Asm0, [], Dict)
+ Fb = fun(Old) -> throw({error,{undefined_label,Old}}) end,
+ beam_utils:replace_labels(Asm0, [], Dict, Fb)
catch
throw:{error,{undefined_label,Lbl}=Reason} ->
io:format("Function ~s/~w refers to undefined label ~w\n",
@@ -197,57 +198,6 @@ function_replace([{function,Name,Arity,Entry,Asm0}|Fs], Dict, Acc) ->
function_replace(Fs, Dict, [{function,Name,Arity,Entry,Asm}|Acc]);
function_replace([], _, Acc) -> Acc.
-replace([{test,Test,{f,Lbl},Ops}|Is], Acc, D) ->
- replace(Is, [{test,Test,{f,label(Lbl, D)},Ops}|Acc], D);
-replace([{test,Test,{f,Lbl},Live,Ops,Dst}|Is], Acc, D) ->
- replace(Is, [{test,Test,{f,label(Lbl, D)},Live,Ops,Dst}|Acc], D);
-replace([{select,I,R,{f,Fail0},Vls0}|Is], Acc, D) ->
- Vls = map(fun ({f,L}) -> {f,label(L, D)};
- (Other) -> Other
- end, Vls0),
- Fail = label(Fail0, D),
- replace(Is, [{select,I,R,{f,Fail},Vls}|Acc], D);
-replace([{'try',R,{f,Lbl}}|Is], Acc, D) ->
- replace(Is, [{'try',R,{f,label(Lbl, D)}}|Acc], D);
-replace([{'catch',R,{f,Lbl}}|Is], Acc, D) ->
- replace(Is, [{'catch',R,{f,label(Lbl, D)}}|Acc], D);
-replace([{jump,{f,Lbl}}|Is], Acc, D) ->
- replace(Is, [{jump,{f,label(Lbl, D)}}|Acc], D);
-replace([{loop_rec,{f,Lbl},R}|Is], Acc, D) ->
- replace(Is, [{loop_rec,{f,label(Lbl, D)},R}|Acc], D);
-replace([{loop_rec_end,{f,Lbl}}|Is], Acc, D) ->
- replace(Is, [{loop_rec_end,{f,label(Lbl, D)}}|Acc], D);
-replace([{wait,{f,Lbl}}|Is], Acc, D) ->
- replace(Is, [{wait,{f,label(Lbl, D)}}|Acc], D);
-replace([{wait_timeout,{f,Lbl},To}|Is], Acc, D) ->
- replace(Is, [{wait_timeout,{f,label(Lbl, D)},To}|Acc], D);
-replace([{bif,Name,{f,Lbl},As,R}|Is], Acc, D) when Lbl =/= 0 ->
- replace(Is, [{bif,Name,{f,label(Lbl, D)},As,R}|Acc], D);
-replace([{gc_bif,Name,{f,Lbl},Live,As,R}|Is], Acc, D) when Lbl =/= 0 ->
- replace(Is, [{gc_bif,Name,{f,label(Lbl, D)},Live,As,R}|Acc], D);
-replace([{call,Ar,{f,Lbl}}|Is], Acc, D) ->
- replace(Is, [{call,Ar,{f,label(Lbl,D)}}|Acc], D);
-replace([{make_fun2,{f,Lbl},U1,U2,U3}|Is], Acc, D) ->
- replace(Is, [{make_fun2,{f,label(Lbl, D)},U1,U2,U3}|Acc], D);
-replace([{bs_init,{f,Lbl},Info,Live,Ss,Dst}|Is], Acc, D) when Lbl =/= 0 ->
- replace(Is, [{bs_init,{f,label(Lbl, D)},Info,Live,Ss,Dst}|Acc], D);
-replace([{bs_put,{f,Lbl},Info,Ss}|Is], Acc, D) when Lbl =/= 0 ->
- replace(Is, [{bs_put,{f,label(Lbl, D)},Info,Ss}|Acc], D);
-replace([{put_map=I,{f,Lbl},Op,Src,Dst,Live,List}|Is], Acc, D)
- when Lbl =/= 0 ->
- replace(Is, [{I,{f,label(Lbl, D)},Op,Src,Dst,Live,List}|Acc], D);
-replace([{get_map_elements=I,{f,Lbl},Src,List}|Is], Acc, D) when Lbl =/= 0 ->
- replace(Is, [{I,{f,label(Lbl, D)},Src,List}|Acc], D);
-replace([I|Is], Acc, D) ->
- replace(Is, [I|Acc], D);
-replace([], Acc, _) -> Acc.
-
-label(Old, D) ->
- case gb_trees:lookup(Old, D) of
- {value,Val} -> Val;
- none -> throw({error,{undefined_label,Old}})
- end.
-
%%%
%%% Final fixup of bs_start_match2/5,bs_save2/bs_restore2 instructions for
%%% new bit syntax matching (introduced in R11B).
diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl
index 4365451356..0bcec9ce19 100644
--- a/lib/compiler/src/beam_jump.erl
+++ b/lib/compiler/src/beam_jump.erl
@@ -71,9 +71,9 @@
%%%
%%% jump L2
%%% . . .
-%%% L1:
%%% L2: ...
%%%
+%%% and all preceding uses of L1 renamed to L2.
%%% If the jump is unreachable, it will be removed according to (1).
%%%
%%% (5) In
@@ -156,41 +156,46 @@ function({function,Name,Arity,CLabel,Asm0}) ->
%%%
share(Is0) ->
- %% We will get more sharing if we never fall through to a label.
- Is = eliminate_fallthroughs(Is0, []),
- share_1(Is, #{}, [], []).
+ Is1 = eliminate_fallthroughs(Is0, []),
+ Is2 = find_fixpoint(fun(Is) ->
+ share_1(Is, #{}, #{}, [], [])
+ end, Is1),
+ reverse(Is2).
-share_1([{label,L}=Lbl|Is], Dict0, [_|_]=Seq, Acc) ->
+share_1([{label,L}=Lbl|Is], Dict0, Lbls0, [_|_]=Seq, Acc) ->
case maps:find(Seq, Dict0) of
error ->
Dict = maps:put(Seq, L, Dict0),
- share_1(Is, Dict, [], [Lbl|Seq ++ Acc]);
+ share_1(Is, Dict, Lbls0, [], [Lbl|Seq ++ Acc]);
{ok,Label} ->
- share_1(Is, Dict0, [], [Lbl,{jump,{f,Label}}|Acc])
+ Lbls = maps:put(L, Label, Lbls0),
+ share_1(Is, Dict0, Lbls, [], [Lbl,{jump,{f,Label}}|Acc])
end;
-share_1([{func_info,_,_,_}=I|Is], _, [], Acc) ->
- reverse(Is, [I|Acc]);
-share_1([{'catch',_,_}=I|Is], Dict0, Seq, Acc) ->
- Dict = clean_non_sharable(Dict0),
- share_1(Is, Dict, [I|Seq], Acc);
-share_1([{'try',_,_}=I|Is], Dict0, Seq, Acc) ->
- Dict = clean_non_sharable(Dict0),
- share_1(Is, Dict, [I|Seq], Acc);
-share_1([{try_case,_}=I|Is], Dict0, Seq, Acc) ->
- Dict = clean_non_sharable(Dict0),
- share_1(Is, Dict, [I|Seq], Acc);
-share_1([{catch_end,_}=I|Is], Dict0, Seq, Acc) ->
- Dict = clean_non_sharable(Dict0),
- share_1(Is, Dict, [I|Seq], Acc);
-share_1([I|Is], Dict, Seq, Acc) ->
+share_1([{func_info,_,_,_}|_]=Is, _, Lbls, [], Acc) when Lbls =/= #{} ->
+ beam_utils:replace_labels(Acc, Is, Lbls, fun(Old) -> Old end);
+share_1([{func_info,_,_,_}|_]=Is, _, Lbls, [], Acc) when Lbls =:= #{} ->
+ reverse(Acc, Is);
+share_1([{'catch',_,_}=I|Is], Dict0, Lbls0, Seq, Acc) ->
+ {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0),
+ share_1(Is, Dict, Lbls, [I|Seq], Acc);
+share_1([{'try',_,_}=I|Is], Dict0, Lbls0, Seq, Acc) ->
+ {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0),
+ share_1(Is, Dict, Lbls, [I|Seq], Acc);
+share_1([{try_case,_}=I|Is], Dict0, Lbls0, Seq, Acc) ->
+ {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0),
+ share_1(Is, Dict, Lbls, [I|Seq], Acc);
+share_1([{catch_end,_}=I|Is], Dict0, Lbls0, Seq, Acc) ->
+ {Dict,Lbls} = clean_non_sharable(Dict0, Lbls0),
+ share_1(Is, Dict, Lbls, [I|Seq], Acc);
+share_1([I|Is], Dict, Lbls, Seq, Acc) ->
case is_unreachable_after(I) of
false ->
- share_1(Is, Dict, [I|Seq], Acc);
+ share_1(Is, Dict, Lbls, [I|Seq], Acc);
true ->
- share_1(Is, Dict, [I], Acc)
+ share_1(Is, Dict, Lbls, [I], Acc)
end.
-clean_non_sharable(Dict) ->
+clean_non_sharable(Dict0, Lbls0) ->
%% We are passing in or out of a 'catch' or 'try' block. Remove
%% sequences that should not be shared over the boundaries of the
%% block. Since the end of the sequence must match, the only
@@ -198,7 +203,17 @@ clean_non_sharable(Dict) ->
%% the 'catch'/'try' block is a sequence that ends with an
%% instruction that causes an exception. Any sequence that causes
%% an exception must contain a line/1 instruction.
- maps:filter(fun(K, _V) -> sharable_with_try(K) end, Dict).
+ Dict1 = maps:to_list(Dict0),
+ Lbls1 = maps:to_list(Lbls0),
+ {Dict2,Lbls2} = foldl(fun({K, V}, {Dict,Lbls}) ->
+ case sharable_with_try(K) of
+ true ->
+ {[{K,V}|Dict],lists:keydelete(V, 2, Lbls)};
+ false ->
+ {Dict,Lbls}
+ end
+ end, {[],Lbls1}, Dict1),
+ {maps:from_list(Dict2),maps:from_list(Lbls2)}.
sharable_with_try([{line,_}|_]) ->
%% This sequence may cause an exception and may potentially
@@ -275,14 +290,15 @@ extract_seq_1(_, _) -> no.
-record(st,
{
entry :: beam_asm:label(), %Entry label (must not be moved).
- mlbl :: #{beam_asm:label() := [beam_asm:label()]}, %Moved labels.
- labels :: cerl_sets:set() %Set of referenced labels.
+ replace :: #{beam_asm:label() := beam_asm:label()}, %Labels to replace.
+ labels :: cerl_sets:set(), %Set of referenced labels.
+ index :: beam_utils:code_index() | {lazy,[beam_utils:instruction()]} %Index built lazily only if needed
}).
opt(Is0, CLabel) ->
find_fixpoint(fun(Is) ->
Lbls = initial_labels(Is),
- St = #st{entry=CLabel,mlbl=#{},labels=Lbls},
+ St = #st{entry=CLabel,replace=#{},labels=Lbls,index={lazy,Is}},
opt(Is, [], St)
end, Is0).
@@ -292,7 +308,7 @@ find_fixpoint(OptFun, Is0) ->
Is -> find_fixpoint(OptFun, Is)
end.
-opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc, St) ->
+opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc0, St0) ->
%% We have
%% Test Label Ops
%% jump Label
@@ -301,10 +317,34 @@ opt([{test,_,{f,L}=Lbl,_}=I|[{jump,{f,L}}|_]=Is], Acc, St) ->
case beam_utils:is_pure_test(I) of
false ->
%% Test is not pure; we must keep it.
- opt(Is, [I|Acc], label_used(Lbl, St));
+ opt(Is, [I|Acc0], label_used(Lbl, St0));
true ->
%% The test is pure and its failure label is the same
%% as in the jump that follows -- thus it is not needed.
+ %% Check if any of the previous instructions could also be eliminated.
+ {Acc,St} = opt_useless_loads(Acc0, L, St0),
+ opt(Is, Acc, St)
+ end;
+opt([{test,_,{f,L}=Lbl,_}=I|[{label,L}|_]=Is], Acc0, St0) ->
+ %% Similar to the above, except we have a fall-through rather than jump
+ %% Test Label Ops
+ %% label Label
+ case beam_utils:is_pure_test(I) of
+ false ->
+ opt(Is, [I|Acc0], label_used(Lbl, St0));
+ true ->
+ {Acc,St} = opt_useless_loads(Acc0, L, St0),
+ opt(Is, Acc, St)
+ end;
+opt([{test,_,{f,L}=Lbl,_}=I|[{label,L}|_]=Is], Acc0, St0) ->
+ %% Similar to the above, except we have a fall-through rather than jump
+ %% Test Label Ops
+ %% label Label
+ case beam_utils:is_pure_test(I) of
+ false ->
+ opt(Is, [I|Acc0], label_used(Lbl, St0));
+ true ->
+ {Acc,St} = opt_useless_loads(Acc0, L, St0),
opt(Is, Acc, St)
end;
opt([{test,Test0,{f,L}=Lbl,Ops}=I|[{jump,To}|Is]=Is0], Acc, St) ->
@@ -326,30 +366,16 @@ opt([{test,_,{f,_}=Lbl,_,_,_}=I|Is], Acc, St) ->
opt(Is, [I|Acc], label_used(Lbl, St));
opt([{select,_,_R,Fail,Vls}=I|Is], Acc, St) ->
skip_unreachable(Is, [I|Acc], label_used([Fail|Vls], St));
-opt([{label,Lbl}=I|Is], Acc, #st{mlbl=Mlbl}=St0) ->
- case maps:find(Lbl, Mlbl) of
- {ok,Lbls} ->
- %% Essential to remove the list of labels from the dictionary,
- %% since we will rescan the inserted labels. We MUST rescan.
- St = St0#st{mlbl=maps:remove(Lbl, Mlbl)},
- insert_labels([Lbl|Lbls], Is, Acc, St);
- error ->
- opt(Is, [I|Acc], St0)
- end;
+opt([{label,From}=I,{label,To}|Is], Acc, #st{replace=Replace}=St) ->
+ opt([I|Is], Acc, St#st{replace=Replace#{To => From}});
opt([{jump,{f,_}=X}|[{label,_},{jump,X}|_]=Is], Acc, St) ->
opt(Is, Acc, St);
opt([{jump,{f,Lbl}}|[{label,Lbl}|_]=Is], Acc, St) ->
opt(Is, Acc, St);
-opt([{jump,{f,L}=Lbl}=I|Is], Acc0, #st{mlbl=Mlbl0}=St0) ->
- %% All labels before this jump instruction should now be
- %% moved to the location of the jump's target.
- {Lbls,Acc} = collect_labels(Acc0, St0),
- St = case Lbls of
- [] -> St0;
- [_|_] ->
- Mlbl = maps_append_list(L, Lbls, Mlbl0),
- St0#st{mlbl=Mlbl}
- end,
+opt([{jump,{f,L}=Lbl}=I|Is], Acc0, St0) ->
+ %% Replace all labels before this jump instruction into the
+ %% location of the jump's target.
+ {Acc,St} = collect_labels(Acc0, L, St0),
skip_unreachable(Is, [I|Acc], label_used(Lbl, St));
%% Optimization: quickly handle some common instructions that don't
%% have any failure labels and where is_unreachable_after(I) =:= false.
@@ -369,36 +395,72 @@ opt([I|Is], Acc, #st{labels=Used0}=St0) ->
true -> skip_unreachable(Is, [I|Acc], St);
false -> opt(Is, [I|Acc], St)
end;
-opt([], Acc, #st{mlbl=Mlbl}) ->
- Code = reverse(Acc),
- insert_fc_labels(Code, Mlbl).
-
-insert_fc_labels([{label,L}=I|Is0], Mlbl) ->
- case maps:find(L, Mlbl) of
- error ->
- [I|insert_fc_labels(Is0, Mlbl)];
- {ok,Lbls} ->
- Is = [{label,Lb} || Lb <- Lbls] ++ Is0,
- [I|insert_fc_labels(Is, maps:remove(L, Mlbl))]
+opt([], Acc, #st{replace=Replace0}) when Replace0 =/= #{} ->
+ Replace = normalize_replace(maps:to_list(Replace0), Replace0, []),
+ beam_utils:replace_labels(Acc, [], Replace, fun(Old) -> Old end);
+opt([], Acc, #st{replace=Replace}) when Replace =:= #{} ->
+ reverse(Acc).
+
+normalize_replace([{From,To0}|Rest], Replace, Acc) ->
+ case Replace of
+ #{To0 := To} ->
+ normalize_replace([{From,To}|Rest], Replace, Acc);
+ _ ->
+ normalize_replace(Rest, Replace, [{From,To0}|Acc])
end;
-insert_fc_labels([_|_]=Is, _) -> Is.
-
-maps_append_list(K,Vs,M) ->
- case M of
- #{K:=Vs0} -> M#{K:=Vs0++Vs}; % same order as dict
- _ -> M#{K => Vs}
- end.
+normalize_replace([], _Replace, Acc) ->
+ maps:from_list(Acc).
+
+%% After eliminating a test, it might happen, that a register was only used
+%% in this test. Let's check if that was the case and if it was so, we can
+%% eliminate the load into the register completely.
+opt_useless_loads([{block,_}|_]=Is, L, #st{index={lazy,FIs}}=St) ->
+ opt_useless_loads(Is, L, St#st{index=beam_utils:index_labels(FIs)});
+opt_useless_loads([{block,Block0}|Is], L, #st{index=Index}=St) ->
+ case opt_useless_block_loads(Block0, L, Index) of
+ [] ->
+ opt_useless_loads(Is, L, St);
+ [_|_]=Block ->
+ {[{block,Block}|Is],St}
+ end;
+%% After eliminating the test and useless blocks, it might happen,
+%% that the previous test could also be eliminated.
+%% It might be that the label was already marked as used, even if ultimately,
+%% it never will be - we can't do much about it at that point, though
+opt_useless_loads([{test,_,{f,L},_}=I|Is], L, St) ->
+ case beam_utils:is_pure_test(I) of
+ false ->
+ {[I|Is],St};
+ true ->
+ opt_useless_loads(Is, L, St)
+ end;
+opt_useless_loads(Is, _L, St) ->
+ {Is,St}.
+
+opt_useless_block_loads([{set,[Dst],_,_}=I|Is], L, Index) ->
+ BlockJump = [{block,Is},{jump,{f,L}}],
+ case beam_utils:is_killed(Dst, BlockJump, Index) of
+ true ->
+ %% The register is killed and not used, we can remove the load
+ opt_useless_block_loads(Is, L, Index);
+ false ->
+ [I|opt_useless_block_loads(Is, L, Index)]
+ end;
+opt_useless_block_loads([I|Is], L, Index) ->
+ [I|opt_useless_block_loads(Is, L, Index)];
+opt_useless_block_loads([], _L, _Index) ->
+ [].
-collect_labels(Is, #st{entry=Entry}) ->
- collect_labels_1(Is, Entry, []).
+collect_labels(Is, Label, #st{entry=Entry,replace=Replace} = St) ->
+ collect_labels_1(Is, Label, Entry, Replace, St).
-collect_labels_1([{label,Entry}|_]=Is, Entry, Acc) ->
+collect_labels_1([{label,Entry}|_]=Is, _Label, Entry, Acc, St) ->
%% Never move the entry label.
- {Acc,Is};
-collect_labels_1([{label,L}|Is], Entry, Acc) ->
- collect_labels_1(Is, Entry, [L|Acc]);
-collect_labels_1(Is, _Entry, Acc) ->
- {Acc,Is}.
+ {Is,St#st{replace=Acc}};
+collect_labels_1([{label,L}|Is], Label, Entry, Acc, St) ->
+ collect_labels_1(Is, Label, Entry, Acc#{L => Label}, St);
+collect_labels_1(Is, _Label, _Entry, Acc, St) ->
+ {Is,St#st{replace=Acc}}.
%% label_defined(Is, Label) -> true | false.
%% Test whether the label Label is defined at the start of the instruction
@@ -418,13 +480,6 @@ invert_test(is_eq_exact) -> is_ne_exact;
invert_test(is_ne_exact) -> is_eq_exact;
invert_test(_) -> not_possible.
-insert_labels([L|Ls], Is, [{jump,{f,L}}|Acc], St) ->
- insert_labels(Ls, [{label,L}|Is], Acc, St);
-insert_labels([L|Ls], Is, Acc, St) ->
- insert_labels(Ls, [{label,L}|Is], Acc, St);
-insert_labels([], Is, Acc, St) ->
- opt(Is, Acc, St).
-
%% skip_unreachable([Instruction], St).
%% Remove all instructions (including definitions of labels
%% that have not been referenced yet) up to the next
diff --git a/lib/compiler/src/beam_listing.erl b/lib/compiler/src/beam_listing.erl
index 94b47cf568..9422f9a78a 100644
--- a/lib/compiler/src/beam_listing.erl
+++ b/lib/compiler/src/beam_listing.erl
@@ -23,14 +23,12 @@
-include("core_parse.hrl").
-include("v3_kernel.hrl").
--include("v3_life.hrl").
-import(lists, [foreach/2]).
-type code() :: cerl:c_module()
| beam_utils:module_code()
| #k_mdef{}
- | {module(),_,_,_} %v3_life
| [_]. %form-based format
-spec module(file:io_device(), code()) -> 'ok'.
@@ -42,13 +40,9 @@ module(File, #k_mdef{}=Kern) ->
%% This is a kernel module.
io:put_chars(File, v3_kernel_pp:format(Kern));
%%io:put_chars(File, io_lib:format("~p~n", [Kern]));
-module(File, {Mod,Exp,Attr,Kern}) ->
- %% This is output from beam_life (v3).
- io:fwrite(File, "~w.~n~p.~n~p.~n", [Mod,Exp,Attr]),
- foreach(fun (F) -> function(File, F) end, Kern);
module(Stream, {Mod,Exp,Attr,Code,NumLabels}) ->
- %% This is output from beam_codegen.
- io:format(Stream, "{module, ~p}. %% version = ~w\n",
+ %% This is output from v3_codegen.
+ io:format(Stream, "{module, ~p}. %% version = ~w\n",
[Mod, beam_opcodes:format_number()]),
io:format(Stream, "\n{exports, ~p}.\n", [Exp]),
io:format(Stream, "\n{attributes, ~p}.\n", [Attr]),
@@ -68,60 +62,3 @@ format_asm([{label,L}|Is]) ->
format_asm([I|Is]) ->
[io_lib:format(" ~p", [I]),".\n"|format_asm(Is)];
format_asm([]) -> [].
-
-function(File, {function,Name,Arity,Args,Body,Vdb,_Anno}) ->
- io:nl(File),
- io:format(File, "function ~p/~p.\n", [Name,Arity]),
- io:format(File, " ~p.\n", [Args]),
- print_vdb(File, Vdb),
- put(beam_listing_nl, false),
- nl(File),
- foreach(fun(F) -> format(File, F, []) end, Body),
- nl(File),
- erase(beam_listing_nl).
-
-format(File, #l{ke=Ke,i=I,vdb=Vdb}, Ind) ->
- nl(File),
- ind_format(File, Ind, "~p ", [I]),
- print_vdb(File, Vdb),
- nl(File),
- format(File, Ke, Ind);
-format(File, Tuple, Ind) when is_tuple(Tuple) ->
- ind_format(File, Ind, "{", []),
- format_list(File, tuple_to_list(Tuple), [$\s|Ind]),
- ind_format(File, Ind, "}", []);
-format(File, List, Ind) when is_list(List) ->
- ind_format(File, Ind, "[", []),
- format_list(File, List, [$\s|Ind]),
- ind_format(File, Ind, "]", []);
-format(File, F, Ind) ->
- ind_format(File, Ind, "~p", [F]).
-
-format_list(File, [F], Ind) ->
- format(File, F, Ind);
-format_list(File, [F|Fs], Ind) ->
- format(File, F, Ind),
- ind_format(File, Ind, ",", []),
- format_list(File, Fs, Ind);
-format_list(_, [], _) -> ok.
-
-
-print_vdb(File, [{Var,F,E}|Vs]) ->
- io:format(File, "~p:~p..~p ", [Var,F,E]),
- print_vdb(File, Vs);
-print_vdb(_, []) -> ok.
-
-ind_format(File, Ind, Format, Args) ->
- case get(beam_listing_nl) of
- true ->
- put(beam_listing_nl, false),
- io:put_chars(File, Ind);
- false -> ok
- end,
- io:format(File, Format, Args).
-
-nl(File) ->
- case put(beam_listing_nl, true) of
- true -> ok;
- false -> io:nl(File)
- end.
diff --git a/lib/compiler/src/beam_peep.erl b/lib/compiler/src/beam_peep.erl
index 6df5c02334..9436c20b36 100644
--- a/lib/compiler/src/beam_peep.erl
+++ b/lib/compiler/src/beam_peep.erl
@@ -89,15 +89,37 @@ peep([{gc_bif,_,_,_,_,Dst}=I|Is], SeenTests0, Acc) ->
peep([{jump,{f,L}},{label,L}=I|Is], _, Acc) ->
%% Sometimes beam_jump has missed this optimization.
peep(Is, gb_sets:empty(), [I|Acc]);
-peep([{select,Op,R,F,Vls0}|Is], _, Acc) ->
+peep([{select,Op,R,F,Vls0}|Is], SeenTests0, Acc0) ->
case prune_redundant_values(Vls0, F) of
[] ->
%% No values left. Must convert to plain jump.
I = {jump,F},
- peep(Is, gb_sets:empty(), [I|Acc]);
+ peep([I|Is], gb_sets:empty(), Acc0);
+ [{atom,_}=Value,Lbl] when Op =:= select_val ->
+ %% Single value left. Convert to regular test and pop redundant tests.
+ Is1 = [{test,is_eq_exact,F,[R,Value]},{jump,Lbl}|Is],
+ case Acc0 of
+ [{test,is_atom,F,[R]}|Acc] ->
+ peep(Is1, SeenTests0, Acc);
+ _ ->
+ peep(Is1, SeenTests0, Acc0)
+ end;
+ [{integer,_}=Value,Lbl] when Op =:= select_val ->
+ %% Single value left. Convert to regular test and pop redundant tests.
+ Is1 = [{test,is_eq_exact,F,[R,Value]},{jump,Lbl}|Is],
+ case Acc0 of
+ [{test,is_integer,F,[R]}|Acc] ->
+ peep(Is1, SeenTests0, Acc);
+ _ ->
+ peep(Is1, SeenTests0, Acc0)
+ end;
+ [Arity,Lbl] when Op =:= select_tuple_arity ->
+ %% Single value left. Convert to regular test
+ Is1 = [{test,test_arity,F,[R,Arity]},{jump,Lbl}|Is],
+ peep(Is1, SeenTests0, Acc0);
[_|_]=Vls ->
I = {select,Op,R,F,Vls},
- peep(Is, gb_sets:empty(), [I|Acc])
+ peep(Is, gb_sets:empty(), [I|Acc0])
end;
peep([{test,Op,_,Ops}=I|Is], SeenTests0, Acc) ->
case beam_utils:is_pure_test(I) of
diff --git a/lib/compiler/src/beam_receive.erl b/lib/compiler/src/beam_receive.erl
index 1403e1e05e..468460eedf 100644
--- a/lib/compiler/src/beam_receive.erl
+++ b/lib/compiler/src/beam_receive.erl
@@ -207,6 +207,8 @@ opt_update_regs({label,Lbl}, R, L) ->
%% A catch label for a previously seen catch instruction is OK.
{R,L}
end;
+opt_update_regs({'try',_,{f,Lbl}}, R, L) ->
+ {R,gb_sets:add(Lbl, L)};
opt_update_regs({try_end,_}, R, L) ->
{R,L};
opt_update_regs({line,_}, R, L) ->
diff --git a/lib/compiler/src/beam_utils.erl b/lib/compiler/src/beam_utils.erl
index e39fbdc3b7..00f396c246 100644
--- a/lib/compiler/src/beam_utils.erl
+++ b/lib/compiler/src/beam_utils.erl
@@ -23,14 +23,19 @@
-module(beam_utils).
-export([is_killed_block/2,is_killed/3,is_killed_at/3,
is_not_used/3,
- empty_label_index/0,index_label/3,index_labels/1,
+ empty_label_index/0,index_label/3,index_labels/1,replace_labels/4,
code_at/2,bif_to_test/3,is_pure_test/1,
live_opt/1,delete_live_annos/1,combine_heap_needs/2,
split_even/1]).
-export_type([code_index/0,module_code/0,instruction/0]).
--import(lists, [member/2,sort/1,reverse/1,splitwith/2]).
+-import(lists, [map/2,member/2,sort/1,reverse/1,splitwith/2]).
+
+-define(is_const(Val), (element(1, Val) =:= integer orelse
+ element(1, Val) =:= float orelse
+ element(1, Val) =:= atom orelse
+ element(1, Val) =:= literal)).
%% instruction() describes all instructions that are used during optimzation
%% (from beam_a to beam_z).
@@ -160,6 +165,18 @@ index_label(Lbl, Is0, Acc) ->
code_at(L, Ll) ->
gb_trees:get(L, Ll).
+%% replace_labels(FunctionIs, Tail, ReplaceDb, Fallback) -> FunctionIs.
+%% Replace all labels in instructions according to the ReplaceDb.
+%% If label is not found the Fallback is called with the label to
+%% produce a new one.
+
+-spec replace_labels([instruction()],
+ [instruction()],
+ #{beam_asm:label() => beam_asm:label()},
+ fun((beam_asm:label()) -> term())) -> [instruction()].
+replace_labels(Is, Acc, D, Fb) ->
+ replace_labels_1(Is, Acc, D, Fb).
+
%% bif_to_test(Bif, [Op], Fail) -> {test,Test,Fail,[Op]}
%% Convert a BIF to a test. Fail if not possible.
@@ -185,10 +202,20 @@ bif_to_test('>', [A,B], Fail) -> {test,is_lt,Fail,[B,A]};
bif_to_test('<', [_,_]=Ops, Fail) -> {test,is_lt,Fail,Ops};
bif_to_test('>=', [_,_]=Ops, Fail) -> {test,is_ge,Fail,Ops};
bif_to_test('==', [A,nil], Fail) -> {test,is_nil,Fail,[A]};
+bif_to_test('==', [nil,A], Fail) -> {test,is_nil,Fail,[A]};
+bif_to_test('==', [C,A], Fail) when ?is_const(C) ->
+ {test,is_eq,Fail,[A,C]};
bif_to_test('==', [_,_]=Ops, Fail) -> {test,is_eq,Fail,Ops};
+bif_to_test('/=', [C,A], Fail) when ?is_const(C) ->
+ {test,is_ne,Fail,[A,C]};
bif_to_test('/=', [_,_]=Ops, Fail) -> {test,is_ne,Fail,Ops};
bif_to_test('=:=', [A,nil], Fail) -> {test,is_nil,Fail,[A]};
+bif_to_test('=:=', [nil,A], Fail) -> {test,is_nil,Fail,[A]};
+bif_to_test('=:=', [C,A], Fail) when ?is_const(C) ->
+ {test,is_eq_exact,Fail,[A,C]};
bif_to_test('=:=', [_,_]=Ops, Fail) -> {test,is_eq_exact,Fail,Ops};
+bif_to_test('=/=', [C,A], Fail) when ?is_const(C) ->
+ {test,is_ne_exact,Fail,[A,C]};
bif_to_test('=/=', [_,_]=Ops, Fail) -> {test,is_ne_exact,Fail,Ops};
bif_to_test(is_record, [_,_,_]=Ops, Fail) -> {test,is_record,Fail,Ops}.
@@ -643,6 +670,58 @@ index_labels_1([], Acc) -> gb_trees:from_orddict(sort(Acc)).
drop_labels([{label,_}|Is]) -> drop_labels(Is);
drop_labels(Is) -> Is.
+
+replace_labels_1([{test,Test,{f,Lbl},Ops}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{test,Test,{f,label(Lbl, D, Fb)},Ops}|Acc], D, Fb);
+replace_labels_1([{test,Test,{f,Lbl},Live,Ops,Dst}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{test,Test,{f,label(Lbl, D, Fb)},Live,Ops,Dst}|Acc], D, Fb);
+replace_labels_1([{select,I,R,{f,Fail0},Vls0}|Is], Acc, D, Fb) ->
+ Vls = map(fun ({f,L}) -> {f,label(L, D, Fb)};
+ (Other) -> Other
+ end, Vls0),
+ Fail = label(Fail0, D, Fb),
+ replace_labels_1(Is, [{select,I,R,{f,Fail},Vls}|Acc], D, Fb);
+replace_labels_1([{'try',R,{f,Lbl}}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{'try',R,{f,label(Lbl, D, Fb)}}|Acc], D, Fb);
+replace_labels_1([{'catch',R,{f,Lbl}}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{'catch',R,{f,label(Lbl, D, Fb)}}|Acc], D, Fb);
+replace_labels_1([{jump,{f,Lbl}}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{jump,{f,label(Lbl, D, Fb)}}|Acc], D, Fb);
+replace_labels_1([{loop_rec,{f,Lbl},R}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{loop_rec,{f,label(Lbl, D, Fb)},R}|Acc], D, Fb);
+replace_labels_1([{loop_rec_end,{f,Lbl}}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{loop_rec_end,{f,label(Lbl, D, Fb)}}|Acc], D, Fb);
+replace_labels_1([{wait,{f,Lbl}}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{wait,{f,label(Lbl, D, Fb)}}|Acc], D, Fb);
+replace_labels_1([{wait_timeout,{f,Lbl},To}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{wait_timeout,{f,label(Lbl, D, Fb)},To}|Acc], D, Fb);
+replace_labels_1([{bif,Name,{f,Lbl},As,R}|Is], Acc, D, Fb) when Lbl =/= 0 ->
+ replace_labels_1(Is, [{bif,Name,{f,label(Lbl, D, Fb)},As,R}|Acc], D, Fb);
+replace_labels_1([{gc_bif,Name,{f,Lbl},Live,As,R}|Is], Acc, D, Fb) when Lbl =/= 0 ->
+ replace_labels_1(Is, [{gc_bif,Name,{f,label(Lbl, D, Fb)},Live,As,R}|Acc], D, Fb);
+replace_labels_1([{call,Ar,{f,Lbl}}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{call,Ar,{f,label(Lbl, D, Fb)}}|Acc], D, Fb);
+replace_labels_1([{make_fun2,{f,Lbl},U1,U2,U3}|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [{make_fun2,{f,label(Lbl, D, Fb)},U1,U2,U3}|Acc], D, Fb);
+replace_labels_1([{bs_init,{f,Lbl},Info,Live,Ss,Dst}|Is], Acc, D, Fb) when Lbl =/= 0 ->
+ replace_labels_1(Is, [{bs_init,{f,label(Lbl, D, Fb)},Info,Live,Ss,Dst}|Acc], D, Fb);
+replace_labels_1([{bs_put,{f,Lbl},Info,Ss}|Is], Acc, D, Fb) when Lbl =/= 0 ->
+ replace_labels_1(Is, [{bs_put,{f,label(Lbl, D, Fb)},Info,Ss}|Acc], D, Fb);
+replace_labels_1([{put_map=I,{f,Lbl},Op,Src,Dst,Live,List}|Is], Acc, D, Fb)
+ when Lbl =/= 0 ->
+ replace_labels_1(Is, [{I,{f,label(Lbl, D, Fb)},Op,Src,Dst,Live,List}|Acc], D, Fb);
+replace_labels_1([{get_map_elements=I,{f,Lbl},Src,List}|Is], Acc, D, Fb) when Lbl =/= 0 ->
+ replace_labels_1(Is, [{I,{f,label(Lbl, D, Fb)},Src,List}|Acc], D, Fb);
+replace_labels_1([I|Is], Acc, D, Fb) ->
+ replace_labels_1(Is, [I|Acc], D, Fb);
+replace_labels_1([], Acc, _, _) -> Acc.
+
+label(Old, D, Fb) ->
+ case D of
+ #{Old := New} -> New;
+ _ -> Fb(Old)
+ end.
+
%% Help functions for combine_heap_needs.
combine_alloc_lists(Al1, Al2) ->
@@ -746,8 +825,14 @@ live_opt([{select,_,Src,Fail,List}=I|Is], Regs0, D, Acc) ->
Regs1 = x_live([Src], Regs0),
Regs = live_join_labels([Fail|List], D, Regs1),
live_opt(Is, Regs, D, [I|Acc]);
-live_opt([{try_case,_}=I|Is], _, D, Acc) ->
- live_opt(Is, live_call(1), D, [I|Acc]);
+live_opt([{try_case,Y}=I|Is], Regs0, D, Acc) ->
+ Regs = live_call(1),
+ case Regs0 of
+ 0 ->
+ live_opt(Is, Regs, D, [{try_end,Y}|Acc]);
+ _ ->
+ live_opt(Is, live_call(1), D, [I|Acc])
+ end;
live_opt([{loop_rec,_Fail,_Dst}=I|Is], _, D, Acc) ->
live_opt(Is, 0, D, [I|Acc]);
live_opt([timeout=I|Is], _, D, Acc) ->
diff --git a/lib/compiler/src/cerl_clauses.erl b/lib/compiler/src/cerl_clauses.erl
index 7d6518c3c6..fa5104c01b 100644
--- a/lib/compiler/src/cerl_clauses.erl
+++ b/lib/compiler/src/cerl_clauses.erl
@@ -353,6 +353,8 @@ match(P, E, Bs) ->
map ->
%% The most we can do is to say "definitely no match" if a
%% map pattern is matched against non-map data.
+ %% (Note: See the document internal_doc/cerl-notes.md for
+ %% information why we don't try to do more here.)
case E of
any ->
{false, Bs};
diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl
index aa2d224bb4..bc519264fc 100644
--- a/lib/compiler/src/compile.erl
+++ b/lib/compiler/src/compile.erl
@@ -706,14 +706,16 @@ core_passes() ->
[{unless,no_copt,
[{core_old_inliner,fun test_old_inliner/1,fun core_old_inliner/2},
{iff,doldinline,{listing,"oldinline"}},
- {pass,sys_core_fold},
+ {unless,no_fold,{pass,sys_core_fold}},
{iff,dcorefold,{listing,"corefold"}},
{core_inline_module,fun test_core_inliner/1,fun core_inline_module/2},
{iff,dinline,{listing,"inline"}},
{core_fold_after_inlining,fun test_any_inliner/1,
fun core_fold_module_after_inlining/2},
+ {iff,dcopt,{listing,"copt"}},
+ {unless,no_alias,{pass,sys_core_alias}},
+ {iff,dalias,{listing,"core_alias"}},
?pass(core_transforms)]},
- {iff,dcopt,{listing,"copt"}},
{iff,'to_core',{done,"core"}}]}
| kernel_passes()].
@@ -731,8 +733,6 @@ kernel_passes() ->
?pass(v3_kernel),
{iff,dkern,{listing,"kernel"}},
{iff,'to_kernel',{done,"kernel"}},
- {pass,v3_life},
- {iff,dlife,{listing,"life"}},
{pass,v3_codegen},
{iff,dcg,{listing,"codegen"}}
| asm_passes()].
@@ -1446,15 +1446,33 @@ save_core_code(Code, St) ->
beam_asm(Code0, #compile{ifile=File,extra_chunks=ExtraChunks,options=CompilerOpts}=St) ->
case debug_info(St) of
{ok,DebugInfo,Opts0} ->
- Source = paranoid_absname(File),
Opts1 = [O || O <- Opts0, effects_code_generation(O)],
Chunks = [{<<"Dbgi">>, DebugInfo} | ExtraChunks],
- {ok,Code} = beam_asm:module(Code0, Chunks, Source, Opts1, CompilerOpts),
+ CompileInfo = compile_info(File, Opts1),
+ {ok,Code} = beam_asm:module(Code0, Chunks, CompileInfo, CompilerOpts),
{ok,Code,St#compile{abstract_code=[]}};
{error,Es} ->
{error,St#compile{errors=St#compile.errors ++ [{File,Es}]}}
end.
+compile_info(File, Opts) ->
+ IsSlim = member(slim, Opts),
+ IsDeterministic = member(deterministic, Opts),
+ Info0 = proplists:get_value(compile_info, Opts, []),
+ Info1 =
+ case paranoid_absname(File) of
+ [_|_] = Source when not IsSlim, not IsDeterministic ->
+ [{source,Source} | Info0];
+ _ ->
+ Info0
+ end,
+ Info2 =
+ case IsDeterministic of
+ false -> [{options,proplists:delete(compile_info, Opts)} | Info1];
+ true -> Info1
+ end,
+ Info2.
+
paranoid_absname(""=File) ->
File;
paranoid_absname(File) ->
@@ -1921,12 +1939,12 @@ pre_load() ->
erl_lint,
erl_parse,
erl_scan,
+ sys_core_alias,
sys_core_bsm,
sys_core_dsetel,
sys_core_fold,
v3_codegen,
v3_core,
- v3_kernel,
- v3_life],
+ v3_kernel],
_ = code:ensure_modules_loaded(L),
ok.
diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src
index 3139d68902..cf32fd251c 100644
--- a/lib/compiler/src/compiler.app.src
+++ b/lib/compiler/src/compiler.app.src
@@ -58,6 +58,7 @@
core_lib,
erl_bifs,
rec_env,
+ sys_core_alias,
sys_core_bsm,
sys_core_dsetel,
sys_core_fold,
@@ -67,8 +68,7 @@
v3_codegen,
v3_core,
v3_kernel,
- v3_kernel_pp,
- v3_life
+ v3_kernel_pp
]},
{registered, []},
{applications, [kernel, stdlib]},
diff --git a/lib/compiler/src/core_pp.erl b/lib/compiler/src/core_pp.erl
index cff6c7098b..2516a9a1e1 100644
--- a/lib/compiler/src/core_pp.erl
+++ b/lib/compiler/src/core_pp.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -464,7 +464,7 @@ indent(#ctxt{indent=N}) ->
N =< 0 ->
"";
true ->
- string:chars($\t, N div ?TAB_WIDTH, spaces(N rem ?TAB_WIDTH))
+ lists:duplicate(N div ?TAB_WIDTH, $\t) ++ spaces(N rem ?TAB_WIDTH)
end.
nl_indent(Ctxt) -> [$\n|indent(Ctxt)].
diff --git a/lib/compiler/src/core_scan.erl b/lib/compiler/src/core_scan.erl
index 9f0676538f..a50a2ffa8d 100644
--- a/lib/compiler/src/core_scan.erl
+++ b/lib/compiler/src/core_scan.erl
@@ -200,8 +200,8 @@ pre_string(eof, Q, _, Sp, SoFar, Pos) ->
pre_string_error(Q, Sp, SoFar, Pos).
pre_string_error(Q, Sp, SoFar, Pos) ->
- S = reverse(string:substr(SoFar, 1, string:chr(SoFar, Q)-1)),
- pre_error({string,Q,string:substr(S, 1, 16)}, Sp, Pos).
+ [S,_] = string:split(SoFar, [Q]),
+ pre_error({string,Q,string:slice(string:reverse(S), 0, 16)}, Sp, Pos).
pre_char([C|Cs], SoFar) -> pre_char(C, Cs, SoFar);
pre_char([], _) -> more;
diff --git a/lib/compiler/src/sys_core_alias.erl b/lib/compiler/src/sys_core_alias.erl
new file mode 100644
index 0000000000..63e2f7488e
--- /dev/null
+++ b/lib/compiler/src/sys_core_alias.erl
@@ -0,0 +1,308 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+%% Purpose : Replace values by aliases from patterns optimisation for Core
+
+%% Replace expressions by aliases from patterns. For example:
+%%
+%% example({ok, Val}) ->
+%% {ok, Val}.
+%%
+%% will become:
+%%
+%% example({ok, Val} = Tuple) ->
+%% Tuple.
+%%
+%% Currently this pass aliases tuple and cons nodes made of literals,
+%% variables and other cons. The tuple/cons may appear anywhere in the
+%% pattern and it will be aliased if used later on.
+%%
+%% Notice a tuple/cons made only of literals is not aliased as it may
+%% be part of the literal pool.
+
+-module(sys_core_alias).
+
+-export([module/2]).
+
+-include("core_parse.hrl").
+
+-define(NOTSET, 0).
+
+-record(sub, {p=#{} :: #{term() => ?NOTSET | atom()}, %% Found pattern substitutions
+ v=cerl_sets:new() :: cerl_sets:set(cerl:var_name()), %% Variables used by patterns
+ t=undefined :: term()}). %% Temporary information from pre to post
+
+-type sub() :: #sub{}.
+
+-spec module(cerl:c_module(), [compile:option()]) ->
+ {'ok',cerl:c_module(),[]}.
+
+module(#c_module{defs=Ds0}=Mod, _Opts) ->
+ Ds1 = [def(D) || D <- Ds0],
+ {ok,Mod#c_module{defs=Ds1},[]}.
+
+def({#c_var{name={F,Arity}}=Name,B0}) ->
+ try
+ put(new_var_num, 0),
+ {B1,_} = cerl_trees:mapfold(fun pre/2, fun post/2, sub_new(undefined), B0),
+ erase(new_var_num),
+ {Name,B1}
+ catch
+ Class:Error ->
+ Stack = erlang:get_stacktrace(),
+ io:fwrite("Function: ~w/~w\n", [F,Arity]),
+ erlang:raise(Class, Error, Stack)
+ end.
+
+pre(#c_let{vars=Vars}=Node, Sub) ->
+ {Node,sub_fold(get_variables(Vars), Sub)};
+
+pre(#c_fun{vars=Vars}=Node, Sub) ->
+ {Node,sub_fold(get_variables(Vars), Sub)};
+
+pre(#c_clause{pats=Pats}=Node, Sub0) ->
+ VarNames = get_variables(Pats),
+ Sub1 = sub_fold(VarNames, Sub0),
+ Keys = get_pattern_keys(Pats),
+ Sub2 = sub_add_keys(Keys, Sub1),
+
+ #sub{v=SubNames,t=Temp} = Sub2,
+ Sub3 = Sub2#sub{v=merge_variables(VarNames, SubNames),
+ t={clause,Pats,Keys,SubNames,Temp}},
+
+ {Node#c_clause{pats=[]},Sub3};
+
+pre(Node, Sub0) ->
+ %% We cache only tuples and cons.
+ case cerl:is_data(Node) andalso not cerl:is_literal(Node) of
+ false ->
+ {Node,Sub0};
+ true ->
+ Kind = cerl:data_type(Node),
+ Es = cerl:data_es(Node),
+ case sub_cache_nodes(Kind, Es, Sub0) of
+ {Name,Sub1} ->
+ {cerl:ann_c_var(cerl:get_ann(Node), Name),Sub1};
+ error ->
+ {Node,Sub0}
+ end
+ end.
+
+post(#c_let{}=Node, Sub) ->
+ {Node,sub_unfold(Sub)};
+
+post(#c_fun{}=Node, Sub) ->
+ {Node,sub_unfold(Sub)};
+
+post(#c_clause{}=Node, #sub{t={clause,Pats0,Keys,V,T}}=Sub0) ->
+ {Sub1,PostKeys} = sub_take_keys(Keys, Sub0),
+ Pats1 = put_pattern_keys(Pats0, PostKeys),
+ Sub2 = sub_unfold(Sub1#sub{v=V,t=T}),
+ {Node#c_clause{pats=Pats1},Sub2};
+
+post(Node, Sub) ->
+ {Node,Sub}.
+
+%% sub_new/1
+%% sub_add_keys/2
+%% sub_take_keys/3
+%% sub_cache_nodes/3
+%%
+%% Manages the substitutions record.
+
+%% Builds a new sub.
+-spec sub_new(term()) -> sub().
+sub_new(Temp) ->
+ #sub{t=Temp}.
+
+%% Folds the sub into a new one if the variables in nodes are not disjoint
+sub_fold(VarNames, #sub{v=SubNames}=Sub) ->
+ case is_disjoint_variables(VarNames, SubNames) of
+ true -> Sub#sub{t={temp,Sub#sub.t}};
+ false -> sub_new({sub,Sub})
+ end.
+
+%% Unfolds the sub in case one was folded in the previous step
+sub_unfold(#sub{t={temp,Temp}}=Sub) ->
+ Sub#sub{t=Temp};
+sub_unfold(#sub{t={sub,Sub}}) ->
+ Sub.
+
+%% Adds the keys extracted from patterns to the state.
+-spec sub_add_keys([term()], sub()) -> sub().
+sub_add_keys(Keys, #sub{p=Pat0}=Sub) ->
+ Pat1 =
+ lists:foldl(fun(Key, Acc) ->
+ false = maps:is_key(Key, Acc), %Assertion.
+ maps:put(Key, ?NOTSET, Acc)
+ end, Pat0, Keys),
+ Sub#sub{p=Pat1}.
+
+%% Take the keys from the map taking into account the keys
+%% that have changed as those must become aliases in the pattern.
+-spec sub_take_keys([term()], sub()) -> {sub(), [{term(), atom()}]}.
+sub_take_keys(Keys, #sub{p=Pat0}=Sub) ->
+ {Pat1,Acc} = sub_take_keys(Keys, Pat0, []),
+ {Sub#sub{p=Pat1},Acc}.
+
+sub_take_keys([K|T], Sub0, Acc) ->
+ case maps:take(K, Sub0) of
+ {?NOTSET,Sub1} ->
+ sub_take_keys(T, Sub1, Acc);
+ {Name,Sub1} ->
+ sub_take_keys(T, Sub1, [{K,Name}|Acc])
+ end;
+sub_take_keys([], Sub, Acc) ->
+ {Sub,Acc}.
+
+%% Check if the node can be cached based on the state information.
+%% If it can be cached and it does not have an alias for it, we
+%% build one.
+-spec sub_cache_nodes(atom(), [cerl:cerl()], sub()) -> {atom(), sub()} | error.
+sub_cache_nodes(Kind, Nodes, #sub{p=Pat}=Sub) ->
+ case nodes_to_key(Kind, Nodes) of
+ {ok, Key} ->
+ case Pat of
+ #{Key := ?NOTSET} ->
+ new_var_name(Key, Sub);
+ #{Key := Name} ->
+ {Name,Sub};
+ #{} ->
+ error
+ end;
+ error ->
+ error
+ end.
+
+new_var_name(Key, #sub{p=Pat}=Sub) ->
+ Counter = get(new_var_num),
+ Name = list_to_atom("@r" ++ integer_to_list(Counter)),
+ put(new_var_num, Counter + 1),
+ {Name,Sub#sub{p=maps:put(Key, Name, Pat)}}.
+
+%% get_variables/1
+%% is_disjoint_variables/2
+%% merge_variables/2
+
+get_variables(NodesList) ->
+ cerl_sets:from_list([Var || Node <- NodesList, Var <- cerl_trees:variables(Node)]).
+
+is_disjoint_variables(Vars1, Vars2) ->
+ cerl_sets:is_disjoint(Vars1, Vars2).
+
+merge_variables(Vars1, Vars2) ->
+ cerl_sets:union(Vars1, Vars2).
+
+%% get_pattern_keys/2
+%% put_pattern_keys/2
+%%
+%% Gets keys from patterns or add them as aliases.
+
+get_pattern_keys(Patterns) ->
+ lists:foldl(fun get_pattern_keys/2, [], Patterns).
+
+get_pattern_keys(#c_tuple{es=Es}, Acc0) ->
+ Acc1 = accumulate_pattern_keys(tuple, Es, Acc0),
+ lists:foldl(fun get_pattern_keys/2, Acc1, Es);
+get_pattern_keys(#c_cons{hd=Hd,tl=Tl}, Acc0) ->
+ Acc1 = accumulate_pattern_keys(cons, [Hd, Tl], Acc0),
+ get_pattern_keys(Tl, get_pattern_keys(Hd, Acc1));
+get_pattern_keys(#c_alias{pat=Pat}, Acc0) ->
+ get_pattern_keys(Pat, Acc0);
+get_pattern_keys(#c_map{es=Es}, Acc0) ->
+ lists:foldl(fun get_pattern_keys/2, Acc0, Es);
+get_pattern_keys(#c_map_pair{val=Val}, Acc0) ->
+ get_pattern_keys(Val, Acc0);
+get_pattern_keys(_, Acc) ->
+ Acc.
+
+accumulate_pattern_keys(Kind, Nodes, Acc) ->
+ case nodes_to_key(Kind, Nodes) of
+ {ok,Key} -> [Key|Acc];
+ error -> Acc
+ end.
+
+put_pattern_keys(Patterns, []) ->
+ Patterns;
+put_pattern_keys(Patterns, Keys) ->
+ {NewPatterns,Map} =
+ lists:mapfoldl(fun alias_pattern_keys/2, maps:from_list(Keys), Patterns),
+ %% Check all aliases have been consumed from the map.
+ 0 = map_size(Map),
+ NewPatterns.
+
+alias_pattern_keys(#c_tuple{anno=Anno,es=Es0}=Node, Acc0) ->
+ {Es1,Acc1} = lists:mapfoldl(fun alias_pattern_keys/2, Acc0, Es0),
+ nodes_to_alias(tuple, Es0, Anno, Node#c_tuple{es=Es1}, Acc1);
+alias_pattern_keys(#c_cons{anno=Anno,hd=Hd0,tl=Tl0}=Node, Acc0) ->
+ {Hd1,Acc1} = alias_pattern_keys(Hd0, Acc0),
+ {Tl1,Acc2} = alias_pattern_keys(Tl0, Acc1),
+ nodes_to_alias(cons, [Hd0, Tl0], Anno, Node#c_cons{hd=Hd1,tl=Tl1}, Acc2);
+alias_pattern_keys(#c_alias{pat=Pat0}=Node, Acc0) ->
+ {Pat1,Acc1} = alias_pattern_keys(Pat0, Acc0),
+ {Node#c_alias{pat=Pat1}, Acc1};
+alias_pattern_keys(#c_map{es=Es0}=Node, Acc0) ->
+ {Es1,Acc1} = lists:mapfoldl(fun alias_pattern_keys/2, Acc0, Es0),
+ {Node#c_map{es=Es1}, Acc1};
+alias_pattern_keys(#c_map_pair{val=Val0}=Node, Acc0) ->
+ {Val1,Acc1} = alias_pattern_keys(Val0, Acc0),
+ {Node#c_map_pair{val=Val1}, Acc1};
+alias_pattern_keys(Pattern, Acc) ->
+ {Pattern,Acc}.
+
+%% Check if a node must become an alias because
+%% its pattern was used later on as an expression.
+nodes_to_alias(Kind, Inner, Anno, Node, Keys0) ->
+ case nodes_to_key(Kind, Inner) of
+ {ok,Key} ->
+ case maps:take(Key, Keys0) of
+ {Name,Keys1} ->
+ Var = cerl:ann_c_var(Anno, Name),
+ {cerl:ann_c_alias(Anno, Var, Node), Keys1};
+ error ->
+ {Node,Keys0}
+ end;
+ error ->
+ {Node,Keys0}
+ end.
+
+%% Builds the key used to check if a value can be
+%% replaced by an alias. It considers literals,
+%% aliases, variables, tuples and cons recursively.
+nodes_to_key(Kind, Nodes) ->
+ nodes_to_key(Nodes, [], Kind).
+
+nodes_to_key([#c_alias{var=Var}|T], Acc, Kind) ->
+ nodes_to_key([Var|T], Acc, Kind);
+nodes_to_key([#c_var{name=Name}|T], Acc, Kind) ->
+ nodes_to_key(T, [[var,Name]|Acc], Kind);
+nodes_to_key([Node|T], Acc0, Kind) ->
+ case cerl:is_data(Node) of
+ false ->
+ error;
+ true ->
+ case nodes_to_key(cerl:data_es(Node), [], cerl:data_type(Node)) of
+ {ok,Key} ->
+ nodes_to_key(T, [Key|Acc0], Kind);
+ error ->
+ error
+ end
+ end;
+nodes_to_key([], Acc, Kind) ->
+ {ok,[Kind|Acc]}.
diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl
index d73060fb7e..df880ff784 100644
--- a/lib/compiler/src/sys_core_fold.erl
+++ b/lib/compiler/src/sys_core_fold.erl
@@ -418,6 +418,15 @@ expr(#c_call{module=M0,name=N0}=Call0, Ctxt, Sub) ->
expr(#c_primop{args=As0}=Prim, _, Sub) ->
As1 = expr_list(As0, value, Sub),
Prim#c_primop{args=As1};
+expr(#c_catch{anno=Anno,body=B}, effect, Sub) ->
+ %% When the return value of the 'catch' is ignored, we can replace it
+ %% with a try/catch to avoid building a stack trace when an exception
+ %% occurs.
+ Var = #c_var{name='catch_value'},
+ Evs = [#c_var{name='Class'},#c_var{name='Reason'},#c_var{name='Stk'}],
+ Try = #c_try{anno=Anno,arg=B,vars=[Var],body=Var,
+ evars=Evs,handler=void()},
+ expr(Try, effect, Sub);
expr(#c_catch{body=B0}=Catch, _, Sub) ->
%% We can remove catch if the value is simple
B1 = body(B0, value, Sub),
@@ -2422,16 +2431,10 @@ move_let_into_expr(#c_let{vars=InnerVs0,body=InnerBody0}=Inner,
Outer#c_let{vars=OuterVs,arg=Arg,
body=Inner#c_let{vars=InnerVs,arg=OuterBody,body=InnerBody}};
move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let,
- #c_case{arg=Cexpr0,clauses=[Ca0,Cb0|Cs]}=Case, Sub0) ->
- %% Test if there are no more clauses than Ca0 and Cb0, or if
- %% Cb0 is guaranteed to match.
- TwoClauses = Cs =:= [] orelse
- case Cb0 of
- #c_clause{pats=[#c_var{}],guard=#c_literal{val=true}} -> true;
- _ -> false
- end,
- case {TwoClauses,is_failing_clause(Ca0),is_failing_clause(Cb0)} of
- {true,false,true} ->
+ #c_case{arg=Cexpr0,clauses=[Ca0|Cs0]}=Case, Sub0) ->
+ case not is_failing_clause(Ca0) andalso
+ are_all_failing_clauses(Cs0) of
+ true ->
%% let <Lvars> = case <Case-expr> of
%% <Cpats> -> <Clause-body>;
%% <OtherCpats> -> erlang:error(...)
@@ -2467,8 +2470,8 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let,
body=Lbody},
Ca = Ca0#c_clause{pats=CaPats,guard=G,body=B},
- Cb = clause(Cb0, Cexpr, value, Sub0),
- Case#c_case{arg=Cexpr,clauses=[Ca,Cb]}
+ Cs = [clause(C, Cexpr, value, Sub0) || C <- Cs0],
+ Case#c_case{arg=Cexpr,clauses=[Ca|Cs]}
catch
nomatch ->
%% This is not a defeat. The code will eventually
@@ -2476,7 +2479,7 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let,
%% optimizations done in this module.
impossible
end;
- {_,_,_} -> impossible
+ false -> impossible
end;
move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let,
#c_seq{arg=Sarg0,body=Sbody0}=Seq, Sub0) ->
@@ -2499,6 +2502,9 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let,
body=Lbody}};
move_let_into_expr(_Let, _Expr, _Sub) -> impossible.
+are_all_failing_clauses(Cs) ->
+ all(fun is_failing_clause/1, Cs).
+
is_failing_clause(#c_clause{body=B}) ->
will_fail(B).
diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl
index 47c1567f10..006a6a82d2 100644
--- a/lib/compiler/src/v3_codegen.erl
+++ b/lib/compiler/src/v3_codegen.erl
@@ -19,25 +19,6 @@
%%
%% Purpose : Code generator for Beam.
-%% The following assumptions have been made:
-%%
-%% 1. Matches, i.e. things with {match,M,Ret} wrappers, only return
-%% values; no variables are exported. If the match would have returned
-%% extra variables then these have been transformed to multiple return
-%% values.
-%%
-%% 2. All BIF's called in guards are gc-safe so there is no need to
-%% put thing on the stack in the guard. While this would in principle
-%% work it would be difficult to keep track of the stack depth when
-%% trimming.
-%%
-%% The code generation uses variable lifetime information added by
-%% the v3_life module to save variables, allocate registers and
-%% move registers to the stack when necessary.
-%%
-%% We try to use a consistent variable name scheme throughout. The
-%% StackReg record is always called Bef,Int<n>,Aft.
-
-module(v3_codegen).
%% The main interface.
@@ -45,12 +26,14 @@
-import(lists, [member/2,keymember/3,keysort/2,keydelete/3,
append/1,flatmap/2,filter/2,foldl/3,foldr/3,mapfoldl/3,
- sort/1,reverse/1,reverse/2]).
--import(v3_life, [vdb_find/2]).
+ sort/1,reverse/1,reverse/2,map/2]).
+-import(ordsets, [add_element/2,intersection/2,union/2]).
-%%-compile([export_all]).
+-include("v3_kernel.hrl").
--include("v3_life.hrl").
+%% These are not defined in v3_kernel.hrl.
+get_kanno(Kthing) -> element(2, Kthing).
+set_kanno(Kthing, Anno) -> setelement(2, Kthing, Anno).
%% Main codegen structure.
-record(cg, {lcount=1, %Label counter
@@ -61,38 +44,273 @@
functable=#{}, %Map of local functions: {Name,Arity}=>Label
in_catch=false, %Inside a catch or not.
need_frame, %Need a stack frame.
- ultimate_failure %Label for ultimate match failure.
- }).
+ ultimate_failure, %Label for ultimate match failure.
+ ctx %Match context.
+ }).
%% Stack/register state record.
-record(sr, {reg=[], %Register table
stk=[], %Stack table
res=[]}). %Reserved regs: [{reserved,I,V}]
--type life_module() :: {module(),_,_,[_]}.
+%% Internal records.
+-record(cg_need_heap, {anno=[] :: term(),
+ h=0 :: integer()}).
+-record(cg_block, {anno=[] :: term(),
+ es=[] :: [term()]}).
--spec module(life_module(), [compile:option()]) -> {'ok',beam_asm:module_code()}.
+-type vdb_entry() :: {atom(),non_neg_integer(),non_neg_integer()}.
-module({Mod,Exp,Attr,Forms}, _Options) ->
- {Fs,St} = functions(Forms, {atom,Mod}),
- {ok,{Mod,Exp,Attr,Fs,St#cg.lcount}}.
+-record(l, {i=0 :: non_neg_integer(), %Op number
+ vdb=[] :: [vdb_entry()], %Variable database
+ a=[] :: [term()]}). %Core annotation
+
+-spec module(#k_mdef{}, [compile:option()]) -> {'ok',beam_asm:module_code()}.
+
+module(#k_mdef{name=Mod,exports=Es,attributes=Attr,body=Forms}, _Opts) ->
+ {Asm,St} = functions(Forms, {atom,Mod}),
+ {ok,{Mod,Es,Attr,Asm,St#cg.lcount}}.
functions(Forms, AtomMod) ->
mapfoldl(fun (F, St) -> function(F, AtomMod, St) end, #cg{lcount=1}, Forms).
-function({function,Name,Arity,Asm0,Vb,Vdb,Anno}, AtomMod, St0) ->
+function(#k_fdef{anno=#k{a=Anno},func=Name,arity=Arity,
+ vars=As,body=Kb}, AtomMod, St0) ->
try
- {Asm,EntryLabel,St} = cg_fun(Vb, Asm0, Vdb, AtomMod,
- {Name,Arity}, Anno, St0),
- Func = {function,Name,Arity,EntryLabel,Asm},
- {Func,St}
+ %% Annotate kernel records with variable usage.
+ #k_match{} = Kb, %Assertion.
+ Vdb0 = init_vars(As),
+ {Body,_,Vdb} = body(Kb, 1, Vdb0),
+
+ %% Generate the BEAM assembly code.
+ {Asm,EntryLabel,St} = cg_fun(Body, As, Vdb, AtomMod,
+ {Name,Arity}, Anno, St0),
+ Func = {function,Name,Arity,EntryLabel,Asm},
+ {Func,St}
catch
- Class:Error ->
- Stack = erlang:get_stacktrace(),
- io:fwrite("Function: ~w/~w\n", [Name,Arity]),
- erlang:raise(Class, Error, Stack)
+ Class:Error ->
+ Stack = erlang:get_stacktrace(),
+ io:fwrite("Function: ~w/~w\n", [Name,Arity]),
+ erlang:raise(Class, Error, Stack)
end.
+%% This pass creates beam format annotated with variable lifetime
+%% information. Each thing is given an index and for each variable we
+%% store the first and last index for its occurrence. The variable
+%% database, VDB, attached to each thing is only relevant internally
+%% for that thing.
+%%
+%% For nested things like matches the numbering continues locally and
+%% the VDB for that thing refers to the variable usage within that
+%% thing. Variables which live through a such a thing are internally
+%% given a very large last index. Internally the indexes continue
+%% after the index of that thing. This creates no problems as the
+%% internal variable info never escapes and externally we only see
+%% variable which are alive both before or after.
+%%
+%% This means that variables never "escape" from a thing and the only
+%% way to get values from a thing is to "return" them, with 'break' or
+%% 'return'. Externally these values become the return values of the
+%% thing. This is no real limitation as most nested things have
+%% multiple threads so working out a common best variable usage is
+%% difficult.
+
+%% body(Kbody, I, Vdb) -> {[Expr],MaxI,Vdb}.
+%% Handle a body.
+
+body(#k_seq{arg=Ke,body=Kb}, I, Vdb0) ->
+ %%ok = io:fwrite("life ~w:~p~n", [?LINE,{Ke,I,Vdb0}]),
+ A = get_kanno(Ke),
+ Vdb1 = use_vars(union(A#k.us, A#k.ns), I, Vdb0),
+ {Es,MaxI,Vdb2} = body(Kb, I+1, Vdb1),
+ E = expr(Ke, I, Vdb2),
+ {[E|Es],MaxI,Vdb2};
+body(Ke, I, Vdb0) ->
+ %%ok = io:fwrite("life ~w:~p~n", [?LINE,{Ke,I,Vdb0}]),
+ A = get_kanno(Ke),
+ Vdb1 = use_vars(union(A#k.us, A#k.ns), I, Vdb0),
+ E = expr(Ke, I, Vdb1),
+ {[E],I,Vdb1}.
+
+%% expr(Kexpr, I, Vdb) -> Expr.
+
+expr(#k_test{anno=A}=Test, I, _Vdb) ->
+ Test#k_test{anno=#l{i=I,a=A#k.a}};
+expr(#k_call{anno=A}=Call, I, _Vdb) ->
+ Call#k_call{anno=#l{i=I,a=A#k.a}};
+expr(#k_enter{anno=A}=Enter, I, _Vdb) ->
+ Enter#k_enter{anno=#l{i=I,a=A#k.a}};
+expr(#k_bif{anno=A}=Bif, I, _Vdb) ->
+ Bif#k_bif{anno=#l{i=I,a=A#k.a}};
+expr(#k_match{anno=A,body=Kb,ret=Rs}, I, Vdb) ->
+ %% Work out imported variables which need to be locked.
+ Mdb = vdb_sub(I, I+1, Vdb),
+ M = match(Kb, A#k.us, I+1, Mdb),
+ L = #l{i=I,vdb=use_vars(A#k.us, I+1, Mdb),a=A#k.a},
+ #k_match{anno=L,body=M,ret=Rs};
+expr(#k_guard_match{anno=A,body=Kb,ret=Rs}, I, Vdb) ->
+ %% Work out imported variables which need to be locked.
+ Mdb = vdb_sub(I, I+1, Vdb),
+ M = match(Kb, A#k.us, I+1, Mdb),
+ L = #l{i=I,vdb=use_vars(A#k.us, I+1, Mdb),a=A#k.a},
+ #k_guard_match{anno=L,body=M,ret=Rs};
+expr(#k_protected{}=Protected, I, Vdb) ->
+ protected(Protected, I, Vdb);
+expr(#k_try{anno=A,arg=Ka,vars=Vs,body=Kb,evars=Evs,handler=Kh}=Try, I, Vdb) ->
+ %% Lock variables that are alive before the catch and used afterwards.
+ %% Don't lock variables that are only used inside the try.
+ Tdb0 = vdb_sub(I, I+1, Vdb),
+ %% This is the tricky bit. Lock variables in Arg that are used in
+ %% the body and handler. Add try tag 'variable'.
+ Ab = get_kanno(Kb),
+ Ah = get_kanno(Kh),
+ Tdb1 = use_vars(union(Ab#k.us, Ah#k.us), I+3, Tdb0),
+ Tdb2 = vdb_sub(I, I+2, Tdb1),
+ Vnames = fun (Kvar) -> Kvar#k_var.name end, %Get the variable names
+ {Aes,_,Adb} = body(Ka, I+2, add_var({catch_tag,I+1}, I+1, locked, Tdb2)),
+ {Bes,_,Bdb} = body(Kb, I+4, new_vars(sort(map(Vnames, Vs)), I+3, Tdb2)),
+ {Hes,_,Hdb} = body(Kh, I+4, new_vars(sort(map(Vnames, Evs)), I+3, Tdb2)),
+ L = #l{i=I,vdb=Tdb1,a=A#k.a},
+ Try#k_try{anno=L,
+ arg=#cg_block{es=Aes,anno=#l{i=I+1,vdb=Adb,a=[]}},
+ vars=Vs,body=#cg_block{es=Bes,anno=#l{i=I+3,vdb=Bdb,a=[]}},
+ evars=Evs,handler=#cg_block{es=Hes,anno=#l{i=I+3,vdb=Hdb,a=[]}}};
+expr(#k_try_enter{anno=A,arg=Ka,vars=Vs,body=Kb,evars=Evs,handler=Kh}, I, Vdb) ->
+ %% Lock variables that are alive before the catch and used afterwards.
+ %% Don't lock variables that are only used inside the try.
+ Tdb0 = vdb_sub(I, I+1, Vdb),
+ %% This is the tricky bit. Lock variables in Arg that are used in
+ %% the body and handler. Add try tag 'variable'.
+ Ab = get_kanno(Kb),
+ Ah = get_kanno(Kh),
+ Tdb1 = use_vars(union(Ab#k.us, Ah#k.us), I+3, Tdb0),
+ Tdb2 = vdb_sub(I, I+2, Tdb1),
+ Vnames = fun (Kvar) -> Kvar#k_var.name end, %Get the variable names
+ {Aes,_,Adb} = body(Ka, I+2, add_var({catch_tag,I+1}, I+1, 1000000, Tdb2)),
+ {Bes,_,Bdb} = body(Kb, I+4, new_vars(sort(map(Vnames, Vs)), I+3, Tdb2)),
+ {Hes,_,Hdb} = body(Kh, I+4, new_vars(sort(map(Vnames, Evs)), I+3, Tdb2)),
+ L = #l{i=I,vdb=Tdb1,a=A#k.a},
+ #k_try_enter{anno=L,
+ arg=#cg_block{es=Aes,anno=#l{i=I+1,vdb=Adb,a=[]}},
+ vars=Vs,body=#cg_block{es=Bes,anno=#l{i=I+3,vdb=Bdb,a=[]}},
+ evars=Evs,handler=#cg_block{es=Hes,anno=#l{i=I+3,vdb=Hdb,a=[]}}};
+expr(#k_catch{anno=A,body=Kb}=Catch, I, Vdb) ->
+ %% Lock variables that are alive before the catch and used afterwards.
+ %% Don't lock variables that are only used inside the catch.
+ %% Add catch tag 'variable'.
+ Cdb0 = vdb_sub(I, I+1, Vdb),
+ {Es,_,Cdb1} = body(Kb, I+1, add_var({catch_tag,I}, I, locked, Cdb0)),
+ L = #l{i=I,vdb=Cdb1,a=A#k.a},
+ Catch#k_catch{anno=L,body=#cg_block{es=Es}};
+expr(#k_receive{anno=A,var=V,body=Kb,action=Ka}=Recv, I, Vdb) ->
+ %% Work out imported variables which need to be locked.
+ Rdb = vdb_sub(I, I+1, Vdb),
+ M = match(Kb, add_element(V#k_var.name, A#k.us), I+1,
+ new_vars([V#k_var.name], I, Rdb)),
+ {Tes,_,Adb} = body(Ka, I+1, Rdb),
+ Le = #l{i=I,vdb=use_vars(A#k.us, I+1, Vdb),a=A#k.a},
+ Recv#k_receive{anno=Le,body=M,
+ action=#cg_block{anno=#l{i=I+1,vdb=Adb,a=[]},es=Tes}};
+expr(#k_receive_accept{anno=A}, I, _Vdb) ->
+ #k_receive_accept{anno=#l{i=I,a=A#k.a}};
+expr(#k_receive_next{anno=A}, I, _Vdb) ->
+ #k_receive_next{anno=#l{i=I,a=A#k.a}};
+expr(#k_put{anno=A}=Put, I, _Vdb) ->
+ Put#k_put{anno=#l{i=I,a=A#k.a}};
+expr(#k_break{anno=A}=Break, I, _Vdb) ->
+ Break#k_break{anno=#l{i=I,a=A#k.a}};
+expr(#k_guard_break{anno=A}=Break, I, Vdb) ->
+ Locked = [V || {V,_,_} <- Vdb],
+ L = #l{i=I,a=A#k.a},
+ Break#k_guard_break{anno=L,locked=Locked};
+expr(#k_return{anno=A}=Ret, I, _Vdb) ->
+ Ret#k_return{anno=#l{i=I,a=A#k.a}}.
+
+%% protected(Kprotected, I, Vdb) -> Protected.
+%% Only used in guards.
+
+protected(#k_protected{anno=A,arg=Ts}=Prot, I, Vdb) ->
+ %% Lock variables that are alive before try and used afterwards.
+ %% Don't lock variables that are only used inside the protected
+ %% expression.
+ Pdb0 = vdb_sub(I, I+1, Vdb),
+ {T,MaxI,Pdb1} = body(Ts, I+1, Pdb0),
+ Pdb2 = use_vars(A#k.ns, MaxI+1, Pdb1), %Save "return" values
+ Prot#k_protected{arg=T,anno=#l{i=I,a=A#k.a,vdb=Pdb2}}.
+
+%% match(Kexpr, [LockVar], I, Vdb) -> Expr.
+%% Convert match tree to old format.
+
+match(#k_alt{anno=A,first=Kf,then=Kt}, Ls, I, Vdb0) ->
+ Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0),
+ F = match(Kf, Ls, I+1, Vdb1),
+ T = match(Kt, Ls, I+1, Vdb1),
+ #k_alt{anno=[],first=F,then=T};
+match(#k_select{anno=A,var=V,types=Kts}=Select, Ls0, I, Vdb0) ->
+ Vanno = get_kanno(V),
+ Ls1 = case member(no_usage, Vanno) of
+ false -> add_element(V#k_var.name, Ls0);
+ true -> Ls0
+ end,
+ Vdb1 = use_vars(union(A#k.us, Ls1), I, Vdb0),
+ Ts = [type_clause(Tc, Ls1, I+1, Vdb1) || Tc <- Kts],
+ Select#k_select{anno=[],types=Ts};
+match(#k_guard{anno=A,clauses=Kcs}, Ls, I, Vdb0) ->
+ Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0),
+ Cs = [guard_clause(G, Ls, I+1, Vdb1) || G <- Kcs],
+ #k_guard{anno=[],clauses=Cs};
+match(Other, Ls, I, Vdb0) ->
+ Vdb1 = use_vars(Ls, I, Vdb0),
+ {B,_,Vdb2} = body(Other, I+1, Vdb1),
+ Le = #l{i=I,vdb=Vdb2,a=[]},
+ #cg_block{anno=Le,es=B}.
+
+type_clause(#k_type_clause{anno=A,type=T,values=Kvs}, Ls, I, Vdb0) ->
+ %%ok = io:format("life ~w: ~p~n", [?LINE,{T,Kvs}]),
+ Vdb1 = use_vars(union(A#k.us, Ls), I+1, Vdb0),
+ Vs = [val_clause(Vc, Ls, I+1, Vdb1) || Vc <- Kvs],
+ #k_type_clause{anno=[],type=T,values=Vs}.
+
+val_clause(#k_val_clause{anno=A,val=V,body=Kb}, Ls0, I, Vdb0) ->
+ New = (get_kanno(V))#k.ns,
+ Bus = (get_kanno(Kb))#k.us,
+ %%ok = io:format("Ls0 = ~p, Used=~p\n New=~p, Bus=~p\n", [Ls0,Used,New,Bus]),
+ Ls1 = union(intersection(New, Bus), Ls0), %Lock for safety
+ Vdb1 = use_vars(union(A#k.us, Ls1), I+1, new_vars(New, I, Vdb0)),
+ B = match(Kb, Ls1, I+1, Vdb1),
+ Le = #l{i=I,vdb=use_vars(Bus, I+1, Vdb1),a=A#k.a},
+ #k_val_clause{anno=Le,val=V,body=B}.
+
+guard_clause(#k_guard_clause{anno=A,guard=Kg,body=Kb}, Ls, I, Vdb0) ->
+ Vdb1 = use_vars(union(A#k.us, Ls), I+2, Vdb0),
+ Gdb = vdb_sub(I+1, I+2, Vdb1),
+ G = protected(Kg, I+1, Gdb),
+ B = match(Kb, Ls, I+2, Vdb1),
+ Le = #l{i=I,vdb=use_vars((get_kanno(Kg))#k.us, I+2, Vdb1),a=A#k.a},
+ #k_guard_clause{anno=Le,guard=G,body=B}.
+
+
+%% Here follows the code generator pass.
+%%
+%% The following assumptions have been made:
+%%
+%% 1. Matches, i.e. things with {match,M,Ret} wrappers, only return
+%% values; no variables are exported. If the match would have returned
+%% extra variables then these have been transformed to multiple return
+%% values.
+%%
+%% 2. All BIF's called in guards are gc-safe so there is no need to
+%% put thing on the stack in the guard. While this would in principle
+%% work it would be difficult to keep track of the stack depth when
+%% trimming.
+%%
+%% The code generation uses variable lifetime information added by
+%% the previous pass to save variables, allocate registers and
+%% move registers to the stack when necessary.
+%%
+%% We try to use a consistent variable name scheme throughout. The
+%% StackReg record is always called Bef,Int<n>,Aft.
+
%% cg_fun([Lkexpr], [HeadVar], Vdb, State) -> {[Ainstr],State}
cg_fun(Les, Hvs, Vdb, AtomMod, NameArity, Anno, St0) ->
@@ -114,14 +332,14 @@ cg_fun(Les, Hvs, Vdb, AtomMod, NameArity, Anno, St0) ->
%% Note that and 'if_end' instruction does not need any
%% live x registers, so it will always be safe to jump to
%% it. (We never ever expect the jump to be taken, and in
- %% must functions there will never be any references to
+ %% most functions there will never be any references to
%% the label in the first place.)
%%
{UltimateMatchFail,St3} = new_label(St2),
%% Create initial stack/register state, clear unused arguments.
- Bef = clear_dead(#sr{reg=foldl(fun ({var,V}, Reg) ->
+ Bef = clear_dead(#sr{reg=foldl(fun (#k_var{name=V}, Reg) ->
put_reg(V, Reg)
end, [], Hvs),
stk=[]}, 0, Vdb),
@@ -136,45 +354,43 @@ cg_fun(Les, Hvs, Vdb, AtomMod, NameArity, Anno, St0) ->
%% cg(Lkexpr, Vdb, StackReg, State) -> {[Ainstr],StackReg,State}.
%% Generate code for a kexpr.
-%% Split function into two steps for clarity, not efficiency.
-
-cg(Le, Vdb, Bef, St) ->
- cg(Le#l.ke, Le, Vdb, Bef, St).
-cg({block,Es}, Le, Vdb, Bef, St) ->
+cg(#cg_block{anno=Le,es=Es}, Vdb, Bef, St) ->
block_cg(Es, Le, Vdb, Bef, St);
-cg({match,M,Rs}, Le, Vdb, Bef, St) ->
+cg(#k_match{anno=Le,body=M,ret=Rs}, Vdb, Bef, St) ->
match_cg(M, Rs, Le, Vdb, Bef, St);
-cg({guard_match,M,Rs}, Le, Vdb, Bef, St) ->
+cg(#k_guard_match{anno=Le,body=M,ret=Rs}, Vdb, Bef, St) ->
guard_match_cg(M, Rs, Le, Vdb, Bef, St);
-cg({call,Func,As,Rs}, Le, Vdb, Bef, St) ->
+cg(#k_call{anno=Le,op=Func,args=As,ret=Rs}, Vdb, Bef, St) ->
call_cg(Func, As, Rs, Le, Vdb, Bef, St);
-cg({enter,Func,As}, Le, Vdb, Bef, St) ->
+cg(#k_enter{anno=Le,op=Func,args=As}, Vdb, Bef, St) ->
enter_cg(Func, As, Le, Vdb, Bef, St);
-cg({bif,Bif,As,Rs}, Le, Vdb, Bef, St) ->
- bif_cg(Bif, As, Rs, Le, Vdb, Bef, St);
-cg({gc_bif,Bif,As,Rs}, Le, Vdb, Bef, St) ->
- gc_bif_cg(Bif, As, Rs, Le, Vdb, Bef, St);
-cg({internal,Bif,As,Rs}, Le, Vdb, Bef, St) ->
- internal_cg(Bif, As, Rs, Le, Vdb, Bef, St);
-cg({receive_loop,Te,Rvar,Rm,Tes,Rs}, Le, Vdb, Bef, St) ->
+cg(#k_bif{anno=Le}=Bif, Vdb, Bef, St) ->
+ bif_cg(Bif, Le, Vdb, Bef, St);
+cg(#k_receive{anno=Le,timeout=Te,var=Rvar,body=Rm,action=Tes,ret=Rs},
+ Vdb, Bef, St) ->
recv_loop_cg(Te, Rvar, Rm, Tes, Rs, Le, Vdb, Bef, St);
-cg(receive_next, Le, Vdb, Bef, St) ->
+cg(#k_receive_next{anno=Le}, Vdb, Bef, St) ->
recv_next_cg(Le, Vdb, Bef, St);
-cg(receive_accept, _Le, _Vdb, Bef, St) -> {[remove_message],Bef,St};
-cg({'try',Ta,Vs,Tb,Evs,Th,Rs}, Le, Vdb, Bef, St) ->
+cg(#k_receive_accept{}, _Vdb, Bef, St) ->
+ {[remove_message],Bef,St};
+cg(#k_try{anno=Le,arg=Ta,vars=Vs,body=Tb,evars=Evs,handler=Th,ret=Rs},
+ Vdb, Bef, St) ->
try_cg(Ta, Vs, Tb, Evs, Th, Rs, Le, Vdb, Bef, St);
-cg({try_enter,Ta,Vs,Tb,Evs,Th}, Le, Vdb, Bef, St) ->
+cg(#k_try_enter{anno=Le,arg=Ta,vars=Vs,body=Tb,evars=Evs,handler=Th},
+ Vdb, Bef, St) ->
try_enter_cg(Ta, Vs, Tb, Evs, Th, Le, Vdb, Bef, St);
-cg({'catch',Cb,R}, Le, Vdb, Bef, St) ->
+cg(#k_catch{anno=Le,body=Cb,ret=[R]}, Vdb, Bef, St) ->
catch_cg(Cb, R, Le, Vdb, Bef, St);
-cg({set,Var,Con}, Le, Vdb, Bef, St) ->
- set_cg(Var, Con, Le, Vdb, Bef, St);
-cg({return,Rs}, Le, Vdb, Bef, St) -> return_cg(Rs, Le, Vdb, Bef, St);
-cg({break,Bs}, Le, Vdb, Bef, St) -> break_cg(Bs, Le, Vdb, Bef, St);
-cg({guard_break,Bs,N}, Le, Vdb, Bef, St) ->
+cg(#k_put{anno=Le,arg=Con,ret=Var}, Vdb, Bef, St) ->
+ put_cg(Var, Con, Le, Vdb, Bef, St);
+cg(#k_return{anno=Le,args=Rs}, Vdb, Bef, St) ->
+ return_cg(Rs, Le, Vdb, Bef, St);
+cg(#k_break{anno=Le,args=Bs}, Vdb, Bef, St) ->
+ break_cg(Bs, Le, Vdb, Bef, St);
+cg(#k_guard_break{anno=Le,args=Bs,locked=N}, Vdb, Bef, St) ->
guard_break_cg(Bs, N, Le, Vdb, Bef, St);
-cg({need_heap,H}, _Le, _Vdb, Bef, St) ->
+cg(#cg_need_heap{h=H}, _Vdb, Bef, St) ->
{[{test_heap,H,max_reg(Bef#sr.reg)}],Bef,St}.
%% cg_list([Kexpr], FirstI, Vdb, StackReg, St) -> {[Ainstr],StackReg,St}.
@@ -191,11 +407,11 @@ cg_list(Kes, I, Vdb, Bef, St0) ->
%% Insert need_heap instructions in Kexpr list. Try to be smart and
%% collect them together as much as possible.
-need_heap(Kes0, I) ->
+need_heap(Kes0, _I) ->
{Kes,H} = need_heap_0(reverse(Kes0), 0, []),
%% Prepend need_heap if necessary.
- need_heap_need(I, H) ++ Kes.
+ need_heap_need(H) ++ Kes.
need_heap_0([Ke|Kes], H0, Acc) ->
{Ns,H} = need_heap_1(Ke, H0),
@@ -203,27 +419,54 @@ need_heap_0([Ke|Kes], H0, Acc) ->
need_heap_0([], H, Acc) ->
{Acc,H}.
-need_heap_1(#l{ke={set,_,{binary,_}},i=I}, H) ->
- {need_heap_need(I, H),0};
-need_heap_1(#l{ke={set,_,{map,_,_,_}},i=I}, H) ->
- {need_heap_need(I, H),0};
-need_heap_1(#l{ke={set,_,Val}}, H) ->
+need_heap_1(#k_put{arg=#k_binary{}}, H) ->
+ {need_heap_need(H),0};
+need_heap_1(#k_put{arg=#k_map{}}, H) ->
+ {need_heap_need(H),0};
+need_heap_1(#k_put{arg=Val}, H) ->
%% Just pass through adding to needed heap.
{[],H + case Val of
- {cons,_} -> 2;
- {tuple,Es} -> 1 + length(Es);
+ #k_cons{} -> 2;
+ #k_tuple{es=Es} -> 1 + length(Es);
_Other -> 0
end};
-need_heap_1(#l{ke={bif,_Bif,_As,_Rs}}, H) ->
- {[],H};
-need_heap_1(#l{i=I}, H) ->
+need_heap_1(#k_bif{}=Bif, H) ->
+ case is_gc_bif(Bif) of
+ false ->
+ {[],H};
+ true ->
+ {need_heap_need(H),0}
+ end;
+need_heap_1(_Ke, H) ->
%% Call or call-like instruction such as set_tuple_element/3.
- {need_heap_need(I, H),0}.
-
-need_heap_need(_I, 0) -> [];
-need_heap_need(I, H) -> [#l{ke={need_heap,H},i=I}].
-
-%% match_cg(Match, [Ret], Le, Vdb, StackReg, State) ->
+ {need_heap_need(H),0}.
+
+need_heap_need(0) -> [];
+need_heap_need(H) -> [#cg_need_heap{h=H}].
+
+%% is_gc_bif(#k_bif{}) -> true|false.
+%% is_gc_bif(Name, Arity) -> true|false.
+%% Determines whether the BIF Name/Arity might do a GC.
+
+is_gc_bif(#k_bif{op=#k_remote{name=#k_atom{val=Name}},args=Args}) ->
+ is_gc_bif(Name, length(Args));
+is_gc_bif(#k_bif{op=#k_internal{}}) ->
+ true.
+
+is_gc_bif(hd, 1) -> false;
+is_gc_bif(tl, 1) -> false;
+is_gc_bif(self, 0) -> false;
+is_gc_bif(node, 0) -> false;
+is_gc_bif(node, 1) -> false;
+is_gc_bif(element, 2) -> false;
+is_gc_bif(get, 1) -> false;
+is_gc_bif(tuple_size, 1) -> false;
+is_gc_bif(Bif, Arity) ->
+ not (erl_internal:bool_op(Bif, Arity) orelse
+ erl_internal:new_type_test(Bif, Arity) orelse
+ erl_internal:comp_op(Bif, Arity)).
+
+%% match_cg(Matc, [Ret], Le, Vdb, StackReg, State) ->
%% {[Ainstr],StackReg,State}.
%% Generate code for a match. First save all variables on the stack
%% that are to survive after the match. We leave saved variables in
@@ -252,7 +495,7 @@ guard_match_cg(M, Rs, Le, Vdb, Bef, St0) ->
clear_dead(Aft#sr{reg=Reg}, I, Vdb),
St2#cg{break=St1#cg.break}}.
-guard_match_regs([{I,gbreakvar}|Rs], [{var,V}|Vs]) ->
+guard_match_regs([{I,gbreakvar}|Rs], [#k_var{name=V}|Vs]) ->
[{I,V}|guard_match_regs(Rs, Vs)];
guard_match_regs([R|Rs], Vs) ->
[R|guard_match_regs(Rs, Vs)];
@@ -264,17 +507,14 @@ guard_match_regs([], []) -> [].
%% down as each level which uses this takes its own internal Vdb not
%% the outer one.
-match_cg(Le, Fail, Bef, St) ->
- match_cg(Le#l.ke, Le, Fail, Bef, St).
-
-match_cg({alt,F,S}, _Le, Fail, Bef, St0) ->
+match_cg(#k_alt{first=F,then=S}, Fail, Bef, St0) ->
{Tf,St1} = new_label(St0),
{Fis,Faft,St2} = match_cg(F, Tf, Bef, St1),
{Sis,Saft,St3} = match_cg(S, Fail, Bef, St2),
Aft = sr_merge(Faft, Saft),
{Fis ++ [{label,Tf}] ++ Sis,Aft,St3};
-match_cg({select,{var,Vname}=V,Scs0}, #l{a=Anno}, Fail, Bef, St) ->
- ReuseForContext = member(reuse_for_context, Anno) andalso
+match_cg(#k_select{var=#k_var{anno=Vanno,name=Vname}=V,types=Scs0}, Fail, Bef, St) ->
+ ReuseForContext = member(reuse_for_context, Vanno) andalso
find_reg(Vname, Bef#sr.reg) =/= error,
Scs = case ReuseForContext of
false -> Scs0;
@@ -283,10 +523,10 @@ match_cg({select,{var,Vname}=V,Scs0}, #l{a=Anno}, Fail, Bef, St) ->
match_fmf(fun (S, F, Sta) ->
select_cg(S, V, F, Fail, Bef, Sta) end,
Fail, St, Scs);
-match_cg({guard,Gcs}, _Le, Fail, Bef, St) ->
+match_cg(#k_guard{clauses=Gcs}, Fail, Bef, St) ->
match_fmf(fun (G, F, Sta) -> guard_clause_cg(G, F, Bef, Sta) end,
Fail, St, Gcs);
-match_cg({block,Es}, Le, _Fail, Bef, St) ->
+match_cg(#cg_block{anno=Le,es=Es}, _Fail, Bef, St) ->
%% Must clear registers and stack of dead variables.
Int = clear_dead(Bef, Le#l.i, Le#l.vdb),
block_cg(Es, Le, Int, St).
@@ -294,8 +534,8 @@ match_cg({block,Es}, Le, _Fail, Bef, St) ->
%% bsm_rename_ctx([Clause], Var) -> [Clause]
%% We know from an annotation that the register for a binary can
%% be reused for the match context because the two are not truly
-%% alive at the same time (even though the conservative life time
-%% information calculated by v3_life says so).
+%% alive at the same time (even though the life time information
+%% says so).
%%
%% The easiest way to have those variables share the same register is
%% to rename the variable with the shortest life-span (the match
@@ -306,12 +546,14 @@ match_cg({block,Es}, Le, _Fail, Bef, St) ->
%% We must also remove all information about the match context
%% variable from all life-time information databases (Vdb).
-bsm_rename_ctx([#l{ke={type_clause,binary,
- [#l{ke={val_clause,{binary,{var,Old}},Ke0}}=L2]}}=L1|Cs], New) ->
+bsm_rename_ctx([#k_type_clause{type=k_binary,values=Vcs}=TC|Cs], New) ->
+ [#k_val_clause{val=#k_binary{segs=#k_var{name=Old}}=Bin,
+ body=Ke0}=VC0] = Vcs,
Ke = bsm_rename_ctx(Ke0, Old, New, false),
- [L1#l{ke={type_clause,binary,
- [L2#l{ke={val_clause,{binary,{var,New}},Ke}}]}}|bsm_rename_ctx(Cs, New)];
-bsm_rename_ctx([C|Cs], New) ->
+ VC = VC0#k_val_clause{val=Bin#k_binary{segs=#k_var{name=New}},
+ body=Ke},
+ [TC#k_type_clause{values=[VC]}|bsm_rename_ctx(Cs, New)];
+bsm_rename_ctx([C|Cs], New) ->
[C|bsm_rename_ctx(Cs, New)];
bsm_rename_ctx([], _) -> [].
@@ -321,34 +563,24 @@ bsm_rename_ctx([], _) -> [].
%% only complicatate things to recurse into blocks not in a protected
%% (the match context variable is not live inside them).
-bsm_rename_ctx(#l{ke={select,{var,V},Cs0}}=L, Old, New, InProt) ->
+bsm_rename_ctx(#k_select{var=#k_var{name=V},types=Cs0}=Sel,
+ Old, New, InProt) ->
Cs = bsm_rename_ctx_list(Cs0, Old, New, InProt),
- L#l{ke={select,{var,bsm_rename_var(V, Old, New)},Cs}};
-bsm_rename_ctx(#l{ke={type_clause,Type,Cs0}}=L, Old, New, InProt) ->
+ Sel#k_select{var=#k_var{name=bsm_rename_var(V, Old, New)},types=Cs};
+bsm_rename_ctx(#k_type_clause{values=Cs0}=TC, Old, New, InProt) ->
Cs = bsm_rename_ctx_list(Cs0, Old, New, InProt),
- L#l{ke={type_clause,Type,Cs}};
-bsm_rename_ctx(#l{ke={val_clause,{bin_end,V},Ke0}}=L, Old, New, InProt) ->
- Ke = bsm_rename_ctx(Ke0, Old, New, InProt),
- L#l{ke={val_clause,{bin_end,bsm_rename_var(V, Old, New)},Ke}};
-bsm_rename_ctx(#l{ke={val_clause,{bin_seg,V,Sz,U,Type,Fl,Vs},Ke0}}=L,
- Old, New, InProt) ->
+ TC#k_type_clause{values=Cs};
+bsm_rename_ctx(#k_val_clause{body=Ke0}=VC, Old, New, InProt) ->
Ke = bsm_rename_ctx(Ke0, Old, New, InProt),
- L#l{ke={val_clause,{bin_seg,bsm_rename_var(V, Old, New),Sz,U,Type,Fl,Vs},Ke}};
-bsm_rename_ctx(#l{ke={val_clause,{bin_int,V,Sz,U,Fl,Val,Vs},Ke0}}=L,
- Old, New, InProt) ->
- Ke = bsm_rename_ctx(Ke0, Old, New, InProt),
- L#l{ke={val_clause,{bin_int,bsm_rename_var(V, Old, New),Sz,U,Fl,Val,Vs},Ke}};
-bsm_rename_ctx(#l{ke={val_clause,Val,Ke0}}=L, Old, New, InProt) ->
- Ke = bsm_rename_ctx(Ke0, Old, New, InProt),
- L#l{ke={val_clause,Val,Ke}};
-bsm_rename_ctx(#l{ke={alt,F0,S0}}=L, Old, New, InProt) ->
+ VC#k_val_clause{body=Ke};
+bsm_rename_ctx(#k_alt{first=F0,then=S0}=Alt, Old, New, InProt) ->
F = bsm_rename_ctx(F0, Old, New, InProt),
S = bsm_rename_ctx(S0, Old, New, InProt),
- L#l{ke={alt,F,S}};
-bsm_rename_ctx(#l{ke={guard,Gcs0}}=L, Old, New, InProt) ->
+ Alt#k_alt{first=F,then=S};
+bsm_rename_ctx(#k_guard{clauses=Gcs0}=Guard, Old, New, InProt) ->
Gcs = bsm_rename_ctx_list(Gcs0, Old, New, InProt),
- L#l{ke={guard,Gcs}};
-bsm_rename_ctx(#l{ke={guard_clause,G0,B0}}=L, Old, New, InProt) ->
+ Guard#k_guard{clauses=Gcs};
+bsm_rename_ctx(#k_guard_clause{guard=G0,body=B0}=GC, Old, New, InProt) ->
G = bsm_rename_ctx(G0, Old, New, InProt),
B = bsm_rename_ctx(B0, Old, New, InProt),
%% A guard clause may cause unsaved variables to be saved on the stack.
@@ -356,49 +588,49 @@ bsm_rename_ctx(#l{ke={guard_clause,G0,B0}}=L, Old, New, InProt) ->
%% same register), it is neither in the stack nor register descriptor
%% lists and we would crash when we didn't find it unless we remove
%% it from the database.
- bsm_forget_var(L#l{ke={guard_clause,G,B}}, Old);
-bsm_rename_ctx(#l{ke={protected,Ts0,Rs}}=L, Old, New, _InProt) ->
+ bsm_forget_var(GC#k_guard_clause{guard=G,body=B}, Old);
+bsm_rename_ctx(#k_protected{arg=Ts0}=Prot, Old, New, _InProt) ->
InProt = true,
Ts = bsm_rename_ctx_list(Ts0, Old, New, InProt),
- bsm_forget_var(L#l{ke={protected,Ts,Rs}}, Old);
-bsm_rename_ctx(#l{ke={match,Ms0,Rs}}=L, Old, New, InProt) ->
+ bsm_forget_var(Prot#k_protected{arg=Ts}, Old);
+bsm_rename_ctx(#k_match{body=Ms0}=Match, Old, New, InProt) ->
Ms = bsm_rename_ctx(Ms0, Old, New, InProt),
- L#l{ke={match,Ms,Rs}};
-bsm_rename_ctx(#l{ke={guard_match,Ms0,Rs}}=L, Old, New, InProt) ->
+ Match#k_match{body=Ms};
+bsm_rename_ctx(#k_guard_match{body=Ms0}=Match, Old, New, InProt) ->
Ms = bsm_rename_ctx(Ms0, Old, New, InProt),
- L#l{ke={guard_match,Ms,Rs}};
-bsm_rename_ctx(#l{ke={test,_,_,_}}=L, _, _, _) -> L;
-bsm_rename_ctx(#l{ke={bif,_,_,_}}=L, _, _, _) -> L;
-bsm_rename_ctx(#l{ke={gc_bif,_,_,_}}=L, _, _, _) -> L;
-bsm_rename_ctx(#l{ke={set,_,_}}=L, _, _, _) -> L;
-bsm_rename_ctx(#l{ke={call,_,_,_}}=L, _, _, _) -> L;
-bsm_rename_ctx(#l{ke={block,_}}=L, Old, _, false) ->
+ Match#k_guard_match{body=Ms};
+bsm_rename_ctx(#k_test{}=Test, _, _, _) -> Test;
+bsm_rename_ctx(#k_bif{}=Bif, _, _, _) -> Bif;
+bsm_rename_ctx(#k_put{}=Put, _, _, _) -> Put;
+bsm_rename_ctx(#k_call{}=Call, _, _, _) -> Call;
+bsm_rename_ctx(#cg_block{}=Block, Old, _, false) ->
%% This block is not inside a protected. The match context variable cannot
%% possibly be live inside the block.
- bsm_forget_var(L, Old);
-bsm_rename_ctx(#l{ke={block,Bl0}}=L, Old, New, true) ->
+ bsm_forget_var(Block, Old);
+bsm_rename_ctx(#cg_block{es=Es0}=Block, Old, New, true) ->
%% A block in a protected. We must recursively rename the variable
%% inside the block.
- Bl = bsm_rename_ctx_list(Bl0, Old, New, true),
- bsm_forget_var(L#l{ke={block,Bl}}, Old);
-bsm_rename_ctx(#l{ke={guard_break,Bs,Locked0}}=L0, Old, _New, _InProt) ->
+ Es = bsm_rename_ctx_list(Es0, Old, New, true),
+ bsm_forget_var(Block#cg_block{es=Es}, Old);
+bsm_rename_ctx(#k_guard_break{locked=Locked0}=Break, Old, _New, _InProt) ->
Locked = Locked0 -- [Old],
- L = L0#l{ke={guard_break,Bs,Locked}},
- bsm_forget_var(L, Old).
+ bsm_forget_var(Break#k_guard_break{locked=Locked}, Old).
bsm_rename_ctx_list([C|Cs], Old, New, InProt) ->
[bsm_rename_ctx(C, Old, New, InProt)|
bsm_rename_ctx_list(Cs, Old, New, InProt)];
bsm_rename_ctx_list([], _, _, _) -> [].
-
+
bsm_rename_var(Old, Old, New) -> New;
bsm_rename_var(V, _, _) -> V.
%% bsm_forget_var(#l{}, Variable) -> #l{}
%% Remove a variable from the variable life-time database.
-bsm_forget_var(#l{vdb=Vdb}=L, V) ->
- L#l{vdb=keydelete(V, 1, Vdb)}.
+bsm_forget_var(Ke, V) ->
+ #l{vdb=Vdb} = L0 = get_kanno(Ke),
+ L = L0#l{vdb=keydelete(V, 1, Vdb)},
+ set_kanno(Ke, L).
%% block_cg([Kexpr], Le, Vdb, StackReg, St) -> {[Ainstr],StackReg,St}.
%% block_cg([Kexpr], Le, StackReg, St) -> {[Ainstr],StackReg,St}.
@@ -421,7 +653,7 @@ cg_block(Kes0, I, Vdb, Bef, St0) ->
case basic_block(Kes0) of
{Kes1,LastI,Args,Rest} ->
Ke = hd(Kes1),
- Fb = Ke#l.i,
+ #l{i=Fb} = get_kanno(Ke),
cg_basic_block(Kes1, Fb, LastI, Args, Vdb, Bef, St0);
{Kes1,Rest} ->
cg_list(Kes1, I, Vdb, Bef, St0)
@@ -431,36 +663,46 @@ cg_block(Kes0, I, Vdb, Bef, St0) ->
basic_block(Kes) -> basic_block(Kes, []).
-basic_block([Le|Les], Acc) ->
- case collect_block(Le#l.ke) of
- include -> basic_block(Les, [Le|Acc]);
+basic_block([Ke|Kes], Acc) ->
+ case collect_block(Ke) of
+ include -> basic_block(Kes, [Ke|Acc]);
{block_end,As} ->
case Acc of
[] ->
- %% If the basic block does not contain any set instructions,
+ %% If the basic block does not contain any #k_put{} instructions,
%% it serves no useful purpose to do basic block optimizations.
- {[Le],Les};
+ {[Ke],Kes};
_ ->
- {reverse(Acc, [Le]),Le#l.i,As,Les}
+ #l{i=I} = get_kanno(Ke),
+ {reverse(Acc, [Ke]),I,As,Kes}
end;
- no_block -> {reverse(Acc, [Le]),Les}
+ no_block -> {reverse(Acc, [Ke]),Kes}
end.
-%% sets that may garbage collect are not allowed in basic blocks.
-
-collect_block({set,_,{binary,_}}) -> no_block;
-collect_block({set,_,{map,_,_,_}}) -> no_block;
-collect_block({set,_,_}) -> include;
-collect_block({call,{var,_}=Var,As,_Rs}) -> {block_end,As++[Var]};
-collect_block({call,Func,As,_Rs}) -> {block_end,As++func_vars(Func)};
-collect_block({enter,{var,_}=Var,As})-> {block_end,As++[Var]};
-collect_block({enter,Func,As}) -> {block_end,As++func_vars(Func)};
-collect_block({return,Rs}) -> {block_end,Rs};
-collect_block({break,Bs}) -> {block_end,Bs};
+%% #k_put{} instructions that may garbage collect are not allowed in basic blocks.
+
+collect_block(#k_put{arg=#k_binary{}}) ->
+ no_block;
+collect_block(#k_put{arg=#k_map{}}) ->
+ no_block;
+collect_block(#k_put{}) ->
+ include;
+collect_block(#k_call{op=#k_var{}=Var,args=As}) ->
+ {block_end,As++[Var]};
+collect_block(#k_call{op=Func,args=As}) ->
+ {block_end,As++func_vars(Func)};
+collect_block(#k_enter{op=#k_var{}=Var,args=As}) ->
+ {block_end,As++[Var]};
+collect_block(#k_enter{op=Func,args=As}) ->
+ {block_end,As++func_vars(Func)};
+collect_block(#k_return{args=Rs}) ->
+ {block_end,Rs};
+collect_block(#k_break{args=Bs}) ->
+ {block_end,Bs};
collect_block(_) -> no_block.
-func_vars({remote,M,F}) when element(1, M) =:= var;
- element(1, F) =:= var ->
+func_vars(#k_remote{mod=M,name=F})
+ when is_record(M, k_var); is_record(F, k_var) ->
[M,F];
func_vars(_) -> [].
@@ -478,18 +720,19 @@ cg_basic_block(Kes, Fb, Lf, As, Vdb, Bef, St0) ->
{Int0,X0_v0,St0}, need_heap(Kes, Fb)),
{Keis,Aft,St1}.
-cg_basic_block(#l{ke={need_heap,_}}=Ke, {Inta,X0v,Sta}, _Lf, Vdb) ->
+cg_basic_block(#cg_need_heap{}=Ke, {Inta,X0v,Sta}, _Lf, Vdb) ->
{Keis,Intb,Stb} = cg(Ke, Vdb, Inta, Sta),
{Keis, {Intb,X0v,Stb}};
cg_basic_block(Ke, {Inta,X0_v1,Sta}, Lf, Vdb) ->
- {Sis,Intb} = save_carefully(Inta, Ke#l.i, Lf+1, Vdb),
- {X0_v2,Intc} = allocate_x0(X0_v1, Ke#l.i, Intb),
+ #l{i=I} = get_kanno(Ke),
+ {Sis,Intb} = save_carefully(Inta, I, Lf+1, Vdb),
+ {X0_v2,Intc} = allocate_x0(X0_v1, I, Intb),
Intd = reserve(Intc),
{Keis,Inte,Stb} = cg(Ke, Vdb, Intd, Sta),
{Sis ++ Keis, {Inte,X0_v2,Stb}}.
make_reservation([], _) -> [];
-make_reservation([{var,V}|As], I) -> [{I,V}|make_reservation(As, I+1)];
+make_reservation([#k_var{name=V}|As], I) -> [{I,V}|make_reservation(As, I+1)];
make_reservation([A|As], I) -> [{I,A}|make_reservation(As, I+1)].
reserve(Sr) -> Sr#sr{reg=reserve(Sr#sr.res, Sr#sr.reg, Sr#sr.stk)}.
@@ -538,7 +781,7 @@ save_carefully([V|Vs], Bef, Acc) ->
end.
x0_vars([], _Fb, _Lf, _Vdb) -> [];
-x0_vars([{var,V}|_], Fb, _Lf, Vdb) ->
+x0_vars([#k_var{name=V}|_], Fb, _Lf, Vdb) ->
{V,F,_L} = VFL = vdb_find(V, Vdb),
x0_vars1([VFL], Fb, F, Vdb);
x0_vars([X0|_], Fb, Lf, Vdb) ->
@@ -640,21 +883,27 @@ turn_yreg(Other, _MaxY) ->
%% wrong. These are different as in the second case there is no need
%% to try the next type, it will always fail.
-select_cg(#l{ke={type_clause,cons,[S]}}, {var,V}, Tf, Vf, Bef, St) ->
+select_cg(#k_type_clause{type=Type,values=Vs}, Var, Tf, Vf, Bef, St) ->
+ #k_var{name=V} = Var,
+ select_cg(Type, Vs, V, Tf, Vf, Bef, St).
+
+select_cg(k_cons, [S], V, Tf, Vf, Bef, St) ->
select_cons(S, V, Tf, Vf, Bef, St);
-select_cg(#l{ke={type_clause,nil,[S]}}, {var,V}, Tf, Vf, Bef, St) ->
+select_cg(k_nil, [S], V, Tf, Vf, Bef, St) ->
select_nil(S, V, Tf, Vf, Bef, St);
-select_cg(#l{ke={type_clause,binary,[S]}}, {var,V}, Tf, Vf, Bef, St) ->
+select_cg(k_binary, [S], V, Tf, Vf, Bef, St) ->
select_binary(S, V, Tf, Vf, Bef, St);
-select_cg(#l{ke={type_clause,bin_seg,S}}, {var,V}, Tf, _Vf, Bef, St) ->
+select_cg(k_bin_seg, S, V, Tf, _Vf, Bef, St) ->
select_bin_segs(S, V, Tf, Bef, St);
-select_cg(#l{ke={type_clause,bin_int,S}}, {var,V}, Tf, _Vf, Bef, St) ->
+select_cg(k_bin_int, S, V, Tf, _Vf, Bef, St) ->
select_bin_segs(S, V, Tf, Bef, St);
-select_cg(#l{ke={type_clause,bin_end,[S]}}, {var,V}, Tf, _Vf, Bef, St) ->
+select_cg(k_bin_end, [S], V, Tf, _Vf, Bef, St) ->
select_bin_end(S, V, Tf, Bef, St);
-select_cg(#l{ke={type_clause,map,S}}, {var,V}, Tf, Vf, Bef, St) ->
+select_cg(k_map, S, V, Tf, Vf, Bef, St) ->
select_map(S, V, Tf, Vf, Bef, St);
-select_cg(#l{ke={type_clause,Type,Scs}}, {var,V}, Tf, Vf, Bef, St0) ->
+select_cg(k_literal, S, V, Tf, Vf, Bef, St) ->
+ select_literal(S, V, Tf, Vf, Bef, St);
+select_cg(Type, Scs, V, Tf, Vf, Bef, St0) ->
{Vis,{Aft,St1}} =
mapfoldl(fun (S, {Int,Sta}) ->
{Val,Is,Inta,Stb} = select_val(S, V, Vf, Bef, Sta),
@@ -664,22 +913,29 @@ select_cg(#l{ke={type_clause,Type,Scs}}, {var,V}, Tf, Vf, Bef, St0) ->
{Vls,Sis,St2} = select_labels(OptVls, St1, [], []),
{select_val_cg(Type, fetch_var(V, Bef), Vls, Tf, Vf, Sis), Aft, St2}.
-select_val_cg(tuple, R, [Arity,{f,Lbl}], Tf, Vf, [{label,Lbl}|Sis]) ->
+select_val_cg(k_tuple, R, [Arity,{f,Lbl}], Tf, Vf, [{label,Lbl}|Sis]) ->
[{test,is_tuple,{f,Tf},[R]},{test,test_arity,{f,Vf},[R,Arity]}|Sis];
-select_val_cg(tuple, R, Vls, Tf, Vf, Sis) ->
+select_val_cg(k_tuple, R, Vls, Tf, Vf, Sis) ->
[{test,is_tuple,{f,Tf},[R]},{select_tuple_arity,R,{f,Vf},{list,Vls}}|Sis];
select_val_cg(Type, R, [Val, {f,Lbl}], Fail, Fail, [{label,Lbl}|Sis]) ->
- [{test,is_eq_exact,{f,Fail},[R,{Type,Val}]}|Sis];
+ [{test,is_eq_exact,{f,Fail},[R,{type(Type),Val}]}|Sis];
select_val_cg(Type, R, [Val, {f,Lbl}], Tf, Vf, [{label,Lbl}|Sis]) ->
[{test,select_type_test(Type),{f,Tf},[R]},
- {test,is_eq_exact,{f,Vf},[R,{Type,Val}]}|Sis];
+ {test,is_eq_exact,{f,Vf},[R,{type(Type),Val}]}|Sis];
select_val_cg(Type, R, Vls0, Tf, Vf, Sis) ->
- Vls1 = [case Value of {f,_Lbl} -> Value; _ -> {Type,Value} end || Value <- Vls0],
+ Vls1 = [case Value of
+ {f,_Lbl} -> Value;
+ _ -> {type(Type),Value}
+ end || Value <- Vls0],
[{test,select_type_test(Type),{f,Tf},[R]}, {select_val,R,{f,Vf},{list,Vls1}}|Sis].
-
-select_type_test(integer) -> is_integer;
-select_type_test(atom) -> is_atom;
-select_type_test(float) -> is_float.
+
+type(k_atom) -> atom;
+type(k_float) -> float;
+type(k_int) -> integer.
+
+select_type_test(k_int) -> is_integer;
+select_type_test(k_atom) -> is_atom;
+select_type_test(k_float) -> is_float.
combine([{Is,Vs1}, {Is,Vs2}|Vis]) -> combine([{Is,Vs1 ++ Vs2}|Vis]);
combine([V|Vis]) -> [V|combine(Vis)];
@@ -695,36 +951,49 @@ add_vls([V|Vs], Lbl, Acc) ->
add_vls(Vs, Lbl, [V, {f,Lbl}|Acc]);
add_vls([], _, Acc) -> Acc.
-select_cons(#l{ke={val_clause,{cons,Es},B},i=I,vdb=Vdb}, V, Tf, Vf, Bef, St0) ->
+select_literal(S, V, Tf, Vf, Bef, St) ->
+ Reg = fetch_var(V, Bef),
+ F = fun(ValClause, Fail, St0) ->
+ {Val,Is,Aft,St1} = select_val(ValClause, V, Vf, Bef, St0),
+ Test = {test,is_eq_exact,{f,Fail},[Reg,{literal,Val}]},
+ {[Test|Is],Aft,St1}
+ end,
+ match_fmf(F, Tf, St, S).
+
+select_cons(#k_val_clause{val=#k_cons{hd=Hd,tl=Tl},body=B,anno=#l{i=I,vdb=Vdb}},
+ V, Tf, Vf, Bef, St0) ->
+ Es = [Hd,Tl],
{Eis,Int,St1} = select_extract_cons(V, Es, I, Vdb, Bef, St0),
{Bis,Aft,St2} = match_cg(B, Vf, Int, St1),
{[{test,is_nonempty_list,{f,Tf},[fetch_var(V, Bef)]}] ++ Eis ++ Bis,Aft,St2}.
-select_nil(#l{ke={val_clause,nil,B}}, V, Tf, Vf, Bef, St0) ->
+select_nil(#k_val_clause{val=#k_nil{},body=B}, V, Tf, Vf, Bef, St0) ->
{Bis,Aft,St1} = match_cg(B, Vf, Bef, St0),
{[{test,is_nil,{f,Tf},[fetch_var(V, Bef)]}] ++ Bis,Aft,St1}.
-select_binary(#l{ke={val_clause,{binary,{var,V}},B},i=I,vdb=Vdb},
- V, Tf, Vf, Bef, St0) ->
+select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=V}},body=B,
+ anno=#l{i=I,vdb=Vdb}}, V, Tf, Vf, Bef, St0) ->
+ #cg{ctx=OldCtx} = St0,
Int0 = clear_dead(Bef#sr{reg=Bef#sr.reg}, I, Vdb),
- {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0),
+ {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0#cg{ctx=V}),
CtxReg = fetch_var(V, Int0),
Live = max_reg(Bef#sr.reg),
Bis1 = [{test,bs_start_match2,{f,Tf},Live,[CtxReg,V],CtxReg},
{bs_save2,CtxReg,{V,V}}|Bis0],
Bis = finish_select_binary(Bis1),
- {Bis,Aft,St1};
-select_binary(#l{ke={val_clause,{binary,{var,Ivar}},B},i=I,vdb=Vdb},
- V, Tf, Vf, Bef, St0) ->
+ {Bis,Aft,St1#cg{ctx=OldCtx}};
+select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=Ivar}},body=B,
+ anno=#l{i=I,vdb=Vdb}}, V, Tf, Vf, Bef, St0) ->
+ #cg{ctx=OldCtx} = St0,
Regs = put_reg(Ivar, Bef#sr.reg),
Int0 = clear_dead(Bef#sr{reg=Regs}, I, Vdb),
- {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0),
+ {Bis0,Aft,St1} = match_cg(B, Vf, Int0, St0#cg{ctx=Ivar}),
CtxReg = fetch_var(Ivar, Int0),
Live = max_reg(Bef#sr.reg),
Bis1 = [{test,bs_start_match2,{f,Tf},Live,[fetch_var(V, Bef),Ivar],CtxReg},
{bs_save2,CtxReg,{Ivar,Ivar}}|Bis0],
Bis = finish_select_binary(Bis1),
- {Bis,Aft,St1}.
+ {Bis,Aft,St1#cg{ctx=OldCtx}}.
finish_select_binary([{bs_save2,R,Point}=I,{bs_restore2,R,Point}|Is]) ->
[I|finish_select_binary(Is)];
@@ -746,9 +1015,16 @@ select_bin_segs(Scs, Ivar, Tf, Bef, St) ->
select_bin_seg(S, Ivar, Fail, Bef, Sta) end,
Tf, St, Scs).
-select_bin_seg(#l{ke={val_clause,{bin_seg,Ctx,Size,U,T,Fs0,Es},B},i=I,vdb=Vdb,a=A},
- Ivar, Fail, Bef, St0) ->
+select_bin_seg(#k_val_clause{val=#k_bin_seg{size=Size,unit=U,type=T,
+ seg=Seg,flags=Fs0,next=Next},
+ body=B,
+ anno=#l{i=I,vdb=Vdb,a=A}}, Ivar, Fail, Bef, St0) ->
+ Ctx = St0#cg.ctx,
Fs = [{anno,A}|Fs0],
+ Es = case Next of
+ [] -> [Seg];
+ _ -> [Seg,Next]
+ end,
{Mis,Int,St1} = select_extract_bin(Es, Size, U, T, Fs, Fail,
I, Vdb, Bef, Ctx, B, St0),
{Bis,Aft,St2} = match_cg(B, Fail, Int, St1),
@@ -761,9 +1037,12 @@ select_bin_seg(#l{ke={val_clause,{bin_seg,Ctx,Size,U,T,Fs0,Es},B},i=I,vdb=Vdb,a=
[{bs_restore2,CtxReg,{Ctx,Ivar}}|Mis++Bis]
end,
{Is,Aft,St2};
-select_bin_seg(#l{ke={val_clause,{bin_int,Ctx,Sz,U,Fs,Val,Es},B},i=I,vdb=Vdb},
- Ivar, Fail, Bef, St0) ->
- {Mis,Int,St1} = select_extract_int(Es, Val, Sz, U, Fs, Fail,
+select_bin_seg(#k_val_clause{val=#k_bin_int{size=Sz,unit=U,flags=Fs,
+ val=Val,next=Next},
+ body=B,
+ anno=#l{i=I,vdb=Vdb}}, Ivar, Fail, Bef, St0) ->
+ Ctx = St0#cg.ctx,
+ {Mis,Int,St1} = select_extract_int(Next, Val, Sz, U, Fs, Fail,
I, Vdb, Bef, Ctx, St0),
{Bis,Aft,St2} = match_cg(B, Fail, Int, St1),
CtxReg = fetch_var(Ctx, Bef),
@@ -784,7 +1063,7 @@ select_bin_seg(#l{ke={val_clause,{bin_int,Ctx,Sz,U,Fs,Val,Es},B},i=I,vdb=Vdb},
end,
{[{bs_restore2,CtxReg,{Ctx,Ivar}}|Is],Aft,St2}.
-select_extract_int([{var,Tl}], Val, {integer,Sz}, U, Fs, Vf,
+select_extract_int(#k_var{name=Tl}, Val, #k_int{val=Sz}, U, Fs, Vf,
I, Vdb, Bef, Ctx, St) ->
Bits = U*Sz,
Bin = case member(big, Fs) of
@@ -805,7 +1084,7 @@ select_extract_int([{var,Tl}], Val, {integer,Sz}, U, Fs, Vf,
end,
{Is,clear_dead(Bef, I, Vdb),St}.
-select_extract_bin([{var,Hd},{var,Tl}], Size0, Unit, Type, Flags, Vf,
+select_extract_bin([#k_var{name=Hd},#k_var{name=Tl}], Size0, Unit, Type, Flags, Vf,
I, Vdb, Bef, Ctx, _Body, St) ->
SizeReg = get_bin_size_reg(Size0, Bef),
{Es,Aft} =
@@ -828,11 +1107,11 @@ select_extract_bin([{var,Hd},{var,Tl}], Size0, Unit, Type, Flags, Vf,
{bs_save2,CtxReg,{Ctx,Tl}}],Int1}
end,
{Es,clear_dead(Aft, I, Vdb),St};
-select_extract_bin([{var,Hd}], Size, Unit, binary, Flags, Vf,
+select_extract_bin([#k_var{name=Hd}], Size, Unit, binary, Flags, Vf,
I, Vdb, Bef, Ctx, Body, St) ->
%% Match the last segment of a binary. We KNOW that the size
%% must be 'all'.
- Size = {atom,all}, %Assertion.
+ #k_atom{val=all} = Size, %Assertion.
{Es,Aft} =
case vdb_find(Hd, Vdb) of
{_,_,Lhd} when Lhd =< I ->
@@ -857,7 +1136,7 @@ select_extract_bin([{var,Hd}], Size, Unit, binary, Flags, Vf,
Name = bs_get_binary2,
Live = max_reg(Bef#sr.reg),
{[{test,Name,{f,Vf},Live,
- [CtxReg,Size,Unit,{field_flags,Flags}],Rhd}],
+ [CtxReg,atomic(Size),Unit,{field_flags,Flags}],Rhd}],
Int1};
true ->
%% Since the matching context will not be used again,
@@ -872,36 +1151,42 @@ select_extract_bin([{var,Hd}], Size, Unit, binary, Flags, Vf,
Name = bs_get_binary2,
Live = max_reg(Int1#sr.reg),
{[{test,Name,{f,Vf},Live,
- [CtxReg,Size,Unit,{field_flags,Flags}],CtxReg}],
+ [CtxReg,atomic(Size),Unit,{field_flags,Flags}],CtxReg}],
Int1}
end
end,
{Es,clear_dead(Aft, I, Vdb),St}.
%% is_context_unused(Ke) -> true | false
-%% Simple heurististic to determine whether the code that follows will
-%% use the current matching context again. (The information of liveness
-%% calculcated by v3_life is too conservative to be useful for this purpose.)
-%% 'true' means that the code that follows will definitely not use the context
-%% again (because it is a block, not guard or matching code); 'false' that we
-%% are not sure (there is either a guard, or more matching, either which may
-%% reference the context again).
-
-is_context_unused(#l{ke=Ke}) -> is_context_unused(Ke);
-is_context_unused({block,_}) -> true;
-is_context_unused(_) -> false.
-
-select_bin_end(#l{ke={val_clause,{bin_end,Ctx},B}},
- Ivar, Tf, Bef, St0) ->
+%% Simple heurististic to determine whether the code that follows
+%% will use the current matching context again. (The liveness
+%% information is too conservative to be useful for this purpose.)
+%% 'true' means that the code that follows will definitely not use
+%% the context again (because it is a block, not guard or matching
+%% code); 'false' that we are not sure (there could be more
+%% matching).
+
+is_context_unused(#k_alt{then=Then}) ->
+ %% #k_alt{} can be used for different purposes. If the Then part
+ %% is a block, it means that matching has finished and is used for a guard
+ %% to choose between the matched clauses.
+ is_context_unused(Then);
+is_context_unused(#cg_block{}) ->
+ true;
+is_context_unused(_) ->
+ false.
+
+select_bin_end(#k_val_clause{val=#k_bin_end{},body=B}, Ivar, Tf, Bef, St0) ->
+ Ctx = St0#cg.ctx,
{Bis,Aft,St2} = match_cg(B, Tf, Bef, St0),
CtxReg = fetch_var(Ctx, Bef),
{[{bs_restore2,CtxReg,{Ctx,Ivar}},
{test,bs_test_tail2,{f,Tf},[CtxReg,0]}|Bis],Aft,St2}.
-get_bin_size_reg({var,V}, Bef) ->
+get_bin_size_reg(#k_var{name=V}, Bef) ->
fetch_var(V, Bef);
get_bin_size_reg(Literal, _Bef) ->
- Literal.
+ atomic(Literal).
build_bs_instr(Type, Vf, CtxReg, Live, SizeReg, Unit, Flags, Rhd) ->
{Format,Name} = case Type of
@@ -935,11 +1220,18 @@ build_skip_instr(Type, Vf, CtxReg, Live, SizeReg, Unit, Flags) ->
{test,Name,{f,Vf},[CtxReg,Live,{field_flags,Flags}]}
end.
-select_val(#l{ke={val_clause,{tuple,Es},B},i=I,vdb=Vdb}, V, Vf, Bef, St0) ->
+select_val(#k_val_clause{val=#k_tuple{es=Es},body=B,anno=#l{i=I,vdb=Vdb}},
+ V, Vf, Bef, St0) ->
{Eis,Int,St1} = select_extract_tuple(V, Es, I, Vdb, Bef, St0),
{Bis,Aft,St2} = match_cg(B, Vf, Int, St1),
{length(Es),Eis ++ Bis,Aft,St2};
-select_val(#l{ke={val_clause,{_,Val},B}}, _V, Vf, Bef, St0) ->
+select_val(#k_val_clause{val=Val0,body=B}, _V, Vf, Bef, St0) ->
+ Val = case Val0 of
+ #k_atom{val=Lit} -> Lit;
+ #k_float{val=Lit} -> Lit;
+ #k_int{val=Lit} -> Lit;
+ #k_literal{val=Lit} -> Lit
+ end,
{Bis,Aft,St1} = match_cg(B, Vf, Bef, St0),
{Val,Bis,Aft,St1}.
@@ -948,7 +1240,7 @@ select_val(#l{ke={val_clause,{_,Val},B}}, _V, Vf, Bef, St0) ->
%% Extract tuple elements, but only if they do not immediately die.
select_extract_tuple(Src, Vs, I, Vdb, Bef, St) ->
- F = fun ({var,V}, {Int0,Elem}) ->
+ F = fun (#k_var{name=V}, {Int0,Elem}) ->
case vdb_find(V, Vdb) of
{V,_,L} when L =< I -> {[], {Int0,Elem+1}};
_Other ->
@@ -965,9 +1257,10 @@ select_extract_tuple(Src, Vs, I, Vdb, Bef, St) ->
select_map(Scs, V, Tf, Vf, Bef, St0) ->
Reg = fetch_var(V, Bef),
{Is,Aft,St1} =
- match_fmf(fun(#l{ke={val_clause,{map,exact,_,Es},B},i=I,vdb=Vdb}, Fail, St1) ->
- select_map_val(V, Es, B, Fail, I, Vdb, Bef, St1)
- end, Vf, St0, Scs),
+ match_fmf(fun(#k_val_clause{val=#k_map{op=exact,es=Es},
+ body=B,anno=#l{i=I,vdb=Vdb}}, Fail, St1) ->
+ select_map_val(V, Es, B, Fail, I, Vdb, Bef, St1)
+ end, Vf, St0, Scs),
{[{test,is_map,{f,Tf},[Reg]}|Is],Aft,St1}.
select_map_val(V, Es, B, Fail, I, Vdb, Bef, St0) ->
@@ -984,29 +1277,32 @@ select_extract_map(Src, Vs, Fail, I, Vdb, Bef, St) ->
%% Assume keys are term-sorted
Rsrc = fetch_var(Src, Bef),
- {{HasKs,GetVs,HasVarKs,GetVarVs},Aft} = lists:foldr(fun
- ({map_pair,{var,K},{var,V}},{{HasKsi,GetVsi,HasVarVsi,GetVarVsi},Int0}) ->
- case vdb_find(V, Vdb) of
- {V,_,L} when L =< I ->
- RK = fetch_var(K,Int0),
- {{HasKsi,GetVsi,[RK|HasVarVsi],GetVarVsi},Int0};
- _Other ->
- Reg1 = put_reg(V, Int0#sr.reg),
- Int1 = Int0#sr{reg=Reg1},
- RK = fetch_var(K,Int0),
- RV = fetch_reg(V,Reg1),
- {{HasKsi,GetVsi,HasVarVsi,[[RK,RV]|GetVarVsi]},Int1}
- end;
- ({map_pair,Key,{var,V}},{{HasKsi,GetVsi,HasVarVsi,GetVarVsi},Int0}) ->
- case vdb_find(V, Vdb) of
- {V,_,L} when L =< I ->
- {{[Key|HasKsi],GetVsi,HasVarVsi,GetVarVsi},Int0};
- _Other ->
- Reg1 = put_reg(V, Int0#sr.reg),
- Int1 = Int0#sr{reg=Reg1},
- {{HasKsi,[Key,fetch_reg(V, Reg1)|GetVsi],HasVarVsi,GetVarVsi},Int1}
- end
- end, {{[],[],[],[]},Bef}, Vs),
+ {{HasKs,GetVs,HasVarKs,GetVarVs},Aft} =
+ foldr(fun(#k_map_pair{key=#k_var{name=K},val=#k_var{name=V}},
+ {{HasKsi,GetVsi,HasVarVsi,GetVarVsi},Int0}) ->
+ case vdb_find(V, Vdb) of
+ {V,_,L} when L =< I ->
+ RK = fetch_var(K,Int0),
+ {{HasKsi,GetVsi,[RK|HasVarVsi],GetVarVsi},Int0};
+ _Other ->
+ Reg1 = put_reg(V, Int0#sr.reg),
+ Int1 = Int0#sr{reg=Reg1},
+ RK = fetch_var(K,Int0),
+ RV = fetch_reg(V,Reg1),
+ {{HasKsi,GetVsi,HasVarVsi,[[RK,RV]|GetVarVsi]},Int1}
+ end;
+ (#k_map_pair{key=Key,val=#k_var{name=V}},
+ {{HasKsi,GetVsi,HasVarVsi,GetVarVsi},Int0}) ->
+ case vdb_find(V, Vdb) of
+ {V,_,L} when L =< I ->
+ {{[atomic(Key)|HasKsi],GetVsi,HasVarVsi,GetVarVsi},Int0};
+ _Other ->
+ Reg1 = put_reg(V, Int0#sr.reg),
+ Int1 = Int0#sr{reg=Reg1},
+ {{HasKsi,[atomic(Key),fetch_reg(V, Reg1)|GetVsi],
+ HasVarVsi,GetVarVsi},Int1}
+ end
+ end, {{[],[],[],[]},Bef}, Vs),
Code = [{test,has_map_fields,{f,Fail},Rsrc,{list,HasKs}} || HasKs =/= []] ++
[{test,has_map_fields,{f,Fail},Rsrc,{list,[K]}} || K <- HasVarKs] ++
@@ -1015,7 +1311,7 @@ select_extract_map(Src, Vs, Fail, I, Vdb, Bef, St) ->
{Code, Aft, St}.
-select_extract_cons(Src, [{var,Hd}, {var,Tl}], I, Vdb, Bef, St) ->
+select_extract_cons(Src, [#k_var{name=Hd}, #k_var{name=Tl}], I, Vdb, Bef, St) ->
{Es,Aft} = case {vdb_find(Hd, Vdb), vdb_find(Tl, Vdb)} of
{{_,_,Lhd}, {_,_,Ltl}} when Lhd =< I, Ltl =< I ->
%% Both head and tail are dead. No need to generate
@@ -1038,7 +1334,7 @@ select_extract_cons(Src, [{var,Hd}, {var,Tl}], I, Vdb, Bef, St) ->
{Es,Aft,St}.
-guard_clause_cg(#l{ke={guard_clause,G,B},vdb=Vdb}, Fail, Bef, St0) ->
+guard_clause_cg(#k_guard_clause{anno=#l{vdb=Vdb},guard=G,body=B}, Fail, Bef, St0) ->
{Gis,Int,St1} = guard_cg(G, Fail, Vdb, Bef, St0),
{Bis,Aft,St} = match_cg(B, Fail, Int, St1),
{Gis ++ Bis,Aft,St}.
@@ -1051,11 +1347,13 @@ guard_clause_cg(#l{ke={guard_clause,G,B},vdb=Vdb}, Fail, Bef, St0) ->
%% the correct exit point. Primops and tests all go to the next
%% instruction on success or jump to a failure label.
-guard_cg(#l{ke={protected,Ts,Rs},i=I,vdb=Pdb}, Fail, _Vdb, Bef, St) ->
+guard_cg(#k_protected{arg=Ts,ret=Rs,anno=#l{i=I,vdb=Pdb}}, Fail, _Vdb, Bef, St) ->
protected_cg(Ts, Rs, Fail, I, Pdb, Bef, St);
-guard_cg(#l{ke={block,Ts},i=I,vdb=Bdb}, Fail, _Vdb, Bef, St) ->
+guard_cg(#cg_block{es=Ts,anno=#l{i=I,vdb=Bdb}}, Fail, _Vdb, Bef, St) ->
guard_cg_list(Ts, Fail, I, Bdb, Bef, St);
-guard_cg(#l{ke={test,Test,As,Inverted},i=I,vdb=_Tdb}, Fail, Vdb, Bef, St0) ->
+guard_cg(#k_test{anno=#l{i=I},op=Test0,args=As,inverted=Inverted},
+ Fail, Vdb, Bef, St0) ->
+ #k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Test}} = Test0,
case Inverted of
false ->
test_cg(Test, As, Fail, I, Vdb, Bef, St0);
@@ -1089,7 +1387,7 @@ protected_cg(Ts, Rs, _Fail, I, Vdb, Bef, St0) ->
St2#cg{bfail=Pfail}),
%%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{Rs,I,Vdb,Aft}]),
%% Set return values to false.
- Mis = [{move,{atom,false},fetch_var(V,Aft)}||{var,V} <- Rs],
+ Mis = [{move,{atom,false},fetch_var(V,Aft)}||#k_var{name=V} <- Rs],
{Tis ++ [{jump,{f,Psucc}},
{label,Pfail}] ++ Mis ++ [{label,Psucc}],
Aft,St3#cg{bfail=St0#cg.bfail}}.
@@ -1114,7 +1412,7 @@ test_cg(is_map, [A], Fail, I, Vdb, Bef, St) ->
Arg = cg_reg_arg_prefer_y(A, Bef),
Aft = clear_dead(Bef, I, Vdb),
{[{test,is_map,{f,Fail},[Arg]}],Aft,St};
-test_cg(is_boolean, [{atom,Val}], Fail, I, Vdb, Bef, St) ->
+test_cg(is_boolean, [#k_atom{val=Val}], Fail, I, Vdb, Bef, St) ->
Aft = clear_dead(Bef, I, Vdb),
Is = case is_boolean(Val) of
true -> [];
@@ -1160,7 +1458,7 @@ match_fmf(F, LastFail, St0, [H|T]) ->
%% frame size. Finally the actual call is made. Call then needs the
%% return values filled in.
-call_cg({var,_V} = Var, As, Rs, Le, Vdb, Bef, St0) ->
+call_cg(#k_var{}=Var, As, Rs, Le, Vdb, Bef, St0) ->
{Sis,Int} = cg_setup_call(As++[Var], Bef, Le#l.i, Vdb),
%% Put return values in registers.
Reg = load_vars(Rs, clear_regs(Int#sr.reg)),
@@ -1169,9 +1467,8 @@ call_cg({var,_V} = Var, As, Rs, Le, Vdb, Bef, St0) ->
{Frees,Aft} = free_dead(clear_dead(Int#sr{reg=Reg}, Le#l.i, Vdb)),
{Sis ++ Frees ++ [line(Le),{call_fun,Arity}],Aft,
need_stack_frame(St0)};
-call_cg({remote,Mod,Name}, As, Rs, Le, Vdb, Bef, St0)
- when element(1, Mod) =:= var;
- element(1, Name) =:= var ->
+call_cg(#k_remote{mod=Mod,name=Name}, As, Rs, Le, Vdb, Bef, St0)
+ when is_record(Mod, k_var); is_record(Name, k_var) ->
{Sis,Int} = cg_setup_call(As++[Mod,Name], Bef, Le#l.i, Vdb),
%% Put return values in registers.
Reg = load_vars(Rs, clear_regs(Int#sr.reg)),
@@ -1189,8 +1486,9 @@ call_cg(Func, As, Rs, Le, Vdb, Bef, St0) ->
%%
%% move {atom,ok} DestReg
%% jump FailureLabel
- {remote,{atom,erlang},{atom,error}} = Func, %Assertion.
- [{var,DestVar}] = Rs,
+ #k_remote{mod=#k_atom{val=erlang},
+ name=#k_atom{val=error}} = Func, %Assertion.
+ [#k_var{name=DestVar}] = Rs,
Int0 = clear_dead(Bef, Le#l.i, Vdb),
Reg = put_reg(DestVar, Int0#sr.reg),
Int = Int0#sr{reg=Reg},
@@ -1209,11 +1507,11 @@ call_cg(Func, As, Rs, Le, Vdb, Bef, St0) ->
{Sis ++ Frees ++ [line(Le)|Call],Aft,St1}
end.
-build_call({remote,{atom,erlang},{atom,'!'}}, 2, St0) ->
+build_call(#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val='!'}}, 2, St0) ->
{[send],need_stack_frame(St0)};
-build_call({remote,{atom,Mod},{atom,Name}}, Arity, St0) ->
+build_call(#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, Arity, St0) ->
{[{call_ext,Arity,{extfunc,Mod,Name,Arity}}],need_stack_frame(St0)};
-build_call(Name, Arity, St0) when is_atom(Name) ->
+build_call(#k_local{name=Name}, Arity, St0) when is_atom(Name) ->
{Lbl,St1} = local_func_label(Name, Arity, need_stack_frame(St0)),
{[{call,Arity,{f,Lbl}}],St1}.
@@ -1229,16 +1527,15 @@ free_dead([Any|Stk], Y, Instr, StkAcc) ->
free_dead(Stk, Y+1, Instr, [Any|StkAcc]);
free_dead([], _, Instr, StkAcc) -> {Instr,reverse(StkAcc)}.
-enter_cg({var,_V} = Var, As, Le, Vdb, Bef, St0) ->
+enter_cg(#k_var{} = Var, As, Le, Vdb, Bef, St0) ->
{Sis,Int} = cg_setup_call(As++[Var], Bef, Le#l.i, Vdb),
%% Build complete code and final stack/register state.
Arity = length(As),
{Sis ++ [line(Le),{call_fun,Arity},return],
clear_dead(Int#sr{reg=clear_regs(Int#sr.reg)}, Le#l.i, Vdb),
need_stack_frame(St0)};
-enter_cg({remote,Mod,Name}, As, Le, Vdb, Bef, St0)
- when element(1, Mod) =:= var;
- element(1, Name) =:= var ->
+enter_cg(#k_remote{mod=Mod,name=Name}, As, Le, Vdb, Bef, St0)
+ when is_record(Mod, k_var); is_record(Name, k_var) ->
{Sis,Int} = cg_setup_call(As++[Mod,Name], Bef, Le#l.i, Vdb),
%% Build complete code and final stack/register state.
Arity = length(As),
@@ -1256,19 +1553,19 @@ enter_cg(Func, As, Le, Vdb, Bef, St0) ->
clear_dead(Int#sr{reg=clear_regs(Int#sr.reg)}, Le#l.i, Vdb),
St1}.
-build_enter({remote,{atom,erlang},{atom,'!'}}, 2, St0) ->
+build_enter(#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val='!'}}, 2, St0) ->
{[send,return],need_stack_frame(St0)};
-build_enter({remote,{atom,Mod},{atom,Name}}, Arity, St0) ->
+build_enter(#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, Arity, St0) ->
St1 = case trap_bif(Mod, Name, Arity) of
true -> need_stack_frame(St0);
false -> St0
end,
{[{call_ext_only,Arity,{extfunc,Mod,Name,Arity}}],St1};
-build_enter(Name, Arity, St0) when is_atom(Name) ->
+build_enter(#k_local{name=Name}, Arity, St0) when is_atom(Name) ->
{Lbl,St1} = local_func_label(Name, Arity, St0),
{[{call_only,Arity,{f,Lbl}}],St1}.
-enter_line({remote,{atom,Mod},{atom,Name}}, Arity, Le) ->
+enter_line(#k_remote{mod=#k_atom{val=Mod},name=#k_atom{val=Name}}, Arity, Le) ->
case erl_bifs:is_safe(Mod, Name, Arity) of
false ->
%% Tail-recursive call, possibly to a BIF.
@@ -1316,6 +1613,22 @@ trap_bif(erlang, group_leader, 2) -> true;
trap_bif(erlang, exit, 2) -> true;
trap_bif(_, _, _) -> false.
+%% bif_cg(#k_bif{}, Le, Vdb, StackReg, State) ->
+%% {[Ainstr],StackReg,State}.
+%% Generate code a BIF.
+
+bif_cg(#k_bif{op=#k_internal{name=Name},args=As,ret=Rs}, Le, Vdb, Bef, St) ->
+ internal_cg(Name, As, Rs, Le, Vdb, Bef, St);
+bif_cg(#k_bif{op=#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Name}},
+ args=As,ret=Rs}, Le, Vdb, Bef, St) ->
+ Ar = length(As),
+ case is_gc_bif(Name, Ar) of
+ false ->
+ bif_cg(Name, As, Rs, Le, Vdb, Bef, St);
+ true ->
+ gc_bif_cg(Name, As, Rs, Le, Vdb, Bef, St)
+ end.
+
%% internal_cg(Bif, [Arg], [Ret], Le, Vdb, StackReg, State) ->
%% {[Ainstr],StackReg,State}.
@@ -1334,8 +1647,8 @@ internal_cg(dsetelement, [Index0,Tuple0,New0], _Rs, Le, Vdb, Bef, St0) ->
clear_dead(Bef, Le#l.i, Vdb), St0};
internal_cg(make_fun, [Func0,Arity0|As], Rs, Le, Vdb, Bef, St0) ->
%% This behaves more like a function call.
- {atom,Func} = Func0,
- {integer,Arity} = Arity0,
+ #k_atom{val=Func} = Func0,
+ #k_int{val=Arity} = Arity0,
{Sis,Int} = cg_setup_call(As, Bef, Le#l.i, Vdb),
Reg = load_vars(Rs, clear_regs(Int#sr.reg)),
{FuncLbl,St1} = local_func_label(Func, Arity, St0),
@@ -1355,7 +1668,7 @@ internal_cg(raise, As, Rs, Le, Vdb, Bef, St) ->
%% bif_cg(Bif, [Arg], [Ret], Le, Vdb, StackReg, State) ->
%% {[Ainstr],StackReg,State}.
-bif_cg(Bif, As, [{var,V}], Le, Vdb, Bef, St0) ->
+bif_cg(Bif, As, [#k_var{name=V}], Le, Vdb, Bef, St0) ->
Ars = cg_reg_args(As, Bef),
%% If we are inside a catch and in a body (not in guard) and the
@@ -1393,7 +1706,7 @@ bif_cg(Bif, As, [{var,V}], Le, Vdb, Bef, St0) ->
%% gc_bif_cg(Bif, [Arg], [Ret], Le, Vdb, StackReg, State) ->
%% {[Ainstr],StackReg,State}.
-gc_bif_cg(Bif, As, [{var,V}], Le, Vdb, Bef, St0) ->
+gc_bif_cg(Bif, As, [#k_var{name=V}], Le, Vdb, Bef, St0) ->
Ars = cg_reg_args(As, Bef),
%% If we are inside a catch and in a body (not in guard) and the
@@ -1439,7 +1752,7 @@ recv_loop_cg(Te, Rvar, Rm, Tes, Rs, Le, Vdb, Bef, St0) ->
%% cg_recv_mesg( ) -> {[Ainstr],Aft,St}.
-cg_recv_mesg({var,R}, Rm, Tl, Bef, St0) ->
+cg_recv_mesg(#k_var{name=R}, Rm, Tl, Bef, St0) ->
Int0 = Bef#sr{reg=put_reg(R, Bef#sr.reg)},
Ret = fetch_reg(R, Int0#sr.reg),
%% Int1 = clear_dead(Int0, I, Rm#l.vdb),
@@ -1449,22 +1762,22 @@ cg_recv_mesg({var,R}, Rm, Tl, Bef, St0) ->
%% cg_recv_wait(Te, Tes, I, Vdb, Int2, St3) -> {[Ainstr],Aft,St}.
-cg_recv_wait({atom,infinity}, Tes, I, Bef, St0) ->
+cg_recv_wait(#k_atom{val=infinity}, #cg_block{anno=Le,es=Tes}, I, Bef, St0) ->
%% We know that the 'after' body will never be executed.
%% But to keep the stack and register information up to date,
%% we will generate the code for the 'after' body, and then discard it.
- Int1 = clear_dead(Bef, I, Tes#l.vdb),
- {_,Int2,St1} = cg_block(Tes#l.ke, Tes#l.i, Tes#l.vdb,
- Int1#sr{reg=clear_regs(Int1#sr.reg)}, St0),
+ Int1 = clear_dead(Bef, I, Le#l.vdb),
+ {_,Int2,St1} = cg_block(Tes, Le#l.i, Le#l.vdb,
+ Int1#sr{reg=clear_regs(Int1#sr.reg)}, St0),
{[{wait,{f,St1#cg.recv}}],Int2,St1};
-cg_recv_wait({integer,0}, Tes, _I, Bef, St0) ->
- {Tis,Int,St1} = cg_block(Tes#l.ke, Tes#l.i, Tes#l.vdb, Bef, St0),
+cg_recv_wait(#k_int{val=0}, #cg_block{anno=Le,es=Tes}, _I, Bef, St0) ->
+ {Tis,Int,St1} = cg_block(Tes, Le#l.i, Le#l.vdb, Bef, St0),
{[timeout|Tis],Int,St1};
-cg_recv_wait(Te, Tes, I, Bef, St0) ->
+cg_recv_wait(Te, #cg_block{anno=Le,es=Tes}, I, Bef, St0) ->
Reg = cg_reg_arg(Te, Bef),
%% Must have empty registers here! Bug if anything in registers.
- Int0 = clear_dead(Bef, I, Tes#l.vdb),
- {Tis,Int,St1} = cg_block(Tes#l.ke, Tes#l.i, Tes#l.vdb,
+ Int0 = clear_dead(Bef, I, Le#l.vdb),
+ {Tis,Int,St1} = cg_block(Tes, Le#l.i, Le#l.vdb,
Int0#sr{reg=clear_regs(Int0#sr.reg)}, St0),
{[{wait_timeout,{f,St1#cg.recv},Reg},timeout] ++ Tis,Int,St1}.
@@ -1482,7 +1795,7 @@ try_cg(Ta, Vs, Tb, Evs, Th, Rs, Le, Vdb, Bef, St0) ->
{B,St1} = new_label(St0), %Body label
{H,St2} = new_label(St1), %Handler label
{E,St3} = new_label(St2), %End label
- TryTag = Ta#l.i,
+ #l{i=TryTag} = get_kanno(Ta),
Int1 = Bef#sr{stk=put_catch(TryTag, Bef#sr.stk)},
TryReg = fetch_stack({catch_tag,TryTag}, Int1#sr.stk),
{Ais,Int2,St4} = cg(Ta, Vdb, Int1, St3#cg{break=B,in_catch=true}),
@@ -1502,7 +1815,7 @@ try_cg(Ta, Vs, Tb, Evs, Th, Rs, Le, Vdb, Bef, St0) ->
try_enter_cg(Ta, Vs, Tb, Evs, Th, Le, Vdb, Bef, St0) ->
{B,St1} = new_label(St0), %Body label
{H,St2} = new_label(St1), %Handler label
- TryTag = Ta#l.i,
+ #l{i=TryTag} = get_kanno(Ta),
Int1 = Bef#sr{stk=put_catch(TryTag, Bef#sr.stk)},
TryReg = fetch_stack({catch_tag,TryTag}, Int1#sr.stk),
{Ais,Int2,St3} = cg(Ta, Vdb, Int1, St2#cg{break=B,in_catch=true}),
@@ -1520,7 +1833,7 @@ try_enter_cg(Ta, Vs, Tb, Evs, Th, Le, Vdb, Bef, St0) ->
%% catch_cg(CatchBlock, Ret, Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}.
-catch_cg(C, {var,R}, Le, Vdb, Bef, St0) ->
+catch_cg(#cg_block{es=C}, #k_var{name=R}, Le, Vdb, Bef, St0) ->
{B,St1} = new_label(St0),
CatchTag = Le#l.i,
Int1 = Bef#sr{stk=put_catch(CatchTag, Bef#sr.stk)},
@@ -1534,8 +1847,8 @@ catch_cg(C, {var,R}, Le, Vdb, Bef, St0) ->
clear_dead(Aft, Le#l.i, Vdb),
St2#cg{break=St1#cg.break,in_catch=St1#cg.in_catch}}.
-%% set_cg([Var], Constr, Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}.
-%% We have to be careful how a 'set' works. First the structure is
+%% put_cg([Var], Constr, Le, Vdb, Bef, St) -> {[Ainstr],Aft,St}.
+%% We have to be careful how a 'put' works. First the structure is
%% built, then it is filled and finally things can be cleared. The
%% annotation must reflect this and make sure that the return
%% variable is allocated first.
@@ -1543,13 +1856,14 @@ catch_cg(C, {var,R}, Le, Vdb, Bef, St0) ->
%% put_list and put_map are atomic instructions, both of
%% which can safely resuse one of the source registers as target.
-set_cg([{var,R}], {cons,Es}, Le, Vdb, Bef, St) ->
- [S1,S2] = cg_reg_args(Es, Bef),
+put_cg([#k_var{name=R}], #k_cons{hd=Hd,tl=Tl}, Le, Vdb, Bef, St) ->
+ [S1,S2] = cg_reg_args([Hd,Tl], Bef),
Int0 = clear_dead(Bef, Le#l.i, Vdb),
Int1 = Int0#sr{reg=put_reg(R, Int0#sr.reg)},
Ret = fetch_reg(R, Int1#sr.reg),
{[{put_list,S1,S2,Ret}], Int1, St};
-set_cg([{var,R}], {binary,Segs}, Le, Vdb, Bef, #cg{bfail=Bfail}=St) ->
+put_cg([#k_var{name=R}], #k_binary{segs=Segs}, Le, Vdb, Bef,
+ #cg{bfail=Bfail}=St) ->
%% At run-time, binaries are constructed in three stages:
%% 1) First the size of the binary is calculated.
%% 2) Then the binary is allocated.
@@ -1577,7 +1891,9 @@ set_cg([{var,R}], {binary,Segs}, Le, Vdb, Bef, #cg{bfail=Bfail}=St) ->
{Sis++Code,Aft,St};
%% Map: single variable key.
-set_cg([{var,R}], {map,Op,Map,[{map_pair,{var,_}=K,V}]}, Le, Vdb, Bef, St0) ->
+put_cg([#k_var{name=R}], #k_map{op=Op,var=Map,
+ es=[#k_map_pair{key=#k_var{}=K,val=V}]},
+ Le, Vdb, Bef, St0) ->
{Sis,Int0} = maybe_adjust_stack(Bef, Le#l.i, Le#l.i+1, Vdb, St0),
SrcReg = cg_reg_arg_prefer_y(Map, Int0),
@@ -1592,22 +1908,23 @@ set_cg([{var,R}], {map,Op,Map,[{map_pair,{var,_}=K,V}]}, Le, Vdb, Bef, St0) ->
Aft = Aft0#sr{reg=put_reg(R, Aft0#sr.reg)},
Target = fetch_reg(R, Aft#sr.reg),
- {Is,St1} = set_cg_map(Line, Op, SrcReg, Target, Live, List, St0),
+ {Is,St1} = put_cg_map(Line, Op, SrcReg, Target, Live, List, St0),
{Sis++Is,Aft,St1};
%% Map: (possibly) multiple literal keys.
-set_cg([{var,R}], {map,Op,Map,Es}, Le, Vdb, Bef, St0) ->
+put_cg([#k_var{name=R}], #k_map{op=Op,var=Map,es=Es}, Le, Vdb, Bef, St0) ->
%% assert key literals
- [] = [Var||{map_pair,{var,_}=Var,_} <- Es],
+ [] = [Var || #k_map_pair{key=#k_var{}=Var} <- Es],
{Sis,Int0} = maybe_adjust_stack(Bef, Le#l.i, Le#l.i+1, Vdb, St0),
SrcReg = cg_reg_arg_prefer_y(Map, Int0),
Line = line(Le#l.a),
%% fetch registers for values to be put into the map
- Pairs = [{K,V} || {_,K,V} <- Es],
- List = flatmap(fun({K,V}) -> [K,cg_reg_arg(V,Int0)] end, Pairs),
+ List = flatmap(fun(#k_map_pair{key=K,val=V}) ->
+ [atomic(K),cg_reg_arg(V, Int0)]
+ end, Es),
Live = max_reg(Bef#sr.reg),
@@ -1616,16 +1933,16 @@ set_cg([{var,R}], {map,Op,Map,Es}, Le, Vdb, Bef, St0) ->
Aft = Aft0#sr{reg=put_reg(R, Aft0#sr.reg)},
Target = fetch_reg(R, Aft#sr.reg),
- {Is,St1} = set_cg_map(Line, Op, SrcReg, Target, Live, List, St0),
+ {Is,St1} = put_cg_map(Line, Op, SrcReg, Target, Live, List, St0),
{Sis++Is,Aft,St1};
%% Everything else.
-set_cg([{var,R}], Con, Le, Vdb, Bef, St) ->
+put_cg([#k_var{name=R}], Con, Le, Vdb, Bef, St) ->
%% Find a place for the return register first.
Int = Bef#sr{reg=put_reg(R, Bef#sr.reg)},
Ret = fetch_reg(R, Int#sr.reg),
Ais = case Con of
- {tuple,Es} ->
+ #k_tuple{es=Es} ->
[{put_tuple,length(Es),Ret}] ++ cg_build_args(Es, Bef);
Other ->
[{move,cg_reg_arg(Other, Int),Ret}]
@@ -1633,7 +1950,7 @@ set_cg([{var,R}], Con, Le, Vdb, Bef, St) ->
{Ais,clear_dead(Int, Le#l.i, Vdb),St}.
-set_cg_map(Line, Op0, SrcReg, Target, Live, List, St0) ->
+put_cg_map(Line, Op0, SrcReg, Target, Live, List, St0) ->
Bfail = St0#cg.bfail,
Fail = {f,St0#cg.bfail},
Op = case Op0 of
@@ -1811,24 +2128,44 @@ cg_gen_binsize([], _, _, _, _, Acc) -> Acc.
%% cg_bin_opt(Code0) -> Code
%% Optimize the size calculations for binary construction.
-cg_bin_opt([{move,Size,D},{bs_append,Fail,D,Extra,Regs,U,Bin,Flags,D}|Is]) ->
- cg_bin_opt([{bs_append,Fail,Size,Extra,Regs,U,Bin,Flags,D}|Is]);
-cg_bin_opt([{move,Size,D},{bs_private_append,Fail,D,U,Bin,Flags,D}|Is]) ->
- cg_bin_opt([{bs_private_append,Fail,Size,U,Bin,Flags,D}|Is]);
-cg_bin_opt([{move,{integer,0},D},{bs_add,_,[D,{integer,_}=S,1],Dst}|Is]) ->
- cg_bin_opt([{move,S,Dst}|Is]);
-cg_bin_opt([{move,{integer,0},D},{bs_add,Fail,[D,S,U],Dst}|Is]) ->
- cg_bin_opt([{bs_add,Fail,[{integer,0},S,U],Dst}|Is]);
-cg_bin_opt([{move,{integer,Bytes},D},{Op,Fail,D,Extra,Regs,Flags,D}|Is])
+cg_bin_opt([{move,S1,{x,X}=D},{gc_bif,Op,Fail,Live0,As,Dst}|Is]) ->
+ Live = if
+ X + 1 =:= Live0 -> X;
+ true -> Live0
+ end,
+ [{gc_bif,Op,Fail,Live,As,D}|cg_bin_opt([{move,S1,Dst}|Is])];
+cg_bin_opt([{move,_,_}=I1,{Op,_,_,_}=I2|Is])
+ when Op =:= bs_utf8_size orelse Op =:= bs_utf16_size ->
+ [I2|cg_bin_opt([I1|Is])];
+cg_bin_opt([{bs_add,_,[{integer,0},Src,1],Dst}|Is]) ->
+ cg_bin_opt_1([{move,Src,Dst}|Is]);
+cg_bin_opt([{bs_add,_,[Src,{integer,0},_],Dst}|Is]) ->
+ cg_bin_opt_1([{move,Src,Dst}|Is]);
+cg_bin_opt(Is) ->
+ cg_bin_opt_1(Is).
+
+cg_bin_opt_1([{move,Size,D},{bs_append,Fail,D,Extra,Regs,U,Bin,Flags,D}|Is]) ->
+ [{bs_append,Fail,Size,Extra,Regs,U,Bin,Flags,D}|cg_bin_opt(Is)];
+cg_bin_opt_1([{move,Size,D},{bs_private_append,Fail,D,U,Bin,Flags,D}|Is]) ->
+ [{bs_private_append,Fail,Size,U,Bin,Flags,D}|cg_bin_opt(Is)];
+cg_bin_opt_1([{move,Size,D},{Op,Fail,D,Extra,Regs,Flags,D}|Is])
when Op =:= bs_init2; Op =:= bs_init_bits ->
- cg_bin_opt([{Op,Fail,Bytes,Extra,Regs,Flags,D}|Is]);
-cg_bin_opt([{move,Src1,Dst},{bs_add,Fail,[Dst,Src2,U],Dst}|Is]) ->
- cg_bin_opt([{bs_add,Fail,[Src1,Src2,U],Dst}|Is]);
-cg_bin_opt([I|Is]) ->
+ Bytes = case Size of
+ {integer,Int} -> Int;
+ _ -> Size
+ end,
+ [{Op,Fail,Bytes,Extra,Regs,Flags,D}|cg_bin_opt(Is)];
+cg_bin_opt_1([{move,S1,D},{bs_add,Fail,[D,S2,U],Dst}|Is]) ->
+ cg_bin_opt([{bs_add,Fail,[S1,S2,U],Dst}|Is]);
+cg_bin_opt_1([{move,S1,D},{bs_add,Fail,[S2,D,U],Dst}|Is]) ->
+ cg_bin_opt([{bs_add,Fail,[S2,S1,U],Dst}|Is]);
+cg_bin_opt_1([I|Is]) ->
[I|cg_bin_opt(Is)];
-cg_bin_opt([]) -> [].
+cg_bin_opt_1([]) ->
+ [].
-cg_bin_put({bin_seg,[],S0,U,T,Fs,[E0,Next]}, Fail, Bef) ->
+cg_bin_put(#k_bin_seg{size=S0,unit=U,type=T,flags=Fs,seg=E0,next=Next},
+ Fail, Bef) ->
S1 = cg_reg_arg(S0, Bef),
E1 = cg_reg_arg(E0, Bef),
{Format,Op} = case T of
@@ -1845,7 +2182,7 @@ cg_bin_put({bin_seg,[],S0,U,T,Fs,[E0,Next]}, Fail, Bef) ->
utf ->
[{Op,Fail,{field_flags,Fs},E1}|cg_bin_put(Next, Fail, Bef)]
end;
-cg_bin_put({bin_end,[]}, _, _) -> [].
+cg_bin_put(#k_bin_end{}, _, _) -> [].
cg_build_args(As, Bef) ->
[{put,cg_reg_arg(A, Bef)} || A <- As].
@@ -1899,11 +2236,11 @@ get_locked_regs([], _) -> [].
cg_reg_args(As, Bef) -> [cg_reg_arg(A, Bef) || A <- As].
-cg_reg_arg({var,V}, Bef) -> fetch_var(V, Bef);
-cg_reg_arg(Literal, _) -> Literal.
+cg_reg_arg(#k_var{name=V}, Bef) -> fetch_var(V, Bef);
+cg_reg_arg(Literal, _) -> atomic(Literal).
-cg_reg_arg_prefer_y({var,V}, Bef) -> fetch_var_prefer_y(V, Bef);
-cg_reg_arg_prefer_y(Literal, _) -> Literal.
+cg_reg_arg_prefer_y(#k_var{name=V}, Bef) -> fetch_var_prefer_y(V, Bef);
+cg_reg_arg_prefer_y(Literal, _) -> atomic(Literal).
%% cg_setup_call([Arg], Bef, Cur, Vdb) -> {[Instr],Aft}.
%% Do the complete setup for a call/enter.
@@ -1941,9 +2278,9 @@ cg_call_args(As, Bef, I, Vdb) ->
load_arg_regs(Regs, As) -> load_arg_regs(Regs, As, 0).
-load_arg_regs([_|Rs], [{var,V}|As], I) -> [{I,V}|load_arg_regs(Rs, As, I+1)];
+load_arg_regs([_|Rs], [#k_var{name=V}|As], I) -> [{I,V}|load_arg_regs(Rs, As, I+1)];
load_arg_regs([_|Rs], [A|As], I) -> [{I,A}|load_arg_regs(Rs, As, I+1)];
-load_arg_regs([], [{var,V}|As], I) -> [{I,V}|load_arg_regs([], As, I+1)];
+load_arg_regs([], [#k_var{name=V}|As], I) -> [{I,V}|load_arg_regs([], As, I+1)];
load_arg_regs([], [A|As], I) -> [{I,A}|load_arg_regs([], As, I+1)];
load_arg_regs(Rs, [], _) -> Rs.
@@ -1979,12 +2316,13 @@ move_unsaved([], _, Regs, Acc) -> {Acc,Regs}.
gen_moves(As, Sr) -> gen_moves(As, Sr, 0, []).
-gen_moves([{var,V}|As], Sr, I, Acc) ->
+gen_moves([#k_var{name=V}|As], Sr, I, Acc) ->
case fetch_var(V, Sr) of
{x,I} -> gen_moves(As, Sr, I+1, Acc);
Reg -> gen_moves(As, Sr, I+1, [{move,Reg,{x,I}}|Acc])
end;
-gen_moves([A|As], Sr, I, Acc) ->
+gen_moves([A0|As], Sr, I, Acc) ->
+ A = atomic(A0),
gen_moves(As, Sr, I+1, [{move,A,{x,I}}|Acc]);
gen_moves([], _, _, Acc) -> lists:keysort(3, Acc).
@@ -2153,7 +2491,7 @@ fetch_var_prefer_y(V, #sr{reg=Reg,stk=Stk}) ->
end.
load_vars(Vs, Regs) ->
- foldl(fun ({var,V}, Rs) -> put_reg(V, Rs) end, Regs, Vs).
+ foldl(fun (#k_var{name=V}, Rs) -> put_reg(V, Rs) end, Regs, Vs).
%% put_reg(Val, Regs) -> Regs.
%% find_reg(Val, Regs) -> {ok,r{R}} | error.
@@ -2254,6 +2592,16 @@ put_catch(Tag, [Other|Stk], Acc) ->
drop_catch(Tag, [{{catch_tag,Tag}}|Stk]) -> [free|Stk];
drop_catch(Tag, [Other|Stk]) -> [Other|drop_catch(Tag, Stk)].
+%% atomic(Klit) -> Lit.
+%% atomic_list([Klit]) -> [Lit].
+
+atomic(#k_literal{val=V}) -> {literal,V};
+atomic(#k_int{val=I}) -> {integer,I};
+atomic(#k_float{val=F}) -> {float,F};
+atomic(#k_atom{val=A}) -> {atom,A};
+%%atomic(#k_char{val=C}) -> {char,C};
+atomic(#k_nil{}) -> nil.
+
%% new_label(St) -> {L,St}.
new_label(#cg{lcount=Next}=St) ->
@@ -2296,3 +2644,86 @@ flatmapfoldl(F, Accu0, [Hd|Tail]) ->
{Rs,Accu2} = flatmapfoldl(F, Accu1, Tail),
{R++Rs,Accu2};
flatmapfoldl(_, Accu, []) -> {[],Accu}.
+
+%% Keep track of life time for variables.
+%%
+%% init_vars([{var,VarName}]) -> Vdb.
+%% new_vars([VarName], I, Vdb) -> Vdb.
+%% use_vars([VarName], I, Vdb) -> Vdb.
+%% add_var(VarName, F, L, Vdb) -> Vdb.
+%%
+%% The list of variable names for new_vars/3 and use_vars/3
+%% must be sorted.
+
+init_vars(Vs) ->
+ vdb_new(Vs).
+
+new_vars([], _, Vdb) -> Vdb;
+new_vars([V], I, Vdb) -> vdb_store_new(V, {V,I,I}, Vdb);
+new_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I).
+
+use_vars([], _, Vdb) ->
+ Vdb;
+use_vars([V], I, Vdb) ->
+ case vdb_find(V, Vdb) of
+ {V,F,L} when I > L -> vdb_update(V, {V,F,I}, Vdb);
+ {V,_,_} -> Vdb;
+ error -> vdb_store_new(V, {V,I,I}, Vdb)
+ end;
+use_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I).
+
+add_var(V, F, L, Vdb) ->
+ vdb_store_new(V, {V,F,L}, Vdb).
+
+%% vdb
+
+vdb_new(Vs) ->
+ ordsets:from_list([{V,0,0} || #k_var{name=V} <- Vs]).
+
+-type var() :: atom().
+
+-spec vdb_find(var(), [vdb_entry()]) -> 'error' | vdb_entry().
+
+vdb_find(V, Vdb) ->
+ case lists:keyfind(V, 1, Vdb) of
+ false -> error;
+ Vd -> Vd
+ end.
+
+vdb_update(V, Update, [{V,_,_}|Vdb]) ->
+ [Update|Vdb];
+vdb_update(V, Update, [Vd|Vdb]) ->
+ [Vd|vdb_update(V, Update, Vdb)].
+
+vdb_store_new(V, New, [{V1,_,_}=Vd|Vdb]) when V > V1 ->
+ [Vd|vdb_store_new(V, New, Vdb)];
+vdb_store_new(V, New, [{V1,_,_}|_]=Vdb) when V < V1 ->
+ [New|Vdb];
+vdb_store_new(_, New, []) -> [New].
+
+vdb_update_vars([V|_]=Vs, [{V1,_,_}=Vd|Vdb], I) when V > V1 ->
+ [Vd|vdb_update_vars(Vs, Vdb, I)];
+vdb_update_vars([V|Vs], [{V1,_,_}|_]=Vdb, I) when V < V1 ->
+ %% New variable.
+ [{V,I,I}|vdb_update_vars(Vs, Vdb, I)];
+vdb_update_vars([V|Vs], [{_,F,L}=Vd|Vdb], I) ->
+ %% Existing variable.
+ if
+ I > L -> [{V,F,I}|vdb_update_vars(Vs, Vdb, I)];
+ true -> [Vd|vdb_update_vars(Vs, Vdb, I)]
+ end;
+vdb_update_vars([V|Vs], [], I) ->
+ %% New variable.
+ [{V,I,I}|vdb_update_vars(Vs, [], I)];
+vdb_update_vars([], Vdb, _) -> Vdb.
+
+%% vdb_sub(Min, Max, Vdb) -> Vdb.
+%% Extract variables which are used before and after Min. Lock
+%% variables alive after Max.
+
+vdb_sub(Min, Max, Vdb) ->
+ [ if L >= Max -> {V,F,locked};
+ true -> Vd
+ end || {V,F,L}=Vd <- Vdb,
+ F < Min,
+ L >= Min ].
diff --git a/lib/compiler/src/v3_core.erl b/lib/compiler/src/v3_core.erl
index ae650546e5..20cb3343fb 100644
--- a/lib/compiler/src/v3_core.erl
+++ b/lib/compiler/src/v3_core.erl
@@ -2505,8 +2505,46 @@ cexpr(#ifun{anno=#a{us=Us0}=A0,name={named,Name},fc=#iclause{pats=Ps}}=Fun0,
end;
cexpr(#iapply{anno=A,op=Op,args=Args}, _As, St) ->
{#c_apply{anno=A#a.anno,op=Op,args=Args},[],A#a.us,St};
-cexpr(#icall{anno=A,module=Mod,name=Name,args=Args}, _As, St) ->
- {#c_call{anno=A#a.anno,module=Mod,name=Name,args=Args},[],A#a.us,St};
+cexpr(#icall{anno=A,module=Mod,name=Name,args=Args}, _As, St0) ->
+ Anno = A#a.anno,
+ case (not cerl:is_c_atom(Mod)) andalso member(tuple_calls, St0#core.opts) of
+ true ->
+ GenAnno = [compiler_generated|Anno],
+
+ %% Generate the clause that matches on the tuple
+ {TupleVar,St1} = new_var(GenAnno, St0),
+ {TupleSizeVar, St2} = new_var(GenAnno, St1),
+ {TupleModVar, St3} = new_var(GenAnno, St2),
+ {TupleArgsVar, St4} = new_var(GenAnno, St3),
+ TryVar = cerl:c_var('Try'),
+
+ TupleGuardExpr =
+ cerl:c_let([TupleSizeVar],
+ c_call_erl(tuple_size, [TupleVar]),
+ c_call_erl('>', [TupleSizeVar, cerl:c_int(0)])),
+
+ TupleGuard =
+ cerl:c_try(TupleGuardExpr, [TryVar], TryVar,
+ [cerl:c_var('T'),cerl:c_var('R')], cerl:c_atom(false)),
+
+ TupleApply =
+ cerl:c_let([TupleModVar],
+ c_call_erl(element, [cerl:c_int(1),TupleVar]),
+ cerl:c_let([TupleArgsVar],
+ cerl:make_list(Args ++ [TupleVar]),
+ c_call_erl(apply, [TupleModVar,Name,TupleArgsVar]))),
+
+ TupleClause = cerl:ann_c_clause(GenAnno, [TupleVar], TupleGuard, TupleApply),
+
+ %% Generate the fallback clause
+ {OtherVar,St5} = new_var(GenAnno, St4),
+ OtherApply = cerl:ann_c_call(GenAnno, OtherVar, Name, Args),
+ OtherClause = cerl:ann_c_clause(GenAnno, [OtherVar], OtherApply),
+
+ {cerl:ann_c_case(GenAnno, Mod, [TupleClause,OtherClause]),[],A#a.us,St5};
+ false ->
+ {#c_call{anno=Anno,module=Mod,name=Name,args=Args},[],A#a.us,St0}
+ end;
cexpr(#iprimop{anno=A,name=Name,args=Args}, _As, St) ->
{#c_primop{anno=A#a.anno,name=Name,args=Args},[],A#a.us,St};
cexpr(#iprotect{anno=A,body=Es}, _As, St0) ->
@@ -2536,6 +2574,9 @@ cfun(#ifun{anno=A,id=Id,vars=Args,clauses=Lcs,fc=Lfc}, _As, St0) ->
clauses=Ccs ++ [Cfc]}},
[],A#a.us,St2}.
+c_call_erl(Fun, Args) ->
+ cerl:c_call(cerl:c_atom(erlang), cerl:c_atom(Fun), Args).
+
%% lit_vars(Literal) -> [Var].
lit_vars(Lit) -> lit_vars(Lit, []).
diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl
index 1fc05109c5..3eea058153 100644
--- a/lib/compiler/src/v3_kernel.erl
+++ b/lib/compiler/src/v3_kernel.erl
@@ -82,7 +82,8 @@
-export([module/2,format_error/1]).
-import(lists, [map/2,foldl/3,foldr/3,mapfoldl/3,splitwith/2,member/2,
- keymember/3,keyfind/3,partition/2,droplast/1,last/1,sort/1]).
+ keymember/3,keyfind/3,partition/2,droplast/1,last/1,sort/1,
+ reverse/1]).
-import(ordsets, [add_element/2,del_element/2,union/2,union/1,subtract/2]).
-import(cerl, [c_tuple/1]).
@@ -159,8 +160,7 @@ function({#c_var{name={F,Arity}=FA},Body}, St0) ->
{#ifun{anno=Ab,vars=Kvs,body=B0},[],St2} = expr(Body, new_sub(), St1),
{B1,_,St3} = ubody(B0, return, St2),
%%B1 = B0, St3 = St2, %Null second pass
- {#k_fdef{anno=#k{us=[],ns=[],a=Ab},
- func=F,arity=Arity,vars=Kvs,body=B1},St3}
+ {make_fdef(#k{us=[],ns=[],a=Ab}, F, Arity, Kvs, B1),St3}
catch
Class:Error ->
Stack = erlang:get_stacktrace(),
@@ -1589,23 +1589,18 @@ match_var([U|Us], Cs0, Def, St) ->
%% according to type, the order is really irrelevant but tries to be
%% smart.
-match_con(Us, [C], Def, St) ->
- %% There is only one clause. We can keep literal tuples and
- %% lists, but we must convert []/integer/float/atom literals
- %% to the proper record (#k_nil{} and so on).
- Cs = [expand_pat_lit_clause(C, false)],
- match_con_1(Us, Cs, Def, St);
match_con(Us, Cs0, Def, St) ->
- %% More than one clause. Remove literals at the top level.
- Cs = [expand_pat_lit_clause(C, true) || C <- Cs0],
+ %% Expand literals at the top level.
+ Cs = [expand_pat_lit_clause(C) || C <- Cs0],
match_con_1(Us, Cs, Def, St).
match_con_1([U|_Us] = L, Cs, Def, St0) ->
%% Extract clauses for different constructors (types).
%%ok = io:format("match_con ~p~n", [Cs]),
- Ttcs = select_types([k_binary], Cs) ++ select_bin_con(Cs) ++
- select_types([k_cons,k_tuple,k_map,k_atom,k_float,k_int,
- k_nil,k_literal], Cs),
+ Ttcs0 = select_types([k_binary], Cs) ++ select_bin_con(Cs) ++
+ select_types([k_cons,k_tuple,k_map,k_atom,k_float,
+ k_int,k_nil], Cs),
+ Ttcs = opt_single_valued(Ttcs0),
%%ok = io:format("ttcs = ~p~n", [Ttcs]),
{Scs,St1} =
mapfoldl(fun ({T,Tcs}, St) ->
@@ -1618,28 +1613,14 @@ match_con_1([U|_Us] = L, Cs, Def, St0) ->
select_types(Types, Cs) ->
[{T,Tcs} || T <- Types, begin Tcs = select(T, Cs), Tcs =/= [] end].
-
-expand_pat_lit_clause(#iclause{pats=[#ialias{pat=#k_literal{anno=A,val=Val}}=Alias|Ps]}=C, B) ->
- P = case B of
- true -> expand_pat_lit(Val, A);
- false -> literal(Val, A)
- end,
+
+expand_pat_lit_clause(#iclause{pats=[#ialias{pat=#k_literal{anno=A,val=Val}}=Alias|Ps]}=C) ->
+ P = expand_pat_lit(Val, A),
C#iclause{pats=[Alias#ialias{pat=P}|Ps]};
-expand_pat_lit_clause(#iclause{pats=[#k_literal{anno=A,val=Val}|Ps]}=C, B) ->
- P = case B of
- true -> expand_pat_lit(Val, A);
- false -> literal(Val, A)
- end,
+expand_pat_lit_clause(#iclause{pats=[#k_literal{anno=A,val=Val}|Ps]}=C) ->
+ P = expand_pat_lit(Val, A),
C#iclause{pats=[P|Ps]};
-expand_pat_lit_clause(#iclause{pats=[#k_binary{anno=A,segs=#k_bin_end{}}|Ps]}=C, B) ->
- case B of
- true ->
- C;
- false ->
- P = #k_literal{anno=A,val = <<>>},
- C#iclause{pats=[P|Ps]}
- end;
-expand_pat_lit_clause(C, _) -> C.
+expand_pat_lit_clause(C) -> C.
expand_pat_lit([H|T], A) ->
#k_cons{anno=A,hd=literal(H, A),tl=literal(T, A)};
@@ -1659,6 +1640,107 @@ literal(Val, A) when is_atom(Val) ->
literal(Val, A) when is_list(Val); is_tuple(Val) ->
#k_literal{anno=A,val=Val}.
+%% opt_singled_valued([{Type,Clauses}]) -> [{Type,Clauses}].
+%% If a type only has one clause and if the pattern is literal,
+%% the matching can be done more efficiently by directly comparing
+%% with the literal (that is especially true for binaries).
+
+opt_single_valued(Ttcs) ->
+ opt_single_valued(Ttcs, [], []).
+
+opt_single_valued([{_,[#iclause{pats=[P0|Ps]}=Tc]}=Ttc|Ttcs], TtcAcc, LitAcc) ->
+ try combine_lit_pat(P0) of
+ P ->
+ LitTtc = Tc#iclause{pats=[P|Ps]},
+ opt_single_valued(Ttcs, TtcAcc, [LitTtc|LitAcc])
+ catch
+ not_possible ->
+ opt_single_valued(Ttcs, [Ttc|TtcAcc], LitAcc)
+ end;
+opt_single_valued([Ttc|Ttcs], TtcAcc, LitAcc) ->
+ opt_single_valued(Ttcs, [Ttc|TtcAcc], LitAcc);
+opt_single_valued([], TtcAcc, []) ->
+ reverse(TtcAcc);
+opt_single_valued([], TtcAcc, LitAcc) ->
+ Literals = {k_literal,reverse(LitAcc)},
+ %% Test the literals as early as possible.
+ case reverse(TtcAcc) of
+ [{k_binary,_}=Bin|Ttcs] ->
+ %% The delayed creation of sub binaries requires
+ %% bs_start_match2 to be the first instruction in the
+ %% function.
+ [Bin,Literals|Ttcs];
+ Ttcs ->
+ [Literals|Ttcs]
+ end.
+
+combine_lit_pat(#ialias{pat=Pat0}=Alias) ->
+ Pat = combine_lit_pat(Pat0),
+ Alias#ialias{pat=Pat};
+combine_lit_pat(Pat) ->
+ case do_combine_lit_pat(Pat) of
+ #k_literal{val=Val} when is_atom(Val) ->
+ throw(not_possible);
+ #k_literal{val=Val} when is_number(Val) ->
+ throw(not_possible);
+ #k_literal{val=[]} ->
+ throw(not_possible);
+ #k_literal{}=Lit ->
+ Lit
+ end.
+
+do_combine_lit_pat(#k_atom{anno=A,val=Val}) ->
+ #k_literal{anno=A,val=Val};
+do_combine_lit_pat(#k_float{anno=A,val=Val}) ->
+ #k_literal{anno=A,val=Val};
+do_combine_lit_pat(#k_int{anno=A,val=Val}) ->
+ #k_literal{anno=A,val=Val};
+do_combine_lit_pat(#k_nil{anno=A}) ->
+ #k_literal{anno=A,val=[]};
+do_combine_lit_pat(#k_binary{anno=A,segs=Segs}) ->
+ Bin = combine_bin_segs(Segs),
+ #k_literal{anno=A,val=Bin};
+do_combine_lit_pat(#k_cons{anno=A,hd=Hd0,tl=Tl0}) ->
+ #k_literal{val=Hd} = do_combine_lit_pat(Hd0),
+ #k_literal{val=Tl} = do_combine_lit_pat(Tl0),
+ #k_literal{anno=A,val=[Hd|Tl]};
+do_combine_lit_pat(#k_literal{}=Lit) ->
+ Lit;
+do_combine_lit_pat(#k_tuple{anno=A,es=Es0}) ->
+ Es = [begin
+ #k_literal{val=Lit} = do_combine_lit_pat(El),
+ Lit
+ end || El <- Es0],
+ #k_literal{anno=A,val=list_to_tuple(Es)};
+do_combine_lit_pat(_) ->
+ throw(not_possible).
+
+combine_bin_segs(#k_bin_seg{size=Size0,unit=Unit,type=integer,
+ flags=[unsigned,big],seg=Seg,next=Next}) ->
+ #k_literal{val=Size1} = do_combine_lit_pat(Size0),
+ #k_literal{val=Int} = do_combine_lit_pat(Seg),
+ Size = Size1 * Unit,
+ if
+ 0 < Size, Size < 64 ->
+ Bin = <<Int:Size>>,
+ case Bin of
+ <<Int:Size>> ->
+ NextBin = combine_bin_segs(Next),
+ <<Bin/bits,NextBin/bits>>;
+ _ ->
+ %% The integer Int does not fit in the segment,
+ %% thus it will not match.
+ throw(not_possible)
+ end;
+ true ->
+ %% Avoid creating huge binary literals.
+ throw(not_possible)
+ end;
+combine_bin_segs(#k_bin_end{}) ->
+ <<>>;
+combine_bin_segs(_) ->
+ throw(not_possible).
+
%% select_bin_con([Clause]) -> [{Type,[Clause]}].
%% Extract clauses for the k_bin_seg constructor. As k_bin_seg
%% matching can overlap, the k_bin_seg constructors cannot be
@@ -2179,9 +2261,8 @@ iletrec_funs_gen(Fs, FreeVs, St) ->
Arity0 = length(Vs),
{Fb1,_,Lst1} = ubody(Fb0, return, Lst0#kern{ff={N,Arity0}}),
Arity = Arity0 + length(FreeVs),
- Fun = #k_fdef{anno=#k{us=[],ns=[],a=Fa},
- func=N,arity=Arity,
- vars=Vs ++ FreeVs,body=Fb1},
+ Fun = make_fdef(#k{us=[],ns=[],a=Fa}, N, Arity,
+ Vs++FreeVs, Fb1),
Lst1#kern{funs=[Fun|Lst1#kern.funs]}
end, St, Fs).
@@ -2325,8 +2406,7 @@ uexpr(#ifun{anno=A,vars=Vs,body=B0}, {break,Rs}, St0) ->
%% No id annotation. Must invent a fun name.
new_fun_name(St1)
end,
- Fun = #k_fdef{anno=#k{us=[],ns=[],a=A},func=Fname,arity=Arity,
- vars=Vs ++ Fvs,body=B1},
+ Fun = make_fdef(#k{us=[],ns=[],a=A}, Fname, Arity, Vs++Fvs, B1),
{#k_bif{anno=#k{us=Free,ns=lit_list_vars(Rs),a=A},
op=#k_internal{name=make_fun,arity=length(Free)+2},
args=[#k_atom{val=Fname},#k_int{val=Arity}|Fvs],
@@ -2343,6 +2423,16 @@ uexpr(Lit, {break,Rs0}, St0) ->
add_local_function(_, #kern{funs=ignore}=St) -> St;
add_local_function(F, #kern{funs=Funs}=St) -> St#kern{funs=[F|Funs]}.
+%% Make a #k_fdef{}, making sure that the body is always a #k_match{}.
+make_fdef(Anno, Name, Arity, Vs, #k_match{}=Body) ->
+ #k_fdef{anno=Anno,func=Name,arity=Arity,vars=Vs,body=Body};
+make_fdef(Anno, Name, Arity, Vs, Body) ->
+ Ka = get_kanno(Body),
+ Match = #k_match{anno=#k{us=Ka#k.us,ns=[],a=Ka#k.a},
+ vars=Vs,body=Body,ret=[]},
+ #k_fdef{anno=Anno,func=Name,arity=Arity,vars=Vs,body=Match}.
+
+
%% handle_reuse_annos([#k_var{}], State) -> State.
%% In general, it is only safe to reuse a variable for a match context
%% if the original value of the variable will no longer be needed.
diff --git a/lib/compiler/src/v3_kernel.hrl b/lib/compiler/src/v3_kernel.hrl
index 7cd30b25a8..87011b7680 100644
--- a/lib/compiler/src/v3_kernel.hrl
+++ b/lib/compiler/src/v3_kernel.hrl
@@ -79,7 +79,7 @@
-record(k_guard_clause, {anno=[],guard,body}).
-record(k_break, {anno=[],args=[]}).
--record(k_guard_break, {anno=[],args=[]}).
+-record(k_guard_break, {anno=[],args=[],locked=[]}).
-record(k_return, {anno=[],args=[]}).
%%k_get_anno(Thing) -> element(2, Thing).
diff --git a/lib/compiler/src/v3_kernel_pp.erl b/lib/compiler/src/v3_kernel_pp.erl
index 53097d0d7d..ac91039ae0 100644
--- a/lib/compiler/src/v3_kernel_pp.erl
+++ b/lib/compiler/src/v3_kernel_pp.erl
@@ -491,7 +491,7 @@ indent(Ctxt) -> indent(Ctxt#ctxt.indent, Ctxt).
indent(N, _Ctxt) when N =< 0 -> "";
indent(N, Ctxt) ->
T = Ctxt#ctxt.tab_width,
- string:chars($\t, N div T, string:chars($\s, N rem T)).
+ lists:duplicate(N div T, $\t) ++ lists:duplicate(N rem T, $\s).
nl_indent(Ctxt) -> [$\n|indent(Ctxt)].
@@ -508,7 +508,7 @@ unindent([$\t|T], N, Ctxt, C) ->
if N >= Tab ->
unindent(T, N - Tab, Ctxt, C);
true ->
- unindent([string:chars($\s, Tab - N)|T], 0, Ctxt, C)
+ unindent([lists:duplicate(Tab - N, $\s)|T], 0, Ctxt, C)
end;
unindent([L|T], N, Ctxt, C) when is_list(L) ->
unindent(L, N, Ctxt, [T|C]);
diff --git a/lib/compiler/src/v3_life.erl b/lib/compiler/src/v3_life.erl
deleted file mode 100644
index be3ade47ff..0000000000
--- a/lib/compiler/src/v3_life.erl
+++ /dev/null
@@ -1,468 +0,0 @@
-%%
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%
-%% %CopyrightEnd%
-%%
-%% Purpose : Convert annotated kernel expressions to annotated beam format.
-
-%% This module creates beam format annotated with variable lifetime
-%% information. Each thing is given an index and for each variable we
-%% store the first and last index for its occurrence. The variable
-%% database, VDB, attached to each thing is only relevant internally
-%% for that thing.
-%%
-%% For nested things like matches the numbering continues locally and
-%% the VDB for that thing refers to the variable usage within that
-%% thing. Variables which live through a such a thing are internally
-%% given a very large last index. Internally the indexes continue
-%% after the index of that thing. This creates no problems as the
-%% internal variable info never escapes and externally we only see
-%% variable which are alive both before or after.
-%%
-%% This means that variables never "escape" from a thing and the only
-%% way to get values from a thing is to "return" them, with 'break' or
-%% 'return'. Externally these values become the return values of the
-%% thing. This is no real limitation as most nested things have
-%% multiple threads so working out a common best variable usage is
-%% difficult.
-
--module(v3_life).
-
--export([module/2]).
-
--export([vdb_find/2]).
-
--import(lists, [member/2,map/2,reverse/1,sort/1]).
--import(ordsets, [add_element/2,intersection/2,union/2]).
-
--include("v3_kernel.hrl").
--include("v3_life.hrl").
-
--type fa() :: {atom(),arity()}.
-
-%% These are not defined in v3_kernel.hrl.
-get_kanno(Kthing) -> element(2, Kthing).
-%%set_kanno(Kthing, Anno) -> setelement(2, Kthing, Anno).
-
--spec module(#k_mdef{}, [compile:option()]) ->
- {'ok',{module(),[fa()],[_],[_]}}.
-
-module(#k_mdef{name=M,exports=Es,attributes=As,body=Fs0}, _Opts) ->
- Fs1 = functions(Fs0, []),
- {ok,{M,Es,As,Fs1}}.
-
-functions([F|Fs], Acc) ->
- functions(Fs, [function(F)|Acc]);
-functions([], Acc) -> reverse(Acc).
-
-%% function(Kfunc) -> Func.
-
-function(#k_fdef{anno=#k{a=Anno},func=F,arity=Ar,vars=Vs,body=Kb}) ->
- try
- As = var_list(Vs),
- Vdb0 = init_vars(As),
- %% Force a top-level match!
- B0 = case Kb of
- #k_match{} -> Kb;
- _ ->
- Ka = get_kanno(Kb),
- #k_match{anno=#k{us=Ka#k.us,ns=[],a=Ka#k.a},
- vars=Vs,body=Kb,ret=[]}
- end,
- {B1,_,Vdb1} = body(B0, 1, Vdb0),
- {function,F,Ar,As,B1,Vdb1,Anno}
- catch
- Class:Error ->
- Stack = erlang:get_stacktrace(),
- io:fwrite("Function: ~w/~w\n", [F,Ar]),
- erlang:raise(Class, Error, Stack)
- end.
-
-%% body(Kbody, I, Vdb) -> {[Expr],MaxI,Vdb}.
-%% Handle a body.
-
-body(#k_seq{arg=Ke,body=Kb}, I, Vdb0) ->
- %%ok = io:fwrite("life ~w:~p~n", [?LINE,{Ke,I,Vdb0}]),
- A = get_kanno(Ke),
- Vdb1 = use_vars(union(A#k.us, A#k.ns), I, Vdb0),
- {Es,MaxI,Vdb2} = body(Kb, I+1, Vdb1),
- E = expr(Ke, I, Vdb2),
- {[E|Es],MaxI,Vdb2};
-body(Ke, I, Vdb0) ->
- %%ok = io:fwrite("life ~w:~p~n", [?LINE,{Ke,I,Vdb0}]),
- A = get_kanno(Ke),
- Vdb1 = use_vars(union(A#k.us, A#k.ns), I, Vdb0),
- E = expr(Ke, I, Vdb1),
- {[E],I,Vdb1}.
-
-%% protected(Kprotected, I, Vdb) -> Protected.
-%% Only used in guards.
-
-protected(#k_protected{anno=A,arg=Ts,ret=Rs}, I, Vdb) ->
- %% Lock variables that are alive before try and used afterwards.
- %% Don't lock variables that are only used inside the protected
- %% expression.
- Pdb0 = vdb_sub(I, I+1, Vdb),
- {T,MaxI,Pdb1} = body(Ts, I+1, Pdb0),
- Pdb2 = use_vars(A#k.ns, MaxI+1, Pdb1), %Save "return" values
- #l{ke={protected,T,var_list(Rs)},i=I,a=A#k.a,vdb=Pdb2}.
-
-%% expr(Kexpr, I, Vdb) -> Expr.
-
-expr(#k_test{anno=A,op=Op,args=As,inverted=Inverted}, I, _Vdb) ->
- #l{ke={test,test_op(Op),atomic_list(As),Inverted},i=I,a=A#k.a};
-expr(#k_call{anno=A,op=Op,args=As,ret=Rs}, I, _Vdb) ->
- #l{ke={call,call_op(Op),atomic_list(As),var_list(Rs)},i=I,a=A#k.a};
-expr(#k_enter{anno=A,op=Op,args=As}, I, _Vdb) ->
- #l{ke={enter,call_op(Op),atomic_list(As)},i=I,a=A#k.a};
-expr(#k_bif{anno=A,op=Op,args=As,ret=Rs}, I, _Vdb) ->
- Bif = k_bif(A, Op, As, Rs),
- #l{ke=Bif,i=I,a=A#k.a};
-expr(#k_match{anno=A,body=Kb,ret=Rs}, I, Vdb) ->
- %% Work out imported variables which need to be locked.
- Mdb = vdb_sub(I, I+1, Vdb),
- M = match(Kb, A#k.us, I+1, [], Mdb),
- #l{ke={match,M,var_list(Rs)},i=I,vdb=use_vars(A#k.us, I+1, Mdb),a=A#k.a};
-expr(#k_guard_match{anno=A,body=Kb,ret=Rs}, I, Vdb) ->
- %% Work out imported variables which need to be locked.
- Mdb = vdb_sub(I, I+1, Vdb),
- M = match(Kb, A#k.us, I+1, [], Mdb),
- #l{ke={guard_match,M,var_list(Rs)},i=I,vdb=use_vars(A#k.us, I+1, Mdb),a=A#k.a};
-expr(#k_try{}=Try, I, Vdb) ->
- body_try(Try, I, Vdb);
-expr(#k_protected{}=Protected, I, Vdb) ->
- protected(Protected, I, Vdb);
-expr(#k_try_enter{anno=A,arg=Ka,vars=Vs,body=Kb,evars=Evs,handler=Kh}, I, Vdb) ->
- %% Lock variables that are alive before the catch and used afterwards.
- %% Don't lock variables that are only used inside the try.
- Tdb0 = vdb_sub(I, I+1, Vdb),
- %% This is the tricky bit. Lock variables in Arg that are used in
- %% the body and handler. Add try tag 'variable'.
- Ab = get_kanno(Kb),
- Ah = get_kanno(Kh),
- Tdb1 = use_vars(union(Ab#k.us, Ah#k.us), I+3, Tdb0),
- Tdb2 = vdb_sub(I, I+2, Tdb1),
- Vnames = fun (Kvar) -> Kvar#k_var.name end, %Get the variable names
- {Aes,_,Adb} = body(Ka, I+2, add_var({catch_tag,I+1}, I+1, 1000000, Tdb2)),
- {Bes,_,Bdb} = body(Kb, I+4, new_vars(sort(map(Vnames, Vs)), I+3, Tdb2)),
- {Hes,_,Hdb} = body(Kh, I+4, new_vars(sort(map(Vnames, Evs)), I+3, Tdb2)),
- #l{ke={try_enter,#l{ke={block,Aes},i=I+1,vdb=Adb,a=[]},
- var_list(Vs),#l{ke={block,Bes},i=I+3,vdb=Bdb,a=[]},
- var_list(Evs),#l{ke={block,Hes},i=I+3,vdb=Hdb,a=[]}},
- i=I,vdb=Tdb1,a=A#k.a};
-expr(#k_catch{anno=A,body=Kb,ret=[R]}, I, Vdb) ->
- %% Lock variables that are alive before the catch and used afterwards.
- %% Don't lock variables that are only used inside the catch.
- %% Add catch tag 'variable'.
- Cdb0 = vdb_sub(I, I+1, Vdb),
- {Es,_,Cdb1} = body(Kb, I+1, add_var({catch_tag,I}, I, locked, Cdb0)),
- #l{ke={'catch',Es,variable(R)},i=I,vdb=Cdb1,a=A#k.a};
-expr(#k_receive{anno=A,var=V,body=Kb,timeout=T,action=Ka,ret=Rs}, I, Vdb) ->
- %% Work out imported variables which need to be locked.
- Rdb = vdb_sub(I, I+1, Vdb),
- M = match(Kb, add_element(V#k_var.name, A#k.us), I+1, [],
- new_vars([V#k_var.name], I, Rdb)),
- {Tes,_,Adb} = body(Ka, I+1, Rdb),
- #l{ke={receive_loop,atomic(T),variable(V),M,
- #l{ke=Tes,i=I+1,vdb=Adb,a=[]},var_list(Rs)},
- i=I,vdb=use_vars(A#k.us, I+1, Vdb),a=A#k.a};
-expr(#k_receive_accept{anno=A}, I, _Vdb) ->
- #l{ke=receive_accept,i=I,a=A#k.a};
-expr(#k_receive_next{anno=A}, I, _Vdb) ->
- #l{ke=receive_next,i=I,a=A#k.a};
-expr(#k_put{anno=A,arg=Arg,ret=Rs}, I, _Vdb) ->
- #l{ke={set,var_list(Rs),literal(Arg, [])},i=I,a=A#k.a};
-expr(#k_break{anno=A,args=As}, I, _Vdb) ->
- #l{ke={break,atomic_list(As)},i=I,a=A#k.a};
-expr(#k_guard_break{anno=A,args=As}, I, Vdb) ->
- Locked = [V || {V,_,_} <- Vdb],
- #l{ke={guard_break,atomic_list(As),Locked},i=I,a=A#k.a};
-expr(#k_return{anno=A,args=As}, I, _Vdb) ->
- #l{ke={return,atomic_list(As)},i=I,a=A#k.a}.
-
-body_try(#k_try{anno=A,arg=Ka,vars=Vs,body=Kb,evars=Evs,handler=Kh,ret=Rs},
- I, Vdb) ->
- %% Lock variables that are alive before the catch and used afterwards.
- %% Don't lock variables that are only used inside the try.
- Tdb0 = vdb_sub(I, I+1, Vdb),
- %% This is the tricky bit. Lock variables in Arg that are used in
- %% the body and handler. Add try tag 'variable'.
- Ab = get_kanno(Kb),
- Ah = get_kanno(Kh),
- Tdb1 = use_vars(union(Ab#k.us, Ah#k.us), I+3, Tdb0),
- Tdb2 = vdb_sub(I, I+2, Tdb1),
- Vnames = fun (Kvar) -> Kvar#k_var.name end, %Get the variable names
- {Aes,_,Adb} = body(Ka, I+2, add_var({catch_tag,I+1}, I+1, locked, Tdb2)),
- {Bes,_,Bdb} = body(Kb, I+4, new_vars(sort(map(Vnames, Vs)), I+3, Tdb2)),
- {Hes,_,Hdb} = body(Kh, I+4, new_vars(sort(map(Vnames, Evs)), I+3, Tdb2)),
- #l{ke={'try',#l{ke={block,Aes},i=I+1,vdb=Adb,a=[]},
- var_list(Vs),#l{ke={block,Bes},i=I+3,vdb=Bdb,a=[]},
- var_list(Evs),#l{ke={block,Hes},i=I+3,vdb=Hdb,a=[]},
- var_list(Rs)},
- i=I,vdb=Tdb1,a=A#k.a}.
-
-%% call_op(Op) -> Op.
-%% test_op(Op) -> Op.
-%% Do any necessary name translations here to munge into beam format.
-
-call_op(#k_local{name=N}) -> N;
-call_op(#k_remote{mod=M,name=N}) -> {remote,atomic(M),atomic(N)};
-call_op(Other) -> variable(Other).
-
-test_op(#k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=N}}) -> N.
-
-%% k_bif(Anno, Op, [Arg], [Ret], Vdb) -> Expr.
-%% Build bifs.
-
-k_bif(_A, #k_internal{name=Name}, As, Rs) ->
- {internal,Name,atomic_list(As),var_list(Rs)};
-k_bif(_A, #k_remote{mod=#k_atom{val=erlang},name=#k_atom{val=Name}}, As, Rs) ->
- Ar = length(As),
- case is_gc_bif(Name, Ar) of
- false ->
- {bif,Name,atomic_list(As),var_list(Rs)};
- true ->
- {gc_bif,Name,atomic_list(As),var_list(Rs)}
- end.
-
-%% match(Kexpr, [LockVar], I, Vdb) -> Expr.
-%% Convert match tree to old format.
-
-match(#k_alt{anno=A,first=Kf,then=Kt}, Ls, I, Ctxt, Vdb0) ->
- Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0),
- F = match(Kf, Ls, I+1, Ctxt, Vdb1),
- T = match(Kt, Ls, I+1, Ctxt, Vdb1),
- #l{ke={alt,F,T},i=I,vdb=Vdb1,a=A#k.a};
-match(#k_select{anno=A,var=V,types=Kts}, Ls0, I, Ctxt, Vdb0) ->
- Vanno = get_kanno(V),
- Ls1 = case member(no_usage, Vanno) of
- false -> add_element(V#k_var.name, Ls0);
- true -> Ls0
- end,
- Anno = case member(reuse_for_context, Vanno) of
- true -> [reuse_for_context|A#k.a];
- false -> A#k.a
- end,
- Vdb1 = use_vars(union(A#k.us, Ls1), I, Vdb0),
- Ts = [type_clause(Tc, Ls1, I+1, Ctxt, Vdb1) || Tc <- Kts],
- #l{ke={select,literal(V, Ctxt),Ts},i=I,vdb=Vdb1,a=Anno};
-match(#k_guard{anno=A,clauses=Kcs}, Ls, I, Ctxt, Vdb0) ->
- Vdb1 = use_vars(union(A#k.us, Ls), I, Vdb0),
- Cs = [guard_clause(G, Ls, I+1, Ctxt, Vdb1) || G <- Kcs],
- #l{ke={guard,Cs},i=I,vdb=Vdb1,a=A#k.a};
-match(Other, Ls, I, _Ctxt, Vdb0) ->
- Vdb1 = use_vars(Ls, I, Vdb0),
- {B,_,Vdb2} = body(Other, I+1, Vdb1),
- #l{ke={block,B},i=I,vdb=Vdb2,a=[]}.
-
-type_clause(#k_type_clause{anno=A,type=T,values=Kvs}, Ls, I, Ctxt, Vdb0) ->
- %%ok = io:format("life ~w: ~p~n", [?LINE,{T,Kvs}]),
- Vdb1 = use_vars(union(A#k.us, Ls), I+1, Vdb0),
- Vs = [val_clause(Vc, Ls, I+1, Ctxt, Vdb1) || Vc <- Kvs],
- #l{ke={type_clause,type(T),Vs},i=I,vdb=Vdb1,a=A#k.a}.
-
-val_clause(#k_val_clause{anno=A,val=V,body=Kb}, Ls0, I, Ctxt0, Vdb0) ->
- New = (get_kanno(V))#k.ns,
- Bus = (get_kanno(Kb))#k.us,
- %%ok = io:format("Ls0 = ~p, Used=~p\n New=~p, Bus=~p\n", [Ls0,Used,New,Bus]),
- Ls1 = union(intersection(New, Bus), Ls0), %Lock for safety
- Vdb1 = use_vars(union(A#k.us, Ls1), I+1, new_vars(New, I, Vdb0)),
- Ctxt = case V of
- #k_binary{segs=#k_var{name=C0}} -> C0;
- _ -> Ctxt0
- end,
- B = match(Kb, Ls1, I+1, Ctxt, Vdb1),
- #l{ke={val_clause,literal(V, Ctxt),B},i=I,vdb=use_vars(Bus, I+1, Vdb1),a=A#k.a}.
-
-guard_clause(#k_guard_clause{anno=A,guard=Kg,body=Kb}, Ls, I, Ctxt, Vdb0) ->
- Vdb1 = use_vars(union(A#k.us, Ls), I+2, Vdb0),
- Gdb = vdb_sub(I+1, I+2, Vdb1),
- G = protected(Kg, I+1, Gdb),
- B = match(Kb, Ls, I+2, Ctxt, Vdb1),
- #l{ke={guard_clause,G,B},
- i=I,vdb=use_vars((get_kanno(Kg))#k.us, I+2, Vdb1),
- a=A#k.a}.
-
-%% type(Ktype) -> Type.
-
-type(k_literal) -> literal;
-type(k_int) -> integer;
-%%type(k_char) -> integer; %Hhhmmm???
-type(k_float) -> float;
-type(k_atom) -> atom;
-type(k_nil) -> nil;
-type(k_cons) -> cons;
-type(k_tuple) -> tuple;
-type(k_binary) -> binary;
-type(k_bin_seg) -> bin_seg;
-type(k_bin_int) -> bin_int;
-type(k_bin_end) -> bin_end;
-type(k_map) -> map.
-
-%% variable(Klit) -> Lit.
-%% var_list([Klit]) -> [Lit].
-
-variable(#k_var{name=N}) -> {var,N}.
-
-var_list(Ks) -> [variable(K) || K <- Ks].
-
-%% atomic(Klit) -> Lit.
-%% atomic_list([Klit]) -> [Lit].
-
-atomic(#k_literal{val=V}) -> {literal,V};
-atomic(#k_var{name=N}) -> {var,N};
-atomic(#k_int{val=I}) -> {integer,I};
-atomic(#k_float{val=F}) -> {float,F};
-atomic(#k_atom{val=N}) -> {atom,N};
-%%atomic(#k_char{val=C}) -> {char,C};
-atomic(#k_nil{}) -> nil.
-
-atomic_list(Ks) -> [atomic(K) || K <- Ks].
-
-%% literal(Klit) -> Lit.
-%% literal_list([Klit]) -> [Lit].
-
-literal(#k_var{name=N}, _) -> {var,N};
-literal(#k_literal{val=I}, _) -> {literal,I};
-literal(#k_int{val=I}, _) -> {integer,I};
-literal(#k_float{val=F}, _) -> {float,F};
-literal(#k_atom{val=N}, _) -> {atom,N};
-%%literal(#k_char{val=C}, _) -> {char,C};
-literal(#k_nil{}, _) -> nil;
-literal(#k_cons{hd=H,tl=T}, Ctxt) ->
- {cons,[literal(H, Ctxt),literal(T, Ctxt)]};
-literal(#k_binary{segs=V}, Ctxt) ->
- {binary,literal(V, Ctxt)};
-literal(#k_bin_seg{size=S,unit=U,type=T,flags=Fs,seg=Seg,next=[]}, Ctxt) ->
- %% Only occurs in patterns.
- {bin_seg,Ctxt,literal(S, Ctxt),U,T,Fs,[literal(Seg, Ctxt)]};
-literal(#k_bin_seg{size=S,unit=U,type=T,flags=Fs,seg=Seg,next=N}, Ctxt) ->
- {bin_seg,Ctxt,literal(S, Ctxt),U,T,Fs,
- [literal(Seg, Ctxt),literal(N, Ctxt)]};
-literal(#k_bin_int{size=S,unit=U,flags=Fs,val=Int,next=N}, Ctxt) ->
- %% Only occurs in patterns.
- {bin_int,Ctxt,literal(S, Ctxt),U,Fs,Int,
- [literal(N, Ctxt)]};
-literal(#k_bin_end{}, Ctxt) ->
- {bin_end,Ctxt};
-literal(#k_tuple{es=Es}, Ctxt) ->
- {tuple,literal_list(Es, Ctxt)};
-literal(#k_map{op=Op,var=Var,es=Es0}, Ctxt) ->
- {map,Op,literal(Var, Ctxt),literal_list(Es0, Ctxt)};
-literal(#k_map_pair{key=K,val=V}, Ctxt) ->
- {map_pair,literal(K, Ctxt),literal(V, Ctxt)}.
-
-literal_list(Ks, Ctxt) ->
- [literal(K, Ctxt) || K <- Ks].
-
-
-%% is_gc_bif(Name, Arity) -> true|false
-%% Determines whether the BIF Name/Arity might do a GC.
-
-is_gc_bif(hd, 1) -> false;
-is_gc_bif(tl, 1) -> false;
-is_gc_bif(self, 0) -> false;
-is_gc_bif(node, 0) -> false;
-is_gc_bif(node, 1) -> false;
-is_gc_bif(element, 2) -> false;
-is_gc_bif(get, 1) -> false;
-is_gc_bif(tuple_size, 1) -> false;
-is_gc_bif(Bif, Arity) ->
- not (erl_internal:bool_op(Bif, Arity) orelse
- erl_internal:new_type_test(Bif, Arity) orelse
- erl_internal:comp_op(Bif, Arity)).
-
-%% Keep track of life time for variables.
-%%
-%% init_vars([{var,VarName}]) -> Vdb.
-%% new_vars([VarName], I, Vdb) -> Vdb.
-%% use_vars([VarName], I, Vdb) -> Vdb.
-%% add_var(VarName, F, L, Vdb) -> Vdb.
-%%
-%% The list of variable names for new_vars/3 and use_vars/3
-%% must be sorted.
-
-init_vars(Vs) ->
- vdb_new(Vs).
-
-new_vars([], _, Vdb) -> Vdb;
-new_vars([V], I, Vdb) -> vdb_store_new(V, {V,I,I}, Vdb);
-new_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I).
-
-use_vars([], _, Vdb) ->
- Vdb;
-use_vars([V], I, Vdb) ->
- case vdb_find(V, Vdb) of
- {V,F,L} when I > L -> vdb_update(V, {V,F,I}, Vdb);
- {V,_,_} -> Vdb;
- error -> vdb_store_new(V, {V,I,I}, Vdb)
- end;
-use_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I).
-
-add_var(V, F, L, Vdb) ->
- vdb_store_new(V, {V,F,L}, Vdb).
-
-%% vdb
-
-vdb_new(Vs) ->
- sort([{V,0,0} || {var,V} <- Vs]).
-
--type var() :: atom().
-
--spec vdb_find(var(), [vdb_entry()]) -> 'error' | vdb_entry().
-
-vdb_find(V, Vdb) ->
- case lists:keyfind(V, 1, Vdb) of
- false -> error;
- Vd -> Vd
- end.
-
-vdb_update(V, Update, [{V,_,_}|Vdb]) ->
- [Update|Vdb];
-vdb_update(V, Update, [Vd|Vdb]) ->
- [Vd|vdb_update(V, Update, Vdb)].
-
-vdb_store_new(V, New, [{V1,_,_}=Vd|Vdb]) when V > V1 ->
- [Vd|vdb_store_new(V, New, Vdb)];
-vdb_store_new(V, New, [{V1,_,_}|_]=Vdb) when V < V1 ->
- [New|Vdb];
-vdb_store_new(_, New, []) -> [New].
-
-vdb_update_vars([V|_]=Vs, [{V1,_,_}=Vd|Vdb], I) when V > V1 ->
- [Vd|vdb_update_vars(Vs, Vdb, I)];
-vdb_update_vars([V|Vs], [{V1,_,_}|_]=Vdb, I) when V < V1 ->
- %% New variable.
- [{V,I,I}|vdb_update_vars(Vs, Vdb, I)];
-vdb_update_vars([V|Vs], [{_,F,L}=Vd|Vdb], I) ->
- %% Existing variable.
- if
- I > L -> [{V,F,I}|vdb_update_vars(Vs, Vdb, I)];
- true -> [Vd|vdb_update_vars(Vs, Vdb, I)]
- end;
-vdb_update_vars([V|Vs], [], I) ->
- %% New variable.
- [{V,I,I}|vdb_update_vars(Vs, [], I)];
-vdb_update_vars([], Vdb, _) -> Vdb.
-
-%% vdb_sub(Min, Max, Vdb) -> Vdb.
-%% Extract variables which are used before and after Min. Lock
-%% variables alive after Max.
-
-vdb_sub(Min, Max, Vdb) ->
- [ if L >= Max -> {V,F,locked};
- true -> Vd
- end || {V,F,L}=Vd <- Vdb, F < Min, L >= Min ].
diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile
index 63763f31b2..da5d207db9 100644
--- a/lib/compiler/test/Makefile
+++ b/lib/compiler/test/Makefile
@@ -22,6 +22,7 @@ MODULES= \
bs_construct_SUITE \
bs_match_SUITE \
bs_utf_SUITE \
+ core_alias_SUITE \
core_fold_SUITE \
compile_SUITE \
compilation_SUITE \
diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl
index 2fe8cd0cff..ad48d4c0b7 100644
--- a/lib/compiler/test/bs_match_SUITE.erl
+++ b/lib/compiler/test/bs_match_SUITE.erl
@@ -39,7 +39,7 @@
match_string_opt/1,select_on_integer/1,
map_and_binary/1,unsafe_branch_caching/1,
bad_literals/1,good_literals/1,constant_propagation/1,
- parse_xml/1,get_payload/1,num_slots_different/1]).
+ parse_xml/1,get_payload/1,escape/1,num_slots_different/1]).
-export([coverage_id/1,coverage_external_ignore/2]).
@@ -71,7 +71,7 @@ groups() ->
match_string_opt,select_on_integer,
map_and_binary,unsafe_branch_caching,
bad_literals,good_literals,constant_propagation,parse_xml,
- get_payload,num_slots_different]}].
+ get_payload,escape,num_slots_different]}].
init_per_suite(Config) ->
@@ -1524,6 +1524,21 @@ do_get_payload(ExtHdr) ->
<<_:13,_:35>> = ExtHdr#ext_header.ext_hdr_opts,
ExtHdrOptions.
+escape(_Config) ->
+ 0 = escape(<<>>, 0),
+ 1 = escape(<<128>>, 0),
+ 2 = escape(<<128,255>>, 0),
+ 42 = escape(<<42>>, 0),
+ 50 = escape(<<42,8>>, 0),
+ ok.
+
+escape(<<Byte, Rest/bits>>, Pos) when Byte >= 127 ->
+ escape(Rest, Pos + 1);
+escape(<<Byte, Rest/bits>>, Pos) ->
+ escape(Rest, Pos + Byte);
+escape(<<_Rest/bits>>, Pos) ->
+ Pos.
+
%% ERL-490
num_slots_different(_Config) ->
Ts = [{<<"de">>, <<"default">>, <<"Remove">>, <<"a">>},
diff --git a/lib/compiler/test/compilation_SUITE_data/opt_crash.erl b/lib/compiler/test/compilation_SUITE_data/opt_crash.erl
index f1607cca68..c65ec31593 100644
--- a/lib/compiler/test/compilation_SUITE_data/opt_crash.erl
+++ b/lib/compiler/test/compilation_SUITE_data/opt_crash.erl
@@ -33,7 +33,7 @@ test() ->
{userinfo,nil},
fun() -> nil end},
nil},
- {'query',nil}}},
+ {query,nil}}},
{absoluteURI,
{scheme,_},
@@ -43,7 +43,7 @@ test() ->
{userinfo,nil},
HostportBefore},
nil},
- {'query',nil}}} = URI_Before,
+ {query,nil}}} = URI_Before,
%% ... some funky code ommitted, not relevant ...
@@ -55,7 +55,7 @@ test() ->
{userinfo,nil},
HostportAfter},
nil},
- {'query',nil}}} = URI_Before,
+ {query,nil}}} = URI_Before,
%% NOTE: I intended to write URI_After instead of URI_Before
%% but the accident revealed that when you add the line below,
%% it causes internal error in v3_codegen on compilation
diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl
index f647a4030d..e88a132d59 100644
--- a/lib/compiler/test/compile_SUITE.erl
+++ b/lib/compiler/test/compile_SUITE.erl
@@ -27,12 +27,12 @@
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
init_per_group/2,end_per_group/2,
app_test/1,appup_test/1,
- debug_info/4, custom_debug_info/1,
+ debug_info/4, custom_debug_info/1, custom_compile_info/1,
file_1/1, forms_2/1, module_mismatch/1, big_file/1, outdir/1,
binary/1, makedep/1, cond_and_ifdef/1, listings/1, listings_big/1,
other_output/1, kernel_listing/1, encrypted_abstr/1,
strict_record/1, utf8_atoms/1, utf8_functions/1, extra_chunks/1,
- cover/1, env/1, core_pp/1,
+ cover/1, env/1, core_pp/1, tuple_calls/1,
core_roundtrip/1, asm/1, optimized_guards/1,
sys_pre_attributes/1, dialyzer/1,
warnings/1, pre_load_check/1, env_compiler_options/1,
@@ -49,11 +49,12 @@ all() ->
test_lib:recompile(?MODULE),
[app_test, appup_test, file_1, forms_2, module_mismatch, big_file, outdir,
binary, makedep, cond_and_ifdef, listings, listings_big,
- other_output, kernel_listing, encrypted_abstr,
+ other_output, kernel_listing, encrypted_abstr, tuple_calls,
strict_record, utf8_atoms, utf8_functions, extra_chunks,
cover, env, core_pp, core_roundtrip, asm, optimized_guards,
sys_pre_attributes, dialyzer, warnings, pre_load_check,
- env_compiler_options, custom_debug_info, bc_options].
+ env_compiler_options, custom_debug_info, bc_options,
+ custom_compile_info].
groups() ->
[].
@@ -374,7 +375,6 @@ do_file_listings(DataDir, PrivDir, [File|Files]) ->
do_listing(Simple, TargetDir, dcbsm, ".core_bsm"),
do_listing(Simple, TargetDir, dsetel, ".dsetel"),
do_listing(Simple, TargetDir, dkern, ".kernel"),
- do_listing(Simple, TargetDir, dlife, ".life"),
do_listing(Simple, TargetDir, dcg, ".codegen"),
do_listing(Simple, TargetDir, dblk, ".block"),
do_listing(Simple, TargetDir, dexcept, ".except"),
@@ -649,6 +649,23 @@ custom_debug_info(Config) when is_list(Config) ->
{ok,{simple,[{debug_info,{debug_info_v1,?MODULE,error}}]}} =
beam_lib:chunks(ErrorBin, [debug_info]).
+custom_compile_info(Config) when is_list(Config) ->
+ Anno = erl_anno:new(1),
+ Forms = [{attribute,Anno,module,custom_compile_info}],
+ Opts = [binary,{compile_info,[{another,version}]}],
+
+ {ok,custom_compile_info,Bin} = compile:forms(Forms, Opts),
+ {ok,{custom_compile_info,[{compile_info,CompileInfo}]}} =
+ beam_lib:chunks(Bin, [compile_info]),
+ version = proplists:get_value(another, CompileInfo),
+ CompileOpts = proplists:get_value(options, CompileInfo),
+ undefined = proplists:get_value(compile_info, CompileOpts),
+
+ {ok,custom_compile_info,DetBin} = compile:forms(Forms, [deterministic|Opts]),
+ {ok,{custom_compile_info,[{compile_info,DetInfo}]}} =
+ beam_lib:chunks(DetBin, [compile_info]),
+ version = proplists:get_value(another, DetInfo).
+
cover(Config) when is_list(Config) ->
io:format("~p\n", [compile:options()]),
ok.
@@ -781,6 +798,37 @@ extra_chunks(Config) when is_list(Config) ->
{ok,{extra_chunks,[{"ExCh",<<"Contents">>}]}} =
beam_lib:chunks(ExtraChunksBinary, ["ExCh"]).
+tuple_calls(Config) when is_list(Config) ->
+ Anno = erl_anno:new(1),
+ Forms = [{attribute,Anno,export,[{size,1},{store,1}]},
+ {function,Anno,size,1,
+ [{clause,Anno,[{var,[],mod}],[],
+ [{call,[],{remote,[],{var,[],mod},{atom,[],size}},[]}]}]},
+ {function,Anno,store,1,
+ [{clause,Anno,[{var,[],mod}],[],
+ [{call,[],{remote,[],{var,[],mod},{atom,[],store}},[{atom,[],key},{atom,[],value}]}]}]}],
+
+ TupleCallsFalse = [{attribute,Anno,module,tuple_calls_false}|Forms],
+ {ok,_,TupleCallsFalseBinary} = compile:forms(TupleCallsFalse, [binary]),
+ code:load_binary(tuple_calls_false, "compile_SUITE.erl", TupleCallsFalseBinary),
+ {'EXIT',{badarg,_}} = (catch tuple_calls_false:store(dict())),
+ {'EXIT',{badarg,_}} = (catch tuple_calls_false:size(dict())),
+ {'EXIT',{badarg,_}} = (catch tuple_calls_false:size(empty_tuple())),
+
+ TupleCallsTrue = [{attribute,Anno,module,tuple_calls_true}|Forms],
+ {ok,_,TupleCallsTrueBinary} = compile:forms(TupleCallsTrue, [binary,tuple_calls]),
+ code:load_binary(tuple_calls_true, "compile_SUITE.erl", TupleCallsTrueBinary),
+ Dict = tuple_calls_true:store(dict()),
+ 1 = tuple_calls_true:size(Dict),
+ {'EXIT',{badarg,_}} = (catch tuple_calls_true:size(empty_tuple())),
+
+ ok.
+
+dict() ->
+ dict:new().
+empty_tuple() ->
+ {}.
+
env(Config) when is_list(Config) ->
{Simple,Target} = get_files(Config, simple, env),
{ok,Cwd} = file:get_cwd(),
diff --git a/lib/compiler/test/core_alias_SUITE.erl b/lib/compiler/test/core_alias_SUITE.erl
new file mode 100644
index 0000000000..f3f15ef0f8
--- /dev/null
+++ b/lib/compiler/test/core_alias_SUITE.erl
@@ -0,0 +1,195 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2007-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(core_alias_SUITE).
+
+-export([all/0, suite/0, groups/0,init_per_suite/1, end_per_suite/1,
+ init_per_group/2, end_per_group/2,
+ tuples/1, cons/1]).
+
+-include_lib("common_test/include/ct.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ test_lib:recompile(?MODULE),
+ [{group,p}].
+
+groups() ->
+ [{p,[parallel],
+ [tuples, cons]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+
+id(X) -> X.
+
+tuples(Config) when is_list(Config) ->
+ Tuple = {ok,id(value)},
+
+ true = erts_debug:same(Tuple, simple_tuple(Tuple)),
+ true = erts_debug:same(Tuple, simple_tuple_in_map(#{hello => Tuple})),
+ true = erts_debug:same(Tuple, simple_tuple_case_repeated(Tuple, Tuple)),
+ true = erts_debug:same(Tuple, simple_tuple_fun_repeated(Tuple, Tuple)),
+ true = erts_debug:same(Tuple, simple_tuple_twice_head(Tuple, Tuple)),
+
+ {Tuple1, Tuple2} = simple_tuple_twice_body(Tuple),
+ true = erts_debug:same(Tuple, Tuple1),
+ true = erts_debug:same(Tuple, Tuple2),
+
+ Nested = {nested,Tuple},
+ true = erts_debug:same(Tuple, nested_tuple_part(Nested)),
+ true = erts_debug:same(Nested, nested_tuple_whole(Nested)),
+ true = erts_debug:same(Nested, nested_tuple_with_alias(Nested)),
+
+ true = erts_debug:same(Tuple, tuple_rebinding_after(Tuple)),
+
+ Tuple = unaliased_tuple_rebinding_before(Tuple),
+ false = erts_debug:same(Tuple, unaliased_tuple_rebinding_before(Tuple)),
+ Nested = unaliased_literal_tuple_head(Nested),
+ false = erts_debug:same(Nested, unaliased_literal_tuple_head(Nested)),
+ Nested = unaliased_literal_tuple_body(Nested),
+ false = erts_debug:same(Nested, unaliased_literal_tuple_body(Nested)),
+ Nested = unaliased_different_var_tuple(Nested, Tuple),
+ false = erts_debug:same(Nested, unaliased_different_var_tuple(Nested, Tuple)).
+
+simple_tuple({ok,X}) ->
+ {ok,X}.
+simple_tuple_twice_head({ok,X}, {ok,X}) ->
+ {ok,X}.
+simple_tuple_twice_body({ok,X}) ->
+ {{ok,X},{ok,X}}.
+simple_tuple_in_map(#{hello := {ok,X}}) ->
+ {ok,X}.
+simple_tuple_fun_repeated({ok,X}, Y) ->
+ io:format("~p~n", [X]),
+ (fun({ok,X}) -> {ok,X} end)(Y).
+simple_tuple_case_repeated({ok,X}, Y) ->
+ io:format("~p~n", [X]),
+ case Y of {ok,X} -> {ok,X} end.
+
+nested_tuple_part({nested,{ok,X}}) ->
+ {ok,X}.
+nested_tuple_whole({nested,{ok,X}}) ->
+ {nested,{ok,X}}.
+nested_tuple_with_alias({nested,{ok,_}=Y}) ->
+ {nested,Y}.
+
+tuple_rebinding_after(Y) ->
+ (fun(X) -> {ok,X} end)(Y),
+ case Y of {ok,X} -> {ok,X} end.
+unaliased_tuple_rebinding_before({ok,X}) ->
+ io:format("~p~n", [X]),
+ (fun(X) -> {ok,X} end)(value).
+unaliased_literal_tuple_head({nested,{ok,value}=X}) ->
+ io:format("~p~n", [X]),
+ {nested,{ok,value}}.
+unaliased_literal_tuple_body({nested,{ok,value}=X}) ->
+ Res = {nested,Y={ok,value}},
+ io:format("~p~n", [[X,Y]]),
+ Res.
+unaliased_different_var_tuple({nested,{ok,value}=X}, Y) ->
+ io:format("~p~n", [X]),
+ {nested,Y}.
+
+cons(Config) when is_list(Config) ->
+ Cons = [ok|id(value)],
+
+ true = erts_debug:same(Cons, simple_cons(Cons)),
+ true = erts_debug:same(Cons, simple_cons_in_map(#{hello => Cons})),
+ true = erts_debug:same(Cons, simple_cons_case_repeated(Cons, Cons)),
+ true = erts_debug:same(Cons, simple_cons_fun_repeated(Cons, Cons)),
+ true = erts_debug:same(Cons, simple_cons_twice_head(Cons, Cons)),
+
+ {Cons1,Cons2} = simple_cons_twice_body(Cons),
+ true = erts_debug:same(Cons, Cons1),
+ true = erts_debug:same(Cons, Cons2),
+
+ Nested = [nested,Cons],
+ true = erts_debug:same(Cons, nested_cons_part(Nested)),
+ true = erts_debug:same(Nested, nested_cons_whole(Nested)),
+ true = erts_debug:same(Nested, nested_cons_with_alias(Nested)),
+ true = erts_debug:same(Cons, cons_rebinding_after(Cons)),
+
+ Unstripped = id([a,b]),
+ Stripped = cons_with_binary([<<>>|Unstripped]),
+ true = erts_debug:same(Unstripped, Stripped),
+
+ Cons = unaliased_cons_rebinding_before(Cons),
+ false = erts_debug:same(Cons, unaliased_cons_rebinding_before(Cons)),
+ Nested = unaliased_literal_cons_head(Nested),
+ false = erts_debug:same(Nested, unaliased_literal_cons_head(Nested)),
+ Nested = unaliased_literal_cons_body(Nested),
+ false = erts_debug:same(Nested, unaliased_literal_cons_body(Nested)),
+ Nested = unaliased_different_var_cons(Nested, Cons),
+ false = erts_debug:same(Nested, unaliased_different_var_cons(Nested, Cons)).
+
+simple_cons([ok|X]) ->
+ [ok|X].
+simple_cons_twice_head([ok|X], [ok|X]) ->
+ [ok|X].
+simple_cons_twice_body([ok|X]) ->
+ {[ok|X],[ok|X]}.
+simple_cons_in_map(#{hello := [ok|X]}) ->
+ [ok|X].
+simple_cons_fun_repeated([ok|X], Y) ->
+ io:format("~p~n", [X]),
+ (fun([ok|X]) -> [ok|X] end)(Y).
+simple_cons_case_repeated([ok|X], Y) ->
+ io:format("~p~n", [X]),
+ case Y of [ok|X] -> [ok|X] end.
+
+nested_cons_part([nested,[ok|X]]) ->
+ [ok|X].
+nested_cons_whole([nested,[ok|X]]) ->
+ [nested,[ok|X]].
+nested_cons_with_alias([nested,[ok|_]=Y]) ->
+ [nested,Y].
+
+cons_with_binary([<<>>,X|Y]) ->
+ cons_with_binary([X|Y]);
+cons_with_binary(A) ->
+ A.
+
+cons_rebinding_after(Y) ->
+ (fun(X) -> [ok|X] end)(Y),
+ case Y of [ok|X] -> [ok|X] end.
+unaliased_cons_rebinding_before([ok|X]) ->
+ io:format("~p~n", [X]),
+ (fun(X) -> [ok|X] end)(value).
+unaliased_literal_cons_head([nested,[ok|value]=X]) ->
+ io:format("~p~n", [X]),
+ [nested,[ok|value]].
+unaliased_literal_cons_body([nested,[ok|value]=X]) ->
+ Res = [nested,Y=[ok|value]],
+ io:format("~p~n", [[X, Y]]),
+ Res.
+unaliased_different_var_cons([nested,[ok|value]=X], Y) ->
+ io:format("~p~n", [X]),
+ [nested,Y].
diff --git a/lib/compiler/test/core_fold_SUITE.erl b/lib/compiler/test/core_fold_SUITE.erl
index 0097e28d4d..262967d03d 100644
--- a/lib/compiler/test/core_fold_SUITE.erl
+++ b/lib/compiler/test/core_fold_SUITE.erl
@@ -26,7 +26,8 @@
unused_multiple_values_error/1,unused_multiple_values/1,
multiple_aliases/1,redundant_boolean_clauses/1,
mixed_matching_clauses/1,unnecessary_building/1,
- no_no_file/1,configuration/1,supplies/1]).
+ no_no_file/1,configuration/1,supplies/1,
+ redundant_stack_frame/1]).
-export([foo/0,foo/1,foo/2,foo/3]).
@@ -45,7 +46,8 @@ groups() ->
unused_multiple_values_error,unused_multiple_values,
multiple_aliases,redundant_boolean_clauses,
mixed_matching_clauses,unnecessary_building,
- no_no_file,configuration,supplies]}].
+ no_no_file,configuration,supplies,
+ redundant_stack_frame]}].
init_per_suite(Config) ->
@@ -527,4 +529,26 @@ supplies(_Config) ->
do_supplies(#{1 := Value}) when byte_size(Value), byte_size(kg) -> working.
+redundant_stack_frame(_Config) ->
+ {1,2} = do_redundant_stack_frame(#{x=>1,y=>2}),
+ {'EXIT',{{badkey,_,x},_}} = (catch do_redundant_stack_frame(#{y=>2})),
+ {'EXIT',{{badkey,_,y},_}} = (catch do_redundant_stack_frame(#{x=>1})),
+ ok.
+
+do_redundant_stack_frame(Map) ->
+ %% There should not be a stack frame for this function.
+ X = case Map of
+ #{x := X0} ->
+ X0;
+ #{} ->
+ erlang:error({badkey, Map, x})
+ end,
+ Y = case Map of
+ #{y := Y0} ->
+ Y0;
+ #{} ->
+ erlang:error({badkey, Map, y})
+ end,
+ {X, Y}.
+
id(I) -> I.
diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl
index ccb9b58225..d96cfdb7ac 100644
--- a/lib/compiler/test/guard_SUITE.erl
+++ b/lib/compiler/test/guard_SUITE.erl
@@ -1291,6 +1291,10 @@ rel_ops(Config) when is_list(Config) ->
true = any_atom /= id(42),
true = [] /= id(42),
+ %% Coverage of beam_utils:bif_to_test/3
+ Empty = id([]),
+ ?T(==, [], Empty),
+
ok.
-undef(TestOp).
diff --git a/lib/compiler/test/match_SUITE.erl b/lib/compiler/test/match_SUITE.erl
index 52b2da05f7..c31695be24 100644
--- a/lib/compiler/test/match_SUITE.erl
+++ b/lib/compiler/test/match_SUITE.erl
@@ -23,7 +23,7 @@
init_per_group/2,end_per_group/2,
pmatch/1,mixed/1,aliases/1,non_matching_aliases/1,
match_in_call/1,untuplify/1,shortcut_boolean/1,letify_guard/1,
- selectify/1,underscore/1,match_map/1,map_vars_used/1,
+ selectify/1,deselectify/1,underscore/1,match_map/1,map_vars_used/1,
coverage/1,grab_bag/1,literal_binary/1]).
-include_lib("common_test/include/ct.hrl").
@@ -38,7 +38,7 @@ groups() ->
[{p,[parallel],
[pmatch,mixed,aliases,non_matching_aliases,
match_in_call,untuplify,
- shortcut_boolean,letify_guard,selectify,
+ shortcut_boolean,letify_guard,selectify,deselectify,
underscore,match_map,map_vars_used,coverage,
grab_bag,literal_binary]}].
@@ -466,6 +466,66 @@ sel_same_value2(V) when V =:= 42; V =:= 43 ->
sel_same_value2(_) ->
error.
+%% Test deconstruction of select_val instructions in beam_peep into
+%% regular tests with just one possible value left. Hitting proper cases
+%% in beam_peep relies on unification of labels by beam_jump.
+
+deselectify(Config) when is_list(Config) ->
+ one_or_other = desel_tuple_arity({1}),
+ two = desel_tuple_arity({1,1}),
+ one_or_other = desel_tuple_arity({1,1,1}),
+
+ one_or_other = dsel_integer(1),
+ two = dsel_integer(2),
+ one_or_other = dsel_integer(3),
+
+ one_or_other = dsel_integer_typecheck(1),
+ two = dsel_integer_typecheck(2),
+ one_or_other = dsel_integer_typecheck(3),
+
+ one_or_other = dsel_atom(one),
+ two = dsel_atom(two),
+ one_or_other = dsel_atom(three),
+
+ one_or_other = dsel_atom_typecheck(one),
+ two = dsel_atom_typecheck(two),
+ one_or_other = dsel_atom_typecheck(three).
+
+desel_tuple_arity(Tuple) when is_tuple(Tuple) ->
+ case Tuple of
+ {_} -> one_or_other;
+ {_,_} -> two;
+ _ -> one_or_other
+ end.
+
+dsel_integer(Val) ->
+ case Val of
+ 1 -> one_or_other;
+ 2 -> two;
+ _ -> one_or_other
+ end.
+
+dsel_integer_typecheck(Val) when is_integer(Val) ->
+ case Val of
+ 1 -> one_or_other;
+ 2 -> two;
+ _ -> one_or_other
+ end.
+
+dsel_atom(Val) ->
+ case Val of
+ one -> one_or_other;
+ two -> two;
+ _ -> one_or_other
+ end.
+
+dsel_atom_typecheck(Val) when is_atom(Val) ->
+ case Val of
+ one -> one_or_other;
+ two -> two;
+ _ -> one_or_other
+ end.
+
underscore(Config) when is_list(Config) ->
case Config of
[] ->
diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl
index 4bd884d86b..b12bcbeeab 100644
--- a/lib/compiler/test/misc_SUITE.erl
+++ b/lib/compiler/test/misc_SUITE.erl
@@ -161,16 +161,17 @@ md5_1(Beam) ->
%% Cover some code that handles internal errors.
silly_coverage(Config) when is_list(Config) ->
- %% sys_core_fold, sys_core_bsm, sys_core_setel, v3_kernel
+ %% sys_core_fold, sys_core_alias, sys_core_bsm, sys_core_setel, v3_kernel
BadCoreErlang = {c_module,[],
name,[],[],
[{{c_var,[],{foo,2}},seriously_bad_body}]},
expect_error(fun() -> sys_core_fold:module(BadCoreErlang, []) end),
+ expect_error(fun() -> sys_core_alias:module(BadCoreErlang, []) end),
expect_error(fun() -> sys_core_bsm:module(BadCoreErlang, []) end),
expect_error(fun() -> sys_core_dsetel:module(BadCoreErlang, []) end),
expect_error(fun() -> v3_kernel:module(BadCoreErlang, []) end),
- %% v3_life
+ %% v3_codegen
BadKernel = {k_mdef,[],?MODULE,
[{foo,0}],
[],
@@ -178,11 +179,7 @@ silly_coverage(Config) when is_list(Config) ->
{k,[],[],[]},
f,0,[],
seriously_bad_body}]},
- expect_error(fun() -> v3_life:module(BadKernel, []) end),
-
- %% v3_codegen
- CodegenInput = {?MODULE,[{foo,0}],[],[{function,foo,0,[a|b],a,b,[]}]},
- expect_error(fun() -> v3_codegen:module(CodegenInput, []) end),
+ expect_error(fun() -> v3_codegen:module(BadKernel, []) end),
%% beam_a
BeamAInput = {?MODULE,[{foo,0}],[],
diff --git a/lib/compiler/test/trycatch_SUITE.erl b/lib/compiler/test/trycatch_SUITE.erl
index a591d6cc93..42dbf7d5f0 100644
--- a/lib/compiler/test/trycatch_SUITE.erl
+++ b/lib/compiler/test/trycatch_SUITE.erl
@@ -324,11 +324,11 @@ eclectic(Conf) when is_list(Conf) ->
{{error,{exit,V},{'EXIT',V}},V} =
eclectic_1({foo,{error,{exit,V}}}, error, {value,V}),
{{value,{value,V},V},
- {'EXIT',{badarith,[{?MODULE,my_add,2,_}|_]}}} =
+ {'EXIT',{badarith,[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]}}} =
eclectic_1({foo,{value,{value,V}}}, undefined, {'add',{0,a}}),
{{'EXIT',V},V} =
eclectic_1({catch_foo,{exit,V}}, undefined, {throw,V}),
- {{error,{'div',{1,0}},{'EXIT',{badarith,[{?MODULE,my_div,2,_}|_]}}},
+ {{error,{'div',{1,0}},{'EXIT',{badarith,[{erlang,'div',[1,0],_},{?MODULE,my_div,2,_}|_]}}},
{'EXIT',V}} =
eclectic_1({foo,{error,{'div',{1,0}}}}, error, {exit,V}),
{{{error,V},{'EXIT',{V,[{?MODULE,foo,1,_}|_]}}},
@@ -345,7 +345,7 @@ eclectic(Conf) when is_list(Conf) ->
eclectic_2({error,{value,V}}, throw, {error,V}),
{{caught,{'EXIT',{badarg,[{erlang,abs,[V],_}|_]}}},V} =
eclectic_2({value,{'abs',V}}, undefined, {value,V}),
- {{caught,{'EXIT',{badarith,[{?MODULE,my_add,2,_}|_]}}},V} =
+ {{caught,{'EXIT',{badarith,[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]}}},V} =
eclectic_2({exit,{'add',{0,a}}}, exit, {value,V}),
{{caught,{'EXIT',V}},undefined} =
eclectic_2({value,{error,V}}, undefined, {exit,V}),
diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml
index 8e2d33c928..3ab8f9c832 100644
--- a/lib/crypto/doc/src/crypto.xml
+++ b/lib/crypto/doc/src/crypto.xml
@@ -789,9 +789,16 @@
<seealso marker="stdlib:rand">random number generation</seealso>,
in order to generate cryptographically strong random numbers
(based on OpenSSL's <c>BN_rand_range</c>),
- and saves it on process dictionary before returning it as well.
+ and saves it in the process dictionary before returning it as well.
See also
- <seealso marker="stdlib:rand#seed-1">rand:seed/1</seealso>.
+ <seealso marker="stdlib:rand#seed-1">rand:seed/1</seealso> and
+ <seealso marker="#rand_seed_s-0">rand_seed_s/0</seealso>.
+ </p>
+ <p>
+ When using the state object from this function the
+ <seealso marker="stdlib:rand">rand</seealso> functions using it
+ may throw exception <c>low_entropy</c> in case the random generator
+ failed due to lack of secure "randomness".
</p>
<p><em>Example</em></p>
<pre>
@@ -813,6 +820,120 @@ _FloatValue = rand:uniform(). % [0.0; 1.0[</pre>
See also
<seealso marker="stdlib:rand#seed_s-1">rand:seed_s/1</seealso>.
</p>
+ <p>
+ When using the state object from this function the
+ <seealso marker="stdlib:rand">rand</seealso> functions using it
+ may throw exception <c>low_entropy</c> in case the random generator
+ failed due to lack of secure "randomness".
+ </p>
+ <note>
+ <p>
+ The state returned from this function can not be used
+ to get a reproducable random sequence as from
+ the other
+ <seealso marker="stdlib:rand">rand</seealso>
+ functions,
+ since reproducability does not match cryptographically safe.
+ </p>
+ <p>
+ The only supported usage is to generate one distinct
+ random sequence from this start state.
+ </p>
+ </note>
+ </desc>
+ </func>
+
+ <func>
+ <name>rand_seed_alg(Alg) -> rand:state()</name>
+ <fsummary>Strong random number generation plugin state</fsummary>
+ <type>
+ <v>Alg = crypto | crypto_cache</v>
+ </type>
+ <desc>
+ <marker id="rand_seed_alg-1" />
+ <p>
+ Creates state object for
+ <seealso marker="stdlib:rand">random number generation</seealso>,
+ in order to generate cryptographically strong random numbers.
+ See also
+ <seealso marker="stdlib:rand#seed-1">rand:seed/1</seealso> and
+ <seealso marker="#rand_seed_alg_s-1">rand_seed_alg_s/1</seealso>.
+ </p>
+ <p>
+ When using the state object from this function the
+ <seealso marker="stdlib:rand">rand</seealso> functions using it
+ may throw exception <c>low_entropy</c> in case the random generator
+ failed due to lack of secure "randomness".
+ </p>
+ <p>
+ The cache size can be changed from its default value using the
+ <seealso marker="crypto_app">
+ crypto app's
+ </seealso> configuration parameter <c>rand_cache_size</c>.
+ </p>
+ <p><em>Example</em></p>
+ <pre>
+_ = crypto:rand_seed_alg(crypto_cache),
+_IntegerValue = rand:uniform(42), % [1; 42]
+_FloatValue = rand:uniform(). % [0.0; 1.0[</pre>
+ </desc>
+ </func>
+
+ <func>
+ <name>rand_seed_alg_s(Alg) -> rand:state()</name>
+ <fsummary>Strong random number generation plugin state</fsummary>
+ <type>
+ <v>Alg = crypto | crypto_cache</v>
+ </type>
+ <desc>
+ <marker id="rand_seed_alg_s-1" />
+ <p>
+ Creates state object for
+ <seealso marker="stdlib:rand">random number generation</seealso>,
+ in order to generate cryptographically strongly random numbers.
+ See also
+ <seealso marker="stdlib:rand#seed_s-1">rand:seed_s/1</seealso>.
+ </p>
+ <p>
+ If <c>Alg</c> is <c>crypto</c> this function behaves exactly like
+ <seealso marker="#rand_seed_s-0">rand_seed_s/0</seealso>.
+ </p>
+ <p>
+ If <c>Alg</c> is <c>crypto_cache</c> this function
+ fetches random data with OpenSSL's <c>RAND_bytes</c>
+ and caches it for speed using an internal word size
+ of 56 bits that makes calculations fast on 64 bit machines.
+ </p>
+ <p>
+ When using the state object from this function the
+ <seealso marker="stdlib:rand">rand</seealso> functions using it
+ may throw exception <c>low_entropy</c> in case the random generator
+ failed due to lack of secure "randomness".
+ </p>
+ <p>
+ The cache size can be changed from its default value using the
+ <seealso marker="crypto_app">
+ crypto app's
+ </seealso> configuration parameter <c>rand_cache_size</c>.
+ </p>
+ <note>
+ <p>
+ The state returned from this function can not be used
+ to get a reproducable random sequence as from
+ the other
+ <seealso marker="stdlib:rand">rand</seealso>
+ functions,
+ since reproducability does not match cryptographically safe.
+ </p>
+ <p>
+ In fact since random data is cached some numbers may
+ get reproduced if you try, but this is unpredictable.
+ </p>
+ <p>
+ The only supported usage is to generate one distinct
+ random sequence from this start state.
+ </p>
+ </note>
</desc>
</func>
diff --git a/lib/crypto/doc/src/crypto_app.xml b/lib/crypto/doc/src/crypto_app.xml
index ba22557480..8296b1bc77 100644
--- a/lib/crypto/doc/src/crypto_app.xml
+++ b/lib/crypto/doc/src/crypto_app.xml
@@ -68,6 +68,24 @@
thus the crypto module will fail to load. This mechanism
prevents the accidental use of non-validated algorithms.</p>
</item>
+ <tag><c>rand_cache_size = integer()</c></tag>
+ <item>
+ <p>
+ Sets the cache size in bytes to use by
+ <seealso marker="crypto#rand_seed_alg-1">
+ <c>crypto:rand_seed_alg(crypto_cache)</c>
+ </seealso> and
+ <seealso marker="crypto#rand_seed_alg_s-1">
+ <c>crypto:rand_seed_alg_s(crypto_cache)</c>
+ </seealso>.
+ This parameter is read when a seed function is called,
+ and then kept in generators state object. It has a rather
+ small default value that causes reads of strong random bytes
+ about once per hundred calls for a random value.
+ The set value is rounded up to an integral number of words
+ of the size these seed functions use.
+ </p>
+ </item>
</taglist>
</section>
diff --git a/lib/crypto/src/crypto.app.src b/lib/crypto/src/crypto.app.src
index 1d3f35e465..492aa10e51 100644
--- a/lib/crypto/src/crypto.app.src
+++ b/lib/crypto/src/crypto.app.src
@@ -24,7 +24,7 @@
crypto_ec_curves]},
{registered, []},
{applications, [kernel, stdlib]},
- {env, [{fips_mode, false}]},
+ {env, [{fips_mode, false}, {rand_cache_size, 896}]},
{runtime_dependencies, ["erts-9.0","stdlib-3.4","kernel-5.3"]}]}.
diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl
index 0d39dcc76e..9ba1623348 100644
--- a/lib/crypto/src/crypto.erl
+++ b/lib/crypto/src/crypto.erl
@@ -30,11 +30,12 @@
-export([hmac/3, hmac/4, hmac_init/2, hmac_update/2, hmac_final/1, hmac_final_n/2]).
-export([cmac/3, cmac/4]).
-export([exor/2, strong_rand_bytes/1, mod_pow/3]).
--export([rand_seed/0]).
--export([rand_seed_s/0]).
+-export([rand_seed/0, rand_seed_alg/1]).
+-export([rand_seed_s/0, rand_seed_alg_s/1]).
-export([rand_plugin_next/1]).
-export([rand_plugin_uniform/1]).
-export([rand_plugin_uniform/2]).
+-export([rand_cache_plugin_next/1]).
-export([rand_uniform/2]).
-export([block_encrypt/3, block_decrypt/3, block_encrypt/4, block_decrypt/4]).
-export([next_iv/2, next_iv/3]).
@@ -315,9 +316,17 @@ stream_decrypt(State, Data0) ->
%%
%% RAND - pseudo random numbers using RN_ and BN_ functions in crypto lib
%%
+-type rand_cache_seed() ::
+ nonempty_improper_list(non_neg_integer(), binary()).
-spec strong_rand_bytes(non_neg_integer()) -> binary().
-spec rand_seed() -> rand:state().
-spec rand_seed_s() -> rand:state().
+-spec rand_seed_alg(Alg :: atom()) ->
+ {rand:alg_handler(),
+ atom() | rand_cache_seed()}.
+-spec rand_seed_alg_s(Alg :: atom()) ->
+ {rand:alg_handler(),
+ atom() | rand_cache_seed()}.
-spec rand_uniform(crypto_integer(), crypto_integer()) ->
crypto_integer().
@@ -333,12 +342,36 @@ rand_seed() ->
rand:seed(rand_seed_s()).
rand_seed_s() ->
+ rand_seed_alg_s(?MODULE).
+
+rand_seed_alg(Alg) ->
+ rand:seed(rand_seed_alg_s(Alg)).
+
+-define(CRYPTO_CACHE_BITS, 56).
+rand_seed_alg_s(?MODULE) ->
{#{ type => ?MODULE,
bits => 64,
next => fun ?MODULE:rand_plugin_next/1,
uniform => fun ?MODULE:rand_plugin_uniform/1,
uniform_n => fun ?MODULE:rand_plugin_uniform/2},
- no_seed}.
+ no_seed};
+rand_seed_alg_s(crypto_cache) ->
+ CacheBits = ?CRYPTO_CACHE_BITS,
+ EnvCacheSize =
+ application:get_env(
+ crypto, rand_cache_size, CacheBits * 16), % Cache 16 * 8 words
+ Bytes = (CacheBits + 7) div 8,
+ CacheSize =
+ case ((EnvCacheSize + (Bytes - 1)) div Bytes) * Bytes of
+ Sz when is_integer(Sz), Bytes =< Sz ->
+ Sz;
+ _ ->
+ Bytes
+ end,
+ {#{ type => crypto_cache,
+ bits => CacheBits,
+ next => fun ?MODULE:rand_cache_plugin_next/1},
+ {CacheBits, CacheSize, <<>>}}.
rand_plugin_next(Seed) ->
{bytes_to_integer(strong_rand_range(1 bsl 64)), Seed}.
@@ -349,6 +382,12 @@ rand_plugin_uniform(State) ->
rand_plugin_uniform(Max, State) ->
{bytes_to_integer(strong_rand_range(Max)) + 1, State}.
+rand_cache_plugin_next({CacheBits, CacheSize, <<>>}) ->
+ rand_cache_plugin_next(
+ {CacheBits, CacheSize, strong_rand_bytes(CacheSize)});
+rand_cache_plugin_next({CacheBits, CacheSize, Cache}) ->
+ <<I:CacheBits, NewCache/binary>> = Cache,
+ {I, {CacheBits, CacheSize, NewCache}}.
strong_rand_range(Range) when is_integer(Range), Range > 0 ->
BinRange = int_to_bin(Range),
@@ -396,7 +435,7 @@ rand_uniform_nif(_From,_To) -> ?nif_stub.
-spec rand_seed(binary()) -> ok.
-rand_seed(Seed) ->
+rand_seed(Seed) when is_binary(Seed) ->
rand_seed_nif(Seed).
rand_seed_nif(_Seed) -> ?nif_stub.
diff --git a/lib/debugger/src/i.erl b/lib/debugger/src/i.erl
index 62ce8d0e20..853fa529a0 100644
--- a/lib/debugger/src/i.erl
+++ b/lib/debugger/src/i.erl
@@ -30,7 +30,7 @@
-import(lists, [sort/1,foreach/2]).
iv() ->
- Vsn = string:substr(filename:basename(code:lib_dir(debugger)), 10),
+ Vsn = string:slice(filename:basename(code:lib_dir(debugger)), 9),
list_to_atom(Vsn).
%% -------------------------------------------
diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl
index 0617be6435..1e06d6e974 100644
--- a/lib/dialyzer/src/dialyzer_cl.erl
+++ b/lib/dialyzer/src/dialyzer_cl.erl
@@ -672,7 +672,7 @@ failed_anal_msg(Reason, LogCache) ->
%%
format_log_cache(LogCache) ->
Str = lists:append(lists:reverse(LogCache)),
- string:join(string:tokens(Str, "\n"), "\n ").
+ lists:join("\n ", string:lexemes(Str, "\n")).
-spec store_warnings(#cl_state{}, [raw_warning()]) -> #cl_state{}.
diff --git a/lib/dialyzer/src/dialyzer_gui_wx.erl b/lib/dialyzer/src/dialyzer_gui_wx.erl
index b4b1872c12..b8414b7d8b 100644
--- a/lib/dialyzer/src/dialyzer_gui_wx.erl
+++ b/lib/dialyzer/src/dialyzer_gui_wx.erl
@@ -475,7 +475,7 @@ gui_loop(#gui_state{backend_pid = BackendPid, doc_plt = DocPlt,
gui_loop(State);
{BackendPid, ext_types, ExtTypes} ->
Map = fun({M,F,A}) -> io_lib:format("~tp:~tp/~p",[M,F,A]) end,
- ExtTypeString = string:join(lists:map(Map, ExtTypes), "\n"),
+ ExtTypeString = lists:join("\n", lists:map(Map, ExtTypes)),
Msg = io_lib:format("The following remote types are being used "
"but information about them is not available.\n"
"The analysis might get more precise by including "
@@ -638,7 +638,7 @@ output_sms(#gui_state{frame = Frame}, Title, Message, Type) ->
free_editor(#gui_state{gui = Wx, frame = Frame}, Title, Contents0) ->
Contents = lists:flatten(Contents0),
- Tokens = string:tokens(Contents, "\n"),
+ Tokens = string:lexemes(Contents, "\n"),
NofLines = length(Tokens),
LongestLine = lists:max([length(X) || X <- Tokens]),
Height0 = NofLines * 25 + 80,
diff --git a/lib/dialyzer/src/dialyzer_races.erl b/lib/dialyzer/src/dialyzer_races.erl
index 7fe64c3e11..7602faa21d 100644
--- a/lib/dialyzer/src/dialyzer_races.erl
+++ b/lib/dialyzer/src/dialyzer_races.erl
@@ -1270,8 +1270,8 @@ filter_named_tables(NamesList) ->
[] -> [];
[Head|Tail] ->
NewHead =
- case string:rstr(Head, "()") of
- 0 -> [Head];
+ case string:find(Head, "()", trailing) of
+ nomatch -> [Head];
_Other -> []
end,
NewHead ++ filter_named_tables(Tail)
@@ -1558,8 +1558,8 @@ any_args(StrList) ->
case StrList of
[] -> false;
[Head|Tail] ->
- case string:rstr(Head, "()") of
- 0 -> any_args(Tail);
+ case string:find(Head, "()", trailing) of
+ nomatch -> any_args(Tail);
_Other -> true
end
end.
@@ -1765,10 +1765,8 @@ ets_list_args(MaybeList) ->
end.
ets_list_argtypes(ListStr) ->
- ListStr1 = string:strip(ListStr, left, $[),
- ListStr2 = string:strip(ListStr1, right, $]),
- ListStr3 = string:strip(ListStr2, right, $.),
- string:strip(ListStr3, right, $,).
+ ListStr1 = string:trim(ListStr, leading, "$["),
+ string:trim(ListStr1, trailing, "$]$.$,").
ets_tuple_args(MaybeTuple) ->
case is_tuple(MaybeTuple) of
@@ -1810,7 +1808,7 @@ ets_tuple_argtypes2_helper(TupleStr, ElemStr, NestingLevel) ->
{[H|ElemStr], NestingLevel, false}
end,
case Return of
- true -> string:tokens(NewElemStr, " |");
+ true -> string:lexemes(NewElemStr, " |");
false ->
ets_tuple_argtypes2_helper(T, NewElemStr, NewNestingLevel)
end
@@ -1889,44 +1887,44 @@ format_args_2(StrArgList, Call) ->
case Call of
whereis ->
lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |"));
+ string:lexemes(lists:nth(2, StrArgList), " |"));
register ->
lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |"));
+ string:lexemes(lists:nth(2, StrArgList), " |"));
unregister ->
lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |"));
+ string:lexemes(lists:nth(2, StrArgList), " |"));
ets_new ->
StrArgList1 = lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |")),
+ string:lexemes(lists:nth(2, StrArgList), " |")),
lists_key_replace(4, StrArgList1,
- string:tokens(ets_list_argtypes(lists:nth(4, StrArgList1)), " |"));
+ string:lexemes(ets_list_argtypes(lists:nth(4, StrArgList1)), " |"));
ets_lookup ->
StrArgList1 = lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |")),
+ string:lexemes(lists:nth(2, StrArgList), " |")),
lists_key_replace(4, StrArgList1,
- string:tokens(lists:nth(4, StrArgList1), " |"));
+ string:lexemes(lists:nth(4, StrArgList1), " |"));
ets_insert ->
StrArgList1 = lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |")),
+ string:lexemes(lists:nth(2, StrArgList), " |")),
lists_key_replace(4, StrArgList1,
ets_tuple_argtypes2(
ets_tuple_argtypes1(lists:nth(4, StrArgList1), [], [], 0),
[]));
mnesia_dirty_read1 ->
lists_key_replace(2, StrArgList,
- [mnesia_tuple_argtypes(T) || T <- string:tokens(
+ [mnesia_tuple_argtypes(T) || T <- string:lexemes(
lists:nth(2, StrArgList), " |")]);
mnesia_dirty_read2 ->
lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |"));
+ string:lexemes(lists:nth(2, StrArgList), " |"));
mnesia_dirty_write1 ->
lists_key_replace(2, StrArgList,
- [mnesia_record_tab(R) || R <- string:tokens(
+ [mnesia_record_tab(R) || R <- string:lexemes(
lists:nth(2, StrArgList), " |")]);
mnesia_dirty_write2 ->
lists_key_replace(2, StrArgList,
- string:tokens(lists:nth(2, StrArgList), " |"));
+ string:lexemes(lists:nth(2, StrArgList), " |"));
function_call -> StrArgList
end.
@@ -1943,18 +1941,16 @@ format_type(Type, State) ->
erl_types:t_to_string(Type, R).
mnesia_record_tab(RecordStr) ->
- case string:str(RecordStr, "#") =:= 1 of
- true ->
- "'" ++
- string:sub_string(RecordStr, 2, string:str(RecordStr, "{") - 1) ++
- "'";
- false -> RecordStr
+ case erl_scan:string(RecordStr) of
+ {ok, [{'#', _}, {atom, _, Name}|_], _} ->
+ io_lib:write_string(atom_to_list(Name), $');
+ _ -> RecordStr
end.
mnesia_tuple_argtypes(TupleStr) ->
- TupleStr1 = string:strip(TupleStr, left, ${),
- [TupleStr2|_T] = string:tokens(TupleStr1, " ,"),
- lists:flatten(string:tokens(TupleStr2, " |")).
+ TupleStr1 = string:trim(TupleStr, leading, "${"),
+ [TupleStr2|_T] = string:lexemes(TupleStr1, " ,"),
+ lists:flatten(string:lexemes(TupleStr2, " |")).
-spec race_var_map(var_to_map1(), var_to_map2(), dict:dict(), op()) ->
dict:dict().
@@ -2237,7 +2233,7 @@ var_type_analysis(FunDefArgs, FunCallTypes, WarnVarArgs, RaceWarnTag,
case lists_key_member_lists(Vars, FunVarArgs) of
0 -> [Vars, WVA2, WVA3, WVA4];
N when is_integer(N) ->
- NewWVA2 = string:tokens(lists:nth(N + 1, FunVarArgs), " |"),
+ NewWVA2 = string:lexemes(lists:nth(N + 1, FunVarArgs), " |"),
[Vars, NewWVA2, WVA3, WVA4]
end;
?WARN_WHEREIS_UNREGISTER ->
@@ -2246,7 +2242,7 @@ var_type_analysis(FunDefArgs, FunCallTypes, WarnVarArgs, RaceWarnTag,
case lists_key_member_lists(Vars, FunVarArgs) of
0 -> [Vars, WVA2];
N when is_integer(N) ->
- NewWVA2 = string:tokens(lists:nth(N + 1, FunVarArgs), " |"),
+ NewWVA2 = string:lexemes(lists:nth(N + 1, FunVarArgs), " |"),
[Vars, NewWVA2]
end;
?WARN_ETS_LOOKUP_INSERT ->
@@ -2256,7 +2252,7 @@ var_type_analysis(FunDefArgs, FunCallTypes, WarnVarArgs, RaceWarnTag,
case lists_key_member_lists(Vars1, FunVarArgs) of
0 -> [Vars1, WVA2];
N1 when is_integer(N1) ->
- NewWVA2 = string:tokens(lists:nth(N1 + 1, FunVarArgs), " |"),
+ NewWVA2 = string:lexemes(lists:nth(N1 + 1, FunVarArgs), " |"),
[Vars1, NewWVA2]
end,
Vars2 =
@@ -2286,10 +2282,10 @@ var_type_analysis(FunDefArgs, FunCallTypes, WarnVarArgs, RaceWarnTag,
NewWVA2 =
case Arity of
1 ->
- [mnesia_record_tab(R) || R <- string:tokens(
+ [mnesia_record_tab(R) || R <- string:lexemes(
lists:nth(2, FunVarArgs), " |")];
2 ->
- string:tokens(lists:nth(N + 1, FunVarArgs), " |")
+ string:lexemes(lists:nth(N + 1, FunVarArgs), " |")
end,
[Vars, NewWVA2|T]
end
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index 9b8fbc67eb..abd89034f3 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -120,92 +120,10 @@ get_core_from_beam(File, Opts) ->
{error, " Could not get Core Erlang code for: " ++ File ++ "\n"}
end;
_ ->
- deprecated_get_core_from_beam(File, Opts)
+ {error, " Could not get Core Erlang code for: " ++ File ++ "\n" ++
+ " Recompile with +debug_info or analyze starting from source code"}
end.
-deprecated_get_core_from_beam(File, Opts) ->
- case get_abstract_code_from_beam(File) of
- error ->
- {error, " Could not get abstract code for: " ++ File ++ "\n" ++
- " Recompile with +debug_info or analyze starting from source code"};
- {ok, AbstrCode} ->
- case get_compile_options_from_beam(File) of
- error ->
- {error, " Could not get compile options for: " ++ File ++ "\n" ++
- " Recompile or analyze starting from source code"};
- {ok, CompOpts} ->
- case get_core_from_abstract_code(AbstrCode, Opts ++ CompOpts) of
- error ->
- {error, " Could not get core Erlang code for: " ++ File};
- {ok, _} = Core ->
- Core
- end
- end
- end.
-
-get_abstract_code_from_beam(File) ->
- case beam_lib:chunks(File, [abstract_code]) of
- {ok, {_, List}} ->
- case lists:keyfind(abstract_code, 1, List) of
- {abstract_code, {raw_abstract_v1, Abstr}} -> {ok, Abstr};
- _ -> error
- end;
- _ ->
- %% No or unsuitable abstract code.
- error
- end.
-
-get_compile_options_from_beam(File) ->
- case beam_lib:chunks(File, [compile_info]) of
- {ok, {_, List}} ->
- case lists:keyfind(compile_info, 1, List) of
- {compile_info, CompInfo} -> compile_info_to_options(CompInfo);
- _ -> error
- end;
- _ ->
- %% No or unsuitable compile info.
- error
- end.
-
-compile_info_to_options(CompInfo) ->
- case lists:keyfind(options, 1, CompInfo) of
- {options, CompOpts} -> {ok, CompOpts};
- _ -> error
- end.
-
-get_core_from_abstract_code(AbstrCode, Opts) ->
- %% We do not want the parse_transforms around since we already
- %% performed them. In some cases we end up in trouble when
- %% performing them again.
- AbstrCode1 = cleanup_parse_transforms(AbstrCode),
- %% Remove parse_transforms (and other options) from compile options.
- Opts2 = cleanup_compile_options(Opts),
- try compile:noenv_forms(AbstrCode1, Opts2 ++ src_compiler_opts()) of
- {ok, _, Core} -> {ok, Core};
- _What -> error
- catch
- error:_ -> error
- end.
-
-cleanup_parse_transforms([{attribute, _, compile, {parse_transform, _}}|Left]) ->
- cleanup_parse_transforms(Left);
-cleanup_parse_transforms([Other|Left]) ->
- [Other|cleanup_parse_transforms(Left)];
-cleanup_parse_transforms([]) ->
- [].
-
-cleanup_compile_options(Opts) ->
- lists:filter(fun keep_compile_option/1, Opts).
-
-%% Using abstract, not asm or core.
-keep_compile_option(from_asm) -> false;
-keep_compile_option(from_core) -> false;
-%% The parse transform will already have been applied, may cause
-%% problems if it is re-applied.
-keep_compile_option({parse_transform, _}) -> false;
-keep_compile_option(warnings_as_errors) -> false;
-keep_compile_option(_) -> true.
-
%% ============================================================================
%%
%% Typed Records
diff --git a/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args b/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args
index 1eb8cd455b..1be0ce0d8c 100644
--- a/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args
+++ b/lib/dialyzer/test/behaviour_SUITE_data/results/gen_server_incorrect_args
@@ -1,5 +1,5 @@
gen_server_incorrect_args.erl:3: Undefined callback function handle_cast/2 (behaviour gen_server)
gen_server_incorrect_args.erl:3: Undefined callback function init/1 (behaviour gen_server)
-gen_server_incorrect_args.erl:7: The inferred return type of handle_call/3 ({'no'} | {'ok'}) has nothing in common with {'noreply',_} | {'noreply',_,'hibernate' | 'infinity' | non_neg_integer()} | {'reply',_,_} | {'stop',_,_} | {'reply',_,_,'hibernate' | 'infinity' | non_neg_integer()} | {'stop',_,_,_}, which is the expected return type for the callback of the gen_server behaviour
+gen_server_incorrect_args.erl:7: The inferred return type of handle_call/3 ({'no'} | {'ok'}) has nothing in common with {'noreply',_} | {'noreply',_,'hibernate' | 'infinity' | non_neg_integer() | {'continue',_}} | {'reply',_,_} | {'stop',_,_} | {'reply',_,_,'hibernate' | 'infinity' | non_neg_integer() | {'continue',_}} | {'stop',_,_,_}, which is the expected return type for the callback of the gen_server behaviour
gen_server_incorrect_args.erl:7: The inferred type for the 2nd argument of handle_call/3 ('boo' | 'foo') is not a supertype of {pid(),_}, which is expected type for this argument in the callback of the gen_server behaviour
diff --git a/lib/dialyzer/test/dialyzer_common.erl b/lib/dialyzer/test/dialyzer_common.erl
index 48083a2731..1a8403f486 100644
--- a/lib/dialyzer/test/dialyzer_common.erl
+++ b/lib/dialyzer/test/dialyzer_common.erl
@@ -221,13 +221,9 @@ get_suites(Dir) ->
end.
suffix(String, Suffix) ->
- case string:rstr(String, Suffix) of
- 0 -> no;
- Index ->
- case string:substr(String, Index) =:= Suffix of
- true -> {yes, string:sub_string(String,1,Index-1)};
- false -> no
- end
+ case string:split(String, Suffix, trailing) of
+ [Prefix,[]] -> {yes, Prefix};
+ _ -> no
end.
-spec create_suite(string()) -> 'ok'.
diff --git a/lib/edoc/src/edoc_layout.erl b/lib/edoc/src/edoc_layout.erl
index eafab0588e..baa147410b 100644
--- a/lib/edoc/src/edoc_layout.erl
+++ b/lib/edoc/src/edoc_layout.erl
@@ -608,7 +608,7 @@ etypef([Cs | L], St, O, R, Opts) ->
app_fix(L, Opts) ->
try
{"//" ++ R1,L2} = app_fix1(L, 1),
- [App, Mod] = string:tokens(R1, "/"),
+ [App, Mod] = string:lexemes(R1, "/"),
"//" ++ atom(App, Opts) ++ "/" ++ atom(Mod, Opts) ++ L2
catch _:_ -> L
end.
@@ -1120,13 +1120,13 @@ ot_integer(E) ->
{integer,0,list_to_integer(get_attrval(value, E))}.
ot_range(E) ->
- [I1, I2] = string:tokens(get_attrval(value, E), "."),
+ [I1, I2] = string:lexemes(get_attrval(value, E), "."),
{type,0,range,[{integer,0,list_to_integer(I1)},
{integer,0,list_to_integer(I2)}]}.
ot_binary(E) ->
{Base, Unit} =
- case string:tokens(get_attrval(value, E), ",:*><") of
+ case string:lexemes(get_attrval(value, E), ",:*><") of
[] ->
{0, 0};
["_",B] ->
diff --git a/lib/edoc/src/edoc_specs.erl b/lib/edoc/src/edoc_specs.erl
index fb04bfce0e..26b7202462 100644
--- a/lib/edoc/src/edoc_specs.erl
+++ b/lib/edoc/src/edoc_specs.erl
@@ -83,7 +83,7 @@ spec(Form, Clause) ->
%% the given Erlang spec and an empty list of arguments.
dummy_spec(Form) ->
{#t_name{name = Name}, Arity, TypeSpecs} = get_spec(Form),
- As = string:join(lists:duplicate(Arity, "_X"), ","),
+ As = lists:join(",", lists:duplicate(Arity, "_X")),
S = lists:flatten(io_lib:format("~p(~s) -> true\n", [Name, As])),
#tag{name = spec, line = get_line(element(2, hd(TypeSpecs))),
origin = code, data = S}.
diff --git a/lib/eldap/src/eldap.erl b/lib/eldap/src/eldap.erl
index 625309271b..2b84872b92 100644
--- a/lib/eldap/src/eldap.erl
+++ b/lib/eldap/src/eldap.erl
@@ -1368,9 +1368,9 @@ rm_leading_slash(Tail) -> Tail.
parse_attributes([$?|Tail]) ->
case split_string(Tail,$?) of
{[],Attributes} ->
- {[],{attributes,string:tokens(Attributes,",")}};
+ {[],{attributes,string:lexemes(Attributes,",")}};
{Attributes,Rest} ->
- {Rest,{attributes,string:tokens(Attributes,",")}}
+ {Rest,{attributes,string:lexemes(Attributes,",")}}
end.
parse_hostport(Str) ->
diff --git a/lib/erl_docgen/priv/dtd/erlref.dtd b/lib/erl_docgen/priv/dtd/erlref.dtd
index 615b88b61a..78d6771f52 100644
--- a/lib/erl_docgen/priv/dtd/erlref.dtd
+++ b/lib/erl_docgen/priv/dtd/erlref.dtd
@@ -33,4 +33,5 @@
<!ATTLIST name name CDATA #IMPLIED
arity CDATA #IMPLIED
clause_i CDATA #IMPLIED
+ anchor CDATA #IMPLIED
n_vars CDATA #IMPLIED>
diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl
index 75614392fb..5b7eae4f73 100644
--- a/lib/erl_docgen/priv/xsl/db_html.xsl
+++ b/lib/erl_docgen/priv/xsl/db_html.xsl
@@ -172,6 +172,7 @@
<xsl:template name="spec_name">
<xsl:variable name="name" select="@name"/>
<xsl:variable name="arity" select="@arity"/>
+ <xsl:variable name="anchor" select="@anchor"/>
<xsl:variable name="spec0">
<xsl:call-template name="find_spec"/>
</xsl:variable>
@@ -198,6 +199,11 @@
</xsl:otherwise>
</xsl:choose>
+ <!-- Insert an anchor for "anchor" attribute -->
+ <xsl:if test="string-length($anchor) > 0">
+ <a name="{$anchor}"></a>
+ </xsl:if>
+
<xsl:variable name="global_types" select="ancestor::erlref/datatypes"/>
<xsl:variable name="local_types"
select="../type[string-length(@name) > 0]"/>
diff --git a/lib/erl_docgen/src/docgen_edoc_xml_cb.erl b/lib/erl_docgen/src/docgen_edoc_xml_cb.erl
index 7cdbb502d9..67e6e33c93 100644
--- a/lib/erl_docgen/src/docgen_edoc_xml_cb.erl
+++ b/lib/erl_docgen/src/docgen_edoc_xml_cb.erl
@@ -11,7 +11,7 @@
%% limitations under the License.
%%
%% Copyright (c) 2001-2016 Richard Carlsson. Parts written by Ericsson
-%% are Copyright (c) Ericsson AB 2001-2012. All Rights Reserved.
+%% are Copyright (c) Ericsson AB 2001-2017. All Rights Reserved.
%%
-module(docgen_edoc_xml_cb).
@@ -113,7 +113,7 @@ root_attributes(Element, Opts) ->
%% epp:default_encoding/0 returns 'utf8'
reformat_encoding(utf8) -> "UTF-8";
reformat_encoding(List) when is_list(List) ->
- case string:to_lower(List) of
+ case string:lowercase(List) of
"utf8" -> "UTF-8";
_ -> List
end;
diff --git a/lib/erl_docgen/src/docgen_otp_specs.erl b/lib/erl_docgen/src/docgen_otp_specs.erl
index 126229ecc9..9f2b401f93 100644
--- a/lib/erl_docgen/src/docgen_otp_specs.erl
+++ b/lib/erl_docgen/src/docgen_otp_specs.erl
@@ -297,7 +297,7 @@ indent(L) ->
app_fix(L) ->
try
{"//" ++ R1,L2} = app_fix(L, 1),
- [App, Mod] = string:tokens(R1, "/"),
+ [App, Mod] = string:lexemes(R1, "/"),
"//" ++ atom(App) ++ "/" ++ atom(Mod) ++ L2
catch _:_ -> L
end.
@@ -585,13 +585,13 @@ ot_integer(E) ->
{integer,0,list_to_integer(get_attrval(value, E))}.
ot_range(E) ->
- [I1, I2] = string:tokens(get_attrval(value, E), "."),
+ [I1, I2] = string:lexemes(get_attrval(value, E), "."),
{type,0,range,[{integer,0,list_to_integer(I1)},
{integer,0,list_to_integer(I2)}]}.
ot_binary(E) ->
{Base, Unit} =
- case string:tokens(get_attrval(value, E), ",:*><") of
+ case string:lexemes(get_attrval(value, E), ",:*><") of
[] ->
{0, 0};
["_",B] ->
diff --git a/lib/et/src/et_collector.erl b/lib/et/src/et_collector.erl
index b0f016a8ea..ffe244324c 100644
--- a/lib/et/src/et_collector.erl
+++ b/lib/et/src/et_collector.erl
@@ -1135,7 +1135,7 @@ handle_info(Info, S) ->
noreply(S).
listen_on_trace_port(Node, Port, S) ->
- [_Name, Host] = string:tokens(atom_to_list(Node), [$@]),
+ [_Name, Host] = string:lexemes(atom_to_list(Node), [$@]),
case catch start_trace_client(self(), ip, {Host, Port}) of
{trace_client_pid, RemotePid} ->
rpc:call(Node, et_selector, change_pattern, [S#state.trace_pattern]),
diff --git a/lib/eunit/src/eunit_lib.erl b/lib/eunit/src/eunit_lib.erl
index e340f50a3c..d1bd160ea1 100644
--- a/lib/eunit/src/eunit_lib.erl
+++ b/lib/eunit/src/eunit_lib.erl
@@ -391,7 +391,7 @@ fun_parent(F) ->
-ifdef(TEST).
fun_parent_test() ->
- {?MODULE,fun_parent_test,0} = fun_parent(fun () -> ok end).
+ {?MODULE,fun_parent_test,0} = fun_parent(fun (A) -> {ok,A} end).
-endif.
%% ---------------------------------------------------------------------
diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl
index abb6c259f6..d8d707c05e 100644
--- a/lib/hipe/cerl/erl_types.erl
+++ b/lib/hipe/cerl/erl_types.erl
@@ -4249,13 +4249,13 @@ t_to_string(?identifier(Set), _RecDict) ->
case Set of
?any -> "identifier()";
_ ->
- string:join([flat_format("~w()", [T]) || T <- set_to_list(Set)], " | ")
+ flat_join([flat_format("~w()", [T]) || T <- set_to_list(Set)], " | ")
end;
t_to_string(?opaque(Set), RecDict) ->
- string:join([opaque_type(Mod, Name, Args, S, RecDict) ||
- #opaque{mod = Mod, name = Name, struct = S, args = Args}
- <- set_to_list(Set)],
- " | ");
+ flat_join([opaque_type(Mod, Name, Args, S, RecDict) ||
+ #opaque{mod = Mod, name = Name, struct = S, args = Args}
+ <- set_to_list(Set)],
+ " | ");
t_to_string(?matchstate(Pres, Slots), RecDict) ->
flat_format("ms(~ts,~ts)", [t_to_string(Pres, RecDict),
t_to_string(Slots,RecDict)]);
@@ -4346,9 +4346,9 @@ t_to_string(?map(Pairs0,DefK,DefV), RecDict) ->
end end,
StrMand = [{Tos(K),Tos(V)}||{K,?mand,V}<-Pairs],
StrOpt = [{Tos(K),Tos(V)}||{K,?opt,V}<-Pairs],
- "#{" ++ string:join([K ++ ":=" ++ V||{K,V}<-StrMand]
- ++ [K ++ "=>" ++ V||{K,V}<-StrOpt]
- ++ ExtraEl, ", ") ++ "}";
+ "#{" ++ flat_join([K ++ ":=" ++ V||{K,V}<-StrMand]
+ ++ [K ++ "=>" ++ V||{K,V}<-StrOpt]
+ ++ ExtraEl, ", ") ++ "}";
t_to_string(?tuple(?any, ?any, ?any), _RecDict) -> "tuple()";
t_to_string(?tuple(Elements, _Arity, ?any), RecDict) ->
"{" ++ comma_sequence(Elements, RecDict) ++ "}";
@@ -4371,7 +4371,7 @@ t_to_string(?var(Id), _RecDict) when is_integer(Id) ->
record_to_string(Tag, [_|Fields], FieldNames, RecDict) ->
FieldStrings = record_fields_to_string(Fields, FieldNames, RecDict, []),
- "#" ++ atom_to_string(Tag) ++ "{" ++ string:join(FieldStrings, ",") ++ "}".
+ "#" ++ atom_to_string(Tag) ++ "{" ++ flat_join(FieldStrings, ",") ++ "}".
record_fields_to_string([F|Fs], [{FName, _Abstr, DefType}|FDefs],
RecDict, Acc) ->
@@ -4397,7 +4397,7 @@ record_field_diffs_to_string(?tuple([_|Fs], Arity, Tag), RecDict) ->
{ok, FieldNames} = lookup_record(TagAtom, Arity-1, RecDict),
%% io:format("RecCElems = ~p\nRecTypes = ~p\n", [Fs, FieldNames]),
FieldDiffs = field_diffs(Fs, FieldNames, RecDict, []),
- string:join(FieldDiffs, " and ").
+ flat_join(FieldDiffs, " and ").
field_diffs([F|Fs], [{FName, _Abstr, DefType}|FDefs], RecDict, Acc) ->
%% Don't care about opacity for now.
@@ -4417,11 +4417,11 @@ comma_sequence(Types, RecDict) ->
true -> "_";
false -> t_to_string(T, RecDict)
end || T <- Types],
- string:join(List, ",").
+ flat_join(List, ",").
union_sequence(Types, RecDict) ->
List = [t_to_string(T, RecDict) || T <- Types],
- string:join(List, " | ").
+ flat_join(List, " | ").
-ifdef(DEBUG).
opaque_type(Mod, Name, _Args, S, RecDict) ->
@@ -5261,7 +5261,7 @@ t_form_to_string({ann_type, _L, [Var, Type]}) ->
t_form_to_string({paren_type, _L, [Type]}) ->
flat_format("(~ts)", [t_form_to_string(Type)]);
t_form_to_string({remote_type, _L, [{atom, _, Mod}, {atom, _, Name}, Args]}) ->
- ArgString = "(" ++ string:join(t_form_to_string_list(Args), ",") ++ ")",
+ ArgString = "(" ++ flat_join(t_form_to_string_list(Args), ",") ++ ")",
flat_format("~w:~tw", [Mod, Name]) ++ ArgString;
t_form_to_string({type, _L, arity, []}) -> "arity()";
t_form_to_string({type, _L, binary, []}) -> "binary()";
@@ -5284,7 +5284,7 @@ t_form_to_string({type, _L, 'fun', []}) -> "fun()";
t_form_to_string({type, _L, 'fun', [{type, _, any}, Range]}) ->
"fun(...) -> " ++ t_form_to_string(Range);
t_form_to_string({type, _L, 'fun', [{type, _, product, Domain}, Range]}) ->
- "fun((" ++ string:join(t_form_to_string_list(Domain), ",") ++ ") -> "
+ "fun((" ++ flat_join(t_form_to_string_list(Domain), ",") ++ ") -> "
++ t_form_to_string(Range) ++ ")";
t_form_to_string({type, _L, iodata, []}) -> "iodata()";
t_form_to_string({type, _L, iolist, []}) -> "iolist()";
@@ -5292,7 +5292,7 @@ t_form_to_string({type, _L, list, [Type]}) ->
"[" ++ t_form_to_string(Type) ++ "]";
t_form_to_string({type, _L, map, any}) -> "map()";
t_form_to_string({type, _L, map, Args}) ->
- "#{" ++ string:join(t_form_to_string_list(Args), ",") ++ "}";
+ "#{" ++ flat_join(t_form_to_string_list(Args), ",") ++ "}";
t_form_to_string({type, _L, map_field_assoc, [Key, Val]}) ->
t_form_to_string(Key) ++ "=>" ++ t_form_to_string(Val);
t_form_to_string({type, _L, map_field_exact, [Key, Val]}) ->
@@ -5304,7 +5304,7 @@ t_form_to_string({type, _L, nonempty_list, [Type]}) ->
"[" ++ t_form_to_string(Type) ++ ",...]";
t_form_to_string({type, _L, nonempty_string, []}) -> "nonempty_string()";
t_form_to_string({type, _L, product, Elements}) ->
- "<" ++ string:join(t_form_to_string_list(Elements), ",") ++ ">";
+ "<" ++ flat_join(t_form_to_string_list(Elements), ",") ++ ">";
t_form_to_string({type, _L, range, [From, To]} = Type) ->
case {erl_eval:partial_eval(From), erl_eval:partial_eval(To)} of
{{integer, _, FromVal}, {integer, _, ToVal}} ->
@@ -5314,7 +5314,7 @@ t_form_to_string({type, _L, range, [From, To]} = Type) ->
t_form_to_string({type, _L, record, [{atom, _, Name}]}) ->
flat_format("#~tw{}", [Name]);
t_form_to_string({type, _L, record, [{atom, _, Name}|Fields]}) ->
- FieldString = string:join(t_form_to_string_list(Fields), ","),
+ FieldString = flat_join(t_form_to_string_list(Fields), ","),
flat_format("#~tw{~ts}", [Name, FieldString]);
t_form_to_string({type, _L, field_type, [{atom, _, Name}, Type]}) ->
flat_format("~tw::~ts", [Name, t_form_to_string(Type)]);
@@ -5322,9 +5322,9 @@ t_form_to_string({type, _L, term, []}) -> "term()";
t_form_to_string({type, _L, timeout, []}) -> "timeout()";
t_form_to_string({type, _L, tuple, any}) -> "tuple()";
t_form_to_string({type, _L, tuple, Args}) ->
- "{" ++ string:join(t_form_to_string_list(Args), ",") ++ "}";
+ "{" ++ flat_join(t_form_to_string_list(Args), ",") ++ "}";
t_form_to_string({type, _L, union, Args}) ->
- string:join(t_form_to_string_list(Args), " | ");
+ flat_join(t_form_to_string_list(Args), " | ");
t_form_to_string({type, _L, Name, []} = T) ->
try
M = mod,
@@ -5342,7 +5342,7 @@ t_form_to_string({type, _L, Name, []} = T) ->
end;
t_form_to_string({user_type, _L, Name, List}) ->
flat_format("~tw(~ts)",
- [Name, string:join(t_form_to_string_list(List), ",")]);
+ [Name, flat_join(t_form_to_string_list(List), ",")]);
t_form_to_string({type, L, Name, List}) ->
%% Compatibility: modules compiled before Erlang/OTP 18.0.
t_form_to_string({user_type, L, Name, List}).
@@ -5603,7 +5603,7 @@ set_to_string(Set) ->
true -> io_lib:write_string(atom_to_list(X), $'); % stupid emacs '
false -> flat_format("~tw", [X])
end || X <- set_to_list(Set)],
- string:join(L, " | ").
+ flat_join(L, " | ").
set_min([H|_]) -> H.
@@ -5613,6 +5613,9 @@ set_max(Set) ->
flat_format(F, S) ->
lists:flatten(io_lib:format(F, S)).
+flat_join(List, Sep) ->
+ lists:flatten(lists:join(Sep, List)).
+
%%=============================================================================
%%
%% Utilities for the binary type
diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl
index 2abecf7f18..167f5c67bb 100644
--- a/lib/hipe/icode/hipe_beam_to_icode.erl
+++ b/lib/hipe/icode/hipe_beam_to_icode.erl
@@ -2296,6 +2296,12 @@ split_code([First|Code], Label, Instr) ->
split_code([Instr|Code], Label, Instr, Prev, As) when Prev =:= Label ->
split_code_final(Code, As); % drop both label and instruction
+split_code([{icode_end_try}|_]=Code, Label, {try_case,_}, Prev, As)
+ when Prev =:= Label ->
+ %% The try_case has been replaced with try_end as an optimization.
+ %% Keep this instruction, since it might be the only try_end instruction
+ %% for this try/catch block.
+ split_code_final(Code, As); % drop label
split_code([Other|_Code], Label, Instr, Prev, _As) when Prev =:= Label ->
?EXIT({missing_instr_after_label, Label, Instr, [Other, Prev | _As]});
split_code([Other|Code], Label, Instr, Prev, As) ->
diff --git a/lib/hipe/llvm/hipe_llvm.erl b/lib/hipe/llvm/hipe_llvm.erl
index 641d3fda0a..ccd40162cb 100644
--- a/lib/hipe/llvm/hipe_llvm.erl
+++ b/lib/hipe/llvm/hipe_llvm.erl
@@ -1005,11 +1005,12 @@ pp_ins(Dev, Ver, I) ->
write(Dev, [" ", adj_stack_offset(I),")\n"]);
#llvm_meta{} ->
write(Dev, ["!", meta_id(I), " = !{ "]),
- write(Dev, string:join([if is_list(Op) -> ["!\"", Op, "\""];
- is_integer(Op) -> ["i32 ", integer_to_list(Op)];
- is_record(Op, llvm_meta) ->
- ["!", meta_id(Op)]
- end || Op <- meta_operands(I)], ", ")),
+ write(Dev, lists:join(", ",
+ [if is_list(Op) -> ["!\"", Op, "\""];
+ is_integer(Op) -> ["i32 ", integer_to_list(Op)];
+ is_record(Op, llvm_meta) ->
+ ["!", meta_id(Op)]
+ end || Op <- meta_operands(I)])),
write(Dev, " }\n");
Other ->
exit({?MODULE, pp_ins, {"Unknown LLVM instruction", Other}})
diff --git a/lib/hipe/llvm/hipe_llvm_main.erl b/lib/hipe/llvm/hipe_llvm_main.erl
index 4eec0c752b..54c435c127 100644
--- a/lib/hipe/llvm/hipe_llvm_main.erl
+++ b/lib/hipe/llvm/hipe_llvm_main.erl
@@ -154,7 +154,7 @@ compiler_target_opt() ->
%% @doc Join options.
fix_opts(Opts) ->
- string:join(Opts, " ").
+ lists:flatten(lists:join(" ", Opts)).
%% @doc Translate optimization-level flag (default is "O2").
trans_optlev_flag(Tool, Options) ->
diff --git a/lib/hipe/llvm/hipe_rtl_to_llvm.erl b/lib/hipe/llvm/hipe_rtl_to_llvm.erl
index 79e1bfd381..934717efc1 100644
--- a/lib/hipe/llvm/hipe_rtl_to_llvm.erl
+++ b/lib/hipe/llvm/hipe_rtl_to_llvm.erl
@@ -1537,7 +1537,7 @@ declare_switch_table({Name, {switch, {TableType, Labels, _, _}, _}}, FunName) ->
LabelList = [mk_jump_label(L) || L <- Labels],
Fun1 = fun(X) -> "i8* blockaddress(@" ++ FunName ++ ", " ++ X ++ ")" end,
List2 = lists:map(Fun1, LabelList),
- List3 = string:join(List2, ",\n"),
+ List3 = lists:flatten(lists:join(",\n", List2)),
List4 = "[\n" ++ List3 ++ "\n]\n",
hipe_llvm:mk_const_decl("@" ++ Name, "constant", TableType, List4).
@@ -1553,7 +1553,7 @@ declare_closure_labels(ClosureLabels, Relocs, Fun) ->
Relocs1 = relocs_store("table_closures", {table_closures, ArityList}, Relocs),
List2 =
["i8* blockaddress(@" ++ FunName ++ ", " ++ L ++ ")" || L <- LabelList],
- List3 = string:join(List2, ",\n"),
+ List3 = lists:flatten(lists:join(",\n", List2)),
List4 = "[\n" ++ List3 ++ "\n]\n",
NrLabels = length(LabelList),
ByteTyPtr = hipe_llvm:mk_pointer(hipe_llvm:mk_int(?BITS_IN_BYTE)),
diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl
index 19b4e8bfe2..f5f5bf5830 100644
--- a/lib/hipe/main/hipe.erl
+++ b/lib/hipe/main/hipe.erl
@@ -1616,11 +1616,11 @@ llvm_support_available() ->
get_llvm_version() ->
OptStr = os:cmd("opt -version"),
SubStr = "LLVM version ", N = length(SubStr),
- case string:str(OptStr, SubStr) of
- 0 -> % No opt available
+ case string:find(OptStr, SubStr) of
+ nomatch -> % No opt available
{0, 0};
S ->
- case string:tokens(string:sub_string(OptStr, S + N), ".") of
+ case string:lexemes(string:slice(S, N), ".") of
[MajorS, MinorS | _] ->
case {string:to_integer(MajorS), string:to_integer(MinorS)} of
{{Major, ""}, {Minor, _}}
diff --git a/lib/hipe/test/basic_SUITE_data/basic_exceptions.erl b/lib/hipe/test/basic_SUITE_data/basic_exceptions.erl
index 229a0516dc..d71b924d22 100644
--- a/lib/hipe/test/basic_SUITE_data/basic_exceptions.erl
+++ b/lib/hipe/test/basic_SUITE_data/basic_exceptions.erl
@@ -6,12 +6,13 @@
%%%-------------------------------------------------------------------
-module(basic_exceptions).
--export([test/0, test_catches/0]).
+-export([test/0]).
%% functions used as arguments to spawn/3
-export([bad_guy/2]).
test() ->
+ ok = test_catches(),
ok = test_catch_exit(42),
ok = test_catch_throw(42),
ok = test_catch_element(),
@@ -22,6 +23,7 @@ test() ->
ok = test_pending_errors(),
ok = test_bad_fun_call(),
ok = test_guard_bif(),
+ ok = test_eclectic(),
ok.
%%--------------------------------------------------------------------
@@ -463,3 +465,117 @@ guard_bif('node/0', X, Y) when node() == Y ->
{'node/0', X, Y};
guard_bif('node/1', X, Y) when node(X) == Y ->
{'node/1', X, Y}.
+
+%%--------------------------------------------------------------------
+%% Taken from trycatch_SUITE.erl (compiler test suite).
+%%
+%% Cases that are commented out contain exception information that was
+%% added to Erlang/OTP in commit e8d45ae14c6c3bdfcbbc7964228b004ef4f11ea6
+%% (May 2017) only in the BEAM emulator. Thus, part of this test fails
+%% when compiled in native code.
+%% The remaining cases are uncommented so that they are properly tested
+%% in native code too.
+%%--------------------------------------------------------------------
+
+test_eclectic() ->
+ V = {make_ref(),3.1415926535,[[]|{}]},
+ {{value,{value,V},V},V} =
+ eclectic_1({foo,{value,{value,V}}}, undefined, {value,V}),
+ {{'EXIT',{V,[{?MODULE,foo,1,_}|_]}},V} =
+ eclectic_1({catch_foo,{error,V}}, undefined, {value,V}),
+ {{error,{exit,V},{'EXIT',V}},V} =
+ eclectic_1({foo,{error,{exit,V}}}, error, {value,V}),
+ %% {{value,{value,V},V},
+ %% {'EXIT',{badarith,[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]}}} =
+ %% eclectic_1({foo,{value,{value,V}}}, undefined, {'add',{0,a}}),
+ {{'EXIT',V},V} =
+ eclectic_1({catch_foo,{exit,V}}, undefined, {throw,V}),
+ %% {{error,{'div',{1,0}},{'EXIT',{badarith,[{erlang,'div',[1,0],_},{?MODULE,my_div,2,_}|_]}}},
+ %% {'EXIT',V}} =
+ %% eclectic_1({foo,{error,{'div',{1,0}}}}, error, {exit,V}),
+ {{{error,V},{'EXIT',{V,[{?MODULE,foo,1,_}|_]}}},
+ {'EXIT',V}} =
+ eclectic_1({catch_foo,{throw,{error,V}}}, undefined, {exit,V}),
+ %%
+ {{value,{value,{value,V},V}},V} =
+ eclectic_2({value,{value,V}}, undefined, {value,V}),
+ {{value,{throw,{value,V},V}},V} =
+ eclectic_2({throw,{value,V}}, throw, {value,V}),
+ {{caught,{'EXIT',V}},undefined} =
+ eclectic_2({value,{value,V}}, undefined, {exit,V}),
+ {{caught,{'EXIT',{V,[{?MODULE,foo,1,_}|_]}}},undefined} =
+ eclectic_2({error,{value,V}}, throw, {error,V}),
+ %% The following fails in native code
+ %% %% {{caught,{'EXIT',{badarg,[{erlang,abs,[V],_}|_]}}},V} =
+ %% %% eclectic_2({value,{'abs',V}}, undefined, {value,V}),
+ %% {{caught,{'EXIT',{badarith,[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]}}},V} =
+ %% eclectic_2({exit,{'add',{0,a}}}, exit, {value,V}),
+ {{caught,{'EXIT',V}},undefined} =
+ eclectic_2({value,{error,V}}, undefined, {exit,V}),
+ {{caught,{'EXIT',{V,[{?MODULE,foo,1,_}|_]}}},undefined} =
+ eclectic_2({throw,{'div',{1,0}}}, throw, {error,V}),
+ ok.
+
+eclectic_1(X, C, Y) ->
+ erase(eclectic),
+ Done = make_ref(),
+ Try =
+ try case X of
+ {catch_foo,V} -> catch {Done,foo(V)};
+ {foo,V} -> {Done,foo(V)}
+ end of
+ {Done,D} -> {value,D,catch foo(D)};
+ {'EXIT',_}=Exit -> Exit;
+ D -> {D,catch foo(D)}
+ catch
+ C:D -> {C,D,catch foo(D)}
+ after
+ put(eclectic, catch foo(Y))
+ end,
+ {Try,erase(eclectic)}.
+
+eclectic_2(X, C, Y) ->
+ Done = make_ref(),
+ erase(eclectic),
+ Catch =
+ case
+ catch
+ {Done,
+ try foo(X) of
+ V -> {value,V,foo(V)}
+ catch
+ C:D -> {C,D,foo(D)}
+ after
+ put(eclectic, foo(Y))
+ end} of
+ {Done,Z} -> {value,Z};
+ Z -> {caught,Z}
+ end,
+ {Catch,erase(eclectic)}.
+
+foo({value,Value}) -> Value;
+foo({'div',{A,B}}) ->
+ my_div(A, B);
+foo({'add',{A,B}}) ->
+ my_add(A, B);
+foo({'abs',X}) ->
+ my_abs(X);
+foo({error,Error}) ->
+ erlang:error(Error);
+foo({throw,Throw}) ->
+ erlang:throw(Throw);
+foo({exit,Exit}) ->
+ erlang:exit(Exit);
+foo({raise,{Class,Reason}}) ->
+ erlang:raise(Class, Reason);
+foo(Term) when not is_atom(Term) -> Term.
+%%foo(Atom) when is_atom(Atom) -> % must not be defined!
+
+my_div(A, B) ->
+ A div B.
+
+my_add(A, B) ->
+ A + B.
+
+my_abs(X) ->
+ abs(X).
diff --git a/lib/hipe/test/hipe_testsuite_driver.erl b/lib/hipe/test/hipe_testsuite_driver.erl
index 88576775ca..ee9c57a908 100644
--- a/lib/hipe/test/hipe_testsuite_driver.erl
+++ b/lib/hipe/test/hipe_testsuite_driver.erl
@@ -29,13 +29,9 @@ get_suites(SuitesWithSuiteSuffix) ->
[S || {yes, S} <- Prefixes].
suffix(String, Suffix) ->
- case string:rstr(String, Suffix) of
- 0 -> no;
- Index ->
- case string:substr(String, Index) =:= Suffix of
- true -> {yes, string:sub_string(String, 1, Index-1)};
- false -> no
- end
+ case string:split(String, Suffix, trailing) of
+ [Prefix,[]] -> {yes, Prefix};
+ _ -> no
end.
-spec file_type(file:filename()) -> {ok, file_type()} | {error, ext_posix()}.
diff --git a/lib/hipe/test/opt_verify_SUITE.erl b/lib/hipe/test/opt_verify_SUITE.erl
index a323c10503..24f43af275 100644
--- a/lib/hipe/test/opt_verify_SUITE.erl
+++ b/lib/hipe/test/opt_verify_SUITE.erl
@@ -59,7 +59,7 @@ call_elim_test_file(Config, FileName, Option) ->
substring_count(Icode, Substring) ->
substring_count(Icode, Substring, 0).
substring_count(Icode, Substring, N) ->
- case string:str(Icode, Substring) of
- 0 -> N;
- I -> substring_count(lists:nthtail(I, Icode), Substring, N+1)
+ case string:find(Icode, Substring) of
+ nomatch -> N;
+ Prefix -> substring_count(string:prefix(Prefix, Substring), Substring, N+1)
end.
diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml
index b674b3ca93..2ab35b9b05 100644
--- a/lib/kernel/doc/src/file.xml
+++ b/lib/kernel/doc/src/file.xml
@@ -41,7 +41,7 @@
<p>Regarding filename encoding, the Erlang VM can operate in
two modes. The current mode can be queried using function
- <seealso marker="#native_name_encoding"><c>native_name_encoding/0</c></seealso>.
+ <seealso marker="#native_name_encoding/0"><c>native_name_encoding/0</c></seealso>.
It returns <c>latin1</c> or <c>utf8</c>.</p>
<p>In <c>latin1</c> mode, the Erlang VM does not change the
@@ -59,7 +59,7 @@
terminal supports UTF-8, otherwise <c>latin1</c>. The default can
be overridden using <c>+fnl</c> (to force <c>latin1</c> mode)
or <c>+fnu</c> (to force <c>utf8</c> mode) when starting
- <seealso marker="erts:erl"><c>erts:erl</c></seealso>.</p>
+ <seealso marker="erts:erl"><c>erl</c></seealso>.</p>
<p>On operating systems with transparent naming, files can be
inconsistently named, for example, some files are encoded in UTF-8 while
@@ -81,6 +81,23 @@
<p>See also section <seealso marker="stdlib:unicode_usage#notes-about-raw-filenames">Notes About Raw Filenames</seealso> in the STDLIB User's Guide.</p>
+ <note><p>
+ File operations used to accept filenames containing
+ null characters (integer value zero). This caused
+ the name to be truncated and in some cases arguments
+ to primitive operations to be mixed up. Filenames
+ containing null characters inside the filename
+ are now <em>rejected</em> and will cause primitive
+ file operations fail.
+ </p></note>
+ <warning><p>
+ Currently null characters at the end of the filename
+ will be accepted by primitive file operations. Such
+ filenames are however still documented as invalid. The
+ implementation will also change in the future and
+ reject such filenames.
+ </p></warning>
+
</description>
<datatypes>
@@ -96,9 +113,21 @@
</datatype>
<datatype>
<name name="filename"/>
+ <desc>
+ <p>
+ See also the documentation of the
+ <seealso marker="#type-name_all"><c>name_all()</c></seealso> type.
+ </p>
+ </desc>
</datatype>
<datatype>
<name name="filename_all"/>
+ <desc>
+ <p>
+ See also the documentation of the
+ <seealso marker="#type-name_all"><c>name_all()</c></seealso> type.
+ </p>
+ </desc>
</datatype>
<datatype>
<name name="io_device"/>
@@ -112,21 +141,23 @@
<name name="name"/>
<desc>
<p>If VM is in Unicode filename mode, <c>string()</c> and <c>char()</c>
- are allowed to be &gt; 255.
+ are allowed to be &gt; 255. See also the documentation of the
+ <seealso marker="#type-name_all"><c>name_all()</c></seealso> type.
</p>
</desc>
</datatype>
<datatype>
<name name="name_all"/>
<desc>
- <p>If VM is in Unicode filename mode, <c>string()</c> and <c>char()</c>
+ <p>If VM is in Unicode filename mode, characters
are allowed to be &gt; 255.
<c><anno>RawFilename</anno></c> is a filename not subject to
Unicode translation,
meaning that it can contain characters not conforming to
the Unicode encoding expected from the file system
(that is, non-UTF-8 characters although the VM is started
- in Unicode filename mode).
+ in Unicode filename mode). Null characters (integer value zero)
+ are <em>not</em> allowed in filenames (not even at the end).
</p>
</desc>
</datatype>
@@ -1825,7 +1856,7 @@ f.txt: {person, "kalle", 25}.
<p>The functions in the module <c>file</c> usually treat binaries
as raw filenames, that is, they are passed "as is" even when the
encoding of the binary does not agree with
- <seealso marker="#native_name_encoding"><c>native_name_encoding()</c></seealso>.
+ <seealso marker="#native_name_encoding/0"><c>native_name_encoding()</c></seealso>.
However, this function expects binaries to be encoded according to the
value returned by <c>native_name_encoding()</c>.</p>
<p>Typical error reasons are:</p>
diff --git a/lib/kernel/doc/src/inet_res.xml b/lib/kernel/doc/src/inet_res.xml
index 4ada4203c0..3454e3c6f9 100644
--- a/lib/kernel/doc/src/inet_res.xml
+++ b/lib/kernel/doc/src/inet_res.xml
@@ -130,7 +130,7 @@ dns_header() = DnsHeader
inet_dns:header(DnsHeader) ->
[ {id, integer()}
| {qr, boolean()}
- | {opcode, 'query' | iquery | status | integer()}
+ | {opcode, query | iquery | status | integer()}
| {aa, boolean()}
| {tc, boolean()}
| {rd, boolean()}
diff --git a/lib/kernel/doc/src/os.xml b/lib/kernel/doc/src/os.xml
index 0e9add4161..0a08e2c78a 100644
--- a/lib/kernel/doc/src/os.xml
+++ b/lib/kernel/doc/src/os.xml
@@ -36,8 +36,99 @@
only run on a specific platform. On the other hand, with careful
use, these functions can be of help in enabling a program to run on
most platforms.</p>
+
+ <note>
+ <p>
+ File operations used to accept filenames containing
+ null characters (integer value zero). This caused
+ the name to be truncated and in some cases arguments
+ to primitive operations to be mixed up. Filenames
+ containing null characters inside the filename
+ are now <em>rejected</em> and will cause primitive
+ file operations to fail.
+ </p>
+ <p>
+ Also environment variable operations used to accept
+ names and values of environment variables containing
+ null characters (integer value zero). This caused
+ operations to silently produce erroneous results.
+ Environment variable names and values containing
+ null characters inside the name or value are now
+ <em>rejected</em> and will cause environment variable
+ operations to fail.
+ </p>
+ </note>
+ <warning>
+ <p>
+ Currently null characters at the end of filenames,
+ environment variable names and values will be accepted
+ by the primitive operations. Such filenames, environment
+ variable names and values are however still documented as
+ invalid. The implementation will also change in the
+ future and reject such filenames, environment variable
+ names and values.
+ </p>
+ </warning>
</description>
+ <datatypes>
+ <datatype>
+ <name name="env_var_name"/>
+ <desc>
+ <p>A string containing valid characters on the specific
+ OS for environment variable names using
+ <seealso marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seealso>
+ encoding. Note that specifically null characters (integer
+ value zero) and <c>$=</c> characters are not allowed.
+ However, note that not all invalid characters necessarily
+ will cause the primitiv operations to fail, but may instead
+ produce invalid results.
+ </p>
+ </desc>
+ </datatype>
+ <datatype>
+ <name name="env_var_value"/>
+ <desc>
+ <p>A string containing valid characters on the specific
+ OS for environment variable values using
+ <seealso marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seealso>
+ encoding. Note that specifically null characters (integer
+ value zero) are not allowed. However, note that not all
+ invalid characters necessarily will cause the primitiv
+ operations to fail, but may instead produce invalid results.
+ </p>
+ </desc>
+ </datatype>
+ <datatype>
+ <name name="env_var_name_value"/>
+ <desc>
+ <p>
+ Assuming that environment variables has been correctly
+ set, a strings containing valid characters on the specific
+ OS for environment variable names and values using
+ <seealso marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seealso>
+ encoding. The first <c>$=</c> characters appearing in
+ the string separates environment variable name (on the
+ left) from environment variable value (on the right).
+ </p>
+ </desc>
+ </datatype>
+ <datatype>
+ <name name="command_input"/>
+ <desc>
+ <p>All characters needs to be valid characters on the
+ specific OS using
+ <seealso marker="file#native_name_encoding/0"><c>file:native_name_encoding()</c></seealso>
+ encoding. Note that specifically null characters (integer
+ value zero) are not allowed. However, note that not all
+ invalid characters not necessarily will cause
+ <seealso marker="#cmd/1"><c>os:cmd/1</c></seealso>
+ to fail, but may instead produce invalid results.
+ </p>
+ </desc>
+ </datatype>
+ </datatypes>
+
<funcs>
<func>
<name name="cmd" arity="1"/>
@@ -49,6 +140,15 @@
result as a string. This function is a replacement of
the previous function <c>unix:cmd/1</c>; they are equivalent on a
Unix platform.</p>
+ <warning><p>Previous implementation used to allow all characters
+ as long as they were integer values greater than or equal to zero.
+ This sometimes lead to unwanted results since null characters
+ (integer value zero) often are interpreted as string termination.
+ Current implementation still accepts null characters at the end
+ of <c><anno>Command</anno></c> even though the documentation
+ states that no null characters are allowed. This will however
+ be changed in the future so that no null characters at all will
+ be accepted.</p></warning>
<p><em>Examples:</em></p>
<code type="none">
LsOut = os:cmd("ls"), % on unix platform
@@ -152,6 +252,15 @@ DirOut = os:cmd("dir"), % on Win32 platform</code>
<p>On Unix platforms, the environment is set using UTF-8 encoding
if Unicode filename translation is in effect. On Windows, the
environment is set using wide character interfaces.</p>
+ <note>
+ <p>
+ <c><anno>VarName</anno></c> is not allowed to contain
+ an <c>$=</c> character. Previous implementations used
+ to just let the <c>$=</c> character through which
+ silently caused erroneous results. Current implementation
+ will instead throw a <c>badarg</c> exception.
+ </p>
+ </note>
</desc>
</func>
diff --git a/lib/kernel/examples/Makefile b/lib/kernel/examples/Makefile
index 26ec58f571..f86e662838 100644
--- a/lib/kernel/examples/Makefile
+++ b/lib/kernel/examples/Makefile
@@ -45,7 +45,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/kernel-$(KERNEL_VSN)/examples
# Pack and install the complete directory structure from
# here (CWD) and down, for all examples.
-EXAMPLES = uds_dist
+EXAMPLES = uds_dist gen_tcp_dist
release_spec:
$(INSTALL_DIR) "$(RELSYSDIR)"
diff --git a/lib/kernel/examples/gen_tcp_dist/Makefile b/lib/kernel/examples/gen_tcp_dist/Makefile
new file mode 100644
index 0000000000..65513a1729
--- /dev/null
+++ b/lib/kernel/examples/gen_tcp_dist/Makefile
@@ -0,0 +1,20 @@
+RM=rm -f
+CP=cp
+EBIN=ebin
+ERLC=erlc
+# Works if building in open source source tree
+KERNEL_INCLUDE=$(ERL_TOP)/lib/kernel/include
+ERLCFLAGS+= -W -I$(KERNEL_INCLUDE)
+
+MODULES=gen_tcp_dist
+
+TARGET_FILES=$(MODULES:%=$(EBIN)/%.beam)
+
+opt: $(TARGET_FILES)
+
+$(EBIN)/%.beam: src/%.erl
+ $(ERLC) $(ERLCFLAGS) -o$(EBIN) $<
+
+clean:
+ $(RM) $(TARGET_FILES)
+
diff --git a/lib/kernel/examples/gen_tcp_dist/ebin/.gitignore b/lib/kernel/examples/gen_tcp_dist/ebin/.gitignore
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/kernel/examples/gen_tcp_dist/ebin/.gitignore
diff --git a/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl
new file mode 100644
index 0000000000..98554ed805
--- /dev/null
+++ b/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl
@@ -0,0 +1,781 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(gen_tcp_dist).
+
+%%
+%% This is an example of how to plug in an arbitrary distribution
+%% carrier for Erlang using distribution processes.
+%%
+%% This example uses gen_tcp for transportation of data, but
+%% you can use whatever underlying protocol you want as long
+%% as your implementation reliably delivers data chunks to the
+%% receiving VM in the order they were sent from the sending
+%% VM.
+%%
+%% This code is a rewrite of the lib/kernel/src/inet_tcp_dist.erl
+%% distribution impementation for TCP used by default. That
+%% implementation use distribution ports instead of distribution
+%% processes and is more efficient compared to this implementation.
+%% This since this implementation more or less gets the
+%% distribution processes in between the VM and the ports without
+%% any gain specific gain.
+%%
+
+-export([listen/1, accept/1, accept_connection/5,
+ setup/5, close/1, select/1, is_node_name/1]).
+
+%% Optional
+-export([setopts/2, getopts/2]).
+
+%% internal exports
+
+-export([dist_cntrlr_setup/1, dist_cntrlr_input_setup/3,
+ dist_cntrlr_tick_handler/1]).
+
+-export([accept_loop/2,do_accept/6,do_setup/6]).
+
+-import(error_logger,[error_msg/2]).
+
+-include("net_address.hrl").
+
+-include("dist.hrl").
+-include("dist_util.hrl").
+
+%% ------------------------------------------------------------
+%% Select this protocol based on node name
+%% select(Node) => Bool
+%% ------------------------------------------------------------
+
+select(Node) ->
+ case split_node(atom_to_list(Node), $@, []) of
+ [_, Host] ->
+ case inet:getaddr(Host, inet) of
+ {ok,_} -> true;
+ _ -> false
+ end;
+ _ -> false
+ end.
+
+%% ------------------------------------------------------------
+%% Create the listen socket, i.e. the port that this erlang
+%% node is accessible through.
+%% ------------------------------------------------------------
+
+listen(Name) ->
+ case do_listen([binary, {active, false}, {packet,2}, {reuseaddr, true}]) of
+ {ok, Socket} ->
+ TcpAddress = get_tcp_address(Socket),
+ {_,Port} = TcpAddress#net_address.address,
+ ErlEpmd = net_kernel:epmd_module(),
+ case ErlEpmd:register_node(Name, Port) of
+ {ok, Creation} ->
+ {ok, {Socket, TcpAddress, Creation}};
+ Error ->
+ Error
+ end;
+ Error ->
+ Error
+ end.
+
+do_listen(Options) ->
+ {First,Last} = case application:get_env(kernel,inet_dist_listen_min) of
+ {ok,N} when is_integer(N) ->
+ case application:get_env(kernel,
+ inet_dist_listen_max) of
+ {ok,M} when is_integer(M) ->
+ {N,M};
+ _ ->
+ {N,N}
+ end;
+ _ ->
+ {0,0}
+ end,
+ do_listen(First, Last, listen_options([{backlog,128}|Options])).
+
+do_listen(First,Last,_) when First > Last ->
+ {error,eaddrinuse};
+do_listen(First,Last,Options) ->
+ case gen_tcp:listen(First, Options) of
+ {error, eaddrinuse} ->
+ do_listen(First+1,Last,Options);
+ Other ->
+ Other
+ end.
+
+listen_options(Opts0) ->
+ Opts1 =
+ case application:get_env(kernel, inet_dist_use_interface) of
+ {ok, Ip} ->
+ [{ip, Ip} | Opts0];
+ _ ->
+ Opts0
+ end,
+ case application:get_env(kernel, inet_dist_listen_options) of
+ {ok,ListenOpts} ->
+ ListenOpts ++ Opts1;
+ _ ->
+ Opts1
+ end.
+
+
+%% ------------------------------------------------------------
+%% Accepts new connection attempts from other Erlang nodes.
+%% ------------------------------------------------------------
+
+accept(Listen) ->
+ spawn_opt(?MODULE, accept_loop, [self(), Listen], [link, {priority, max}]).
+
+accept_loop(Kernel, Listen) ->
+ ?trace("~p~n",[{?MODULE, accept_loop, self()}]),
+ case gen_tcp:accept(Listen) of
+ {ok, Socket} ->
+ DistCtrl = spawn_dist_cntrlr(Socket),
+ ?trace("~p~n",[{?MODULE, accept_loop, accepted, Socket, DistCtrl, self()}]),
+ flush_controller(DistCtrl, Socket),
+ gen_tcp:controlling_process(Socket, DistCtrl),
+ flush_controller(DistCtrl, Socket),
+ Kernel ! {accept,self(),DistCtrl,inet,tcp},
+ receive
+ {Kernel, controller, Pid} ->
+ call_ctrlr(DistCtrl, {supervisor, Pid}),
+ Pid ! {self(), controller};
+ {Kernel, unsupported_protocol} ->
+ exit(unsupported_protocol)
+ end,
+ accept_loop(Kernel, Listen);
+ Error ->
+ exit(Error)
+ end.
+
+flush_controller(Pid, Socket) ->
+ receive
+ {tcp, Socket, Data} ->
+ Pid ! {tcp, Socket, Data},
+ flush_controller(Pid, Socket);
+ {tcp_closed, Socket} ->
+ Pid ! {tcp_closed, Socket},
+ flush_controller(Pid, Socket)
+ after 0 ->
+ ok
+ end.
+
+%% ------------------------------------------------------------
+%% Accepts a new connection attempt from another Erlang node.
+%% Performs the handshake with the other side.
+%% ------------------------------------------------------------
+
+accept_connection(AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) ->
+ spawn_opt(?MODULE, do_accept,
+ [self(), AcceptPid, DistCtrl, MyNode, Allowed, SetupTime],
+ [link, {priority, max}]).
+
+do_accept(Kernel, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) ->
+ ?trace("~p~n",[{?MODULE, do_accept, self(), MyNode}]),
+ receive
+ {AcceptPid, controller} ->
+ Timer = dist_util:start_timer(SetupTime),
+ case check_ip(DistCtrl) of
+ true ->
+ HSData0 = hs_data_common(DistCtrl),
+ HSData = HSData0#hs_data{kernel_pid = Kernel,
+ this_node = MyNode,
+ socket = DistCtrl,
+ timer = Timer,
+ this_flags = 0,
+ allowed = Allowed},
+ dist_util:handshake_other_started(HSData);
+ {false,IP} ->
+ error_msg("** Connection attempt from "
+ "disallowed IP ~w ** ~n", [IP]),
+ ?shutdown(no_node)
+ end
+ end.
+
+%% we may not always want the nodelay behaviour
+%% for performance reasons
+
+nodelay() ->
+ case application:get_env(kernel, dist_nodelay) of
+ undefined ->
+ {nodelay, true};
+ {ok, true} ->
+ {nodelay, true};
+ {ok, false} ->
+ {nodelay, false};
+ _ ->
+ {nodelay, true}
+ end.
+
+%% ------------------------------------------------------------
+%% Setup a new connection to another Erlang node.
+%% Performs the handshake with the other side.
+%% ------------------------------------------------------------
+
+setup(Node, Type, MyNode, LongOrShortNames,SetupTime) ->
+ spawn_opt(?MODULE, do_setup,
+ [self(), Node, Type, MyNode, LongOrShortNames, SetupTime],
+ [link, {priority, max}]).
+
+do_setup(Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
+ ?trace("~p~n",[{?MODULE, do_setup, self(), Node}]),
+ [Name, Address] = splitnode(Node, LongOrShortNames),
+ case inet:getaddr(Address, inet) of
+ {ok, Ip} ->
+ Timer = dist_util:start_timer(SetupTime),
+ ErlEpmd = net_kernel:epmd_module(),
+ case ErlEpmd:port_please(Name, Ip) of
+ {port, TcpPort, Version} ->
+ ?trace("port_please(~p) -> version ~p~n",
+ [Node,Version]),
+ dist_util:reset_timer(Timer),
+ case
+ gen_tcp:connect(
+ Ip, TcpPort,
+ connect_options([binary, {active, false}, {packet, 2}]))
+ of
+ {ok, Socket} ->
+ DistCtrl = spawn_dist_cntrlr(Socket),
+ call_ctrlr(DistCtrl, {supervisor, self()}),
+ flush_controller(DistCtrl, Socket),
+ gen_tcp:controlling_process(Socket, DistCtrl),
+ flush_controller(DistCtrl, Socket),
+ HSData0 = hs_data_common(DistCtrl),
+ HSData = HSData0#hs_data{kernel_pid = Kernel,
+ other_node = Node,
+ this_node = MyNode,
+ socket = DistCtrl,
+ timer = Timer,
+ this_flags = 0,
+ other_version = Version,
+ request_type = Type},
+ dist_util:handshake_we_started(HSData);
+ _ ->
+ %% Other Node may have closed since
+ %% port_please !
+ ?trace("other node (~p) "
+ "closed since port_please.~n",
+ [Node]),
+ ?shutdown(Node)
+ end;
+ _ ->
+ ?trace("port_please (~p) "
+ "failed.~n", [Node]),
+ ?shutdown(Node)
+ end;
+ _Other ->
+ ?trace("inet_getaddr(~p) "
+ "failed (~p).~n", [Node,_Other]),
+ ?shutdown(Node)
+ end.
+
+connect_options(Opts) ->
+ case application:get_env(kernel, inet_dist_connect_options) of
+ {ok,ConnectOpts} ->
+ ConnectOpts ++ Opts;
+ _ ->
+ Opts
+ end.
+
+%%
+%% Close a socket.
+%%
+close(Listen) ->
+ gen_tcp:close(Listen).
+
+
+%% If Node is illegal terminate the connection setup!!
+splitnode(Node, LongOrShortNames) ->
+ case split_node(atom_to_list(Node), $@, []) of
+ [Name|Tail] when Tail =/= [] ->
+ Host = lists:append(Tail),
+ case split_node(Host, $., []) of
+ [_] when LongOrShortNames =:= longnames ->
+ case inet:parse_address(Host) of
+ {ok, _} ->
+ [Name, Host];
+ _ ->
+ error_msg("** System running to use "
+ "fully qualified "
+ "hostnames **~n"
+ "** Hostname ~ts is illegal **~n",
+ [Host]),
+ ?shutdown(Node)
+ end;
+ L when length(L) > 1, LongOrShortNames =:= shortnames ->
+ error_msg("** System NOT running to use fully qualified "
+ "hostnames **~n"
+ "** Hostname ~ts is illegal **~n",
+ [Host]),
+ ?shutdown(Node);
+ _ ->
+ [Name, Host]
+ end;
+ [_] ->
+ error_msg("** Nodename ~p illegal, no '@' character **~n",
+ [Node]),
+ ?shutdown(Node);
+ _ ->
+ error_msg("** Nodename ~p illegal **~n", [Node]),
+ ?shutdown(Node)
+ end.
+
+split_node([Chr|T], Chr, Ack) -> [lists:reverse(Ack)|split_node(T, Chr, [])];
+split_node([H|T], Chr, Ack) -> split_node(T, Chr, [H|Ack]);
+split_node([], _, Ack) -> [lists:reverse(Ack)].
+
+%% ------------------------------------------------------------
+%% Fetch local information about a Socket.
+%% ------------------------------------------------------------
+get_tcp_address(Socket) ->
+ {ok, Address} = inet:sockname(Socket),
+ {ok, Host} = inet:gethostname(),
+ #net_address {
+ address = Address,
+ host = Host,
+ protocol = tcp,
+ family = inet
+ }.
+
+%% ------------------------------------------------------------
+%% Do only accept new connection attempts from nodes at our
+%% own LAN, if the check_ip environment parameter is true.
+%% ------------------------------------------------------------
+check_ip(DistCtrl) ->
+ case application:get_env(check_ip) of
+ {ok, true} ->
+ case get_ifs(DistCtrl) of
+ {ok, IFs, IP} ->
+ check_ip(IFs, IP);
+ _ ->
+ ?shutdown(no_node)
+ end;
+ _ ->
+ true
+ end.
+
+get_ifs(DistCtrl) ->
+ Socket = call_ctrlr(DistCtrl, socket),
+ case inet:peername(Socket) of
+ {ok, {IP, _}} ->
+ case inet:getif(Socket) of
+ {ok, IFs} -> {ok, IFs, IP};
+ Error -> Error
+ end;
+ Error ->
+ Error
+ end.
+
+check_ip([{OwnIP, _, Netmask}|IFs], PeerIP) ->
+ case {inet_tcp:mask(Netmask, PeerIP), inet_tcp:mask(Netmask, OwnIP)} of
+ {M, M} -> true;
+ _ -> check_ip(IFs, PeerIP)
+ end;
+check_ip([], PeerIP) ->
+ {false, PeerIP}.
+
+is_node_name(Node) when is_atom(Node) ->
+ case split_node(atom_to_list(Node), $@, []) of
+ [_, _Host] -> true;
+ _ -> false
+ end;
+is_node_name(_Node) ->
+ false.
+
+hs_data_common(DistCtrl) ->
+ TickHandler = call_ctrlr(DistCtrl, tick_handler),
+ Socket = call_ctrlr(DistCtrl, socket),
+ #hs_data{f_send = send_fun(),
+ f_recv = recv_fun(),
+ f_setopts_pre_nodeup = setopts_pre_nodeup_fun(),
+ f_setopts_post_nodeup = setopts_post_nodeup_fun(),
+ f_getll = getll_fun(),
+ f_handshake_complete = handshake_complete_fun(),
+ f_address = address_fun(),
+ mf_setopts = setopts_fun(DistCtrl, Socket),
+ mf_getopts = getopts_fun(DistCtrl, Socket),
+ mf_getstat = getstat_fun(DistCtrl, Socket),
+ mf_tick = tick_fun(DistCtrl, TickHandler)}.
+
+%%% ------------------------------------------------------------
+%%% Distribution controller processes
+%%% ------------------------------------------------------------
+
+%%
+%% There will be five parties working together when the
+%% connection is up:
+%% - The gen_tcp socket. Providing a tcp/ip connection
+%% to the other node.
+%% - The output handler. It will dispatch all outgoing
+%% traffic from the VM to the gen_tcp socket. This
+%% process is registered as distribution controller
+%% for this channel with the VM.
+%% - The input handler. It will dispatch all incoming
+%% traffic from the gen_tcp socket to the VM. This
+%% process is also the socket owner and receives
+%% incoming traffic using active-N.
+%% - The tick handler. Dispatches asynchronous tick
+%% requests to the socket. It executes on max priority
+%% since it is important to get ticks through to the
+%% other end.
+%% - The channel supervisor (provided by dist_util). It
+%% monitors traffic. Issue tick requests to the tick
+%% handler when no outgoing traffic is seen and bring
+%% the connection down if no incoming traffic is seen.
+%% This process also executes on max priority.
+%%
+%% These parties are linked togheter so should one
+%% of them fail, all of them are terminated and the
+%% connection is taken down.
+%%
+
+%% In order to avoid issues with lingering signal binaries
+%% we enable off-heap message queue data as well as fullsweep
+%% after 0. The fullsweeps will be cheap since we have more
+%% or less no live data.
+-define(DIST_CNTRL_COMMON_SPAWN_OPTS,
+ [{message_queue_data, off_heap},
+ {fullsweep_after, 0}]).
+
+tick_fun(DistCtrl, TickHandler) ->
+ fun (Ctrl) when Ctrl == DistCtrl ->
+ TickHandler ! tick
+ end.
+
+getstat_fun(DistCtrl, Socket) ->
+ fun (Ctrl) when Ctrl == DistCtrl ->
+ case inet:getstat(Socket, [recv_cnt, send_cnt, send_pend]) of
+ {ok, Stat} ->
+ split_stat(Stat,0,0,0);
+ Error ->
+ Error
+ end
+ end.
+
+split_stat([{recv_cnt, R}|Stat], _, W, P) ->
+ split_stat(Stat, R, W, P);
+split_stat([{send_cnt, W}|Stat], R, _, P) ->
+ split_stat(Stat, R, W, P);
+split_stat([{send_pend, P}|Stat], R, W, _) ->
+ split_stat(Stat, R, W, P);
+split_stat([], R, W, P) ->
+ {ok, R, W, P}.
+
+setopts_fun(DistCtrl, Socket) ->
+ fun (Ctrl, Opts) when Ctrl == DistCtrl ->
+ setopts(Socket, Opts)
+ end.
+
+getopts_fun(DistCtrl, Socket) ->
+ fun (Ctrl, Opts) when Ctrl == DistCtrl ->
+ getopts(Socket, Opts)
+ end.
+
+setopts(S, Opts) ->
+ case [Opt || {K,_}=Opt <- Opts,
+ K =:= active orelse K =:= deliver orelse K =:= packet] of
+ [] -> inet:setopts(S,Opts);
+ Opts1 -> {error, {badopts,Opts1}}
+ end.
+
+getopts(S, Opts) ->
+ inet:getopts(S, Opts).
+
+send_fun() ->
+ fun (Ctrlr, Packet) ->
+ call_ctrlr(Ctrlr, {send, Packet})
+ end.
+
+recv_fun() ->
+ fun (Ctrlr, Length, Timeout) ->
+ case call_ctrlr(Ctrlr, {recv, Length, Timeout}) of
+ {ok, Bin} when is_binary(Bin) ->
+ {ok, binary_to_list(Bin)};
+ Other ->
+ Other
+ end
+ end.
+
+getll_fun() ->
+ fun (Ctrlr) ->
+ call_ctrlr(Ctrlr, getll)
+ end.
+
+address_fun() ->
+ fun (Ctrlr, Node) ->
+ case call_ctrlr(Ctrlr, {address, Node}) of
+ {error, no_node} -> %% No '@' or more than one '@' in node name.
+ ?shutdown(no_node);
+ Res ->
+ Res
+ end
+ end.
+
+setopts_pre_nodeup_fun() ->
+ fun (Ctrlr) ->
+ call_ctrlr(Ctrlr, pre_nodeup)
+ end.
+
+setopts_post_nodeup_fun() ->
+ fun (Ctrlr) ->
+ call_ctrlr(Ctrlr, post_nodeup)
+ end.
+
+handshake_complete_fun() ->
+ fun (Ctrlr, Node, DHandle) ->
+ call_ctrlr(Ctrlr, {handshake_complete, Node, DHandle})
+ end.
+
+call_ctrlr(Ctrlr, Msg) ->
+ Ref = erlang:monitor(process, Ctrlr),
+ Ctrlr ! {Ref, self(), Msg},
+ receive
+ {Ref, Res} ->
+ erlang:demonitor(Ref, [flush]),
+ Res;
+ {'DOWN', Ref, process, Ctrlr, Reason} ->
+ exit({dist_controller_exit, Reason})
+ end.
+
+%%
+%% The tick handler process writes a tick to the
+%% socket when it receives a 'tick' message from
+%% the connection supervisor.
+%%
+%% We are not allowed to block the connection
+%% superviser when writing a tick and we also want
+%% the tick to go through even during a heavily
+%% loaded system. gen_tcp does not have a
+%% non-blocking send operation exposed in its API
+%% and we don't want to run the distribution
+%% controller under high priority. Therefore this
+%% sparate process with max prio that dispatches
+%% ticks.
+%%
+dist_cntrlr_tick_handler(Socket) ->
+ receive
+ tick ->
+ %% May block due to busy port...
+ sock_send(Socket, "");
+ _ ->
+ ok
+ end,
+ dist_cntrlr_tick_handler(Socket).
+
+spawn_dist_cntrlr(Socket) ->
+ spawn_opt(?MODULE, dist_cntrlr_setup, [Socket],
+ [{priority, max}] ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS).
+
+dist_cntrlr_setup(Socket) ->
+ TickHandler = spawn_opt(?MODULE, dist_cntrlr_tick_handler,
+ [Socket],
+ [link, {priority, max}]
+ ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS),
+ dist_cntrlr_setup_loop(Socket, TickHandler, undefined).
+
+%%
+%% During the handshake phase we loop in dist_cntrlr_setup().
+%% When the connection is up we spawn an input handler and
+%% continue as output handler.
+%%
+dist_cntrlr_setup_loop(Socket, TickHandler, Sup) ->
+ receive
+ {tcp_closed, Socket} ->
+ exit(connection_closed);
+
+ {Ref, From, {supervisor, Pid}} ->
+ Res = link(Pid),
+ From ! {Ref, Res},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Pid);
+
+ {Ref, From, tick_handler} ->
+ From ! {Ref, TickHandler},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, socket} ->
+ From ! {Ref, Socket},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, {send, Packet}} ->
+ Res = gen_tcp:send(Socket, Packet),
+ From ! {Ref, Res},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, {recv, Length, Timeout}} ->
+ Res = gen_tcp:recv(Socket, Length, Timeout),
+ From ! {Ref, Res},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, getll} ->
+ From ! {Ref, {ok, self()}},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, {address, Node}} ->
+ Res = case inet:peername(Socket) of
+ {ok, Address} ->
+ case split_node(atom_to_list(Node), $@, []) of
+ [_,Host] ->
+ #net_address{address=Address,host=Host,
+ protocol=tcp, family=inet};
+ _ ->
+ {error, no_node}
+ end
+ end,
+ From ! {Ref, Res},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, pre_nodeup} ->
+ Res = inet:setopts(Socket,
+ [{active, false},
+ {packet, 4},
+ nodelay()]),
+ From ! {Ref, Res},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, post_nodeup} ->
+ Res = inet:setopts(Socket,
+ [{active, false},
+ {packet, 4},
+ nodelay()]),
+ From ! {Ref, Res},
+ dist_cntrlr_setup_loop(Socket, TickHandler, Sup);
+
+ {Ref, From, {handshake_complete, _Node, DHandle}} ->
+ From ! {Ref, ok},
+ %% Handshake complete! Begin dispatching traffic...
+
+ %% We use separate process for dispatching input. This
+ %% is not necessary, but it enables parallel execution
+ %% of independent work loads at the same time as it
+ %% simplifies the the implementation...
+ InputHandler = spawn_opt(?MODULE, dist_cntrlr_input_setup,
+ [DHandle, Socket, Sup],
+ [link] ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS),
+
+ flush_controller(InputHandler, Socket),
+ gen_tcp:controlling_process(Socket, InputHandler),
+ flush_controller(InputHandler, Socket),
+
+ ok = erlang:dist_ctrl_input_handler(DHandle, InputHandler),
+
+ InputHandler ! DHandle,
+
+ %% From now on we execute on normal priority
+ process_flag(priority, normal),
+ erlang:dist_ctrl_get_data_notification(DHandle),
+ dist_cntrlr_output_loop(DHandle, Socket)
+ end.
+
+%% We use active 10 for good throughput while still
+%% maintaining back-pressure if the input controller
+%% isn't able to handle all incoming messages...
+-define(ACTIVE_INPUT, 10).
+
+dist_cntrlr_input_setup(DHandle, Socket, Sup) ->
+ link(Sup),
+ %% Ensure we don't try to put data before registerd
+ %% as input handler...
+ receive
+ DHandle ->
+ dist_cntrlr_input_loop(DHandle, Socket, 0)
+ end.
+
+dist_cntrlr_input_loop(DHandle, Socket, N) when N =< ?ACTIVE_INPUT/2 ->
+ inet:setopts(Socket, [{active, ?ACTIVE_INPUT - N}]),
+ dist_cntrlr_input_loop(DHandle, Socket, ?ACTIVE_INPUT);
+dist_cntrlr_input_loop(DHandle, Socket, N) ->
+ receive
+ {tcp_closed, Socket} ->
+ %% Connection to remote node terminated...
+ exit(connection_closed);
+
+ {tcp, Socket, Data} ->
+ %% Incoming data from remote node...
+ try erlang:dist_ctrl_put_data(DHandle, Data)
+ catch _ : _ -> death_row()
+ end,
+ dist_cntrlr_input_loop(DHandle, Socket, N-1);
+
+ _ ->
+ %% Ignore...
+ dist_cntrlr_input_loop(DHandle, Socket, N)
+ end.
+
+dist_cntrlr_send_data(DHandle, Socket) ->
+ case erlang:dist_ctrl_get_data(DHandle) of
+ none ->
+ erlang:dist_ctrl_get_data_notification(DHandle);
+ Data ->
+ sock_send(Socket, Data),
+ dist_cntrlr_send_data(DHandle, Socket)
+ end.
+
+
+dist_cntrlr_output_loop(DHandle, Socket) ->
+ receive
+ dist_data ->
+ %% Outgoing data from this node...
+ try dist_cntrlr_send_data(DHandle, Socket)
+ catch _ : _ -> death_row()
+ end,
+ dist_cntrlr_output_loop(DHandle, Socket);
+
+ {send, From, Ref, Data} ->
+ %% This is for testing only!
+ %%
+ %% Needed by some OTP distribution
+ %% test suites...
+ sock_send(Socket, Data),
+ From ! {Ref, ok},
+ dist_cntrlr_output_loop(DHandle, Socket);
+
+ _ ->
+ %% Drop garbage message...
+ dist_cntrlr_output_loop(DHandle, Socket)
+
+ end.
+
+sock_send(Socket, Data) ->
+ try gen_tcp:send(Socket, Data) of
+ ok -> ok;
+ {error, Reason} -> death_row({send_error, Reason})
+ catch
+ Type : Reason -> death_row({send_error, {Type, Reason}})
+ end.
+
+death_row() ->
+ death_row(connection_closed).
+
+death_row(normal) ->
+ %% We do not want to exit with normal
+ %% exit reason since it wont bring down
+ %% linked processes...
+ death_row();
+death_row(Reason) ->
+ %% When the connection is on its way down operations
+ %% begin to fail. We catch the failures and call
+ %% this function waiting for termination. We should
+ %% be terminated by one of our links to the other
+ %% involved parties that began bringing the
+ %% connection down. By waiting for termination we
+ %% avoid altering the exit reason for the connection
+ %% teardown. We however limit the wait to 5 seconds
+ %% and bring down the connection ourselves if not
+ %% terminated...
+ receive after 5000 -> exit(Reason) end.
diff --git a/lib/kernel/include/dist.hrl b/lib/kernel/include/dist.hrl
index d6bccdf474..db4a5eaebc 100644
--- a/lib/kernel/include/dist.hrl
+++ b/lib/kernel/include/dist.hrl
@@ -40,3 +40,33 @@
-define(DFLAG_UTF8_ATOMS, 16#10000).
-define(DFLAG_MAP_TAG, 16#20000).
-define(DFLAG_BIG_CREATION, 16#40000).
+-define(DFLAG_SEND_SENDER, 16#80000).
+
+%% DFLAGs that require strict ordering or:ed together...
+-define(DFLAGS_STRICT_ORDER_DELIVERY,
+ ?DFLAG_DIST_HDR_ATOM_CACHE).
+
+
+%% Also update dflag2str() in ../src/dist_util.erl
+%% when adding flags...
+
+-define(DFLAGS_ALL,
+ (?DFLAG_PUBLISHED
+ bor ?DFLAG_ATOM_CACHE
+ bor ?DFLAG_EXTENDED_REFERENCES
+ bor ?DFLAG_DIST_MONITOR
+ bor ?DFLAG_FUN_TAGS
+ bor ?DFLAG_DIST_MONITOR_NAME
+ bor ?DFLAG_HIDDEN_ATOM_CACHE
+ bor ?DFLAG_NEW_FUN_TAGS
+ bor ?DFLAG_EXTENDED_PIDS_PORTS
+ bor ?DFLAG_EXPORT_PTR_TAG
+ bor ?DFLAG_BIT_BINARIES
+ bor ?DFLAG_NEW_FLOATS
+ bor ?DFLAG_UNICODE_IO
+ bor ?DFLAG_DIST_HDR_ATOM_CACHE
+ bor ?DFLAG_SMALL_ATOM_TAGS
+ bor ?DFLAG_UTF8_ATOMS
+ bor ?DFLAG_MAP_TAG
+ bor ?DFLAG_BIG_CREATION
+ bor ?DFLAG_SEND_SENDER)).
diff --git a/lib/kernel/include/dist_util.hrl b/lib/kernel/include/dist_util.hrl
index e3d2fe0eb6..eeb0f8dd43 100644
--- a/lib/kernel/include/dist_util.hrl
+++ b/lib/kernel/include/dist_util.hrl
@@ -29,9 +29,9 @@
-endif.
-ifdef(dist_trace).
--define(trace(Fmt,Args), io:format("~p ~p:~s",[erlang:timestamp(),node(),lists:flatten(io_lib:format(Fmt, Args))])).
+-define(trace(Fmt,Args), io:format("~p ~p:~s",[erlang:convert_time_unit(erlang:monotonic_time()-erlang:system_info(start_time), native, microsecond),node(),lists:flatten(io_lib:format(Fmt, Args))])).
% Use the one below for config-file (early boot) connection tracing
-%-define(trace(Fmt,Args), erlang:display([erlang:now(),node(),lists:flatten(io_lib:format(Fmt, Args))])).
+%-define(trace(Fmt,Args), erlang:display([erlang:convert_time_unit(erlang:monotonic_time()-erlang:system_info(start_time), native, microsecond),node(),lists:flatten(io_lib:format(Fmt, Args))])).
-define(trace_factor,8).
-else.
-define(trace(Fmt,Args), ok).
@@ -78,7 +78,13 @@
%% New in kernel-5.1 (OTP 19.1):
mf_setopts, %% netkernel:setopts on active connection
- mf_getopts %% netkernel:getopts on active connection
+ mf_getopts, %% netkernel:getopts on active connection
+
+ %% New in kernel-6.0 (OTP 21.0)
+ f_handshake_complete, %% Notify handshake complete
+ add_flags, %% dflags to add
+ reject_flags, %% dflags not to use (not all can be rejected)
+ require_flags %% dflags that are required
}).
diff --git a/lib/kernel/src/dist_util.erl b/lib/kernel/src/dist_util.erl
index b3507e5d13..08bd5946cd 100644
--- a/lib/kernel/src/dist_util.erl
+++ b/lib/kernel/src/dist_util.erl
@@ -74,6 +74,48 @@
ticked = 0
}).
+dflag2str(?DFLAG_PUBLISHED) ->
+ "PUBLISHED";
+dflag2str(?DFLAG_ATOM_CACHE) ->
+ "ATOM_CACHE";
+dflag2str(?DFLAG_EXTENDED_REFERENCES) ->
+ "EXTENDED_REFERENCES";
+dflag2str(?DFLAG_DIST_MONITOR) ->
+ "DIST_MONITOR";
+dflag2str(?DFLAG_FUN_TAGS) ->
+ "FUN_TAGS";
+dflag2str(?DFLAG_DIST_MONITOR_NAME) ->
+ "DIST_MONITOR_NAME";
+dflag2str(?DFLAG_HIDDEN_ATOM_CACHE) ->
+ "HIDDEN_ATOM_CACHE";
+dflag2str(?DFLAG_NEW_FUN_TAGS) ->
+ "NEW_FUN_TAGS";
+dflag2str(?DFLAG_EXTENDED_PIDS_PORTS) ->
+ "EXTENDED_PIDS_PORTS";
+dflag2str(?DFLAG_EXPORT_PTR_TAG) ->
+ "EXPORT_PTR_TAG";
+dflag2str(?DFLAG_BIT_BINARIES) ->
+ "BIT_BINARIES";
+dflag2str(?DFLAG_NEW_FLOATS) ->
+ "NEW_FLOATS";
+dflag2str(?DFLAG_UNICODE_IO) ->
+ "UNICODE_IO";
+dflag2str(?DFLAG_DIST_HDR_ATOM_CACHE) ->
+ "DIST_HDR_ATOM_CACHE";
+dflag2str(?DFLAG_SMALL_ATOM_TAGS) ->
+ "SMALL_ATOM_TAGS";
+dflag2str(?DFLAG_UTF8_ATOMS) ->
+ "UTF8_ATOMS";
+dflag2str(?DFLAG_MAP_TAG) ->
+ "MAP_TAG";
+dflag2str(?DFLAG_BIG_CREATION) ->
+ "BIG_CREATION";
+dflag2str(?DFLAG_SEND_SENDER) ->
+ "SEND_SENDER";
+dflag2str(_) ->
+ "UNKNOWN".
+
+
remove_flag(Flag, Flags) ->
case Flags band Flag of
0 ->
@@ -82,13 +124,13 @@ remove_flag(Flag, Flags) ->
Flags - Flag
end.
-adjust_flags(ThisFlags, OtherFlags) ->
+adjust_flags(ThisFlags, OtherFlags, RejectFlags) ->
case (?DFLAG_PUBLISHED band ThisFlags) band OtherFlags of
0 ->
{remove_flag(?DFLAG_PUBLISHED, ThisFlags),
remove_flag(?DFLAG_PUBLISHED, OtherFlags)};
_ ->
- {ThisFlags, OtherFlags}
+ {ThisFlags, OtherFlags band (bnot RejectFlags)}
end.
publish_flag(hidden, _) ->
@@ -101,36 +143,71 @@ publish_flag(_, OtherNode) ->
0
end.
-make_this_flags(RequestType, OtherNode) ->
- publish_flag(RequestType, OtherNode) bor
- %% The parenthesis below makes the compiler generate better code.
- (?DFLAG_EXPORT_PTR_TAG bor
- ?DFLAG_EXTENDED_PIDS_PORTS bor
- ?DFLAG_EXTENDED_REFERENCES bor
- ?DFLAG_DIST_MONITOR bor
- ?DFLAG_FUN_TAGS bor
- ?DFLAG_DIST_MONITOR_NAME bor
- ?DFLAG_HIDDEN_ATOM_CACHE bor
- ?DFLAG_NEW_FUN_TAGS bor
- ?DFLAG_BIT_BINARIES bor
- ?DFLAG_NEW_FLOATS bor
- ?DFLAG_UNICODE_IO bor
- ?DFLAG_DIST_HDR_ATOM_CACHE bor
- ?DFLAG_SMALL_ATOM_TAGS bor
- ?DFLAG_UTF8_ATOMS bor
- ?DFLAG_MAP_TAG bor
- ?DFLAG_BIG_CREATION).
-
-handshake_other_started(#hs_data{request_type=ReqType}=HSData0) ->
+-define(DFLAGS_REMOVABLE,
+ (?DFLAG_DIST_HDR_ATOM_CACHE
+ bor ?DFLAG_HIDDEN_ATOM_CACHE
+ bor ?DFLAG_ATOM_CACHE)).
+
+-define(DFLAGS_ADDABLE,
+ (?DFLAGS_ALL
+ band (bnot (?DFLAG_PUBLISHED
+ bor ?DFLAG_HIDDEN_ATOM_CACHE
+ bor ?DFLAG_ATOM_CACHE)))).
+
+-define(DFLAGS_THIS_DEFAULT,
+ (?DFLAG_EXPORT_PTR_TAG
+ bor ?DFLAG_EXTENDED_PIDS_PORTS
+ bor ?DFLAG_EXTENDED_REFERENCES
+ bor ?DFLAG_DIST_MONITOR
+ bor ?DFLAG_FUN_TAGS
+ bor ?DFLAG_DIST_MONITOR_NAME
+ bor ?DFLAG_NEW_FUN_TAGS
+ bor ?DFLAG_BIT_BINARIES
+ bor ?DFLAG_NEW_FLOATS
+ bor ?DFLAG_UNICODE_IO
+ bor ?DFLAG_DIST_HDR_ATOM_CACHE
+ bor ?DFLAG_SMALL_ATOM_TAGS
+ bor ?DFLAG_UTF8_ATOMS
+ bor ?DFLAG_MAP_TAG
+ bor ?DFLAG_BIG_CREATION
+ bor ?DFLAG_SEND_SENDER)).
+
+make_this_flags(RequestType, AddFlags, RemoveFlags, OtherNode) ->
+ case RemoveFlags band (bnot ?DFLAGS_REMOVABLE) of
+ 0 -> ok;
+ Rerror -> exit({"Rejecting non rejectable flags", Rerror})
+ end,
+ case AddFlags band (bnot ?DFLAGS_ADDABLE) of
+ 0 -> ok;
+ Aerror -> exit({"Adding non addable flags", Aerror})
+ end,
+ Flgs0 = ?DFLAGS_THIS_DEFAULT,
+ Flgs1 = Flgs0 bor publish_flag(RequestType, OtherNode),
+ Flgs2 = Flgs1 bor AddFlags,
+ Flgs3 = Flgs2 band (bnot (?DFLAG_HIDDEN_ATOM_CACHE
+ bor ?DFLAG_ATOM_CACHE)),
+ Flgs3 band (bnot RemoveFlags).
+
+handshake_other_started(#hs_data{request_type=ReqType,
+ add_flags=AddFlgs0,
+ reject_flags=RejFlgs0,
+ require_flags=ReqFlgs0}=HSData0) ->
+ AddFlgs = convert_flags(AddFlgs0),
+ RejFlgs = convert_flags(RejFlgs0),
+ ReqFlgs = convert_flags(ReqFlgs0),
{PreOtherFlags,Node,Version} = recv_name(HSData0),
- PreThisFlags = make_this_flags(ReqType, Node),
+ PreThisFlags = make_this_flags(ReqType, AddFlgs, RejFlgs, Node),
{ThisFlags, OtherFlags} = adjust_flags(PreThisFlags,
- PreOtherFlags),
+ PreOtherFlags,
+ RejFlgs),
HSData = HSData0#hs_data{this_flags=ThisFlags,
other_flags=OtherFlags,
other_version=Version,
other_node=Node,
- other_started=true},
+ other_started=true,
+ add_flags=AddFlgs,
+ reject_flags=RejFlgs,
+ require_flags=ReqFlgs},
check_dflags(HSData),
is_allowed(HSData),
?debug({"MD5 connection from ~p (V~p)~n",
@@ -165,23 +242,18 @@ is_allowed(#hs_data{other_node = Node,
end.
%%
-%% Check that both nodes can handle the same types of extended
-%% node containers. If they can not, abort the connection.
+%% Check mandatory flags...
%%
check_dflags(#hs_data{other_node = Node,
other_flags = OtherFlags,
- other_started = OtherStarted} = HSData) ->
-
- Mandatory = [{?DFLAG_EXTENDED_REFERENCES, "EXTENDED_REFERENCES"},
- {?DFLAG_EXTENDED_PIDS_PORTS, "EXTENDED_PIDS_PORTS"},
- {?DFLAG_UTF8_ATOMS, "UTF8_ATOMS"}],
- Missing = lists:filtermap(fun({Bit, Str}) ->
- case Bit band OtherFlags of
- Bit -> false;
- 0 -> {true, Str}
- end
- end,
- Mandatory),
+ other_started = OtherStarted,
+ require_flags = RequiredFlags} = HSData) ->
+ Mandatory = ((?DFLAG_EXTENDED_REFERENCES
+ bor ?DFLAG_EXTENDED_PIDS_PORTS
+ bor ?DFLAG_UTF8_ATOMS)
+ bor RequiredFlags),
+ Missing = check_mandatory(0, ?DFLAGS_ALL, Mandatory,
+ OtherFlags, []),
case Missing of
[] ->
ok;
@@ -201,6 +273,22 @@ check_dflags(#hs_data{other_node = Node,
?shutdown2(Node, {check_dflags_failed, Missing})
end.
+check_mandatory(_Bit, 0, _Mandatory, _OtherFlags, Missing) ->
+ Missing;
+check_mandatory(Bit, Left, Mandatory, OtherFlags, Missing) ->
+ DFlag = (1 bsl Bit),
+ NewLeft = Left band (bnot DFlag),
+ NewMissing = case {DFlag band Mandatory,
+ DFlag band OtherFlags} of
+ {DFlag, 0} ->
+ %% Mandatory and missing...
+ [dflag2str(DFlag) | Missing];
+ _ ->
+ %% Not mandatory or present...
+ Missing
+ end,
+ check_mandatory(Bit+1, NewLeft, Mandatory, OtherFlags, NewMissing).
+
%% No nodedown will be sent if we fail before this process has
%% succeeded to mark the node as pending.
@@ -314,13 +402,24 @@ flush_down() ->
end.
handshake_we_started(#hs_data{request_type=ReqType,
- other_node=Node}=PreHSData) ->
- PreThisFlags = make_this_flags(ReqType, Node),
- HSData = PreHSData#hs_data{this_flags=PreThisFlags},
+ other_node=Node,
+ add_flags=AddFlgs0,
+ reject_flags=RejFlgs0,
+ require_flags=ReqFlgs0}=PreHSData) ->
+ AddFlgs = convert_flags(AddFlgs0),
+ RejFlgs = convert_flags(RejFlgs0),
+ ReqFlgs = convert_flags(ReqFlgs0),
+ PreThisFlags = make_this_flags(ReqType, AddFlgs, RejFlgs, Node),
+ HSData = PreHSData#hs_data{this_flags = PreThisFlags,
+ add_flags = AddFlgs,
+ reject_flags = RejFlgs,
+ require_flags = ReqFlgs},
send_name(HSData),
recv_status(HSData),
{PreOtherFlags,ChallengeA} = recv_challenge(HSData),
- {ThisFlags,OtherFlags} = adjust_flags(PreThisFlags, PreOtherFlags),
+ {ThisFlags,OtherFlags} = adjust_flags(PreThisFlags,
+ PreOtherFlags,
+ RejFlgs),
NewHSData = HSData#hs_data{this_flags = ThisFlags,
other_flags = OtherFlags,
other_started = false},
@@ -336,15 +435,16 @@ handshake_we_started(#hs_data{request_type=ReqType,
handshake_we_started(OldHsData) when element(1,OldHsData) =:= hs_data ->
handshake_we_started(convert_old_hsdata(OldHsData)).
-convert_old_hsdata({hs_data, KP, ON, TN, S, T, TF, A, OV, OF, OS, FS, FR,
- FS_PRE, FS_POST, FG, FA, MFT, MFG, RT}) ->
- #hs_data{
- kernel_pid = KP, other_node = ON, this_node = TN, socket = S, timer = T,
- this_flags = TF, allowed = A, other_version = OV, other_flags = OF,
- other_started = OS, f_send = FS, f_recv = FR, f_setopts_pre_nodeup = FS_PRE,
- f_setopts_post_nodeup = FS_POST, f_getll = FG, f_address = FA,
- mf_tick = MFT, mf_getstat = MFG, request_type = RT}.
+convert_old_hsdata(OldHsData) ->
+ OHSDL = tuple_to_list(OldHsData),
+ NoMissing = tuple_size(#hs_data{}) - tuple_size(OldHsData),
+ true = NoMissing > 0,
+ list_to_tuple(OHSDL ++ lists:duplicate(NoMissing, undefined)).
+convert_flags(Flags) when is_integer(Flags) ->
+ Flags;
+convert_flags(_Undefined) ->
+ 0.
%% --------------------------------------------------------------
%% The connection has been established.
@@ -359,15 +459,20 @@ connection(#hs_data{other_node = Node,
PType = publish_type(HSData#hs_data.other_flags),
case FPreNodeup(Socket) of
ok ->
- do_setnode(HSData), % Succeeds or exits the process.
+ DHandle = do_setnode(HSData), % Succeeds or exits the process.
Address = FAddress(Socket,Node),
mark_nodeup(HSData,Address),
case FPostNodeup(Socket) of
ok ->
+ case HSData#hs_data.f_handshake_complete of
+ undefined -> ok;
+ HsComplete -> HsComplete(Socket, Node, DHandle)
+ end,
con_loop({HSData#hs_data.kernel_pid,
Node,
Socket,
PType,
+ DHandle,
HSData#hs_data.mf_tick,
HSData#hs_data.mf_getstat,
HSData#hs_data.mf_setopts,
@@ -425,18 +530,16 @@ do_setnode(#hs_data{other_node = Node, socket = Socket,
[Node, Port, {publish_type(Flags),
'(', Flags, ')',
Version}]),
- case (catch
- erlang:setnode(Node, Port,
- {Flags, Version, '', ''})) of
- {'EXIT', {system_limit, _}} ->
+ try
+ erlang:setnode(Node, Port, {Flags, Version, '', ''})
+ catch
+ error:system_limit ->
error_msg("** Distribution system limit reached, "
"no table space left for node ~w ** ~n",
[Node]),
?shutdown(Node);
- {'EXIT', Other} ->
- exit(Other);
- _Else ->
- ok
+ error:Other ->
+ exit({Other, erlang:get_stacktrace()})
end;
_ ->
error_msg("** Distribution connection error, "
@@ -468,7 +571,13 @@ mark_nodeup(#hs_data{kernel_pid = Kernel,
?shutdown(Node)
end.
-con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=ConData,
+getstat(DHandle, _Socket, undefined) ->
+ erlang:dist_get_stat(DHandle);
+getstat(_DHandle, Socket, MFGetstat) ->
+ MFGetstat(Socket).
+
+con_loop({Kernel, Node, Socket, Type, DHandle, MFTick, MFGetstat,
+ MFSetOpts, MFGetOpts}=ConData,
Tick) ->
receive
{tcp_closed, Socket} ->
@@ -476,7 +585,7 @@ con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=C
{Kernel, disconnect} ->
?shutdown2(Node, disconnected);
{Kernel, aux_tick} ->
- case MFGetstat(Socket) of
+ case getstat(DHandle, Socket, MFGetstat) of
{ok, _, _, PendWrite} ->
send_tick(Socket, PendWrite, MFTick);
_ ->
@@ -484,7 +593,7 @@ con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=C
end,
con_loop(ConData, Tick);
{Kernel, tick} ->
- case send_tick(Socket, Tick, Type,
+ case send_tick(DHandle, Socket, Tick, Type,
MFTick, MFGetstat) of
{ok, NewTick} ->
con_loop(ConData, NewTick);
@@ -497,7 +606,7 @@ con_loop({Kernel, Node, Socket, Type, MFTick, MFGetstat, MFSetOpts, MFGetOpts}=C
?shutdown2(Node, send_net_tick_failed)
end;
{From, get_status} ->
- case MFGetstat(Socket) of
+ case getstat(DHandle, Socket, MFGetstat) of
{ok, Read, Write, _} ->
From ! {self(), get_status, {ok, Read, Write}},
con_loop(ConData, Tick);
@@ -735,14 +844,14 @@ send_status(#hs_data{socket = Socket, other_node = Node,
%% we haven't read anything as a hidden node only ticks when it receives
%% a TICK !!
-send_tick(Socket, Tick, Type, MFTick, MFGetstat) ->
+send_tick(DHandle, Socket, Tick, Type, MFTick, MFGetstat) ->
#tick{tick = T0,
read = Read,
write = Write,
ticked = Ticked} = Tick,
T = T0 + 1,
T1 = T rem 4,
- case MFGetstat(Socket) of
+ case getstat(DHandle, Socket, MFGetstat) of
{ok, Read, _, _} when Ticked =:= T ->
{error, not_responding};
{ok, Read, W, Pend} when Type =:= hidden ->
@@ -771,11 +880,10 @@ send_tick(Socket, Tick, Type, MFTick, MFGetstat) ->
Error
end.
-send_tick(Socket, 0, MFTick) ->
- MFTick(Socket);
-send_tick(_, _Pend, _) ->
- %% Dont send tick if pending write.
- ok.
+send_tick(_, Pend, _) when Pend /= false, Pend /= 0 ->
+ ok; %% Dont send tick if pending write.
+send_tick(Socket, _Pend, MFTick) ->
+ MFTick(Socket).
%% ------------------------------------------------------------
%% Connection setup timeout timer.
diff --git a/lib/kernel/src/erl_boot_server.erl b/lib/kernel/src/erl_boot_server.erl
index ac81cc9689..2a38266579 100644
--- a/lib/kernel/src/erl_boot_server.erl
+++ b/lib/kernel/src/erl_boot_server.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -253,9 +253,9 @@ handle_info({udp, U, IP, Port, Data}, S0) ->
"~w is not a valid address ** ~n", [IP]),
{noreply,S0};
{true,_,_} ->
- case catch string:substr(Data, 1, length(?EBOOT_REQUEST)) of
+ case catch string:slice(Data, 0, length(?EBOOT_REQUEST)) of
?EBOOT_REQUEST ->
- Vsn = string:substr(Data, length(?EBOOT_REQUEST)+1, length(Data)),
+ Vsn = string:slice(Data, length(?EBOOT_REQUEST), length(Data)),
error_logger:error_msg("** Illegal boot server connection attempt: "
"client version is ~s ** ~n", [Vsn]);
_ ->
diff --git a/lib/kernel/src/erl_epmd.erl b/lib/kernel/src/erl_epmd.erl
index 7bc9e2ede3..f96bc88913 100644
--- a/lib/kernel/src/erl_epmd.erl
+++ b/lib/kernel/src/erl_epmd.erl
@@ -79,7 +79,13 @@ port_please(Node, EpmdAddr, Timeout) ->
port_please1(Node,HostName, Timeout) ->
- case inet:gethostbyname(HostName, inet, Timeout) of
+ Family = case inet_db:res_option(inet6) of
+ true ->
+ inet6;
+ false ->
+ inet
+ end,
+ case inet:gethostbyname(HostName, Family, Timeout) of
{ok,{hostent, _Name, _ , _Af, _Size, [EpmdAddr | _]}} ->
get_port(Node, EpmdAddr, Timeout);
Else ->
diff --git a/lib/kernel/src/erl_reply.erl b/lib/kernel/src/erl_reply.erl
index e1e046cbb4..e1c4ffe839 100644
--- a/lib/kernel/src/erl_reply.erl
+++ b/lib/kernel/src/erl_reply.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ reply(_) ->
%% convert ip number to tuple
ip_string_to_tuple(Ip) ->
- [Ip1,Ip2,Ip3,Ip4] = string:tokens(Ip,"."),
+ [Ip1,Ip2,Ip3,Ip4] = string:lexemes(Ip,"."),
{list_to_integer(Ip1),
list_to_integer(Ip2),
list_to_integer(Ip3),
diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl
index 480db6814e..2887014c1c 100644
--- a/lib/kernel/src/erts_debug.erl
+++ b/lib/kernel/src/erts_debug.erl
@@ -378,16 +378,11 @@ df(Mod, Func, Arity) when is_atom(Mod), is_atom(Func) ->
catch _:_ -> {undef,Mod}
end.
-dff(File, Fs) when is_pid(File), is_list(Fs) ->
- lists:foreach(fun(Mfa) ->
- disassemble_function(File, Mfa),
- io:nl(File)
- end, Fs);
-dff(Name, Fs) when is_list(Name) ->
- case file:open(Name, [write]) of
+dff(Name, Fs) ->
+ case file:open(Name, [write,raw,delayed_write]) of
{ok,F} ->
try
- dff(F, Fs)
+ dff_1(F, Fs)
after
_ = file:close(F)
end;
@@ -395,12 +390,18 @@ dff(Name, Fs) when is_list(Name) ->
{error,{badopen,Reason}}
end.
+dff_1(File, Fs) ->
+ lists:foreach(fun(Mfa) ->
+ disassemble_function(File, Mfa),
+ file:write(File, "\n")
+ end, Fs).
+
disassemble_function(File, {_,_,_}=MFA) ->
cont_dis(File, erts_debug:disassemble(MFA), MFA).
cont_dis(_, false, _) -> ok;
cont_dis(File, {Addr,Str,MFA}, MFA) ->
- io:put_chars(File, binary_to_list(Str)),
+ ok = file:write(File, Str),
cont_dis(File, erts_debug:disassemble(Addr), MFA);
cont_dis(_, {_,_,_}, _) -> ok.
diff --git a/lib/kernel/src/group.erl b/lib/kernel/src/group.erl
index bf785959ff..e1198d2587 100644
--- a/lib/kernel/src/group.erl
+++ b/lib/kernel/src/group.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -261,11 +261,11 @@ io_request({put_chars,latin1,M,F,As}, Drv, From, Buf) ->
end;
io_request({get_chars,Encoding,Prompt,N}, Drv, _From, Buf) ->
- get_chars(Prompt, io_lib, collect_chars, N, Drv, Buf, Encoding);
+ get_chars_n(Prompt, io_lib, collect_chars, N, Drv, Buf, Encoding);
io_request({get_line,Encoding,Prompt}, Drv, _From, Buf) ->
- get_chars(Prompt, io_lib, collect_line, [], Drv, Buf, Encoding);
+ get_chars_line(Prompt, io_lib, collect_line, [], Drv, Buf, Encoding);
io_request({get_until,Encoding, Prompt,M,F,As}, Drv, _From, Buf) ->
- get_chars(Prompt, io_lib, get_until, {M,F,As}, Drv, Buf, Encoding);
+ get_chars_line(Prompt, io_lib, get_until, {M,F,As}, Drv, Buf, Encoding);
io_request({get_password,_Encoding},Drv,_From,Buf) ->
get_password_chars(Drv, Buf);
io_request({setopts,Opts}, Drv, _From, Buf) when is_list(Opts) ->
@@ -434,7 +434,7 @@ getopts(Drv,Buf) ->
{ok,[Exp,Echo,Bin,Uni],Buf}.
-%% get_chars(Prompt, Module, Function, XtraArgument, Drv, Buffer)
+%% get_chars_*(Prompt, Module, Function, XtraArgument, Drv, Buffer)
%% Gets characters from the input Drv until as the applied function
%% returns {stop,Result,Rest}. Does not block output until input has been
%% received.
@@ -452,12 +452,21 @@ get_password_chars(Drv,Buf) ->
{exit, terminated}
end.
-get_chars(Prompt, M, F, Xa, Drv, Buf, Encoding) ->
+get_chars_n(Prompt, M, F, Xa, Drv, Buf, Encoding) ->
+ Pbs = prompt_bytes(Prompt, Encoding),
+ case get(echo) of
+ true ->
+ get_chars_loop(Pbs, M, F, Xa, Drv, Buf, start, Encoding);
+ false ->
+ get_chars_n_loop(Pbs, M, F, Xa, Drv, Buf, start, Encoding)
+ end.
+
+get_chars_line(Prompt, M, F, Xa, Drv, Buf, Encoding) ->
Pbs = prompt_bytes(Prompt, Encoding),
get_chars_loop(Pbs, M, F, Xa, Drv, Buf, start, Encoding).
get_chars_loop(Pbs, M, F, Xa, Drv, Buf0, State, Encoding) ->
- Result = case get(echo) of
+ Result = case get(echo) of
true ->
get_line(Buf0, Pbs, Drv, Encoding);
false ->
@@ -466,8 +475,8 @@ get_chars_loop(Pbs, M, F, Xa, Drv, Buf0, State, Encoding) ->
get_line_echo_off(Buf0, Pbs, Drv)
end,
case Result of
- {done,Line,Buf1} ->
- get_chars_apply(Pbs, M, F, Xa, Drv, Buf1, State, Line, Encoding);
+ {done,Line,Buf} ->
+ get_chars_apply(Pbs, M, F, Xa, Drv, Buf, State, Line, Encoding);
interrupted ->
{error,{error,interrupted},[]};
terminated ->
@@ -476,12 +485,29 @@ get_chars_loop(Pbs, M, F, Xa, Drv, Buf0, State, Encoding) ->
get_chars_apply(Pbs, M, F, Xa, Drv, Buf, State0, Line, Encoding) ->
case catch M:F(State0, cast(Line,get(read_mode), Encoding), Encoding, Xa) of
- {stop,Result,Rest} ->
- {ok,Result,append(Rest, Buf, Encoding)};
- {'EXIT',_} ->
- {error,{error,err_func(M, F, Xa)},[]};
- State1 ->
- get_chars_loop(Pbs, M, F, Xa, Drv, Buf, State1, Encoding)
+ {stop,Result,Rest} ->
+ {ok,Result,append(Rest, Buf, Encoding)};
+ {'EXIT',_} ->
+ {error,{error,err_func(M, F, Xa)},[]};
+ State1 ->
+ get_chars_loop(Pbs, M, F, Xa, Drv, Buf, State1, Encoding)
+ end.
+
+get_chars_n_loop(Pbs, M, F, Xa, Drv, Buf0, State, Encoding) ->
+ try M:F(State, cast(Buf0, get(read_mode), Encoding), Encoding, Xa) of
+ {stop,Result,Rest} ->
+ {ok, Result, Rest};
+ State1 ->
+ case get_chars_echo_off(Pbs, Drv) of
+ interrupted ->
+ {error,{error,interrupted},[]};
+ terminated ->
+ {exit,terminated};
+ Buf ->
+ get_chars_n_loop(Pbs, M, F, Xa, Drv, Buf, State1, Encoding)
+ end
+ catch _:_ ->
+ {error,{error,err_func(M, F, Xa)},[]}
end.
%% Convert error code to make it look as before
@@ -684,6 +710,29 @@ get_line_echo_off1({Chars,[]}, Drv) ->
get_line_echo_off1({Chars,Rest}, _Drv) ->
{done,lists:reverse(Chars),case Rest of done -> []; _ -> Rest end}.
+get_chars_echo_off(Pbs, Drv) ->
+ send_drv_reqs(Drv, [{put_chars, unicode,Pbs}]),
+ get_chars_echo_off1(Drv).
+
+get_chars_echo_off1(Drv) ->
+ receive
+ {Drv, {data, Cs}} ->
+ Cs;
+ {Drv, eof} ->
+ eof;
+ {io_request,From,ReplyAs,Req} when is_pid(From) ->
+ io_request(Req, From, ReplyAs, Drv, []),
+ get_chars_echo_off1(Drv);
+ {reply,{{From,ReplyAs},Reply}} when From =/= undefined ->
+ %% We take care of replies from puts here as well
+ io_reply(From, ReplyAs, Reply),
+ get_chars_echo_off1(Drv);
+ {'EXIT',Drv,interrupt} ->
+ interrupted;
+ {'EXIT',Drv,_} ->
+ terminated
+ end.
+
%% We support line editing for the ICANON mode except the following
%% line editing characters, which already has another meaning in
%% echo-on mode (See Advanced Programming in the Unix Environment, 2nd ed,
@@ -793,9 +842,9 @@ search_up_stack(Stack, Substr) ->
case up_stack(Stack) of
{none,NewStack} -> {none,NewStack};
{L, NewStack} ->
- case string:str(L, Substr) of
- 0 -> search_up_stack(NewStack, Substr);
- _ -> {string:strip(L,right,$\n), NewStack}
+ case string:find(L, Substr) of
+ nomatch -> search_up_stack(NewStack, Substr);
+ _ -> {string:trim(L, trailing, "$\n"), NewStack}
end
end.
@@ -803,9 +852,9 @@ search_down_stack(Stack, Substr) ->
case down_stack(Stack) of
{none,NewStack} -> {none,NewStack};
{L, NewStack} ->
- case string:str(L, Substr) of
- 0 -> search_down_stack(NewStack, Substr);
- _ -> {string:strip(L,right,$\n), NewStack}
+ case string:find(L, Substr) of
+ nomatch -> search_down_stack(NewStack, Substr);
+ _ -> {string:trim(L, trailing, "$\n"), NewStack}
end
end.
diff --git a/lib/kernel/src/inet_config.erl b/lib/kernel/src/inet_config.erl
index 4bbc520449..9f76360b8b 100644
--- a/lib/kernel/src/inet_config.erl
+++ b/lib/kernel/src/inet_config.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -369,7 +369,7 @@ win32_load1(Reg,Type,HFileKey) ->
end.
win32_split_line(Line,nt) -> inet_parse:split_line(Line);
-win32_split_line(Line,windows) -> string:tokens(Line, ",").
+win32_split_line(Line,windows) -> string:lexemes(Line, ",").
win32_get_strings(Reg, Names) ->
win32_get_strings(Reg, Names, []).
diff --git a/lib/kernel/src/inet_dns.erl b/lib/kernel/src/inet_dns.erl
index d5f982cc51..f1f58bc872 100644
--- a/lib/kernel/src/inet_dns.erl
+++ b/lib/kernel/src/inet_dns.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1997-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -29,7 +29,7 @@
-export([decode/1, encode/1]).
--import(lists, [reverse/1, reverse/2, nthtail/2]).
+-import(lists, [reverse/1]).
-include("inet_int.hrl").
-include("inet_dns.hrl").
@@ -473,7 +473,7 @@ decode_data(<<Order:16,Preference:16,Data0/binary>>, _, ?S_NAPTR, Buffer) ->
{Data2,Services} = decode_string(Data1),
{Data,Regexp} = decode_characters(Data2, utf8),
Replacement = decode_domain(Data, Buffer),
- {Order,Preference,string:to_lower(Flags),string:to_lower(Services),
+ {Order,Preference,string:lowercase(Flags),string:lowercase(Services),
Regexp,Replacement};
%% ?S_OPT falls through to default
decode_data(Data, _, ?S_TXT, _) ->
diff --git a/lib/kernel/src/inet_parse.erl b/lib/kernel/src/inet_parse.erl
index 29804dc50b..e9685c6554 100644
--- a/lib/kernel/src/inet_parse.erl
+++ b/lib/kernel/src/inet_parse.erl
@@ -95,7 +95,7 @@ hosts(Fname,File) ->
%% interface with a %if suffix. These kind of
%% addresses maybe need to be gracefully handled
%% throughout inet* and inet_drv.
- case string:tokens(Address, "%") of
+ case string:lexemes(Address, "%") of
[Addr,_] ->
{ok,_} = address(Addr),
skip;
@@ -407,7 +407,7 @@ is_dom1([C | Cs]) when C >= $a, C =< $z -> is_dom_ldh(Cs);
is_dom1([C | Cs]) when C >= $A, C =< $Z -> is_dom_ldh(Cs);
is_dom1([C | Cs]) when C >= $0, C =< $9 ->
case is_dom_ldh(Cs) of
- true -> is_dom2(string:tokens([C | Cs],"."));
+ true -> is_dom2(string:lexemes([C | Cs],"."));
false -> false
end;
is_dom1(_) -> false.
diff --git a/lib/kernel/src/inet_res.erl b/lib/kernel/src/inet_res.erl
index 90e49ddfdf..49aa5f8bda 100644
--- a/lib/kernel/src/inet_res.erl
+++ b/lib/kernel/src/inet_res.erl
@@ -859,15 +859,17 @@ query_ns(S0, Id, Buffer, IP, Port, Timer, Retry, I,
{ok,S} ->
Timeout =
inet:timeout( (Tm * (1 bsl I)) div Retry, Timer),
- {S,
case query_udp(
S, Id, Buffer, IP, Port, Timeout, Verbose) of
{ok,#dns_rec{header=H}} when H#dns_header.tc ->
TcpTimeout = inet:timeout(Tm*5, Timer),
- query_tcp(
- TcpTimeout, Id, Buffer, IP, Port, Verbose);
- Reply -> Reply
- end};
+ {S, query_tcp(
+ TcpTimeout, Id, Buffer, IP, Port, Verbose)};
+ {error, econnrefused} = Err ->
+ ok = udp_close(S),
+ {#sock{}, Err};
+ Reply -> {S, Reply}
+ end;
Error ->
{S0,Error}
end
diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src
index 2a88cc7e26..080b11fc4d 100644
--- a/lib/kernel/src/kernel.app.src
+++ b/lib/kernel/src/kernel.app.src
@@ -120,6 +120,6 @@
{applications, []},
{env, [{error_logger, tty}]},
{mod, {kernel, []}},
- {runtime_dependencies, ["erts-9.1", "stdlib-3.4", "sasl-3.0"]}
+ {runtime_dependencies, ["erts-10.0", "stdlib-3.5", "sasl-3.0"]}
]
}.
diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src
index f1ef70a373..fc5417597f 100644
--- a/lib/kernel/src/kernel.appup.src
+++ b/lib/kernel/src/kernel.appup.src
@@ -18,7 +18,7 @@
%% %CopyrightEnd%
{"%VSN%",
%% Up from - max one major revision back
- [{<<"5\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-19.*, OTP-20.0
+ [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.*
%% Down to - max one major revision back
- [{<<"5\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-19.*, OTP-20.0
+ [{<<"5\\.3(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.*
}.
diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl
index 7da89dd7cb..f36b4f1e6a 100644
--- a/lib/kernel/src/net_kernel.erl
+++ b/lib/kernel/src/net_kernel.erl
@@ -423,8 +423,8 @@ handle_call({connect, Type, Node}, From, State) ->
{ok, SetupPid} ->
Owners = [{SetupPid, Node} | State#state.conn_owners],
{noreply,State#state{conn_owners=Owners}};
- _ ->
- ?connect_failure(Node, {setup_call, failed}),
+ _Error ->
+ ?connect_failure(Node, {setup_call, failed, _Error}),
async_reply({reply, false, State}, From)
end
end;
diff --git a/lib/kernel/src/os.erl b/lib/kernel/src/os.erl
index 0250783632..b5f19d4b99 100644
--- a/lib/kernel/src/os.erl
+++ b/lib/kernel/src/os.erl
@@ -25,6 +25,8 @@
-include("file.hrl").
+-export_type([env_var_name/0, env_var_value/0, env_var_name_value/0, command_input/0]).
+
%%% BIFs
-export([getenv/0, getenv/1, getenv/2, getpid/0,
@@ -32,21 +34,29 @@
putenv/2, set_signal/2, system_time/0, system_time/1,
timestamp/0, unsetenv/1]).
--spec getenv() -> [string()].
+-type env_var_name() :: nonempty_string().
+
+-type env_var_value() :: string().
+
+-type env_var_name_value() :: nonempty_string().
+
+-type command_input() :: atom() | io_lib:chars().
+
+-spec getenv() -> [env_var_name_value()].
getenv() -> erlang:nif_error(undef).
-spec getenv(VarName) -> Value | false when
- VarName :: string(),
- Value :: string().
+ VarName :: env_var_name(),
+ Value :: env_var_value().
getenv(_) ->
erlang:nif_error(undef).
-spec getenv(VarName, DefaultValue) -> Value when
- VarName :: string(),
- DefaultValue :: string(),
- Value :: string().
+ VarName :: env_var_name(),
+ DefaultValue :: env_var_value(),
+ Value :: env_var_value().
getenv(VarName, DefaultValue) ->
case os:getenv(VarName) of
@@ -75,8 +85,8 @@ perf_counter(Unit) ->
erlang:convert_time_unit(os:perf_counter(), perf_counter, Unit).
-spec putenv(VarName, Value) -> true when
- VarName :: string(),
- Value :: string().
+ VarName :: env_var_name(),
+ Value :: env_var_value().
putenv(_, _) ->
erlang:nif_error(undef).
@@ -99,7 +109,7 @@ timestamp() ->
erlang:nif_error(undef).
-spec unsetenv(VarName) -> true when
- VarName :: string().
+ VarName :: env_var_name().
unsetenv(_) ->
erlang:nif_error(undef).
@@ -178,7 +188,7 @@ verify_executable(Name0, [Ext|Rest], OrigExtensions) ->
end;
verify_executable(Name, [], OrigExtensions) when OrigExtensions =/= [""] -> %% Windows
%% Will only happen on windows, hence case insensitivity
- case can_be_full_name(string:to_lower(Name),OrigExtensions) of
+ case can_be_full_name(string:lowercase(Name),OrigExtensions) of
true ->
verify_executable(Name,[""],[""]);
_ ->
@@ -232,10 +242,9 @@ extensions() ->
%% Executes the given command in the default shell for the operating system.
-spec cmd(Command) -> string() when
- Command :: atom() | io_lib:chars().
+ Command :: os:command_input().
cmd(Cmd) ->
- validate(Cmd),
- {SpawnCmd, SpawnOpts, SpawnInput, Eot} = mk_cmd(os:type(), Cmd),
+ {SpawnCmd, SpawnOpts, SpawnInput, Eot} = mk_cmd(os:type(), validate(Cmd)),
Port = open_port({spawn, SpawnCmd}, [binary, stderr_to_stdout,
stream, in, hide | SpawnOpts]),
MonRef = erlang:monitor(port, Port),
@@ -255,8 +264,6 @@ mk_cmd({win32,Wtype}, Cmd) ->
{Cspec,_} -> lists:concat([Cspec," /c",Cmd])
end,
{Command, [], [], <<>>};
-mk_cmd(OsType,Cmd) when is_atom(Cmd) ->
- mk_cmd(OsType, atom_to_list(Cmd));
mk_cmd(_,Cmd) ->
%% Have to send command in like this in order to make sh commands like
%% cd and ulimit available
@@ -279,17 +286,33 @@ mk_cmd(_,Cmd) ->
<<$\^D>>}.
validate(Atom) when is_atom(Atom) ->
- ok;
+ validate(atom_to_list(Atom));
validate(List) when is_list(List) ->
- validate1(List).
+ case validate1(List) of
+ false ->
+ List;
+ true ->
+ %% Had zeros at end; remove them...
+ string:trim(List, trailing, [0])
+ end.
-validate1([C|Rest]) when is_integer(C) ->
+validate1([0|Rest]) ->
+ validate2(Rest);
+validate1([C|Rest]) when is_integer(C), C > 0 ->
validate1(Rest);
validate1([List|Rest]) when is_list(List) ->
- validate1(List),
- validate1(Rest);
+ validate1(List) or validate1(Rest);
validate1([]) ->
- ok.
+ false.
+
+%% Ensure that the rest is zero only...
+validate2([]) ->
+ true;
+validate2([0|Rest]) ->
+ validate2(Rest);
+validate2([List|Rest]) when is_list(List) ->
+ validate2(List),
+ validate2(Rest).
get_data(Port, MonRef, Eot, Sofar) ->
receive
diff --git a/lib/kernel/test/file_name_SUITE.erl b/lib/kernel/test/file_name_SUITE.erl
index 899102c908..f23529fec9 100644
--- a/lib/kernel/test/file_name_SUITE.erl
+++ b/lib/kernel/test/file_name_SUITE.erl
@@ -302,7 +302,9 @@ check_normal(Mod) ->
{ok, BC} = Mod:read(FD,1024),
ok = file:close(FD)
end || {regular,Name,Content} <- NormalDir ],
+ {error, badarg} = Mod:rename("fil1\0tmp_fil2","tmp_fil1"),
Mod:rename("fil1","tmp_fil1"),
+ {error, badarg} = Mod:read_file("tmp_fil1\0.txt"),
{ok, <<"fil1">>} = Mod:read_file("tmp_fil1"),
{error,enoent} = Mod:read_file("fil1"),
Mod:rename("tmp_fil1","fil1"),
diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl
index 331864b5de..e47023d201 100644
--- a/lib/kernel/test/gen_tcp_misc_SUITE.erl
+++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl
@@ -1572,52 +1572,56 @@ fill_sendq(Config) when is_list(Config) ->
Master = self(),
Server =
spawn_link(fun () ->
- {ok,L} = gen_tcp:listen
- (0, [{active,false},binary,
- {reuseaddr,true},{packet,0}]),
+ {ok,L} = gen_tcp:listen(0, [{active,false},binary,
+ {reuseaddr,true},{packet,0}]),
{ok,Port} = inet:port(L),
Master ! {self(),client,
fill_sendq_client(Port, Master)},
fill_sendq_srv(L, Master)
end),
io:format("~p Server~n", [Server]),
- receive {Server,client,Client} ->
- io:format("~p Client~n", [Client]),
- receive {Server,reader,Reader} ->
- io:format("~p Reader~n", [Reader]),
- fill_sendq_loop(Server, Client, Reader)
+ receive
+ {Server,client,Client} ->
+ io:format("~p Client~n", [Client]),
+ receive
+ {Server,reader,Reader} ->
+ io:format("~p Reader~n", [Reader]),
+ fill_sendq_loop(Server, Client, Reader)
end
end.
fill_sendq_loop(Server, Client, Reader) ->
%% Master
%%
- receive {Server,send} ->
+ receive
+ {Server,send} ->
fill_sendq_loop(Server, Client, Reader)
after 2000 ->
%% Send queue full, sender blocked -> close client.
io:format("Send timeout, closing Client...~n", []),
Client ! {self(),close},
- receive {Server,[{error,closed}]} ->
- io:format("Got server closed.~n"),
- receive {Reader,[{error,closed}]} ->
- io:format
- ("Got reader closed.~n"),
- ok
- after 3000 ->
- ct:fail({timeout,{closed,reader}})
- end;
- {Reader,[{error,closed}]} ->
- io:format("Got reader closed.~n"),
- receive {Server,[{error,closed}]} ->
- io:format("Got server closed~n"),
- ok
- after 3000 ->
- ct:fail({timeout,{closed,server}})
- end
- after 3000 ->
- ct:fail({timeout,{closed,[server,reader]}})
- end
+ receive
+ {Server,[{error,closed}]} ->
+ io:format("Got server closed.~n"),
+ receive
+ {Reader,[{error,closed}]} ->
+ io:format("Got reader closed.~n"),
+ ok
+ after 3000 ->
+ ct:fail({timeout,{closed,reader}})
+ end;
+ {Reader,[{error,closed}]} ->
+ io:format("Got reader closed.~n"),
+ receive
+ {Server,[{error,closed}]} ->
+ io:format("Got server closed~n"),
+ ok
+ after 3000 ->
+ ct:fail({timeout,{closed,server}})
+ end
+ after 3000 ->
+ ct:fail({timeout,{closed,[server,reader]}})
+ end
end.
fill_sendq_srv(L, Master) ->
diff --git a/lib/kernel/test/gen_udp_SUITE.erl b/lib/kernel/test/gen_udp_SUITE.erl
index aa616d43d6..96e495505a 100644
--- a/lib/kernel/test/gen_udp_SUITE.erl
+++ b/lib/kernel/test/gen_udp_SUITE.erl
@@ -288,58 +288,56 @@ bad_address(Config) when is_list(Config) ->
%%
%% Starts a slave node that on command sends a bunch of messages
%% to our UDP port. The receiving process just receives and
-%% ignores the incoming messages, but counts them.
-%% A tracing process traces the receiving process for
-%% 'receive' and scheduling events. From the trace,
-%% message contents is verified; and, how many messages
-%% are received per in/out scheduling, which should be
-%% the same as the read_packets parameter.
-%%
-%% What happens on the SMP emulator remains to be seen...
-%%
+%% ignores the incoming messages.
+%% A tracing process traces the receiving port for
+%% 'send' and scheduling events. From the trace,
+%% how many messages are received per in/out scheduling,
+%% which should never be more than the read_packet parameter.
%% OTP-6249 UDP option for number of packet reads.
read_packets(Config) when is_list(Config) ->
- case erlang:system_info(smp_support) of
- false ->
- read_packets_1();
- true ->
- %% We would need some new sort of tracing to test this
- %% option reliably in an SMP emulator.
- {skip,"SMP emulator"}
- end.
-
-read_packets_1() ->
N1 = 5,
- N2 = 7,
+ N2 = 1,
+ Msgs = 30000,
{ok,R} = gen_udp:open(0, [{read_packets,N1}]),
{ok,RP} = inet:port(R),
{ok,Node} = start_node(gen_udp_SUITE_read_packets),
Die = make_ref(),
- Loop = erlang:spawn_link(fun () -> infinite_loop(Die) end),
%%
- Msgs1 = [erlang:integer_to_list(M) || M <- lists:seq(1, N1*3)],
- [V1|_] = read_packets_test(R, RP, Msgs1, Node),
+ {V1, Trace1} = read_packets_test(R, RP, Msgs, Node),
{ok,[{read_packets,N1}]} = inet:getopts(R, [read_packets]),
%%
ok = inet:setopts(R, [{read_packets,N2}]),
- Msgs2 = [erlang:integer_to_list(M) || M <- lists:seq(1, N2*3)],
- [V2|_] = read_packets_test(R, RP, Msgs2, Node),
+ {V2, Trace2} = read_packets_test(R, RP, Msgs, Node),
{ok,[{read_packets,N2}]} = inet:getopts(R, [read_packets]),
%%
stop_node(Node),
- Mref = erlang:monitor(process, Loop),
- Loop ! Die,
- receive
- {'DOWN',Mref,_,_, normal} ->
- case {V1,V2} of
- {N1,N2} ->
- ok;
- _ when V1 =/= N1, V2 =/= N2 ->
- ok
- end
+ ct:log("N1=~p, V1=~p vs N2=~p, V2=~p",[N1,V1,N2,V2]),
+
+ dump_terms(Config, "trace1.terms", Trace2),
+ dump_terms(Config, "trace2.terms", Trace2),
+
+ %% Because of the inherit racy-ness of the feature it is
+ %% hard to test that it behaves correctly.
+ %% Right now (OTP 21) a port task takes 5% of the
+ %% allotted port task reductions to execute, so
+ %% the max number of executions a port is allowed to
+ %% do before being re-scheduled is N * 20
+
+ if
+ V1 > (N1 * 20) ->
+ ct:fail("Got ~p msgs, max was ~p", [V1, N1]);
+ V2 > (N2 * 20) ->
+ ct:fail("Got ~p msgs, max was ~p", [V2, N2]);
+ true ->
+ ok
end.
+dump_terms(Config, Name, Terms) ->
+ FName = filename:join(proplists:get_value(priv_dir, Config),Name),
+ file:write_file(FName, term_to_binary(Terms)),
+ ct:log("Logged terms to ~s",[FName]).
+
infinite_loop(Die) ->
receive
Die ->
@@ -350,7 +348,6 @@ infinite_loop(Die) ->
end.
read_packets_test(R, RP, Msgs, Node) ->
- Len = length(Msgs),
Receiver = self(),
Tracer =
spawn_link(
@@ -375,24 +372,24 @@ read_packets_test(R, RP, Msgs, Node) ->
[link,{priority,high}]),
receive
{Sender,{port,SP}} ->
- erlang:trace(self(), true,
- [running,'receive',{tracer,Tracer}]),
+ erlang:trace(R, true,
+ [running_ports,'send',{tracer,Tracer}]),
erlang:yield(),
Sender ! {Receiver,go},
- read_packets_recv(Len),
- erlang:trace(self(), false, [all]),
+ read_packets_recv(Msgs),
+ erlang:trace(R, false, [all]),
Tracer ! {Receiver,get_trace},
receive
{Tracer,{trace,Trace}} ->
- read_packets_verify(R, SP, Msgs, Trace)
+ {read_packets_verify(R, SP, Trace), Trace}
end
end.
-read_packets_send(S, RP, [Msg|Msgs]) ->
- ok = gen_udp:send(S, localhost, RP, Msg),
- read_packets_send(S, RP, Msgs);
-read_packets_send(_S, _RP, []) ->
- ok.
+read_packets_send(_S, _RP, 0) ->
+ ok;
+read_packets_send(S, RP, Msgs) ->
+ ok = gen_udp:send(S, localhost, RP, "UDP FLOOOOOOD"),
+ read_packets_send(S, RP, Msgs - 1).
read_packets_recv(0) ->
ok;
@@ -404,23 +401,24 @@ read_packets_recv(N) ->
timeout
end.
-read_packets_verify(R, SP, Msg, Trace) ->
- lists:reverse(
- lists:sort(read_packets_verify(R, SP, Msg, Trace, 0))).
-
-read_packets_verify(R, SP, Msgs, [{trace,Self,OutIn,_}|Trace], M)
- when Self =:= self(), OutIn =:= out;
- Self =:= self(), OutIn =:= in ->
- push(M, read_packets_verify(R, SP, Msgs, Trace, 0));
-read_packets_verify(R, SP, [Msg|Msgs],
- [{trace,Self,'receive',{udp,R,{127,0,0,1},SP,Msg}}
- |Trace], M)
+read_packets_verify(R, SP, Trace) ->
+ [Max | _] = Pkts = lists:reverse(lists:sort(read_packets_verify(R, SP, Trace, 0))),
+ ct:pal("~p",[lists:sublist(Pkts,10)]),
+ Max.
+
+read_packets_verify(R, SP, [{trace,R,OutIn,_}|Trace], M)
+ when OutIn =:= out; OutIn =:= in ->
+ push(M, read_packets_verify(R, SP, Trace, 0));
+read_packets_verify(R, SP, [{trace, R,'receive',timeout}|Trace], M) ->
+ push(M, read_packets_verify(R, SP, Trace, 0));
+read_packets_verify(R, SP,
+ [{trace,R,'send',{udp,R,{127,0,0,1},SP,_Msg}, Self} | Trace], M)
when Self =:= self() ->
- read_packets_verify(R, SP, Msgs, Trace, M+1);
-read_packets_verify(_R, _SP, [], [], M) ->
+ read_packets_verify(R, SP, Trace, M+1);
+read_packets_verify(_R, _SP, [], M) ->
push(M, []);
-read_packets_verify(_R, _SP, Msgs, Trace, M) ->
- ct:fail({read_packets_verify,mismatch,Msgs,Trace,M}).
+read_packets_verify(_R, _SP, Trace, M) ->
+ ct:fail({read_packets_verify,mismatch,Trace,M}).
push(0, Vs) ->
Vs;
diff --git a/lib/kernel/test/os_SUITE.erl b/lib/kernel/test/os_SUITE.erl
index 53a9e168ef..8056321448 100644
--- a/lib/kernel/test/os_SUITE.erl
+++ b/lib/kernel/test/os_SUITE.erl
@@ -22,7 +22,8 @@
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
init_per_group/2,end_per_group/2,
init_per_testcase/2,end_per_testcase/2]).
--export([space_in_cwd/1, quoting/1, cmd_unicode/1, space_in_name/1, bad_command/1,
+-export([space_in_cwd/1, quoting/1, cmd_unicode/1,
+ null_in_command/1, space_in_name/1, bad_command/1,
find_executable/1, unix_comment_in_command/1, deep_list_command/1,
large_output_command/1, background_command/0, background_command/1,
message_leak/1, close_stdin/0, close_stdin/1, perf_counter_api/1]).
@@ -34,7 +35,8 @@ suite() ->
{timetrap,{minutes,1}}].
all() ->
- [space_in_cwd, quoting, cmd_unicode, space_in_name, bad_command,
+ [space_in_cwd, quoting, cmd_unicode, null_in_command,
+ space_in_name, bad_command,
find_executable, unix_comment_in_command, deep_list_command,
large_output_command, background_command, message_leak,
close_stdin, perf_counter_api].
@@ -125,6 +127,14 @@ cmd_unicode(Config) when is_list(Config) ->
[] = receive_all(),
ok.
+null_in_command(Config) ->
+ {Ok, Error} = case os:type() of
+ {win32,_} -> {"dir", "di\0r"};
+ _ -> {"ls", "l\0s"}
+ end,
+ true = is_list(try os:cmd(Ok) catch Class0:_ -> Class0 end),
+ error = try os:cmd(Error) catch Class1:_ -> Class1 end,
+ ok.
%% Test that program with a space in its name can be executed.
space_in_name(Config) when is_list(Config) ->
diff --git a/lib/kernel/test/zlib_SUITE.erl b/lib/kernel/test/zlib_SUITE.erl
index 1afcd155b3..7be7e503df 100644
--- a/lib/kernel/test/zlib_SUITE.erl
+++ b/lib/kernel/test/zlib_SUITE.erl
@@ -1022,32 +1022,27 @@ sub_heap_binaries(Config) when is_list(Config) ->
%% Check concurrent access to zlib driver.
smp(Config) ->
- case erlang:system_info(smp_support) of
- true ->
- NumOfProcs = lists:min([8,erlang:system_info(schedulers)]),
- io:format("smp starting ~p workers\n",[NumOfProcs]),
-
- %% Tests to run in parallel.
- Funcs =
- [zip_usage, gz_usage, compress_usage, dictionary_usage,
- crc, adler],
-
- %% We get all function arguments here to avoid repeated parallel
- %% file read access.
- UsageArgs =
- list_to_tuple([{F, ?MODULE:F({get_arg,Config})} || F <- Funcs]),
- Parent = self(),
-
- WorkerFun =
- fun() ->
- worker(rand:uniform(9999), UsageArgs, Parent)
- end,
-
- Pids = [spawn_link(WorkerFun) || _ <- lists:seq(1, NumOfProcs)],
- wait_pids(Pids);
- false ->
- {skipped,"No smp support"}
- end.
+ NumOfProcs = lists:min([8,erlang:system_info(schedulers)]),
+ io:format("smp starting ~p workers\n",[NumOfProcs]),
+
+ %% Tests to run in parallel.
+ Funcs =
+ [zip_usage, gz_usage, compress_usage, dictionary_usage,
+ crc, adler],
+
+ %% We get all function arguments here to avoid repeated parallel
+ %% file read access.
+ UsageArgs =
+ list_to_tuple([{F, ?MODULE:F({get_arg,Config})} || F <- Funcs]),
+ Parent = self(),
+
+ WorkerFun =
+ fun() ->
+ worker(rand:uniform(9999), UsageArgs, Parent)
+ end,
+
+ Pids = [spawn_link(WorkerFun) || _ <- lists:seq(1, NumOfProcs)],
+ wait_pids(Pids).
worker(Seed, FnATpl, Parent) ->
io:format("smp worker ~p, seed=~p~n",[self(),Seed]),
diff --git a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc
index 62759c624b..0265e0efa0 100644
--- a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc
+++ b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc
@@ -226,8 +226,10 @@
not known beforehand, all fragments are searched for
matching records.</p>
<p>Notice that in <c>ordered_set</c> tables, the records
- are ordered per fragment, and the the order is undefined in
- results returned by <c>select</c> and <c>match_object</c>.</p>
+ are ordered per fragment, and the order is undefined in
+ results returned by <c>select</c> and <c>match_object</c>,
+ as well as <c>first</c>, <c>next</c>, <c>prev</c> and
+ <c>last</c>.</p>
<p>The following code illustrates how a <c>Mnesia</c> table is
converted to be a fragmented table and how more fragments
are added later:</p>
diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl
index 0fabdc7929..78dbe7ffde 100644
--- a/lib/mnesia/test/mnesia_test_lib.erl
+++ b/lib/mnesia/test/mnesia_test_lib.erl
@@ -774,7 +774,7 @@ init_nodes([], _File, _Line) ->
%% Returns [Name, Host]
node_to_name_and_host(Node) ->
- string:tokens(atom_to_list(Node), [$@]).
+ string:lexemes(atom_to_list(Node), [$@]).
lookup_config(Key,Config) ->
case lists:keysearch(Key,1,Config) of
diff --git a/lib/observer/src/cdv_dist_cb.erl b/lib/observer/src/cdv_dist_cb.erl
index 2b4c9f56d1..aeb34e5baf 100644
--- a/lib/observer/src/cdv_dist_cb.erl
+++ b/lib/observer/src/cdv_dist_cb.erl
@@ -78,7 +78,7 @@ init_gen_page(Parent, Info) ->
cdv_info_wx:start_link(Parent,{Fields,Info,[]}).
format({creations,Creations}) ->
- string:join([integer_to_list(C) || C <- Creations],",");
+ lists:flatten(lists:join(",",[integer_to_list(C) || C <- Creations]));
format(D) ->
D.
diff --git a/lib/observer/src/cdv_mem_cb.erl b/lib/observer/src/cdv_mem_cb.erl
index abeddc7335..925487786c 100644
--- a/lib/observer/src/cdv_mem_cb.erl
+++ b/lib/observer/src/cdv_mem_cb.erl
@@ -49,9 +49,7 @@ gen_mem_info_fields([]) ->
[].
upper(Key) ->
- string:join([string:to_upper([H]) ++ T ||
- [H|T] <- string:tokens(Key,"_")]," ").
-
+ lists:join(" ", [string:titlecase(Word) || Word <- string:split(Key, "_", all)]).
%%%-----------------------------------------------------------------
%%% Allocated areas page
diff --git a/lib/observer/src/cdv_wx.erl b/lib/observer/src/cdv_wx.erl
index c3f36cd689..0e33c9e618 100644
--- a/lib/observer/src/cdv_wx.erl
+++ b/lib/observer/src/cdv_wx.erl
@@ -130,8 +130,9 @@ init(File0) ->
{ok,File} ->
%% Set window title
T1 = "Crashdump Viewer: ",
+ FileLength = string:length(File),
Title =
- if length(File) > 70 ->
+ if FileLength > 70 ->
T1 ++ filename:basename(File);
true ->
T1 ++ File
@@ -428,8 +429,9 @@ do_load_dump(Frame,FileName) ->
ok ->
%% Set window title
T1 = "Crashdump Viewer: ",
+ FileLength = string:length(FileName),
Title =
- if length(FileName) > 70 ->
+ if FileLength > 70 ->
T1 ++ filename:basename(FileName);
true ->
T1 ++ FileName
diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl
index 40450a2873..50e9102ff0 100644
--- a/lib/observer/src/crashdump_viewer.erl
+++ b/lib/observer/src/crashdump_viewer.erl
@@ -866,7 +866,7 @@ do_read_file(File) ->
end.
check_dump_version(Vsn) ->
- DumpVsn = [list_to_integer(L) || L<-string:tokens(Vsn,".")],
+ DumpVsn = [list_to_integer(L) || L<-string:lexemes(Vsn,".")],
if DumpVsn > ?max_dump_version ->
Info =
"This Crashdump Viewer is too old for the given "
@@ -1350,12 +1350,12 @@ parse_monitor("{"++Str) ->
%% Named process
{Name,Node,Rest1} = parse_name_node(Str,[]),
Pid = get_pid_from_name(Name,Node),
- case parse_link(string:strip(Rest1,left,$,),[]) of
+ case parse_link(string:trim(Rest1,leading,","),[]) of
{Ref,"}"++Rest2} ->
%% Bug in break.c - prints an extra "}" for remote
%% nodes... thus the strip
{{Pid,"{"++Name++","++Node++"} ("++Ref++")"},
- string:strip(Rest2,left,$})};
+ string:trim(Rest2,leading,"}")};
{Ref,[]} ->
{{Pid,"{"++Name++","++Node++"} ("++Ref++")"},[]}
end;
@@ -1635,7 +1635,7 @@ get_ports(File) ->
%% Converting port string to tuple to secure correct sorting. This is
%% converted back in cdv_port_cb:format/1.
port_to_tuple("#Port<"++Port) ->
- [I1,I2] = string:tokens(Port,".>"),
+ [I1,I2] = string:lexemes(Port,".>"),
{list_to_integer(I1),list_to_integer(I2)}.
get_portinfo(Fd,Port) ->
@@ -1658,9 +1658,9 @@ get_portinfo(Fd,Port) ->
"Registered as" ->
get_portinfo(Fd,Port#port{name=string(Fd)});
"Monitors" ->
- Monitors0 = string:tokens(bytes(Fd),"()"),
+ Monitors0 = string:lexemes(bytes(Fd),"()"),
Monitors = [begin
- [Pid,Ref] = string:tokens(Mon,","),
+ [Pid,Ref] = string:lexemes(Mon,","),
{Pid,Pid++" ("++Ref++")"}
end || Mon <- Monitors0],
get_portinfo(Fd,Port#port{monitors=Monitors});
@@ -1721,7 +1721,7 @@ get_etsinfo(Fd,EtsTable = #ets_table{details=Ds},WS) ->
"Buckets" ->
%% A bug in erl_db_hash.c prints a space after the buckets
%% - need to strip the string to make list_to_integer/1 happy.
- Buckets = list_to_integer(string:strip(bytes(Fd))),
+ Buckets = list_to_integer(string:trim(bytes(Fd),both,"\s")),
get_etsinfo(Fd,EtsTable#ets_table{buckets=Buckets},WS);
"Objects" ->
get_etsinfo(Fd,EtsTable#ets_table{size=list_to_integer(bytes(Fd))},WS);
@@ -1739,7 +1739,9 @@ get_etsinfo(Fd,EtsTable = #ets_table{details=Ds},WS) ->
Val = bytes(Fd),
get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{chain_min=>Val}},WS);
"Chain Length Avg" ->
- Val = try list_to_float(string:strip(bytes(Fd))) catch _:_ -> "-" end,
+ Val = try list_to_float(string:trim(bytes(Fd),both,"\s"))
+ catch _:_ -> "-"
+ end,
get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{chain_avg=>Val}},WS);
"Chain Length Max" ->
Val = bytes(Fd),
@@ -1896,7 +1898,7 @@ get_nodeinfo(Fd,Nod) ->
Creations = lists:flatmap(fun(C) -> try [list_to_integer(C)]
catch error:badarg -> []
end
- end, string:tokens(bytes(Fd)," ")),
+ end, string:lexemes(bytes(Fd)," ")),
get_nodeinfo(Fd,Nod#nod{creation={creations,Creations}});
"Remote link" ->
Procs = bytes(Fd), % e.g. "<0.31.0> <4322.54.0>"
@@ -2342,7 +2344,7 @@ get_size_value(Key,Data) ->
%% and Value is the sum over all allocator instances of each type.
sort_allocator_types([{Name,Data}|Allocators],Acc,DoTotal) ->
Type =
- case string:tokens(Name,"[]") of
+ case string:lexemes(Name,"[]") of
[T,_Id] -> T;
[Name] -> Name;
Other -> Other
diff --git a/lib/observer/src/observer_html_lib.erl b/lib/observer/src/observer_html_lib.erl
index 22b4714d63..8202f91030 100644
--- a/lib/observer/src/observer_html_lib.erl
+++ b/lib/observer/src/observer_html_lib.erl
@@ -278,24 +278,24 @@ href_proc_port("['#CDVPort'"++T,Acc,LTB) ->
%% Port written by crashdump_viewer:parse_term(...)
{Port0,Rest} = split($],T),
PortStr=
- case string:tokens(Port0,",.|") of
+ case string:lexemes(Port0,",.|") of
[X,Y] ->
Port = "#Port&lt;"++X++"."++Y++"&gt;",
href(Port,Port);
Ns ->
- "#Port&lt;" ++ string:join(Ns,".") ++"...&gt;"
+ "#Port&lt;" ++ lists:join($.,Ns) ++"...&gt;"
end,
href_proc_port(Rest,[PortStr|Acc],LTB);
href_proc_port("['#CDVPid'"++T,Acc,LTB) ->
%% Pid written by crashdump_viewer:parse_term(...)
{Pid0,Rest} = split($],T),
PidStr =
- case string:tokens(Pid0,",.|") of
+ case string:lexemes(Pid0,",.|") of
[X,Y,Z] ->
Pid = "&lt;"++X++"."++Y++"."++Z++"&gt;",
href(Pid,Pid);
Ns ->
- "&lt;" ++ string:join(Ns,".") ++ "...&gt;"
+ "&lt;" ++ lists:join($.,Ns) ++ "...&gt;"
end,
href_proc_port(Rest,[PidStr|Acc],LTB);
href_proc_port("'#CDVIncompleteHeap'"++T,Acc,LTB)->
@@ -332,7 +332,7 @@ href_proc_port([],Acc,_) ->
href_proc_bin(From, T, Acc, LTB) ->
{OffsetSizePos,Rest} = split($],T),
BinStr =
- case string:tokens(OffsetSizePos,",.| \n") of
+ case string:lexemes(OffsetSizePos,",.| \n") of
[Offset,SizeStr,Pos] when From =:= cdv ->
Size = list_to_integer(SizeStr),
PreviewSize = min(Size,10),
diff --git a/lib/observer/src/observer_lib.erl b/lib/observer/src/observer_lib.erl
index 94d199e688..52e6c3e52a 100644
--- a/lib/observer/src/observer_lib.erl
+++ b/lib/observer/src/observer_lib.erl
@@ -478,7 +478,7 @@ create_box(Parent, Data) ->
link_entry(Panel,Value);
_ ->
Value = to_str(Value0),
- case string:sub_word(lists:sublist(Value, 80),1,$\n) of
+ case string:nth_lexeme(lists:sublist(Value, 80),1, [$\n]) of
Value ->
%% Short string, no newlines - show all
wxStaticText:new(Panel, ?wxID_ANY, Value);
diff --git a/lib/observer/src/observer_procinfo.erl b/lib/observer/src/observer_procinfo.erl
index fb02ae2728..4ce38e439d 100644
--- a/lib/observer/src/observer_procinfo.erl
+++ b/lib/observer/src/observer_procinfo.erl
@@ -454,7 +454,8 @@ local_pid_str(Pid) ->
global_pid_node_pref(Pid) ->
%% Global PID node prefix : X of <X.Y.Z>
- string:strip(string:sub_word(pid_to_list(Pid),1,$.),left,$<).
+ [NodePrefix|_] = string:lexemes(pid_to_list(Pid),"<."),
+ NodePrefix.
io_get_data(Pid) ->
Pid ! {self(), get_data_and_close},
diff --git a/lib/observer/src/observer_traceoptions_wx.erl b/lib/observer/src/observer_traceoptions_wx.erl
index 4f46426cf6..fbcf6d7fe9 100644
--- a/lib/observer/src/observer_traceoptions_wx.erl
+++ b/lib/observer/src/observer_traceoptions_wx.erl
@@ -619,7 +619,7 @@ create_styled_txtctrl(Parent) ->
keyWords() ->
L = ["after","begin","case","try","cond","catch","andalso","orelse",
- "end","fun","if","let","of","query","receive","when","bnot","not",
+ "end","fun","if","let","of","receive","when","bnot","not",
"div","rem","band","and","bor","bxor","bsl","bsr","or","xor"],
lists:flatten([K ++ " " || K <- L] ++ [0]).
diff --git a/lib/observer/src/observer_wx.erl b/lib/observer/src/observer_wx.erl
index be93b1d5f1..453e3bdc2d 100644
--- a/lib/observer/src/observer_wx.erl
+++ b/lib/observer/src/observer_wx.erl
@@ -732,7 +732,7 @@ get_nodes() ->
{Nodes, lists:reverse(Menues)}.
epmd_nodes(Names) ->
- [_, Host] = string:tokens(atom_to_list(node()),"@"),
+ [_, Host] = string:lexemes(atom_to_list(node()),"@"),
[list_to_atom(Name ++ [$@|Host]) || {Name, _} <- Names].
update_node_list(State = #state{menubar=MenuBar}) ->
diff --git a/lib/observer/src/ttb.erl b/lib/observer/src/ttb.erl
index 940fdc9818..29a572d7fe 100644
--- a/lib/observer/src/ttb.erl
+++ b/lib/observer/src/ttb.erl
@@ -100,7 +100,7 @@ do_tracer(Clients,PI,Traci) ->
{ok, H} = inet:gethostname(),
H;
_ ->
- [_,H] = string:tokens(atom_to_list(N),"@"),
+ [_,H] = string:lexemes(atom_to_list(N),"@"),
H
end,
case catch dbg:tracer(N,port,dbg:trace_port(ip,IpPortSpec)) of
@@ -976,7 +976,7 @@ decode_filename(3,Bin,latin1) ->
File.
host(Node) ->
- [_name,Host] = string:tokens(atom_to_list(Node),"@"),
+ [_name,Host] = string:lexemes(atom_to_list(Node),"@"),
Host.
wait_for_fetch([]) ->
@@ -1064,7 +1064,7 @@ collect_files(Dirs) ->
lists:map(fun(Dir) ->
MetaFiles = filelib:wildcard(filename:join(Dir,"*.ti")),
lists:map(fun(M) ->
- Sub = string:left(M,length(M)-3),
+ Sub = filename:rootname(M,".ti"),
case filelib:is_file(Sub) of
true -> Sub;
false -> Sub++".*.wrp"
diff --git a/lib/observer/test/ttb_SUITE.erl b/lib/observer/test/ttb_SUITE.erl
index e8c2f9e37d..ed62efbb58 100644
--- a/lib/observer/test/ttb_SUITE.erl
+++ b/lib/observer/test/ttb_SUITE.erl
@@ -222,7 +222,7 @@ file_fetch(Config) when is_list(Config) ->
?line ?t:capture_stop(),
?line [StoreString] = ?t:capture_get(),
?line UploadDir =
- lists:last(string:tokens(lists:flatten(StoreString),"$ \n")),
+ lists:last(string:lexemes(lists:flatten(StoreString),"$ \n")),
%% check that files are no longer in original directories...
?line ok = check_gone(ThisDir,atom_to_list(Node)++"-file_fetch"),
@@ -1035,8 +1035,8 @@ logfile_name_in_fetch_dir(Config) when is_list(Config) ->
?line {ServerNode, ClientNode} = start_client_and_server(),
?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
?line {_,Dir} = ttb:stop([return_fetch_dir]),
- ?line P1 = lists:nth(3, string:tokens(filename:basename(Dir), "_")),
- ?line P2 = hd(string:tokens(P1, "-")),
+ ?line P1 = lists:nth(3, string:lexemes(filename:basename(Dir), "_")),
+ ?line P2 = hd(string:lexemes(P1, "-")),
?line _File = P2.
logfile_name_in_fetch_dir(cleanup,_Config) ->
?line stop_client_and_server().
diff --git a/lib/os_mon/src/cpu_sup.erl b/lib/os_mon/src/cpu_sup.erl
index e758b63d19..c1c972b5b1 100644
--- a/lib/os_mon/src/cpu_sup.erl
+++ b/lib/os_mon/src/cpu_sup.erl
@@ -244,7 +244,7 @@ get_uint32_measurement(Request, #internal{os_type = {unix, Sys}}) when Sys == ir
%% Get the load average using uptime.
%% "8:01pm up 2 days, 22:12, 4 users, load average: 0.70, 0.58, 0.43"
D = os:cmd("uptime") -- "\n",
- Avg = lists:reverse(hd(string:tokens(lists:reverse(D), ":"))),
+ Avg = lists:reverse(hd(string:lexemes(lists:reverse(D), ":"))),
{ok, [L1, L5, L15], _} = io_lib:fread("~f, ~f, ~f", Avg),
case Request of
?avg1 -> sunify(L1);
diff --git a/lib/os_mon/src/disksup.erl b/lib/os_mon/src/disksup.erl
index 044604b000..aeec335ba7 100644
--- a/lib/os_mon/src/disksup.erl
+++ b/lib/os_mon/src/disksup.erl
@@ -32,7 +32,7 @@
terminate/2, code_change/3]).
%% Other exports
--export([format_status/2]).
+-export([format_status/2, parse_df/2]).
-record(state, {threshold, timeout, os, diskdata = [],port}).
@@ -294,8 +294,8 @@ check_disks_solaris("", _Threshold) ->
check_disks_solaris("\n", _Threshold) ->
[];
check_disks_solaris(Str, Threshold) ->
- case io_lib:fread("~s~d~d~d~d%~s", Str) of
- {ok, [_FS, KB, _Used, _Avail, Cap, MntOn], RestStr} ->
+ case parse_df(Str, posix) of
+ {ok, {KB, Cap, MntOn}, RestStr} ->
if
Cap >= Threshold ->
set_alarm({disk_almost_full, MntOn}, []);
@@ -308,14 +308,102 @@ check_disks_solaris(Str, Threshold) ->
check_disks_solaris(skip_to_eol(Str),Threshold)
end.
+%% @private
+%% @doc Predicate to take a word from the input string until a space or
+%% a percent '%' sign (the Capacity field is followed by a %)
+parse_df_is_not_space($ ) -> false;
+parse_df_is_not_space($%) -> false;
+parse_df_is_not_space(_) -> true.
+
+%% @private
+%% @doc Predicate to take spaces away from string. Stops on a non-space
+parse_df_is_space($ ) -> true;
+parse_df_is_space(_) -> false.
+
+%% @private
+%% @doc Predicate to consume remaining characters until end of line.
+parse_df_is_not_eol($\r) -> false;
+parse_df_is_not_eol($\n) -> false;
+parse_df_is_not_eol(_) -> true.
+
+%% @private
+%% @doc Trims leading non-spaces (the word) from the string then trims spaces.
+parse_df_skip_word(Input) ->
+ Remaining = lists:dropwhile(fun parse_df_is_not_space/1, Input),
+ lists:dropwhile(fun parse_df_is_space/1, Remaining).
+
+%% @private
+%% @doc Takes all non-spaces and then drops following spaces.
+parse_df_take_word(Input) ->
+ {Word, Remaining0} = lists:splitwith(fun parse_df_is_not_space/1, Input),
+ Remaining1 = lists:dropwhile(fun parse_df_is_space/1, Remaining0),
+ {Word, Remaining1}.
+
+%% @private
+%% @doc Takes all non-spaces and then drops the % after it and the spaces.
+parse_df_take_word_percent(Input) ->
+ {Word, Remaining0} = lists:splitwith(fun parse_df_is_not_space/1, Input),
+ %% Drop the leading % or do nothing
+ Remaining1 = case Remaining0 of
+ [$% | R1] -> R1;
+ _ -> Remaining0 % Might be no % or empty list even
+ end,
+ Remaining2 = lists:dropwhile(fun parse_df_is_space/1, Remaining1),
+ {Word, Remaining2}.
+
+%% @private
+%% @doc Given a line of 'df' POSIX/SUSv3 output split it into fields:
+%% a string (mounted device), 4 integers (kilobytes, used, available
+%% and capacity), skip % sign, (optionally for susv3 can also skip IUsed, IFree
+%% and ICap% fields) then take remaining characters as the mount path
+-spec parse_df(string(), posix | susv3) ->
+ {error, parse_df} | {ok, {integer(), integer(), integer()}, string()}.
+parse_df(Input0, Flavor) ->
+ %% Format of Posix/Linux df output looks like Header + Lines
+ %% Filesystem 1024-blocks Used Available Capacity Mounted on
+ %% udev 2467108 0 2467108 0% /dev
+ Input1 = parse_df_skip_word(Input0), % skip device path field
+ {KbStr, Input2} = parse_df_take_word(Input1), % take Kb field
+ Input3 = parse_df_skip_word(Input2), % skip Used field
+ Input4 = parse_df_skip_word(Input3), % skip Avail field
+
+ % take Capacity% field; drop a % sign following the capacity
+ {CapacityStr, Input5} = parse_df_take_word_percent(Input4),
+
+ %% Format of OS X/SUSv3 df looks similar to POSIX but has 3 extra columns
+ %% Filesystem 1024-blocks Used Available Capacity iused ifree %iused Mounted
+ %% /dev/disk1 243949060 2380 86690680 65% 2029724 37555 0% /
+ Input6 = case Flavor of
+ posix -> Input5;
+ susv3 -> % there are 3 extra integers we want to skip
+ Input5a = parse_df_skip_word(Input5), % skip IUsed field
+ Input5b = parse_df_skip_word(Input5a), % skip IFree field
+ %% skip the value of ICap + '%' field
+ {_, Input5c} = parse_df_take_word_percent(Input5b),
+ Input5c
+ end,
+
+ % path is the remaining string till end of line
+ {MountPath, Input7} = lists:splitwith(fun parse_df_is_not_eol/1, Input6),
+ % Trim the newlines
+ Remaining = lists:dropwhile(fun(X) -> not parse_df_is_not_eol(X) end,
+ Input7),
+ try
+ Kb = erlang:list_to_integer(KbStr),
+ Capacity = erlang:list_to_integer(CapacityStr),
+ {ok, {Kb, Capacity, MountPath}, Remaining}
+ catch error:badarg ->
+ {error, parse_df}
+ end.
+
% Parse per SUSv3 specification, notably recent OS X
check_disks_susv3("", _Threshold) ->
[];
check_disks_susv3("\n", _Threshold) ->
[];
check_disks_susv3(Str, Threshold) ->
- case io_lib:fread("~s~d~d~d~d%~d~d~d%~s", Str) of
- {ok, [_FS, KB, _Used, _Avail, Cap, _IUsed, _IFree, _ICap, MntOn], RestStr} ->
+ case parse_df(Str, susv3) of
+ {ok, {KB, Cap, MntOn}, RestStr} ->
if
Cap >= Threshold ->
set_alarm({disk_almost_full, MntOn}, []);
diff --git a/lib/os_mon/src/memsup.erl b/lib/os_mon/src/memsup.erl
index 0a9a883390..95cb798ba5 100644
--- a/lib/os_mon/src/memsup.erl
+++ b/lib/os_mon/src/memsup.erl
@@ -705,7 +705,7 @@ get_os_wordsize_with_uname() ->
_ -> 32
end.
-clean_string(String) -> lists:flatten(string:tokens(String,"\r\n\t ")).
+clean_string(String) -> lists:flatten(string:lexemes(String,"\r\n\t ")).
%%--Replying to pending clients-----------------------------------------
diff --git a/lib/os_mon/test/disksup_SUITE.erl b/lib/os_mon/test/disksup_SUITE.erl
index ad61985014..d7f2626160 100644
--- a/lib/os_mon/test/disksup_SUITE.erl
+++ b/lib/os_mon/test/disksup_SUITE.erl
@@ -30,7 +30,7 @@
-export([port/1]).
-export([terminate/1, unavailable/1, restart/1]).
-export([otp_5910/1]).
--export([posix_only/1]).
+-export([posix_only/1, parse_df_output_posix/1, parse_df_output_susv3/1]).
init_per_suite(Config) when is_list(Config) ->
ok = application:start(os_mon),
@@ -59,7 +59,8 @@ suite() ->
all() ->
Bugs = [otp_5910],
- Always = [api, config, alarm, port, posix_only, unavailable] ++ Bugs,
+ Always = [api, config, alarm, port, posix_only, unavailable,
+ parse_df_output_posix, parse_df_output_susv3] ++ Bugs,
case test_server:os_type() of
{unix, _OSname} -> Always;
{win32, _OSname} -> Always;
@@ -413,3 +414,36 @@ get_disk_data([{"none",0,0}=E]) -> [E];
get_disk_data([{_,_,0}|Es]) -> get_disk_data(Es);
get_disk_data([E|Es]) -> [E|get_disk_data(Es)];
get_disk_data([]) -> [].
+
+%% @doc Test various expected inputs to 'df' command output (Linux/POSIX)
+parse_df_output_posix(Config) when is_list(Config) ->
+ PosixHdr = "Filesystem 1K-blocks Used Available Use% Mounted on\n",
+ {error, _} = disksup:parse_df(PosixHdr, posix),
+ {error, _} = disksup:parse_df("", posix),
+ {error, _} = disksup:parse_df("\n\n", posix),
+
+ %% Have a simple example with no funny spaces in mount path
+ Posix1 = "tmpfs 498048 7288 490760 2% /run\n",
+ {ok, {498048, 2, "/run"}, ""} = disksup:parse_df(Posix1, posix),
+
+ %% Have a mount path with some spaces in it
+ Posix2 = "tmpfs 498048 7288 490760 2% /spaces 1 2\n",
+ {ok, {498048, 2, "/spaces 1 2"}, ""} = disksup:parse_df(Posix2, posix).
+
+%% @doc Test various expected inputs to 'df' command output (Darwin/SUSv3)
+parse_df_output_susv3(Config) when is_list(Config) ->
+ DarwinHdr = "Filesystem 1024-blocks Used Available Capacity " ++
+ "iused ifree %iused Mounted on",
+ {error, _} = disksup:parse_df(DarwinHdr, susv3),
+ {error, _} = disksup:parse_df("", susv3),
+ {error, _} = disksup:parse_df("\n\n", susv3),
+
+ %% Have a simple example with no funny spaces in mount path
+ Darwin1 = "/dev/disk1 243949060 157002380 86690680 65% 2029724 " ++
+ "4292937555 0% /\n",
+ {ok, {243949060, 65, "/"}, ""} = disksup:parse_df(Darwin1, susv3),
+
+ %% Have a mount path with some spaces in it
+ Darwin2 = "/dev/disk1 243949060 157002380 86690680 65% 2029724 " ++
+ "4292937555 0% /spaces 1 2\n",
+ {ok, {243949060, 65, "/spaces 1 2"}, ""} = disksup:parse_df(Darwin2, susv3).
diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl
index e2e7d7359f..8a4a5e8d86 100644
--- a/lib/parsetools/src/leex.erl
+++ b/lib/parsetools/src/leex.erl
@@ -37,7 +37,6 @@
-import(lists, [member/2,reverse/1,sort/1,delete/2,
keysort/2,keydelete/3,
map/2,foldl/3,foreach/2,flatmap/2]).
--import(string, [substr/2,substr/3,span/2]).
-import(ordsets, [is_element/2,add_element/2,union/2]).
-import(orddict, [store/3]).
@@ -251,10 +250,10 @@ is_filename(T) ->
shorten_filename(Name0) ->
{ok,Cwd} = file:get_cwd(),
- case lists:prefix(Cwd, Name0) of
- false -> Name0;
- true ->
- case lists:nthtail(length(Cwd), Name0) of
+ case string:prefix(Name0, Cwd) of
+ nomatch -> Name0;
+ Rest ->
+ case unicode:characters_to_list(Rest) of
"/"++N -> N;
N -> N
end
@@ -490,12 +489,9 @@ parse_rules_end(_, NextLine, REAs, As, St) ->
%% action has been read. Keep track of line number.
collect_rule(Ifile, Chars, L0) ->
- %% Erlang strings are 1 based, but re 0 :-(
- {match,[{St0,Len}|_]} = re:run(Chars, "[^ \t\r\n]+", [unicode]),
- St = St0 + 1,
- %%io:fwrite("RE = ~p~n", [substr(Chars, St, Len)]),
- case collect_action(Ifile, substr(Chars, St+Len), L0, []) of
- {ok,[{':',_}|Toks],L1} -> {ok,substr(Chars, St, Len),Toks,L1};
+ {RegExp,Rest} = string:take(Chars, " \t\r\n", true),
+ case collect_action(Ifile, Rest, L0, []) of
+ {ok,[{':',_}|Toks],L1} -> {ok,RegExp,Toks,L1};
{ok,_,_} -> {error,{L0,leex,bad_rule}};
{eof,L1} -> {error,{L1,leex,bad_rule}};
{error,E,_} -> {error,E}
@@ -549,7 +545,7 @@ var_used(Name, Toks) ->
parse_rule_regexp(RE0, [{M,Exp}|Ms], St) ->
Split= re:split(RE0, "\\{" ++ M ++ "\\}", [{return,list},unicode]),
- RE1 = string:join(Split, Exp),
+ RE1 = lists:append(lists:join(Exp, Split)),
parse_rule_regexp(RE1, Ms, St);
parse_rule_regexp(RE, [], St) ->
%%io:fwrite("RE = ~p~n", [RE]),
@@ -589,9 +585,9 @@ nextline(Ifile, L, St) ->
eof -> {eof,L};
{error, _} -> add_error({L+1, leex, cannot_parse}, St);
Chars ->
- case substr(Chars, span(Chars, " \t\n")+1) of
- [$%|_Rest] -> nextline(Ifile, L+1, St);
- [] -> nextline(Ifile, L+1, St);
+ case string:take(Chars, " \t\n") of
+ {_, [$%|_Rest]} -> nextline(Ifile, L+1, St);
+ {_, []} -> nextline(Ifile, L+1, St);
_Other -> {ok,Chars,L+1}
end
end.
@@ -824,7 +820,7 @@ re_char_class(Cs, Cc, _) -> {reverse(Cc),Cs}. % Preserve order
%% posix_cc("space" ++ Cs) -> {space,Cs};
%% posix_cc("upper" ++ Cs) -> {upper,Cs};
%% posix_cc("xdigit" ++ Cs) -> {xdigit,Cs};
-%% posix_cc(Cs) -> parse_error({posix_cc,substr(Cs, 1, 5)}).
+%% posix_cc(Cs) -> parse_error({posix_cc,string:slice(Cs, 0, 5)}).
escape_char($n) -> $\n; % \n = LF
escape_char($r) -> $\r; % \r = CR
@@ -863,7 +859,7 @@ escape_char(C) -> C. % Pass it straight through
%% re_number(Cs, Acc) -> {Acc,Cs}.
string_between(Cs1, Cs2) ->
- substr(Cs1, 1, length(Cs1)-length(Cs2)).
+ string:slice(Cs1, 0, string:length(Cs1)-string:length(Cs2)).
%% We use standard methods, Thompson's construction and subset
%% construction, to create first an NFA and then a DFA from the
@@ -1343,7 +1339,7 @@ out_file(Ifile, Ofile, St, DFA, DF, Actions, Code, L) ->
eof -> output_file_directive(Ofile, St#leex.ifile, L);
{error, _} -> add_error(St#leex.ifile, {L, leex, cannot_parse}, St);
Line ->
- case substr(Line, 1, 5) of
+ case string:slice(Line, 0, 5) of
"##mod" -> out_module(Ofile, St);
"##cod" -> out_erlang_code(Ofile, St, Code, L);
"##dfa" -> out_dfa(Ofile, St, DFA, Code, DF, L);
@@ -1523,7 +1519,7 @@ prep_out_actions(As) ->
Name = list_to_atom(lists:concat([yyaction_,A])),
[Chars,Len,Line,_,_] = Vars,
Args = [V || V <- [Chars,Len,Line], V =/= "_"],
- ArgsChars = string:join(Args, ", "),
+ ArgsChars = lists:join(", ", Args),
{A,Code,Vars,Name,Args,ArgsChars}
end, As).
diff --git a/lib/parsetools/src/yecc.erl b/lib/parsetools/src/yecc.erl
index 36e33b52a4..b4e1cfe5e3 100644
--- a/lib/parsetools/src/yecc.erl
+++ b/lib/parsetools/src/yecc.erl
@@ -365,10 +365,10 @@ is_filename(T) ->
shorten_filename(Name0) ->
{ok,Cwd} = file:get_cwd(),
- case lists:prefix(Cwd, Name0) of
- false -> Name0;
- true ->
- case lists:nthtail(length(Cwd), Name0) of
+ case string:prefix(Name0, Cwd) of
+ nomatch -> Name0;
+ Rest ->
+ case unicode:characters_to_list(Rest) of
"/"++N -> N;
N -> N
end
@@ -2196,8 +2196,8 @@ output_reduce(St0, State, Terminal,
St20;
true ->
Ns = "Nss",
- Tmp = string:join(lists:duplicate(NmbrOfDaughters - 1, "_"),
- ","),
+ Tmp = lists:join(",",
+ lists:duplicate(NmbrOfDaughters - 1, "_")),
fwrite(St20, <<" [~s|Nss] = Ss,\n">>, [Tmp])
end,
St40 = case tokens(RuleNmbr, St30) of
diff --git a/lib/reltool/src/reltool.hrl b/lib/reltool/src/reltool.hrl
index 9c8aae6b7e..d133762818 100644
--- a/lib/reltool/src/reltool.hrl
+++ b/lib/reltool/src/reltool.hrl
@@ -220,7 +220,8 @@
{
name :: rel_name(),
vsn :: rel_vsn(),
- rel_apps :: [#rel_app{}]
+ rel_apps :: [#rel_app{}],
+ load_dot_erlang = true :: boolean()
}).
-record(sys,
@@ -300,6 +301,7 @@
-define(STANDALONE_INCL_SYS_FILTERS, ["^bin/(erl|epmd)(|\\.exe|\\.ini)\$",
"^bin/start(|_clean).boot\$",
+ "^bin/no_dot_erlang\\.boot\$",
"^erts.*/bin",
"^lib\$"]).
-define(STANDALONE_EXCL_SYS_FILTERS,
diff --git a/lib/reltool/src/reltool_mod_win.erl b/lib/reltool/src/reltool_mod_win.erl
index 2d56d74563..dcd802a5de 100644
--- a/lib/reltool/src/reltool_mod_win.erl
+++ b/lib/reltool/src/reltool_mod_win.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2009-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2009-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -833,7 +833,7 @@ load_code(Ed, Code) when is_binary(Code) ->
keyWords() ->
L = ["after","begin","case","try","cond","catch","andalso","orelse",
- "end","fun","if","let","of","query","receive","when","bnot","not",
+ "end","fun","if","let","of","receive","when","bnot","not",
"div","rem","band","and","bor","bxor","bsl","bsr","or","xor"],
lists:flatten([K ++ " " || K <- L] ++ [0]).
diff --git a/lib/reltool/src/reltool_target.erl b/lib/reltool/src/reltool_target.erl
index 1b1461178e..503e1971b9 100644
--- a/lib/reltool/src/reltool_target.erl
+++ b/lib/reltool/src/reltool_target.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2009-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2009-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -424,7 +424,7 @@ gen_script(Rel, Sys, PathFlag, Variables) ->
{error, Text}
end.
-do_gen_script(#rel{name = RelName, vsn = RelVsn},
+do_gen_script(#rel{name = RelName, vsn = RelVsn, load_dot_erlang=LoadErlangRc},
#sys{apps = Apps},
MergedApps,
PathFlag,
@@ -474,9 +474,11 @@ do_gen_script(#rel{name = RelName, vsn = RelVsn},
Type =/= none,
Type =/= load,
not lists:member(Name, InclApps)],
-
%% Apply user specific customizations
- {apply, {c, erlangrc, []}},
+ case LoadErlangRc of
+ true -> {apply, {c, erlangrc, []}};
+ false -> []
+ end,
{progress, started}
],
{ok, {script, {RelName, RelVsn}, lists:flatten(DeepList)}}.
diff --git a/lib/reltool/src/reltool_utils.erl b/lib/reltool/src/reltool_utils.erl
index 3891b5ae4d..060a0912f9 100644
--- a/lib/reltool/src/reltool_utils.erl
+++ b/lib/reltool/src/reltool_utils.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2009-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2009-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -154,7 +154,12 @@ default_rels() ->
rel_apps = []},
#rel{name = "start_sasl",
vsn = "1.0",
- rel_apps = [#rel_app{name = sasl}]}
+ rel_apps = [#rel_app{name = sasl}]},
+ #rel{name = "no_dot_erlang", %% Needed by escript and erlc
+ vsn = "1.0",
+ rel_apps = [],
+ load_dot_erlang = false
+ }
].
choose_default(Tag, Profile, InclDefs)
diff --git a/lib/reltool/test/reltool_server_SUITE.erl b/lib/reltool/test/reltool_server_SUITE.erl
index 49efe5aeda..3622f6650c 100644
--- a/lib/reltool/test/reltool_server_SUITE.erl
+++ b/lib/reltool/test/reltool_server_SUITE.erl
@@ -247,6 +247,7 @@ get_config(_Config) ->
{app,stdlib,[{incl_cond,include},{vsn,undefined},
{lib_dir,StdLibDir}]},
{boot_rel,"start_clean"},
+ {rel,"no_dot_erlang","1.0",[]},
{rel,"start_clean","1.0",[]},
{rel,"start_sasl","1.0",[sasl]},
{emu_name,"beam"},
@@ -277,6 +278,7 @@ get_config(_Config) ->
{app,stdlib,[{incl_cond,include},{vsn,StdVsn},
{lib_dir,StdLibDir},{mod,_,[]}|_]},
{boot_rel,"start_clean"},
+ {rel,"no_dot_erlang","1.0",[]},
{rel,"start_clean","1.0",[]},
{rel,"start_sasl","1.0",[sasl]},
{emu_name,"beam"},
@@ -2104,6 +2106,7 @@ save_config(Config) ->
{app,stdlib,[{incl_cond,include},{vsn,undefined},
{lib_dir,undefined}]},
{boot_rel,"start_clean"},
+ {rel,"no_dot_erlang","1.0",[]},
{rel,"start_clean","1.0",[]},
{rel,"start_sasl","1.0",[sasl]},
{emu_name,"beam"},
@@ -2144,6 +2147,7 @@ save_config(Config) ->
{app,stdlib,[{incl_cond,include},{vsn,StdVsn},
{lib_dir,StdLibDir},{mod,_,[]}|_]},
{boot_rel,"start_clean"},
+ {rel,"no_dot_erlang","1.0",[]},
{rel,"start_clean","1.0",[]},
{rel,"start_sasl","1.0",[sasl]},
{emu_name,"beam"},
@@ -2545,7 +2549,7 @@ undefined_regexp(_Config) ->
%% Library functions
erl_libs() ->
- string:tokens(os:getenv("ERL_LIBS", ""), ":;").
+ string:lexemes(os:getenv("ERL_LIBS", ""), ":;").
datadir(Config) ->
%% Removes the trailing slash...
@@ -2555,7 +2559,7 @@ latest(App) ->
AppStr = atom_to_list(App),
AppDirs = filelib:wildcard(filename:join(code:lib_dir(),AppStr++"-*")),
[LatestAppDir|_] = lists:reverse(AppDirs),
- [_,Vsn] = string:tokens(filename:basename(LatestAppDir),"-"),
+ [_,Vsn] = string:lexemes(filename:basename(LatestAppDir),"-"),
Vsn.
rm_missing_app(Apps) ->
@@ -2631,16 +2635,11 @@ os_cmd(Cmd) when is_list(Cmd) ->
Return->
%% Find the position of the status code wich is last in the string
%% prepended with #
- case string:rchr(Return, $#) of
-
- %% This happens only if the sh command pipe is somehow interrupted
- 0->
- {98, Return};
-
- Position->
- Result = string:left(Return,Position - 1),
- Status = string:substr(Return,Position + 1, length(Return) - Position - 1),
- {list_to_integer(Status), Result}
+ case string:split(Return, "$#", trailing) of
+ [_] -> %% This happens only if the sh command pipe is somehow interrupted
+ {98, Return};
+ [Result, Status0] ->
+ {list_to_integer(string:trim(Status0)), Result}
end
end.
diff --git a/lib/runtime_tools/src/erts_alloc_config.erl b/lib/runtime_tools/src/erts_alloc_config.erl
index 514530332c..342363d08d 100644
--- a/lib/runtime_tools/src/erts_alloc_config.erl
+++ b/lib/runtime_tools/src/erts_alloc_config.erl
@@ -621,7 +621,7 @@ format_header(FTO) ->
[Y, Mo, D, H, Mi, S]),
fcp(FTO,
"~s was used when generating the configuration.",
- [string:strip(erlang:system_info(system_version), both, $\n)]),
+ [string:trim(erlang:system_info(system_version), both, "$\n")]),
case erlang:system_info(schedulers) of
1 -> ok;
Schdlrs ->
@@ -698,28 +698,32 @@ fc(IODev, Frmt, Args) ->
fc(IODev, lists:flatten(io_lib:format(Frmt, Args))).
fc(IODev, String) ->
- fc_aux(IODev, string:tokens(String, " "), 0).
+ fc_aux(IODev, string:lexemes(String, " "), 0).
fc_aux(_IODev, [], 0) ->
ok;
fc_aux(IODev, [], _Len) ->
format(IODev, "~n");
fc_aux(IODev, [T|Ts], 0) ->
- Len = 2 + length(T),
+ Len = 2 + string:length(T),
format(IODev, "# ~s", [T]),
fc_aux(IODev, Ts, Len);
-fc_aux(IODev, [T|_Ts] = ATs, Len) when (length(T) + Len) >= ?PRINT_WITDH ->
- format(IODev, "~n"),
- fc_aux(IODev, ATs, 0);
-fc_aux(IODev, [T|Ts], Len) ->
- NewLen = Len + 1 + length(T),
- format(IODev, " ~s", [T]),
- fc_aux(IODev, Ts, NewLen).
+fc_aux(IODev, [T|Ts] = ATs, Len) ->
+ TLength = string:length(T),
+ case (TLength + Len) >= ?PRINT_WITDH of
+ true ->
+ format(IODev, "~n"),
+ fc_aux(IODev, ATs, 0);
+ false ->
+ NewLen = Len + 1 + TLength,
+ format(IODev, " ~s", [T]),
+ fc_aux(IODev, Ts, NewLen)
+ end.
%% fcl: format comment line
fcl(FTO) ->
EndStr = "# ",
- Precision = length(EndStr),
+ Precision = string:length(EndStr),
FieldWidth = -1*(?PRINT_WITDH),
format(FTO, "~*.*.*s~n", [FieldWidth, Precision, $-, EndStr]).
@@ -727,6 +731,6 @@ fcl(FTO, A) when is_atom(A) ->
fcl(FTO, atom_to_list(A));
fcl(FTO, Str) when is_list(Str) ->
Str2 = "# --- " ++ Str ++ " ",
- Precision = length(Str2),
+ Precision = string:length(Str2),
FieldWidth = -1*(?PRINT_WITDH),
format(FTO, "~*.*.*s~n", [FieldWidth, Precision, $-, Str2]).
diff --git a/lib/runtime_tools/src/msacc.erl b/lib/runtime_tools/src/msacc.erl
index 0d9b2690e5..37887cee72 100644
--- a/lib/runtime_tools/src/msacc.erl
+++ b/lib/runtime_tools/src/msacc.erl
@@ -46,7 +46,8 @@
system := float()}}.
--type msacc_type() :: scheduler | aux | async.
+-type msacc_type() :: aux | async | dirty_cpu_scheduler
+ | dirty_io_scheduler | poll | scheduler.
-type msacc_id() :: non_neg_integer().
-type msacc_state() :: alloc | aux | bif | busy_wait | check_io |
emulator | ets | gc | gc_fullsweep | nif |
diff --git a/lib/runtime_tools/src/system_information.erl b/lib/runtime_tools/src/system_information.erl
index df25297eb9..92109c9a74 100644
--- a/lib/runtime_tools/src/system_information.erl
+++ b/lib/runtime_tools/src/system_information.erl
@@ -590,12 +590,12 @@ vsnstr2vsn(VsnStr) ->
list_to_tuple(lists:map(fun (Part) ->
list_to_integer(Part)
end,
- string:tokens(VsnStr, "."))).
+ string:lexemes(VsnStr, "."))).
rtdepstrs2rtdeps([]) ->
[];
rtdepstrs2rtdeps([RTDep | RTDeps]) ->
- [AppStr, VsnStr] = string:tokens(RTDep, "-"),
+ [AppStr, VsnStr] = string:lexemes(RTDep, "-"),
[{list_to_atom(AppStr), vsnstr2vsn(VsnStr)} | rtdepstrs2rtdeps(RTDeps)].
build_app_table([], AppTab) ->
diff --git a/lib/runtime_tools/test/dbg_SUITE.erl b/lib/runtime_tools/test/dbg_SUITE.erl
index 4b0864858c..cfe8412e33 100644
--- a/lib/runtime_tools/test/dbg_SUITE.erl
+++ b/lib/runtime_tools/test/dbg_SUITE.erl
@@ -23,7 +23,7 @@
-export([all/0, suite/0,
big/1, tiny/1, simple/1, message/1, distributed/1, port/1,
send/1, recv/1,
- ip_port/1, file_port/1, file_port2/1, file_port_schedfix/1,
+ ip_port/1, file_port/1, file_port2/1,
ip_port_busy/1, wrap_port/1, wrap_port_time/1,
with_seq_trace/1, dead_suspend/1, local_trace/1,
saved_patterns/1, tracer_exit_on_stop/1,
@@ -41,7 +41,7 @@ suite() ->
all() ->
[big, tiny, simple, message, distributed, port, ip_port,
send, recv,
- file_port, file_port2, file_port_schedfix, ip_port_busy,
+ file_port, file_port2, ip_port_busy,
wrap_port, wrap_port_time, with_seq_trace, dead_suspend,
local_trace, saved_patterns, tracer_exit_on_stop,
erl_tracer, distributed_erl_tracer].
@@ -623,99 +623,6 @@ file_port2(Config) when is_list(Config) ->
end,
ok.
-%% Test that the scheduling timestamp fix for trace flag 'running' works.
-file_port_schedfix(Config) when is_list(Config) ->
- case (catch erlang:system_info(smp_support)) of
- true ->
- {skip, "No schedule fix on SMP"};
- _ ->
- try
- file_port_schedfix1(Config)
- after
- dbg:stop()
- end
- end.
-file_port_schedfix1(Config) when is_list(Config) ->
- stop(),
- {A,B,C} = erlang:now(),
- FTMP = atom_to_list(?MODULE) ++ integer_to_list(A) ++
- "-" ++ integer_to_list(B) ++ "-" ++ integer_to_list(C),
- FName = filename:join([proplists:get_value(data_dir, Config), FTMP]),
- %%
- Port = dbg:trace_port(file, {FName, wrap, ".wraplog", 8*1024, 4}),
- {ok, _} = dbg:tracer(port, Port),
- {ok,[{matched,_node,0}]} = dbg:p(new,[running,procs,send,timestamp]),
- %%
- %% Generate the trace data
- %%
- %% This starts 3 processes that sends a message to each other in a ring,
- %% 4 laps. Prior to sending the message to the next in the ring, each
- %% process send 8 messages to itself, just to generate some trace data,
- %% and to lower the possibility that the trace log wraps just after
- %% a schedule out message (which would not burden any process and hence
- %% not show up in the result)
- %%
- %% The wrap file trace is used because it burns a lot of time when the
- %% driver swaps files, a lot more than the regular file trace. The test
- %% case is dimensioned so that the log fills two files and just starts
- %% on the third (out of four wrap files). This gives two file swaps,
- %% and there are three processes, so one process will NOT be burdened.
- %% The criterion for trace success is then that the max process
- %% execution time must not be more than twice the min process
- %% execution time. Wallclock. A normal result is about 10 times more
- %% without schedule in - schedule out compensation (OTP-3938).
- %%
- ok = token_volleyball(3, 4, 8),
- %%
- {ok,[{matched,_,_}]} = dbg:p(all, [clear]),
- stop(),
- %%
- %% Get the trace result
- %%
- Tag = make_ref(),
- dbg:trace_client(file, {FName, wrap, ".wraplog"},
- {fun schedstat_handler/2, {self(), Tag, []}}),
- Result =
- receive
- {Tag, D} ->
- lists:map(
- fun({Pid, {A1, B1, C1}}) ->
- {Pid, C1/1000000 + B1 + A1*1000000}
- end,
- D)
- end,
- ok = io:format("Result=~p", [Result]),
- % erlang:display({?MODULE, ?LINE, Result}),
- %%
- %% Analyze the result
- %%
- {Min, Max} = lists:foldl(fun({_Pid, M}, {Mi, Ma}) ->
- {if M < Mi -> M; true -> Mi end,
- if M > Ma -> M; true -> Ma end}
- end,
- {void, 0},
- Result),
- % More PaN debug
- io:format("Min = ~f, Max = ~f~n",[Min,Max]),
- %%
- %% Cleanup
- %%
- ToBeDeleted = filelib:wildcard(FName++"*"++".wraplog"),
- lists:map(fun file:delete/1, ToBeDeleted),
- % io:format("ToBeDeleted=~p", [ToBeDeleted]),
- %%
- %% Present the result
- %%
- P = (Max / Min - 1) * 100,
- BottomLine = lists:flatten(io_lib:format("~.2f %", [P])),
- if P > 100 ->
- Reason = {BottomLine, '>', "100%"},
- erlang:display({file_port_schedfix, fail, Reason}),
- ct:fail(Reason);
- true ->
- {comment, BottomLine}
- end.
-
%% Test tracing to wrapping file port
wrap_port(Config) when is_list(Config) ->
Self = self(),
diff --git a/lib/runtime_tools/test/dyntrace_SUITE.erl b/lib/runtime_tools/test/dyntrace_SUITE.erl
index 7be2f49a8b..7ffbe54446 100644
--- a/lib/runtime_tools/test/dyntrace_SUITE.erl
+++ b/lib/runtime_tools/test/dyntrace_SUITE.erl
@@ -51,11 +51,7 @@ init_per_suite(Config) ->
case erlang:system_info(debug_compiled) of
false -> "";
true -> ".debug"
- end ++
- case erlang:system_info(smp_support) of
- false -> "";
- true -> ".smp"
- end,
+ end ++ ".smp",
[{emu_name,N}|Config].
end_per_suite(_Config) ->
diff --git a/lib/sasl/src/Makefile b/lib/sasl/src/Makefile
index ac7ee51100..45cd814bf8 100644
--- a/lib/sasl/src/Makefile
+++ b/lib/sasl/src/Makefile
@@ -37,7 +37,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/sasl-$(VSN)
MODULES= alarm_handler sasl sasl_report \
sasl_report_file_h sasl_report_tty_h format_lib_supp \
misc_supp rb rb_format_supp release_handler \
- release_handler_1 si si_sasl_supp systools \
+ release_handler_1 systools \
systools_make systools_rc systools_relup systools_lib \
erlsrv
diff --git a/lib/sasl/src/erlsrv.erl b/lib/sasl/src/erlsrv.erl
index 29d40d362f..e0bbd37ee3 100644
--- a/lib/sasl/src/erlsrv.erl
+++ b/lib/sasl/src/erlsrv.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1998-2016. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -34,7 +34,7 @@ erlsrv(EVer) ->
filename:join([Root, "erts-" ++ EVer, "bin", "erlsrv.exe"]).
current_version() ->
- hd(string:tokens(erlang:system_info(version),"_ ")).
+ hd(string:lexemes(erlang:system_info(version),"_ ")).
%%% Returns {ok, Output} | failed | {error, Reason}
run_erlsrv(Command) ->
@@ -107,7 +107,7 @@ get_all_services() ->
[];
{ok, [_H|T]} ->
F = fun(X) ->
- hd(string:tokens(X,"\t "))
+ hd(string:lexemes(X,"\t "))
end,
lists:map(F,T);
_ ->
@@ -191,8 +191,8 @@ get_service(EVer, ServiceName) ->
%%% have in the environment list...
EnvParts = lists:map(
fun(S) ->
- X = string:strip(S,left,$\t),
- case hd(string:tokens(X,"=")) of
+ X = string:trim(S, leading, "$\t"),
+ case hd(string:lexemes(X,"=")) of
X ->
%% Can this happen?
{X,""};
@@ -371,7 +371,7 @@ split_arglist([H|T]) ->
parse_arglist(Str) ->
lists:reverse(parse_arglist(Str,[])).
parse_arglist(Str,Accum) ->
- Stripped = string:strip(Str,left),
+ Stripped = string:trim(Str, leading),
case length(Stripped) of
0 ->
Accum;
@@ -432,14 +432,9 @@ split_by_env(Data) ->
splitline(Line) ->
- case string:chr(Line,$:) of
- 0 ->
+ case string:split(Line, ":") of
+ [_] ->
{Line, ""};
- N ->
- case length(string:substr(Line,N)) of
- 1 ->
- {string:substr(Line,1,N-1),""};
- _ ->
- {string:substr(Line,1,N-1),string:substr(Line,N+2)}
- end
+ [N, V] ->
+ {N, string:slice(V, 1)}
end.
diff --git a/lib/sasl/src/format_lib_supp.erl b/lib/sasl/src/format_lib_supp.erl
index 00ce1b4e33..2d37dfe117 100644
--- a/lib/sasl/src/format_lib_supp.erl
+++ b/lib/sasl/src/format_lib_supp.erl
@@ -102,13 +102,13 @@ print_newlines(Device, N) when N > 0 ->
print_one_line(Device, Line, Key, Value) ->
Modifier = misc_supp:modifier(Device),
StrKey = term_to_string(Key,Modifier),
- KeyLen = lists:min([length(StrKey), Line]),
+ KeyLen = lists:min([string:length(StrKey), Line]),
ValueLen = Line - KeyLen,
Format1 = lists:concat(["~-", KeyLen, Modifier, "s"]),
Format2 = lists:concat(["~", ValueLen, Modifier, "s~n"]),
io:format(Device, Format1, [StrKey]),
Try = term_to_string(Value,Modifier),
- Length = length(Try),
+ Length = string:length(Try),
if
Length < ValueLen ->
io:format(Device, Format2, [Try]);
@@ -119,7 +119,7 @@ print_one_line(Device, Line, Key, Value) ->
end.
term_to_string(Value,Modifier) ->
- lists:flatten(io_lib:format(get_format(Value,Modifier), [Value])).
+ io_lib:format(get_format(Value,Modifier), [Value]).
get_format([],_) ->
"~p";
diff --git a/lib/sasl/src/rb.erl b/lib/sasl/src/rb.erl
index 6595d29a9c..28829132a1 100644
--- a/lib/sasl/src/rb.erl
+++ b/lib/sasl/src/rb.erl
@@ -586,14 +586,14 @@ find_widths([], _Modifier, DescrWidth, DateWidth, Data) ->
{DescrWidth+1, DateWidth+1, lists:reverse(Data)};
find_widths([H|T], Modifier, DescrWidth, DateWidth, Data) ->
DescrTerm = element(3,H),
- Descr = lists:flatten(io_lib:format("~"++Modifier++"w", [DescrTerm])),
- DescrTry = length(Descr),
+ Descr = io_lib:format("~"++Modifier++"w", [DescrTerm]),
+ DescrTry = string:length(Descr),
NewDescrWidth =
if
DescrTry > DescrWidth -> DescrTry;
true -> DescrWidth
end,
- DateTry = length(element(4, H)),
+ DateTry = string:length(element(4, H)),
NewDateWitdh =
if
DateTry > DateWidth -> DateTry;
diff --git a/lib/sasl/src/rb_format_supp.erl b/lib/sasl/src/rb_format_supp.erl
index 1eda43dae4..b5b7aba151 100644
--- a/lib/sasl/src/rb_format_supp.erl
+++ b/lib/sasl/src/rb_format_supp.erl
@@ -108,7 +108,7 @@ print(Date, Report, Device) ->
format_h(Line, Header, Pid, Date) ->
NHeader = lists:flatten(io_lib:format("~s ~w", [Header, Pid])),
- DateLen = length(Date),
+ DateLen = string:length(Date),
HeaderLen = Line - DateLen,
Format = lists:concat(["~-", HeaderLen, "s~", DateLen, "s"]),
io_lib:format(Format, [NHeader, Date]).
diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl
index d0a7c7332d..bfa49fc05d 100644
--- a/lib/sasl/src/release_handler.erl
+++ b/lib/sasl/src/release_handler.erl
@@ -420,7 +420,7 @@ upgrade_app(App, NewDir) ->
%% located in the ebin dir of the _current_ version
%%-----------------------------------------------------------------
downgrade_app(App, OldDir) ->
- case string:tokens(filename:basename(OldDir), "-") of
+ case string:lexemes(filename:basename(OldDir), "-") of
[_AppS, OldVsn] ->
downgrade_app(App, OldVsn, OldDir);
_ ->
@@ -1174,8 +1174,8 @@ new_emulator_rm_tmp_release(_,_,_,_,Releases,_) ->
%% Rename the tempoarary service (for erts ugprade) to the real ToVsn
rename_tmp_service(EVsn,TmpVsn,NewVsn) ->
- FromName = hd(string:tokens(atom_to_list(node()),"@")) ++ "_" ++ TmpVsn,
- ToName = hd(string:tokens(atom_to_list(node()),"@")) ++ "_" ++ NewVsn,
+ FromName = hd(string:lexemes(atom_to_list(node()),"@")) ++ "_" ++ TmpVsn,
+ ToName = hd(string:lexemes(atom_to_list(node()),"@")) ++ "_" ++ NewVsn,
case erlsrv:get_service(EVsn,ToName) of
{error, _Error} ->
ok;
@@ -1207,9 +1207,9 @@ rename_service(EVsn,FromName,ToName) ->
%%% in which case we try to rename the old service to the new name and try
%%% to update heart's view of what service we are really running.
do_make_services_permanent(PermanentVsn,Vsn, PermanentEVsn, EVsn) ->
- PermName = hd(string:tokens(atom_to_list(node()),"@"))
+ PermName = hd(string:lexemes(atom_to_list(node()),"@"))
++ "_" ++ PermanentVsn,
- Name = hd(string:tokens(atom_to_list(node()),"@"))
+ Name = hd(string:lexemes(atom_to_list(node()),"@"))
++ "_" ++ Vsn,
case erlsrv:get_service(EVsn,Name) of
{error, _Error} ->
@@ -1296,7 +1296,7 @@ do_make_permanent(#state{releases = Releases,
do_back_service(OldVersion, CurrentVersion,OldEVsn,CurrentEVsn) ->
- NN = hd(string:tokens(atom_to_list(node()),"@")),
+ NN = hd(string:lexemes(atom_to_list(node()),"@")),
OldName = NN ++ "_" ++ OldVersion,
CurrentName = NN ++ "_" ++ CurrentVersion,
UpdData = case erlsrv:get_service(CurrentEVsn,CurrentName) of
@@ -1385,7 +1385,7 @@ do_remove_service(Vsn) ->
%% Very unconditionally remove the service.
%% Note that the service could already have been removed when
%% making another release permanent.
- ServiceName = hd(string:tokens(atom_to_list(node()),"@"))
+ ServiceName = hd(string:lexemes(atom_to_list(node()),"@"))
++ "_" ++ Vsn,
case erlsrv:get_service(ServiceName) of
{error, _Error} ->
@@ -1670,9 +1670,9 @@ flush() ->
prepare_restart_nt(#release{erts_vsn = EVsn, vsn = Vsn},
#release{erts_vsn = PermEVsn, vsn = PermVsn},
DataFileName) ->
- CurrentServiceName = hd(string:tokens(atom_to_list(node()),"@"))
+ CurrentServiceName = hd(string:lexemes(atom_to_list(node()),"@"))
++ "_" ++ PermVsn,
- FutureServiceName = hd(string:tokens(atom_to_list(node()),"@"))
+ FutureServiceName = hd(string:lexemes(atom_to_list(node()),"@"))
++ "_" ++ Vsn,
CurrentService = case erlsrv:get_service(PermEVsn,CurrentServiceName) of
{error, _} = Error1 ->
diff --git a/lib/sasl/src/sasl.app.src b/lib/sasl/src/sasl.app.src
index b7bb9133ff..1e8e58a978 100644
--- a/lib/sasl/src/sasl.app.src
+++ b/lib/sasl/src/sasl.app.src
@@ -32,8 +32,6 @@
sasl_report,
sasl_report_tty_h,
sasl_report_file_h,
- si,
- si_sasl_supp,
systools,
systools_make,
systools_rc,
diff --git a/lib/sasl/src/sasl.appup.src b/lib/sasl/src/sasl.appup.src
index dcb568c413..94af164b20 100644
--- a/lib/sasl/src/sasl.appup.src
+++ b/lib/sasl/src/sasl.appup.src
@@ -18,11 +18,7 @@
%% %CopyrightEnd%
{"%VSN%",
%% Up from - max one major revision back
- [{<<"3\\.0((\\.[0-3])(\\.[0-9]+)*)?">>,[restart_new_emulator]}, % OTP-19.*
- {<<"3\\.0\\.[4-9](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0*
- {<<"3\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.1*
+ [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.*
%% Down to - max one major revision back
- [{<<"3\\.0((\\.[0-3])(\\.[0-9]+)*)?">>,[restart_new_emulator]}, % OTP-19.*
- {<<"3\\.0\\.[4-9](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-20.0*
- {<<"3\\.1(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.1*
+ [{<<"3\\.0\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.*
}.
diff --git a/lib/sasl/src/si.erl b/lib/sasl/src/si.erl
deleted file mode 100644
index 275c6d508b..0000000000
--- a/lib/sasl/src/si.erl
+++ /dev/null
@@ -1,169 +0,0 @@
-%%
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%
-%% %CopyrightEnd%
-%%
-%%-----------------------------------------------------------------
-%% l(format_lib_supp), l(si_sasl_supp), l(si), l(si_ms_aos_supp), l(misc_supp).
-%% c(format_lib_supp), c(si_sasl_supp), c(si), c(si_ms_aos_supp), c(misc_supp).
-%%-----------------------------------------------------------------
-
-
-%%--------------------------------------------------
-%% Description:
-%% Status Inspection, main module.
-%%--------------------------------------------------
-
--module(si).
-
-
-%% External exports
--export([h/0, help/0, start/0, start/1, start_log/1, stop_log/0,
- abbrevs/0, pi/1, pi/2, pi/3, pi/4, ppi/1, ppi/3, stop/0]).
-
-%% Internal exports
--export([pi_impl/2, test/0]).
-
-
-%%--------------------------------------------------
-%% Table of contents
-%% 1. Interface
-%% 2. Implementation
-
-
--import(si_sasl_supp, [status_info/1, make_pid/1, p/1]).
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% 1. Interface
-%%--------------------------------------------------
-
-h() -> print_help().
-help() -> print_help().
-
-start() -> si_sasl_supp:start().
-start(Options) -> si_sasl_supp:start(Options).
-
-stop() -> si_sasl_supp:stop().
-
-start_log(FileName) ->
- si_sasl_supp:start_log(FileName).
-
-stop_log() ->
- si_sasl_supp:stop_log().
-
-%%%-----------------------------------------------------------------
-%%% All functions can be called with an option 'normal' or 'all';
-%%% default is 'normal'.
-%%%-----------------------------------------------------------------
-
-abbrevs() ->
- io:format("~p", [lists:append(si_sasl_supp:process_abbrevs(),
- process_abbrevs())]).
-
-%%-----------------------------------------------------------------
-%% Process Info that tries to determine processtype (=Module), then
-%% it uses this Module:format_info to format data from status_info/1.
-%%-----------------------------------------------------------------
-pi(XPid) ->
- si_sasl_supp:si_exec({si, pi_impl}, [normal, XPid]).
-
-pi(Opt, XPid) ->
- si_sasl_supp:si_exec({si, pi_impl}, [si_sasl_supp:valid_opt(Opt), XPid]).
-
-pi(A, B, C) when is_integer(A), is_integer(B), is_integer(C) ->
- si_sasl_supp:si_exec({si, pi_impl}, [normal, {A, B, C}]).
-
-pi(Opt, A, B, C) when is_integer(A), is_integer(B), is_integer(C) ->
- si_sasl_supp:si_exec({si, pi_impl}, [si_sasl_supp:valid_opt(Opt), {A, B, C}]).
-
-%%-----------------------------------------------------------------
-%% Pretty print Process_Info.
-%%-----------------------------------------------------------------
-ppi(XPid) ->
- si_sasl_supp:ppi(XPid).
-ppi(A, B, C) ->
- si_sasl_supp:ppi(A, B, C).
-
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% 2. Implementation
-%%--------------------------------------------------
-
-print_help() ->
- p("~nStatus Inspection tool - usage"),
- p("=============================="),
- p(" For all these functions, Opt is an optional argument"),
- p(" which can be 'normal' or 'all'; default is 'normal'."),
- p(" If 'all', all information will be printed."),
- p(" A Pid can be: \"<A.B.C>\", {A, B, C}, B, a registered_name or an abbrev."),
- p("ANY PROCESS"),
- p("si:pi([Opt,] Pid) - Formatted information about any process that"),
- p(" SI recognises."),
- p("si:pi([Opt,] A,B,C) - Same as si:pi({A, B, C})."),
- p("si:ppi(Pid) - Pretty formating of process_info."),
- p(" Works for any process."),
- p("MISC"),
- p("si:abbrevs() - Lists valid abbreviations."),
- p("si:start_log(Filename) - Logging to file."),
- p("si:stop_log()"),
- p("si:start() - Starts Status Inspection (the si_server)."),
- p("si:start([{start_log, FileName}])"),
- p("si:stop() - Shut down SI.").
-
-
-%%--------------------------------------------------
-%% Copied (and modified) code from si_sasl_supp.
-%%--------------------------------------------------
-pi_impl(Opt, XPid) ->
- case make_pid(try_local_expand_abbrev(XPid)) of
- Pid when is_pid(Pid) ->
- case status_info(Pid) of
- {status_info, Pid, {module, Module}, Data} ->
- si_sasl_supp:do_best_printout(Opt, Pid, Module, Data);
- {error, Reason} ->
- _ = si_sasl_supp:ppi_impl(Pid),
- {error, {"can not get status info from process:",
- XPid,
- Reason}};
- Else ->
- {error, {"unknown status info", Else}}
- end;
- {error, Reason} ->
- {error, Reason}
- end.
-
-%%--------------------------------------------------
-%% Functions for handling of abbreviations
-%%--------------------------------------------------
-try_local_expand_abbrev(Abbrev) ->
- case si_sasl_supp:expand_abbrev(Abbrev, process_abbrevs()) of
- {value, {_, RealName}} -> RealName;
- _ -> Abbrev
- end.
-
-process_abbrevs() ->
- [].
-
-%% Test get_status_info/format_status_info for all implemented servers.
-test() ->
- lists:foreach(fun test_all_registered/1,
- lists:append(si_sasl_supp:process_abbrevs(),
- process_abbrevs())).
-
-test_all_registered({Al, _Ful}) ->
- si:pi(all, Al).
diff --git a/lib/sasl/src/si_sasl_supp.erl b/lib/sasl/src/si_sasl_supp.erl
deleted file mode 100644
index cce628f8c4..0000000000
--- a/lib/sasl/src/si_sasl_supp.erl
+++ /dev/null
@@ -1,380 +0,0 @@
-%%
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 1996-2016. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%
-%% %CopyrightEnd%
-%%
--module(si_sasl_supp).
-
--behaviour(gen_server).
-
-%%%---------------------------------------------------------------------------
-%%% Description:
-%%% This module contains the BOS specific parts of the Status Inspection Tool.
-%%%---------------------------------------------------------------------------
-
-
-%% user interface
--export([h/0, help/0, start_log/1, stop_log/0, abbrevs/0, pi/1, pi/2, pi/3,
- pi/4, ppi/1, ppi/3, start/0, start/1, stop/0, start_link/1]).
-
-%% intermodule exports
--export([make_pid/1, make_pid/3, process_abbrevs/0, expand_abbrev/2,
- status_info/1, valid_opt/1, p/1, do_best_printout/4,
- si_exec/2, handle_call/3, terminate/2]).
-
-%% exports for use within module
--export([init/1, start_log_impl/1, pi_impl/2, ppi_impl/1]).
-
-%% other gen_server callbacks (not used)
--export([handle_cast/2, handle_info/2, code_change/3]).
-
-%%--------------------------------------------------
-%% Table of contents
-%% 1. Interface
-%% 2. SI - Server
-%% 3. Code
-%% 4. Selectors
-%%--------------------------------------------------
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% 1. Interface
-%% -----------------------------------------------------
-
-h() -> print_help().
-help() -> print_help().
-
-si_exec(Fun, Args) -> call({si_exec, Fun, Args}).
-
-start_log(FileName) ->
- call({start_log, FileName}).
-
-stop_log() ->
- call(stop_log).
-
-abbrevs() ->
- io:format("~p", [process_abbrevs()]).
-
-%%-----------------------------------------------------------------
-%% All functions can be called with an option 'normal' or 'all';
-%% default is 'normal'.
-%%-----------------------------------------------------------------
-%% Process Info that tries to determine processtype (=Module), then
-%% it uses this Module:format_info to format data from status_info/1.
-%%-----------------------------------------------------------------
-pi(XPid) ->
- si_exec({si_sasl_supp, pi_impl}, [normal, XPid]).
-
-pi(Opt, XPid) ->
- si_exec({si_sasl_supp, pi_impl}, [valid_opt(Opt), XPid]).
-
-pi(A, B, C) when is_integer(A), is_integer(B), is_integer(C) ->
- si_exec({si_sasl_supp, pi_impl}, [normal, {A, B, C}]).
-
-pi(Opt, A, B, C) when is_integer(A), is_integer(B), is_integer(C) ->
- si_exec({si_sasl_supp, pi_impl}, [valid_opt(Opt), {A, B, C}]).
-
-%%-----------------------------------------------------------------
-%% Pretty print Process_Info.
-%%-----------------------------------------------------------------
-ppi(XPid) ->
- case whereis(si_server) of
- undefined -> % You can always run ppi.
- ppi_impl(XPid); % if si_server is down, use standard_io
- _ ->
- si_exec({si_sasl_supp, ppi_impl}, [XPid])
- end.
-ppi(A, B, C) ->
- case whereis(si_server) of
- undefined -> % You can always run ppi.
- ppi_impl({A, B, C}); % if si_server is down, use standard_io
- _ ->
- si_exec({si_sasl_supp, ppi_impl}, [{A, B, C}])
- end.
-
-
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% 2. SI - Server
-%%--------------------------------------------------
--record(state, {}).
-
-start() -> start([]).
-start(Options) ->
- supervisor:start_child(sasl_sup,
- {si_server, {si_sasl_supp, start_link, [Options]},
- temporary, brutal_kill, worker, [si_sasl_supp]}).
-
-start_link(_Options) ->
- gen_server:start_link({local, si_server}, si_sasl_supp, [], []).
-
-stop() ->
- call(stop),
- supervisor:delete_child(sasl_sup, si_server).
-
-
-init(Options) ->
- process_flag(trap_exit, true),
- start_log_impl(get_option(Options, start_log, standard_io)),
- {ok, #state{}}.
-
-%%-----------------------------------------------------------------
-%% If an error occurs and we're logging to file: write the error
-%% to the file.
-%% Always return the error.
-%% The only data held by the si_server is the device in its process dictionary.
-%%-----------------------------------------------------------------
-handle_call({si_exec, Fun, Args}, _From, State) ->
- case catch apply(Fun, Args) of
- {'EXIT', Reason} ->
- print_error(get(device),
- "SI internal error. Reason: ~w~n",
- [Reason]),
- {stop, shutdown, {internal_error, Reason}, State};
- {error, Reason} ->
- print_error(get(device), "~nSI error: ~w~n", [Reason]),
- {reply, {error, Reason}, State};
- X ->
- {reply, X, State}
- end;
-handle_call({start_log, FileName}, _From, State) ->
- start_log_impl(FileName),
- {reply, ok, State};
-handle_call(stop_log, _From, State) ->
- start_log_impl(standard_io),
- {reply, ok, State};
-handle_call(stop, _From, State) ->
- start_log_impl(standard_io),
- {stop, normal, stopped, State}.
-
-terminate(_Reason, _State) ->
- _ = close_device(get(device)),
- ok.
-
-handle_cast(_Msg, State) ->
- {noreply, State}.
-handle_info(_Info, State) ->
- {noreply, State}.
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
-close_device(standard_io) -> ok;
-close_device(Fd) -> file:close(Fd).
-
-print_error(standard_io, _, _) -> ok;
-print_error(Device, Format, Args) ->
- io:format(Device, Format, Args).
-
-get_option(Options, Key, Default) ->
- case lists:keysearch(Key, 1, Options) of
- {value, {_Key, Value}} -> Value;
- _ -> Default
- end.
-
-open_log_file(undefined, NewFile) ->
- open_log_file(NewFile);
-open_log_file(standard_io, NewFile) ->
- open_log_file(NewFile);
-open_log_file(OldFile, NewFile) ->
- _ = file:close(OldFile),
- open_log_file(NewFile).
-
-open_log_file(standard_io) -> standard_io;
-open_log_file(FileName) ->
- case file:open(FileName, [write]) of
- {ok, Fd} -> Fd;
- Error ->
- io:format("si_sasl_supp: Cannot open file '~s' (~w).~n",
- [FileName, Error]),
- io:format("si_sasl_supp: Using standard_io~n"),
- standard_io
- end.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% 3. Code
-%%--------------------------------------------------
-
-%%-----------------------------------------------------------------
-%% call(Request) -> Term
-%%-----------------------------------------------------------------
-call(Req) ->
- gen_server:call(si_server, Req, infinity).
-
-%%--------------------------------------------------
-%% Makes a Pid of almost anything.
-%% Returns: Pid|{error, Reason}
-%% Fails: Never.
-%%--------------------------------------------------
-make_pid(A,B,C) when is_integer(A), is_integer(B), is_integer(C) ->
- list_to_pid(lists:concat(["<",A,".",B,".",C,">"])).
-make_pid(P) when is_pid(P) -> P;
-make_pid(undefined) -> undefined;
-make_pid(P) when is_atom(P) ->
- case whereis(P) of
- undefined ->
- case expand_abbrev(P, process_abbrevs()) of
- {error, Reason} -> {error, Reason};
- {value, {_Abbrev, FullName}} ->
- case whereis(FullName) of
- undefined ->
- {error, {'process not registered', P}};
- Pid -> Pid
- end
- end;
- Pid -> Pid
- end;
-make_pid(P) when is_list(P) -> list_to_pid(P);
-make_pid({A, B, C}) -> make_pid(A, B, C);
-make_pid(X) -> {error, {'can not make a pid of', X}}.
-
-process_abbrevs() ->
- [{init, init},
- {fs, file_server}].
-
-%%--------------------------------------------------
-%% Args: Abbrevs is an assoc-list of {Abbrev, RealName}
-%% Returns: {value, {Abbrev, FullName}}|{error, Reason}
-%%--------------------------------------------------
-expand_abbrev(ProcessName, Abbrevs) ->
- case lists:keysearch(ProcessName, 1, Abbrevs) of
- {value, {Abbrev, FullName}} ->
- {value, {Abbrev, FullName}};
- _ ->
- case lists:keysearch(ProcessName, 2, Abbrevs) of
- {value, {Abbrev, FullName}} ->
- {value, {Abbrev, FullName}};
- _ ->
- {error, {'invalid process name', ProcessName}}
- end
- end.
-
-%%-----------------------------------------------------------------
-%% This is the function that actually gets the information out
-%% of the agent/server/...
-%% Returns: {status_info, Pid, Type, Data}
-%% | {error, Reason}
-%%-----------------------------------------------------------------
-status_info(Pid) when is_pid(Pid) ->
- case catch sys:get_status(Pid, 5000) of
- {status, Pid, Type, Info} ->
- {status_info, Pid, Type, Info};
- _ ->
- {error, {'process does not respond', Pid}}
- end;
-
-status_info(X) ->
- {error, {'not a pid', X}}.
-
-%%--------------------------------------------------
-%% Implementation starts here.
-%%--------------------------------------------------
-start_log_impl(FileName) ->
- put(device, open_log_file(get(device), FileName)).
-
-valid_opt(all) -> all;
-valid_opt(_Opt) -> normal.
-
-
-print_help() ->
- p("- - - - - - - - PROCESSES - - - - - - - - - "),
- p("si_sasl_supp:pi([Opt,] Pid) - Formatted information about any process that"),
- p(" SI recognises."),
- p("si_sasl_supp:pi([Opt,] A,B,C) - Same as si_sasl_supp:pi({A, B, C})."),
- p("si_sasl_supp:ppi(Pid) - Pretty formating of process_info."),
- p(" Works for any process."),
- p("- - - - - - - - MISC - - - - - - - - - - - "),
- p("si_sasl_supp:abbrevs() - Lists valid abbreviations."),
- p("si_sasl_supp:start_log(FileNname)"),
- p("si_sasl_supp:stop_log()"),
- p("si_sasl_supp:start() - Starts Status Inspection (the si_server)."),
- p("si_sasl_supp:start([{start_log, FileName}])"),
- p("si_sasl_supp:stop() - Shut down SI.").
-
-
-
-%% Convenient shorthand
-p(X) ->
- io:format(lists:append(X, "~n")).
-
-pi_impl(Opt, XPid) ->
- case make_pid(XPid) of
- Pid when is_pid(Pid) ->
- case status_info(Pid) of
- {status_info, Pid, {module, Module}, Data} ->
- do_best_printout(Opt, Pid, Module, Data);
- {error, Reason} ->
- _ = ppi_impl(Pid),
- {error, {"can not get status info from process:",
- XPid,
- Reason}}
- end;
- {error, Reason} ->
- {error, Reason}
- end.
-
-%%--------------------------------------------------
-%% Is there a format_info for this process? In that case, run it!
-%% Return ok|{error, Reason}
-%% Fails: Never.
-%%--------------------------------------------------
-do_best_printout(Opt, Pid, Mod, Data) when is_pid(Pid) ->
- case print_info(get(device), Pid, {Mod, format_status}, Opt, Data) of
- ok -> ok;
- {error, Reason} ->
- _ = ppi_impl(Pid),
- {error, Reason}
- end.
-
-ppi_impl(XPid) ->
- case make_pid(XPid) of
- P when is_pid(P) ->
- case process_info(P) of
- undefined ->
- {error, {'dead process', P}};
- PI ->
- Device = case get(device) of
- undefined -> standard_io;
- X -> X
- end,
- io:format(Device, "~nPretty Process Info~n", []),
- io:format(Device, "-------------------~n", []),
- io:format(Device, "~p~n", [PI])
- end;
- _ -> {error, {no_pid, XPid}}
- end.
-
-print_info(Device, Pid, {Module, Func}, Opt, Data) ->
- case erlang:function_exported(Module, Func, 2) of
- true ->
- case catch apply(Module, Func, [Opt, Data]) of
- Format when is_list(Format) ->
- format_lib_supp:print_info(Device, 79,
- add_pid_to_format(Pid, Format)),
- ok;
- Other -> {error, {'invalid format', Other}}
- end;
- _ ->
- {error, {no_such_function, Module, Func}}
- end.
-
-add_pid_to_format(Pid, [{header, H} | T]) ->
- [{header, H}, {data, [{"Pid", Pid}]} | T];
-add_pid_to_format(Pid, List) ->
- [{data, [{"Pid", Pid}]} | List].
-
-
diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl
index 50932e89e4..63b48e7a4e 100644
--- a/lib/sasl/test/release_handler_SUITE.erl
+++ b/lib/sasl/test/release_handler_SUITE.erl
@@ -1835,24 +1835,32 @@ otp_10463_upgrade_script_regexp(cleanup,Config) ->
code:del_path(filename:join([DataDir,regexp_appup,app1,ebin])),
ok.
-no_dot_erlang(Conf) ->
- PrivDir = ?config(data_dir,Conf),
- {ok, OrigWd} = file:get_cwd(),
- try
- ok = file:set_cwd(PrivDir),
-
- {ok, Wd} = file:get_cwd(),
- io:format("Dir ~ts~n", [Wd]),
+no_dot_erlang(_Conf) ->
+ case init:get_argument(home) of
+ {ok,[[Home]]} when is_list(Home) ->
+ no_dot_erlang_1(Home);
+ _ -> ok
+ end.
+no_dot_erlang_1(Home) ->
+ DotErlang = filename:join(Home, ".erlang"),
+ BupErlang = filename:join(Home, ".erlang_testbup"),
+ try
+ {ok, Wd} = file:get_cwd(),
+ case filelib:is_file(DotErlang) of
+ true -> {ok, _} = file:copy(DotErlang, BupErlang);
+ false -> ok
+ end,
Erl0 = filename:join([code:root_dir(),"bin","erl"]),
Erl = filename:nativename(Erl0),
Quote = "\"",
Args = " -noinput -run c pwd -run erlang halt",
- ok = file:write_file(".erlang", <<"io:put_chars(\"DOT_ERLANG_READ\\n\").\n">>),
+ ok = file:write_file(DotErlang, <<"io:put_chars(\"DOT_ERLANG_READ\\n\").\n">>),
CMD1 = Quote ++ Erl ++ Quote ++ Args ,
case os:cmd(CMD1) of
- "DOT_ERLANG_READ" ++ _ -> ok;
+ "DOT_ERLANG_READ" ++ _ ->
+ io:format("~p: Success~n", [?LINE]);
Other1 ->
io:format("Failed: ~ts~n",[CMD1]),
io:format("Expected: ~s ++ _~n",["DOT_ERLANG_READ "]),
@@ -1862,7 +1870,7 @@ no_dot_erlang(Conf) ->
NO_DOT_ERL = " -boot no_dot_erlang",
CMD2 = Quote ++ Erl ++ Quote ++ NO_DOT_ERL ++ Args,
case lists:prefix(Wd, Other2 = os:cmd(CMD2)) of
- true -> ok;
+ true -> io:format("~p: Success~n", [?LINE]);
false ->
io:format("Failed: ~ts~n",[CMD2]),
io:format("Expected: ~s~n",["TESTOK"]),
@@ -1870,9 +1878,13 @@ no_dot_erlang(Conf) ->
exit({failed_to_start, no_dot_erlang})
end
after
- _ = file:delete(".erlang"),
- ok = file:set_cwd(OrigWd),
- ok
+ case filelib:is_file(BupErlang) of
+ true ->
+ {ok, _} = file:copy(BupErlang, DotErlang),
+ _ = file:delete(BupErlang);
+ false ->
+ _ = file:delete(DotErlang)
+ end
end.
%%%-----------------------------------------------------------------
diff --git a/lib/sasl/test/test_lib.hrl b/lib/sasl/test/test_lib.hrl
index 9a54937f96..f5210d4f27 100644
--- a/lib/sasl/test/test_lib.hrl
+++ b/lib/sasl/test/test_lib.hrl
@@ -1,3 +1,3 @@
-define(ertsvsn,"4.4").
--define(kernelvsn,"5.0").
--define(stdlibvsn,"3.0").
+-define(kernelvsn,"5.3").
+-define(stdlibvsn,"3.4").
diff --git a/lib/ssl/src/Makefile b/lib/ssl/src/Makefile
index 2e7df9792e..8eba5cf347 100644
--- a/lib/ssl/src/Makefile
+++ b/lib/ssl/src/Makefile
@@ -1,7 +1,7 @@
#
# %CopyrightBegin%
#
-# Copyright Ericsson AB 1999-2016. All Rights Reserved.
+# Copyright Ericsson AB 1999-2017. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -87,8 +87,7 @@ MODULES= \
ssl_v2 \
ssl_v3 \
tls_v1 \
- dtls_v1 \
- ssl_tls_dist_proxy
+ dtls_v1
INTERNAL_HRL_FILES = \
ssl_alert.hrl ssl_cipher.hrl \
diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl
index 7d2605e013..387d632ef8 100644
--- a/lib/ssl/src/dtls_connection.erl
+++ b/lib/ssl/src/dtls_connection.erl
@@ -551,7 +551,7 @@ handle_info(new_cookie_secret, StateName,
CookieInfo#{current_cookie_secret => dtls_v1:cookie_secret(),
previous_cookie_secret => Secret}}};
handle_info(Msg, StateName, State) ->
- ssl_connection:handle_info(Msg, StateName, State).
+ ssl_connection:StateName(info, Msg, State, ?MODULE).
handle_call(Event, From, StateName, State) ->
ssl_connection:handle_call(Event, From, StateName, State, ?MODULE).
diff --git a/lib/ssl/src/dtls_udp_listener.erl b/lib/ssl/src/dtls_udp_listener.erl
index c789a32087..c9e04767aa 100644
--- a/lib/ssl/src/dtls_udp_listener.erl
+++ b/lib/ssl/src/dtls_udp_listener.erl
@@ -35,7 +35,7 @@
-record(state,
{port,
- listner,
+ listener,
dtls_options,
emulated_options,
dtls_msq_queues = kv_new(),
@@ -81,7 +81,7 @@ init([Port, EmOpts, InetOptions, DTLSOptions]) ->
first = true,
dtls_options = DTLSOptions,
emulated_options = EmOpts,
- listner = Socket,
+ listener = Socket,
close = false}}
catch _:_ ->
{error, closed}
@@ -91,7 +91,7 @@ handle_call({accept, _}, _, #state{close = true} = State) ->
handle_call({accept, Accepter}, From, #state{first = true,
accepters = Accepters,
- listner = Socket} = State0) ->
+ listener = Socket} = State0) ->
next_datagram(Socket),
State = State0#state{first = false,
accepters = queue:in({Accepter, From}, Accepters)},
@@ -100,7 +100,7 @@ handle_call({accept, Accepter}, From, #state{first = true,
handle_call({accept, Accepter}, From, #state{accepters = Accepters} = State0) ->
State = State0#state{accepters = queue:in({Accepter, From}, Accepters)},
{noreply, State};
-handle_call(sockname, _, #state{listner = Socket} = State) ->
+handle_call(sockname, _, #state{listener = Socket} = State) ->
Reply = inet:sockname(Socket),
{reply, Reply, State};
handle_call(close, _, #state{dtls_processes = Processes,
@@ -114,7 +114,7 @@ handle_call(close, _, #state{dtls_processes = Processes,
end, queue:to_list(Accepters)),
{reply, ok, State#state{close = true, accepters = queue:new()}}
end;
-handle_call({get_sock_opts, {SocketOptNames, EmOptNames}}, _, #state{listner = Socket,
+handle_call({get_sock_opts, {SocketOptNames, EmOptNames}}, _, #state{listener = Socket,
emulated_options = EmOpts} = State) ->
case get_socket_opts(Socket, SocketOptNames) of
{ok, Opts} ->
@@ -125,7 +125,7 @@ handle_call({get_sock_opts, {SocketOptNames, EmOptNames}}, _, #state{listner = S
handle_call(get_all_opts, _, #state{dtls_options = DTLSOptions,
emulated_options = EmOpts} = State) ->
{reply, {ok, EmOpts, DTLSOptions}, State};
-handle_call({set_sock_opts, {SocketOpts, NewEmOpts}}, _, #state{listner = Socket, emulated_options = EmOpts0} = State) ->
+handle_call({set_sock_opts, {SocketOpts, NewEmOpts}}, _, #state{listener = Socket, emulated_options = EmOpts0} = State) ->
set_socket_opts(Socket, SocketOpts),
EmOpts = do_set_emulated_opts(NewEmOpts, EmOpts0),
{reply, ok, State#state{emulated_options = EmOpts}}.
@@ -134,7 +134,7 @@ handle_cast({active_once, Client, Pid}, State0) ->
State = handle_active_once(Client, Pid, State0),
{noreply, State}.
-handle_info({udp, Socket, IP, InPortNo, _} = Msg, #state{listner = Socket} = State0) ->
+handle_info({udp, Socket, IP, InPortNo, _} = Msg, #state{listener = Socket} = State0) ->
State = handle_datagram({IP, InPortNo}, Msg, State0),
next_datagram(Socket),
{noreply, State};
@@ -142,11 +142,11 @@ handle_info({udp, Socket, IP, InPortNo, _} = Msg, #state{listner = Socket} = Sta
%% UDP socket does not have a connection and should not receive an econnreset
%% This does however happens on on some windows versions. Just ignoring it
%% appears to make things work as expected!
-handle_info({udp_error, Socket, econnreset = Error}, #state{listner = Socket} = State) ->
+handle_info({udp_error, Socket, econnreset = Error}, #state{listener = Socket} = State) ->
Report = io_lib:format("Ignore SSL UDP Listener: Socket error: ~p ~n", [Error]),
error_logger:info_report(Report),
{noreply, State};
-handle_info({udp_error, Socket, Error}, #state{listner = Socket} = State) ->
+handle_info({udp_error, Socket, Error}, #state{listener = Socket} = State) ->
Report = io_lib:format("SSL UDP Listener shutdown: Socket error: ~p ~n", [Error]),
error_logger:info_report(Report),
{noreply, State#state{close=true}};
@@ -225,10 +225,10 @@ setup_new_connection(User, From, Client, Msg, #state{dtls_processes = Processes,
dtls_msq_queues = MsgQueues,
dtls_options = DTLSOpts,
port = Port,
- listner = Socket,
+ listener = Socket,
emulated_options = EmOpts} = State) ->
ConnArgs = [server, "localhost", Port, {self(), {Client, Socket}},
- {DTLSOpts, EmOpts, udp_listner}, User, dtls_socket:default_cb_info()],
+ {DTLSOpts, EmOpts, udp_listener}, User, dtls_socket:default_cb_info()],
case dtls_connection_sup:start_child(ConnArgs) of
{ok, Pid} ->
erlang:monitor(process, Pid),
diff --git a/lib/ssl/src/inet6_tls_dist.erl b/lib/ssl/src/inet6_tls_dist.erl
index ffd7296f93..96ce4d493a 100644
--- a/lib/ssl/src/inet6_tls_dist.erl
+++ b/lib/ssl/src/inet6_tls_dist.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2015. All Rights Reserved.
+%% Copyright Ericsson AB 2015-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -21,7 +21,8 @@
%%
-module(inet6_tls_dist).
--export([childspecs/0, listen/1, accept/1, accept_connection/5,
+-export([childspecs/0]).
+-export([listen/1, accept/1, accept_connection/5,
setup/5, close/1, select/1]).
childspecs() ->
@@ -43,4 +44,4 @@ setup(Node, Type, MyNode, LongOrShortNames,SetupTime) ->
inet_tls_dist:gen_setup(inet6_tcp, Node, Type, MyNode, LongOrShortNames,SetupTime).
close(Socket) ->
- inet_tls_dist:close(Socket).
+ inet_tls_dist:gen_close(inet6_tcp, Socket).
diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl
index 78094c474b..96782dcfc0 100644
--- a/lib/ssl/src/inet_tls_dist.erl
+++ b/lib/ssl/src/inet_tls_dist.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2011-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2011-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -21,17 +21,26 @@
%%
-module(inet_tls_dist).
--export([childspecs/0, listen/1, accept/1, accept_connection/5,
+-export([childspecs/0]).
+-export([listen/1, accept/1, accept_connection/5,
setup/5, close/1, select/1, is_node_name/1]).
%% Generalized dist API
-export([gen_listen/2, gen_accept/2, gen_accept_connection/6,
- gen_setup/6, gen_select/2]).
+ gen_setup/6, gen_close/2, gen_select/2]).
+
+-export([split_node/1, nodelay/0]).
+
+-export([dbg/0]). % Debug
-include_lib("kernel/include/net_address.hrl").
-include_lib("kernel/include/dist.hrl").
-include_lib("kernel/include/dist_util.hrl").
+-include("ssl_api.hrl").
+
+%% -------------------------------------------------------------------------
+
childspecs() ->
{ok, [{ssl_dist_sup,{ssl_dist_sup, start_link, []},
permanent, infinity, supervisor, [ssl_dist_sup]}]}.
@@ -40,111 +49,366 @@ select(Node) ->
gen_select(inet_tcp, Node).
gen_select(Driver, Node) ->
- case split_node(atom_to_list(Node), $@, []) of
- [_, Host] ->
- case inet:getaddr(Host, Driver:family()) of
+ case split_node(Node) of
+ false ->
+ false;
+ Host ->
+ case Driver:getaddr(Host) of
{ok, _} -> true;
_ -> false
- end;
- _ ->
- false
+ end
end.
-is_node_name(Node) when is_atom(Node) ->
- select(Node);
-is_node_name(_) ->
- false.
+%% -------------------------------------------------------------------------
+
+is_node_name(Node) ->
+ case split_node(Node) of
+ false ->
+ false;
+ _Host ->
+ true
+ end.
+
+%% -------------------------------------------------------------------------
+
+hs_data_common(#sslsocket{pid = DistCtrl} = SslSocket) ->
+ #hs_data{
+ f_send =
+ fun (Ctrl, Packet) when Ctrl == DistCtrl ->
+ f_send(SslSocket, Packet)
+ end,
+ f_recv =
+ fun (Ctrl, Length, Timeout) when Ctrl == DistCtrl ->
+ f_recv(SslSocket, Length, Timeout)
+ end,
+ f_setopts_pre_nodeup =
+ fun (Ctrl) when Ctrl == DistCtrl ->
+ f_setopts_pre_nodeup(SslSocket)
+ end,
+ f_setopts_post_nodeup =
+ fun (Ctrl) when Ctrl == DistCtrl ->
+%%% sys:trace(Ctrl, true),
+ f_setopts_post_nodeup(SslSocket)
+ end,
+ f_getll =
+ fun (Ctrl) when Ctrl == DistCtrl ->
+ f_getll(DistCtrl)
+ end,
+ f_address =
+ fun (Ctrl, Node) when Ctrl == DistCtrl ->
+ f_address(SslSocket, Node)
+ end,
+ mf_tick =
+ fun (Ctrl) when Ctrl == DistCtrl ->
+ mf_tick(DistCtrl)
+ end,
+ mf_getstat =
+ fun (Ctrl) when Ctrl == DistCtrl ->
+ mf_getstat(SslSocket)
+ end,
+ mf_setopts =
+ fun (Ctrl, Opts) when Ctrl == DistCtrl ->
+ mf_setopts(SslSocket, Opts)
+ end,
+ mf_getopts =
+ fun (Ctrl, Opts) when Ctrl == DistCtrl ->
+ mf_getopts(SslSocket, Opts)
+ end,
+ f_handshake_complete =
+ fun (Ctrl, Node, DHandle) when Ctrl == DistCtrl ->
+ f_handshake_complete(DistCtrl, Node, DHandle)
+ end}.
+
+f_send(SslSocket, Packet) ->
+ ssl:send(SslSocket, Packet).
+
+f_recv(SslSocket, Length, Timeout) ->
+ case ssl:recv(SslSocket, Length, Timeout) of
+ {ok, Bin} when is_binary(Bin) ->
+ {ok, binary_to_list(Bin)};
+ Other ->
+ Other
+ end.
+
+f_setopts_pre_nodeup(_SslSocket) ->
+ ok.
+
+f_setopts_post_nodeup(_SslSocket) ->
+ ok.
+
+f_getll(DistCtrl) ->
+ {ok, DistCtrl}.
+
+f_address(SslSocket, Node) ->
+ case ssl:peername(SslSocket) of
+ {ok, Address} ->
+ case split_node(Node) of
+ false ->
+ {error, no_node};
+ Host ->
+ #net_address{
+ address=Address, host=Host,
+ protocol=tls, family=inet}
+ end
+ end.
+
+mf_tick(DistCtrl) ->
+ DistCtrl ! tick,
+ ok.
+
+mf_getstat(SslSocket) ->
+ case ssl:getstat(
+ SslSocket, [recv_cnt, send_cnt, send_pend]) of
+ {ok, Stat} ->
+ split_stat(Stat,0,0,0);
+ Error ->
+ Error
+ end.
+
+mf_setopts(SslSocket, Opts) ->
+ case setopts_filter(Opts) of
+ [] ->
+ ssl:setopts(SslSocket, Opts);
+ Opts1 ->
+ {error, {badopts,Opts1}}
+ end.
+
+mf_getopts(SslSocket, Opts) ->
+ ssl:getopts(SslSocket, Opts).
+
+f_handshake_complete(DistCtrl, Node, DHandle) ->
+ ssl_connection:handshake_complete(DistCtrl, Node, DHandle).
+
+
+setopts_filter(Opts) ->
+ [Opt || {K,_} = Opt <- Opts,
+ K =:= active orelse K =:= deliver orelse K =:= packet].
+
+split_stat([{recv_cnt, R}|Stat], _, W, P) ->
+ split_stat(Stat, R, W, P);
+split_stat([{send_cnt, W}|Stat], R, _, P) ->
+ split_stat(Stat, R, W, P);
+split_stat([{send_pend, P}|Stat], R, W, _) ->
+ split_stat(Stat, R, W, P);
+split_stat([], R, W, P) ->
+ {ok, R, W, P}.
+
+%% -------------------------------------------------------------------------
listen(Name) ->
gen_listen(inet_tcp, Name).
gen_listen(Driver, Name) ->
- ssl_tls_dist_proxy:listen(Driver, Name).
+ case inet_tcp_dist:gen_listen(Driver, Name) of
+ {ok, {Socket, Address, Creation}} ->
+ inet:setopts(Socket, [{packet, 4}]),
+ {ok, {Socket, Address#net_address{protocol=tls}, Creation}};
+ Other ->
+ Other
+ end.
+
+%% -------------------------------------------------------------------------
accept(Listen) ->
gen_accept(inet_tcp, Listen).
gen_accept(Driver, Listen) ->
- ssl_tls_dist_proxy:accept(Driver, Listen).
+ Kernel = self(),
+ monitor_pid(
+ spawn_opt(
+ fun () ->
+ accept_loop(Driver, Listen, Kernel)
+ end,
+ [link, {priority, max}])).
-accept_connection(AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
- gen_accept_connection(inet_tcp, AcceptPid, Socket, MyNode, Allowed, SetupTime).
+accept_loop(Driver, Listen, Kernel) ->
+ case Driver:accept(Listen) of
+ {ok, Socket} ->
+ Opts = get_ssl_options(server),
+ wait_for_code_server(),
+ case ssl:ssl_accept(
+ Socket, [{active, false}, {packet, 4}] ++ Opts,
+ net_kernel:connecttime()) of
+ {ok, #sslsocket{pid = DistCtrl} = SslSocket} ->
+ monitor_pid(DistCtrl),
+ trace(
+ Kernel !
+ {accept, self(), DistCtrl,
+ Driver:family(), tls}),
+ receive
+ {Kernel, controller, Pid} ->
+ ok = ssl:controlling_process(SslSocket, Pid),
+ trace(
+ Pid ! {self(), controller});
+ {Kernel, unsupported_protocol} ->
+ exit(trace(unsupported_protocol))
+ end,
+ accept_loop(Driver, Listen, Kernel);
+ {error, {options, _}} = Error ->
+ %% Bad options: that's probably our fault.
+ %% Let's log that.
+ error_logger:error_msg(
+ "Cannot accept TLS distribution connection: ~s~n",
+ [ssl:format_error(Error)]),
+ _ = trace(Error),
+ gen_tcp:close(Socket);
+ Other ->
+ _ = trace(Other),
+ gen_tcp:close(Socket)
+ end;
+ Error ->
+ exit(trace(Error))
+ end,
+ accept_loop(Driver, Listen, Kernel).
-gen_accept_connection(Driver, AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
+wait_for_code_server() ->
+ %% This is an ugly hack. Upgrading a socket to TLS requires the
+ %% crypto module to be loaded. Loading the crypto module triggers
+ %% its on_load function, which calls code:priv_dir/1 to find the
+ %% directory where its NIF library is. However, distribution is
+ %% started earlier than the code server, so the code server is not
+ %% necessarily started yet, and code:priv_dir/1 might fail because
+ %% of that, if we receive an incoming connection on the
+ %% distribution port early enough.
+ %%
+ %% If the on_load function of a module fails, the module is
+ %% unloaded, and the function call that triggered loading it fails
+ %% with 'undef', which is rather confusing.
+ %%
+ %% Thus, the accept process will terminate, and be
+ %% restarted by ssl_dist_sup. However, it won't have any memory
+ %% of being asked by net_kernel to listen for incoming
+ %% connections. Hence, the node will believe that it's open for
+ %% distribution, but it actually isn't.
+ %%
+ %% So let's avoid that by waiting for the code server to start.
+ case whereis(code_server) of
+ undefined ->
+ timer:sleep(10),
+ wait_for_code_server();
+ Pid when is_pid(Pid) ->
+ ok
+ end.
+
+%% -------------------------------------------------------------------------
+
+accept_connection(AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) ->
+ gen_accept_connection(
+ inet_tcp, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime).
+
+gen_accept_connection(
+ Driver, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) ->
Kernel = self(),
- spawn_link(fun() -> do_accept(Driver, Kernel, AcceptPid, Socket,
- MyNode, Allowed, SetupTime) end).
+ monitor_pid(
+ spawn_opt(
+ fun() ->
+ do_accept(
+ Driver, Kernel, AcceptPid, DistCtrl,
+ MyNode, Allowed, SetupTime)
+ end,
+ [link, {priority, max}])).
+
+do_accept(Driver, Kernel, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) ->
+ SslSocket = ssl_connection:get_sslsocket(DistCtrl),
+ receive
+ {AcceptPid, controller} ->
+ Timer = dist_util:start_timer(SetupTime),
+ case check_ip(Driver, SslSocket) of
+ true ->
+ HSData0 = hs_data_common(SslSocket),
+ HSData =
+ HSData0#hs_data{
+ kernel_pid = Kernel,
+ this_node = MyNode,
+ socket = DistCtrl,
+ timer = Timer,
+ this_flags = 0,
+ allowed = Allowed},
+ dist_util:handshake_other_started(trace(HSData));
+ {false,IP} ->
+ error_logger:error_msg(
+ "** Connection attempt from "
+ "disallowed IP ~w ** ~n", [IP]),
+ ?shutdown(trace(no_node))
+ end
+ end.
+
+
-setup(Node, Type, MyNode, LongOrShortNames,SetupTime) ->
- gen_setup(inet_tcp, Node, Type, MyNode, LongOrShortNames,SetupTime).
+setup(Node, Type, MyNode, LongOrShortNames, SetupTime) ->
+ gen_setup(inet_tcp, Node, Type, MyNode, LongOrShortNames, SetupTime).
-gen_setup(Driver, Node, Type, MyNode, LongOrShortNames,SetupTime) ->
+gen_setup(Driver, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
Kernel = self(),
- spawn_opt(fun() -> do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) end, [link, {priority, max}]).
-
+ monitor_pid(
+ spawn_opt(
+ fun() ->
+ do_setup(
+ Driver, Kernel, Node, Type,
+ MyNode, LongOrShortNames, SetupTime)
+ end,
+ [link, {priority, max}])).
+
do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
[Name, Address] = splitnode(Driver, Node, LongOrShortNames),
- case inet:getaddr(Address, Driver:family()) of
+ case Driver:getaddr(Address) of
{ok, Ip} ->
- Timer = dist_util:start_timer(SetupTime),
+ Timer = trace(dist_util:start_timer(SetupTime)),
ErlEpmd = net_kernel:epmd_module(),
case ErlEpmd:port_please(Name, Ip) of
{port, TcpPort, Version} ->
- ?trace("port_please(~p) -> version ~p~n",
- [Node,Version]),
+ Opts = trace(connect_options(get_ssl_options(client))),
dist_util:reset_timer(Timer),
- case ssl_tls_dist_proxy:connect(Driver, Address, TcpPort) of
- {ok, Socket} ->
- HSData = connect_hs_data(Kernel, Node, MyNode, Socket,
- Timer, Version, Ip, TcpPort, Address,
- Type),
- dist_util:handshake_we_started(HSData);
+ case ssl:connect(
+ Address, TcpPort,
+ [binary, {active, false}, {packet, 4},
+ Driver:family(), nodelay()] ++ Opts,
+ net_kernel:connecttime()) of
+ {ok, #sslsocket{pid = DistCtrl} = SslSocket} ->
+ monitor_pid(DistCtrl),
+ ok = ssl:controlling_process(SslSocket, self()),
+ HSData0 = hs_data_common(SslSocket),
+ HSData =
+ HSData0#hs_data{
+ kernel_pid = Kernel,
+ other_node = Node,
+ this_node = MyNode,
+ socket = DistCtrl,
+ timer = Timer,
+ this_flags = 0,
+ other_version = Version,
+ request_type = Type},
+ dist_util:handshake_we_started(trace(HSData));
Other ->
%% Other Node may have closed since
%% port_please !
- ?trace("other node (~p) "
- "closed since port_please.~n",
- [Node]),
- ?shutdown2(Node, {shutdown, {connect_failed, Other}})
+ ?shutdown2(
+ Node,
+ trace({shutdown, {connect_failed, Other}}))
end;
Other ->
- ?trace("port_please (~p) "
- "failed.~n", [Node]),
- ?shutdown2(Node, {shutdown, {port_please_failed, Other}})
+ ?shutdown2(
+ Node,
+ trace({shutdown, {port_please_failed, Other}}))
end;
Other ->
- ?trace("inet_getaddr(~p) "
- "failed (~p).~n", [Node,Other]),
- ?shutdown2(Node, {shutdown, {inet_getaddr_failed, Other}})
+ ?shutdown2(Node, trace({shutdown, {getaddr_failed, Other}}))
end.
close(Socket) ->
- gen_tcp:close(Socket),
- ok.
+ gen_close(inet, Socket).
+
+gen_close(Driver, Socket) ->
+ trace(Driver:close(Socket)).
-do_accept(Driver, Kernel, AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
- process_flag(priority, max),
- receive
- {AcceptPid, controller} ->
- Timer = dist_util:start_timer(SetupTime),
- case check_ip(Driver, Socket) of
- true ->
- HSData = accept_hs_data(Kernel, MyNode, Socket, Timer, Allowed),
- dist_util:handshake_other_started(HSData);
- {false,IP} ->
- error_logger:error_msg("** Connection attempt from "
- "disallowed IP ~w ** ~n", [IP]),
- ?shutdown(no_node)
- end
- end.
%% ------------------------------------------------------------
%% Do only accept new connection attempts from nodes at our
%% own LAN, if the check_ip environment parameter is true.
%% ------------------------------------------------------------
-check_ip(Driver, Socket) ->
+check_ip(Driver, SslSocket) ->
case application:get_env(check_ip) of
{ok, true} ->
- case get_ifs(Socket) of
+ case get_ifs(SslSocket) of
{ok, IFs, IP} ->
check_ip(Driver, IFs, IP);
_ ->
@@ -154,9 +418,18 @@ check_ip(Driver, Socket) ->
true
end.
-get_ifs(Socket) ->
+check_ip(Driver, [{OwnIP, _, Netmask}|IFs], PeerIP) ->
+ case {Driver:mask(Netmask, PeerIP), Driver:mask(Netmask, OwnIP)} of
+ {M, M} -> true;
+ _ -> check_ip(IFs, PeerIP)
+ end;
+check_ip(_Driver, [], PeerIP) ->
+ {false, PeerIP}.
+
+get_ifs(#sslsocket{fd = {gen_tcp, Socket, _}}) ->
case inet:peername(Socket) of
{ok, {IP, _}} ->
+ %% XXX this is seriously broken for IPv6
case inet:getif(Socket) of
{ok, IFs} -> {ok, IFs, IP};
Error -> Error
@@ -165,14 +438,6 @@ get_ifs(Socket) ->
Error
end.
-check_ip(Driver, [{OwnIP, _, Netmask}|IFs], PeerIP) ->
- case {Driver:mask(Netmask, PeerIP), Driver:mask(Netmask, OwnIP)} of
- {M, M} -> true;
- _ -> check_ip(IFs, PeerIP)
- end;
-check_ip(_Driver, [], PeerIP) ->
- {false, PeerIP}.
-
%% If Node is illegal terminate the connection setup!!
splitnode(Driver, Node, LongOrShortNames) ->
@@ -181,11 +446,13 @@ splitnode(Driver, Node, LongOrShortNames) ->
Host = lists:append(Tail),
check_node(Driver, Name, Node, Host, LongOrShortNames);
[_] ->
- error_logger:error_msg("** Nodename ~p illegal, no '@' character **~n",
- [Node]),
+ error_logger:error_msg(
+ "** Nodename ~p illegal, no '@' character **~n",
+ [Node]),
?shutdown(Node);
_ ->
- error_logger:error_msg("** Nodename ~p illegal **~n", [Node]),
+ error_logger:error_msg(
+ "** Nodename ~p illegal **~n", [Node]),
?shutdown(Node)
end.
@@ -196,23 +463,34 @@ check_node(Driver, Name, Node, Host, LongOrShortNames) ->
{ok, _} ->
[Name, Host];
_ ->
- error_logger:error_msg("** System running to use "
- "fully qualified "
- "hostnames **~n"
- "** Hostname ~s is illegal **~n",
- [Host]),
+ error_logger:error_msg(
+ "** System running to use "
+ "fully qualified hostnames **~n"
+ "** Hostname ~s is illegal **~n",
+ [Host]),
?shutdown(Node)
end;
[_, _ | _] when LongOrShortNames == shortnames ->
- error_logger:error_msg("** System NOT running to use fully qualified "
- "hostnames **~n"
- "** Hostname ~s is illegal **~n",
- [Host]),
+ error_logger:error_msg(
+ "** System NOT running to use "
+ "fully qualified hostnames **~n"
+ "** Hostname ~s is illegal **~n",
+ [Host]),
?shutdown(Node);
_ ->
[Name, Host]
end.
+split_node(Node) when is_atom(Node) ->
+ case split_node(atom_to_list(Node), $@, []) of
+ [_, Host] ->
+ Host;
+ _ ->
+ false
+ end;
+split_node(_) ->
+ false.
+%%
split_node([Chr|T], Chr, Ack) ->
[lists:reverse(Ack)|split_node(T, Chr, [])];
split_node([H|T], Chr, Ack) ->
@@ -220,70 +498,154 @@ split_node([H|T], Chr, Ack) ->
split_node([], _, Ack) ->
[lists:reverse(Ack)].
-connect_hs_data(Kernel, Node, MyNode, Socket, Timer, Version, Ip, TcpPort, Address, Type) ->
- common_hs_data(Kernel, MyNode, Socket, Timer,
- #hs_data{other_node = Node,
- other_version = Version,
- f_address =
- fun(_,_) ->
- #net_address{address = {Ip,TcpPort},
- host = Address,
- protocol = proxy,
- family = inet}
- end,
- request_type = Type
- }).
-
-accept_hs_data(Kernel, MyNode, Socket, Timer, Allowed) ->
- common_hs_data(Kernel, MyNode, Socket, Timer, #hs_data{
- allowed = Allowed,
- f_address = fun get_remote_id/2
- }).
-
-common_hs_data(Kernel, MyNode, Socket, Timer, HsData) ->
- HsData#hs_data{
- kernel_pid = Kernel,
- this_node = MyNode,
- socket = Socket,
- timer = Timer,
- this_flags = 0,
- f_send =
- fun(S,D) ->
- gen_tcp:send(S,D)
- end,
- f_recv =
- fun(S,N,T) ->
- gen_tcp:recv(S,N,T)
- end,
- f_setopts_pre_nodeup =
- fun(S) ->
- inet:setopts(S, [{active, false}, {packet, 4}])
- end,
- f_setopts_post_nodeup =
- fun(S) ->
- inet:setopts(S, [{deliver, port},{active, true}])
- end,
- f_getll =
- fun(S) ->
- inet:getll(S)
- end,
- mf_tick =
- fun(S) ->
- gen_tcp:send(S, <<>>)
- end,
- mf_getstat =
- fun(S) ->
- {ok, Stats} = inet:getstat(S, [recv_cnt, send_cnt, send_pend]),
- R = proplists:get_value(recv_cnt, Stats, 0),
- W = proplists:get_value(send_cnt, Stats, 0),
- P = proplists:get_value(send_pend, Stats, 0),
- {ok, R,W,P}
- end}.
-
-get_remote_id(Socket, _Node) ->
- case ssl_tls_dist_proxy:get_tcp_address(Socket) of
- {ok, Address} ->
- Address;
- {error, _Reason} ->
- ?shutdown(no_node)
+%% -------------------------------------------------------------------------
+
+connect_options(Opts) ->
+ case application:get_env(kernel, inet_dist_connect_options) of
+ {ok,ConnectOpts} ->
+ lists:ukeysort(1, ConnectOpts ++ Opts);
+ _ ->
+ Opts
end.
+
+%% we may not always want the nodelay behaviour
+%% for performance reasons
+nodelay() ->
+ case application:get_env(kernel, dist_nodelay) of
+ undefined ->
+ {nodelay, true};
+ {ok, true} ->
+ {nodelay, true};
+ {ok, false} ->
+ {nodelay, false};
+ _ ->
+ {nodelay, true}
+ end.
+
+
+get_ssl_options(Type) ->
+ case init:get_argument(ssl_dist_opt) of
+ {ok, Args} ->
+ [{erl_dist, true} | ssl_options(Type, lists:append(Args))];
+ _ ->
+ [{erl_dist, true}]
+ end.
+
+ssl_options(_,[]) ->
+ [];
+ssl_options(server, ["client_" ++ _, _Value |T]) ->
+ ssl_options(server,T);
+ssl_options(client, ["server_" ++ _, _Value|T]) ->
+ ssl_options(client,T);
+ssl_options(server, ["server_certfile", Value|T]) ->
+ [{certfile, Value} | ssl_options(server,T)];
+ssl_options(client, ["client_certfile", Value | T]) ->
+ [{certfile, Value} | ssl_options(client,T)];
+ssl_options(server, ["server_cacertfile", Value|T]) ->
+ [{cacertfile, Value} | ssl_options(server,T)];
+ssl_options(client, ["client_cacertfile", Value|T]) ->
+ [{cacertfile, Value} | ssl_options(client,T)];
+ssl_options(server, ["server_keyfile", Value|T]) ->
+ [{keyfile, Value} | ssl_options(server,T)];
+ssl_options(client, ["client_keyfile", Value|T]) ->
+ [{keyfile, Value} | ssl_options(client,T)];
+ssl_options(server, ["server_password", Value|T]) ->
+ [{password, Value} | ssl_options(server,T)];
+ssl_options(client, ["client_password", Value|T]) ->
+ [{password, Value} | ssl_options(client,T)];
+ssl_options(server, ["server_verify", Value|T]) ->
+ [{verify, atomize(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_verify", Value|T]) ->
+ [{verify, atomize(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_verify_fun", Value|T]) ->
+ [{verify_fun, verify_fun(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_verify_fun", Value|T]) ->
+ [{verify_fun, verify_fun(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_crl_check", Value|T]) ->
+ [{crl_check, atomize(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_crl_check", Value|T]) ->
+ [{crl_check, atomize(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_crl_cache", Value|T]) ->
+ [{crl_cache, termify(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_crl_cache", Value|T]) ->
+ [{crl_cache, termify(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_reuse_sessions", Value|T]) ->
+ [{reuse_sessions, atomize(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_reuse_sessions", Value|T]) ->
+ [{reuse_sessions, atomize(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_secure_renegotiate", Value|T]) ->
+ [{secure_renegotiate, atomize(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_secure_renegotiate", Value|T]) ->
+ [{secure_renegotiate, atomize(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_depth", Value|T]) ->
+ [{depth, list_to_integer(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_depth", Value|T]) ->
+ [{depth, list_to_integer(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_hibernate_after", Value|T]) ->
+ [{hibernate_after, list_to_integer(Value)} | ssl_options(server,T)];
+ssl_options(client, ["client_hibernate_after", Value|T]) ->
+ [{hibernate_after, list_to_integer(Value)} | ssl_options(client,T)];
+ssl_options(server, ["server_ciphers", Value|T]) ->
+ [{ciphers, Value} | ssl_options(server,T)];
+ssl_options(client, ["client_ciphers", Value|T]) ->
+ [{ciphers, Value} | ssl_options(client,T)];
+ssl_options(server, ["server_dhfile", Value|T]) ->
+ [{dhfile, Value} | ssl_options(server,T)];
+ssl_options(server, ["server_fail_if_no_peer_cert", Value|T]) ->
+ [{fail_if_no_peer_cert, atomize(Value)} | ssl_options(server,T)];
+ssl_options(Type, Opts) ->
+ error(malformed_ssl_dist_opt, [Type, Opts]).
+
+atomize(List) when is_list(List) ->
+ list_to_atom(List);
+atomize(Atom) when is_atom(Atom) ->
+ Atom.
+
+termify(String) when is_list(String) ->
+ {ok, Tokens, _} = erl_scan:string(String ++ "."),
+ {ok, Term} = erl_parse:parse_term(Tokens),
+ Term.
+
+verify_fun(Value) ->
+ case termify(Value) of
+ {Mod, Func, State} when is_atom(Mod), is_atom(Func) ->
+ Fun = fun Mod:Func/3,
+ {Fun, State};
+ _ ->
+ error(malformed_ssl_dist_opt, [Value])
+ end.
+
+%% -------------------------------------------------------------------------
+
+%% Trace point
+trace(Term) -> Term.
+
+%% Keep an eye on distribution Pid:s we know of
+monitor_pid(Pid) ->
+ %%spawn(
+ %% fun () ->
+ %% MRef = erlang:monitor(process, Pid),
+ %% receive
+ %% {'DOWN', MRef, _, _, normal} ->
+ %% error_logger:error_report(
+ %% [dist_proc_died,
+ %% {reason, normal},
+ %% {pid, Pid}]);
+ %% {'DOWN', MRef, _, _, Reason} ->
+ %% error_logger:info_report(
+ %% [dist_proc_died,
+ %% {reason, Reason},
+ %% {pid, Pid}])
+ %% end
+ %% end),
+ Pid.
+
+dbg() ->
+ dbg:stop(),
+ dbg:tracer(),
+ dbg:p(all, c),
+ dbg:tpl(?MODULE, cx),
+ dbg:tpl(erlang, dist_ctrl_get_data_notification, cx),
+ dbg:tpl(erlang, dist_ctrl_get_data, cx),
+ dbg:tpl(erlang, dist_ctrl_put_data, cx),
+ ok.
diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src
index 51407ef3b9..c5b55641a1 100644
--- a/lib/ssl/src/ssl.app.src
+++ b/lib/ssl/src/ssl.app.src
@@ -37,7 +37,6 @@
%% Erlang Distribution over SSL/TLS
inet_tls_dist,
inet6_tls_dist,
- ssl_tls_dist_proxy,
ssl_dist_sup,
ssl_dist_connection_sup,
ssl_dist_admin_sup,
@@ -63,7 +62,5 @@
{applications, [crypto, public_key, kernel, stdlib]},
{env, []},
{mod, {ssl_app, []}},
- {runtime_dependencies, ["stdlib-3.2","public_key-1.5","kernel-3.0",
- "erts-7.0","crypto-3.3", "inets-5.10.7"]}]}.
-
-
+ {runtime_dependencies, ["stdlib-3.2","public_key-1.5","kernel-6.0",
+ "erts-10.0","crypto-3.3", "inets-5.10.7"]}]}.
diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl
index 50c5f0d755..b6cd22dd13 100644
--- a/lib/ssl/src/ssl_cipher.erl
+++ b/lib/ssl/src/ssl_cipher.erl
@@ -375,30 +375,38 @@ psk_suites({3, N}) ->
psk_suites(N)
when N >= 3 ->
[
+ ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384,
?TLS_DHE_PSK_WITH_AES_256_GCM_SHA384,
?TLS_RSA_PSK_WITH_AES_256_GCM_SHA384,
?TLS_PSK_WITH_AES_256_GCM_SHA384,
+ ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384,
?TLS_DHE_PSK_WITH_AES_256_CBC_SHA384,
?TLS_RSA_PSK_WITH_AES_256_CBC_SHA384,
?TLS_PSK_WITH_AES_256_CBC_SHA384,
+ ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256,
?TLS_DHE_PSK_WITH_AES_128_GCM_SHA256,
?TLS_RSA_PSK_WITH_AES_128_GCM_SHA256,
?TLS_PSK_WITH_AES_128_GCM_SHA256,
+ ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256,
?TLS_DHE_PSK_WITH_AES_128_CBC_SHA256,
?TLS_RSA_PSK_WITH_AES_128_CBC_SHA256,
?TLS_PSK_WITH_AES_128_CBC_SHA256
] ++ psk_suites(0);
psk_suites(_) ->
- [?TLS_DHE_PSK_WITH_AES_256_CBC_SHA,
+ [?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA,
+ ?TLS_DHE_PSK_WITH_AES_256_CBC_SHA,
?TLS_RSA_PSK_WITH_AES_256_CBC_SHA,
?TLS_PSK_WITH_AES_256_CBC_SHA,
+ ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA,
?TLS_DHE_PSK_WITH_AES_128_CBC_SHA,
?TLS_RSA_PSK_WITH_AES_128_CBC_SHA,
?TLS_PSK_WITH_AES_128_CBC_SHA,
+ ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA,
?TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA,
?TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA,
?TLS_PSK_WITH_3DES_EDE_CBC_SHA,
+ ?TLS_ECDHE_PSK_WITH_RC4_128_SHA,
?TLS_DHE_PSK_WITH_RC4_128_SHA,
?TLS_RSA_PSK_WITH_RC4_128_SHA,
?TLS_PSK_WITH_RC4_128_SHA].
@@ -565,6 +573,15 @@ suite_definition(?TLS_RSA_PSK_WITH_AES_128_CBC_SHA) ->
suite_definition(?TLS_RSA_PSK_WITH_AES_256_CBC_SHA) ->
{rsa_psk, aes_256_cbc, sha, default_prf};
+%%% PSK NULL Cipher Suites RFC 4785
+
+suite_definition(?TLS_PSK_WITH_NULL_SHA) ->
+ {psk, null, sha, default_prf};
+suite_definition(?TLS_DHE_PSK_WITH_NULL_SHA) ->
+ {dhe_psk, null, sha, default_prf};
+suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA) ->
+ {rsa_psk, null, sha, default_prf};
+
%%% TLS 1.2 PSK Cipher Suites RFC 5487
suite_definition(?TLS_PSK_WITH_AES_128_GCM_SHA256) ->
@@ -606,6 +623,36 @@ suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA256) ->
suite_definition(?TLS_RSA_PSK_WITH_NULL_SHA384) ->
{rsa_psk, null, sha384, default_prf};
+%%% ECDHE PSK Cipher Suites RFC 5489
+
+suite_definition(?TLS_ECDHE_PSK_WITH_RC4_128_SHA) ->
+ {ecdhe_psk, rc4_128, sha, default_prf};
+suite_definition(?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA) ->
+ {ecdhe_psk, '3des_ede_cbc', sha, default_prf};
+suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA) ->
+ {ecdhe_psk, aes_128_cbc, sha, default_prf};
+suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA) ->
+ {ecdhe_psk, aes_256_cbc, sha, default_prf};
+suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256) ->
+ {ecdhe_psk, aes_128_cbc, sha256, default_prf};
+suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384) ->
+ {ecdhe_psk, aes_256_cbc, sha384, default_prf};
+suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA256) ->
+ {ecdhe_psk, null, sha256, default_prf};
+suite_definition(?TLS_ECDHE_PSK_WITH_NULL_SHA384) ->
+ {ecdhe_psk, null, sha384, default_prf};
+
+%%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05
+
+suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256) ->
+ {ecdhe_psk, aes_128_gcm, null, sha256};
+suite_definition(?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384) ->
+ {ecdhe_psk, aes_256_gcm, null, sha384};
+%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256) ->
+%% {ecdhe_psk, aes_128_ccm, null, sha256};
+%% suite_definition(?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256) ->
+%% {ecdhe_psk, aes_256_ccm, null, sha256};
+
%%% SRP Cipher Suites RFC 5054
suite_definition(?TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA) ->
@@ -867,6 +914,15 @@ suite({rsa_psk, aes_128_cbc,sha}) ->
suite({rsa_psk, aes_256_cbc,sha}) ->
?TLS_RSA_PSK_WITH_AES_256_CBC_SHA;
+%%% PSK NULL Cipher Suites RFC 4785
+
+suite({psk, null, sha}) ->
+ ?TLS_PSK_WITH_NULL_SHA;
+suite({dhe_psk, null, sha}) ->
+ ?TLS_DHE_PSK_WITH_NULL_SHA;
+suite({rsa_psk, null, sha}) ->
+ ?TLS_RSA_PSK_WITH_NULL_SHA;
+
%%% TLS 1.2 PSK Cipher Suites RFC 5487
suite({psk, aes_128_gcm, null, sha256}) ->
@@ -908,6 +964,36 @@ suite({rsa_psk, null, sha256}) ->
suite({rsa_psk, null, sha384}) ->
?TLS_RSA_PSK_WITH_NULL_SHA384;
+%%% ECDHE PSK Cipher Suites RFC 5489
+
+suite({ecdhe_psk, rc4_128,sha}) ->
+ ?TLS_ECDHE_PSK_WITH_RC4_128_SHA;
+suite({ecdhe_psk, '3des_ede_cbc',sha}) ->
+ ?TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA;
+suite({ecdhe_psk, aes_128_cbc,sha}) ->
+ ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA;
+suite({ecdhe_psk, aes_256_cbc,sha}) ->
+ ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA;
+suite({ecdhe_psk, aes_128_cbc, sha256}) ->
+ ?TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256;
+suite({ecdhe_psk, aes_256_cbc, sha384}) ->
+ ?TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384;
+suite({ecdhe_psk, null, sha256}) ->
+ ?TLS_ECDHE_PSK_WITH_NULL_SHA256;
+suite({ecdhe_psk, null, sha384}) ->
+ ?TLS_ECDHE_PSK_WITH_NULL_SHA384;
+
+%%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05
+
+suite({ecdhe_psk, aes_128_gcm, null, sha256}) ->
+ ?TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256;
+suite({ecdhe_psk, aes_256_gcm, null, sha384}) ->
+ ?TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384;
+%% suite({ecdhe_psk, aes_128_ccm, null, sha256}) ->
+%% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256;
+%% suite({ecdhe_psk, aes_256_ccm, null, sha256}) ->
+%% ?TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256;
+
%%% SRP Cipher Suites RFC 5054
suite({srp_anon, '3des_ede_cbc', sha}) ->
@@ -1467,7 +1553,8 @@ is_acceptable_keyexchange(dhe_dss, Algos) ->
is_acceptable_keyexchange(dhe_rsa, Algos) ->
proplists:get_bool(dh, Algos) andalso
proplists:get_bool(rsa, Algos);
-is_acceptable_keyexchange(ecdh_anon, Algos) ->
+is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_anon;
+ KeyExchange == ecdhe_psk ->
proplists:get_bool(ecdh, Algos);
is_acceptable_keyexchange(KeyExchange, Algos) when KeyExchange == ecdh_ecdsa;
KeyExchange == ecdhe_ecdsa ->
diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl
index 8e8f3d9c67..e5462d8402 100644
--- a/lib/ssl/src/ssl_cipher.hrl
+++ b/lib/ssl/src/ssl_cipher.hrl
@@ -399,6 +399,17 @@
%% TLS_RSA_PSK_WITH_AES_256_CBC_SHA = { 0x00, 0x95 };
-define(TLS_RSA_PSK_WITH_AES_256_CBC_SHA, <<?BYTE(16#00), ?BYTE(16#95)>>).
+%%% PSK NULL Cipher Suites RFC 4785
+
+%% TLS_PSK_WITH_NULL_SHA = { 0x00, 0x2C };
+-define(TLS_PSK_WITH_NULL_SHA, <<?BYTE(16#00), ?BYTE(16#2C)>>).
+
+%% TLS_DHE_PSK_WITH_NULL_SHA = { 0x00, 0x2D };
+-define(TLS_DHE_PSK_WITH_NULL_SHA, <<?BYTE(16#00), ?BYTE(16#2D)>>).
+
+%% TLS_RSA_PSK_WITH_NULL_SHA = { 0x00, 0x2E };
+-define(TLS_RSA_PSK_WITH_NULL_SHA, <<?BYTE(16#00), ?BYTE(16#2E)>>).
+
%%% TLS 1.2 PSK Cipher Suites RFC 5487
%% TLS_PSK_WITH_AES_128_GCM_SHA256 = {0x00,0xA8};
@@ -455,6 +466,46 @@
%% TLS_RSA_PSK_WITH_NULL_SHA384 = {0x00,0xB9};
-define(TLS_RSA_PSK_WITH_NULL_SHA384, <<?BYTE(16#00), ?BYTE(16#B9)>>).
+%%% ECDHE PSK Cipher Suites RFC 5489
+
+%% TLS_ECDHE_PSK_WITH_RC4_128_SHA = {0xC0,0x33};
+-define(TLS_ECDHE_PSK_WITH_RC4_128_SHA, <<?BYTE(16#C0), ?BYTE(16#33)>>).
+
+%% TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = {0xC0,0x34};
+-define(TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, <<?BYTE(16#C0), ?BYTE(16#34)>>).
+
+%% TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA = {0xC0,0x35};
+-define(TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA, <<?BYTE(16#C0), ?BYTE(16#35)>>).
+
+%% TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA = {0xC0,0x36};
+-define(TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA, <<?BYTE(16#C0), ?BYTE(16#36)>>).
+
+%% TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 = {0xC0,0x37};
+-define(TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256, <<?BYTE(16#C0), ?BYTE(16#37)>>).
+
+%% TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 = {0xC0,0x38};
+-define(TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384, <<?BYTE(16#C0), ?BYTE(16#38)>>).
+
+%% TLS_ECDHE_PSK_WITH_NULL_SHA256 = {0xC0,0x3A};
+-define(TLS_ECDHE_PSK_WITH_NULL_SHA256, <<?BYTE(16#C0), ?BYTE(16#3A)>>).
+
+%% TLS_ECDHE_PSK_WITH_NULL_SHA384 = {0xC0,0x3B};
+-define(TLS_ECDHE_PSK_WITH_NULL_SHA384, <<?BYTE(16#C0), ?BYTE(16#3B)>>).
+
+%%% ECDHE_PSK with AES-GCM and AES-CCM Cipher Suites, draft-ietf-tls-ecdhe-psk-aead-05
+
+%% TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256 = {0xTBD; 0xTBD} {0xD0,0x01};
+-define(TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256, <<?BYTE(16#D0), ?BYTE(16#01)>>).
+
+%% TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384 = {0xTBD; 0xTBD} {0xD0,0x02};
+-define(TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384, <<?BYTE(16#D0), ?BYTE(16#02)>>).
+
+%% TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256 = {0xTBD; 0xTBD} {0xD0,0x03};
+-define(TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256, <<?BYTE(16#D0), ?BYTE(16#03)>>).
+
+%% TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256 = {0xTBD; 0xTBD} {0xD0,0x05};
+-define(TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256, <<?BYTE(16#D0), ?BYTE(16#05)>>).
+
%%% SRP Cipher Suites RFC 5054
%% TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA = { 0xC0,0x1A };
diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 2146a9272e..d83c9cb59f 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -44,12 +44,14 @@
-export([send/2, recv/3, close/2, shutdown/2,
new_user/2, get_opts/2, set_opts/2,
peer_certificate/1, renegotiation/1, negotiated_protocol/1, prf/5,
+ get_sslsocket/1, handshake_complete/3,
connection_information/2, handle_common_event/5
]).
%% General gen_statem state functions with extra callback argument
%% to determine if it is an SSL/TLS or DTLS gen_statem machine
--export([init/4, hello/4, abbreviated/4, certify/4, cipher/4, connection/4, downgrade/4]).
+-export([init/4, hello/4, abbreviated/4, certify/4, cipher/4,
+ connection/4, death_row/4, downgrade/4]).
%% gen_statem callbacks
-export([terminate/3, format_status/2]).
@@ -146,8 +148,8 @@ socket_control(Connection, Socket, Pid, Transport) ->
-spec socket_control(tls_connection | dtls_connection, port(), pid(), atom(), pid()| undefined) ->
{ok, #sslsocket{}} | {error, reason()}.
%%--------------------------------------------------------------------
-socket_control(Connection, Socket, Pid, Transport, udp_listner) ->
- %% dtls listner process must have the socket control
+socket_control(Connection, Socket, Pid, Transport, udp_listener) ->
+ %% dtls listener process must have the socket control
{ok, Connection:socket(Pid, Transport, Socket, Connection, undefined)};
socket_control(tls_connection = Connection, Socket, Pid, Transport, ListenTracker) ->
@@ -262,6 +264,13 @@ peer_certificate(ConnectionPid) ->
renegotiation(ConnectionPid) ->
call(ConnectionPid, renegotiate).
+
+get_sslsocket(ConnectionPid) ->
+ call(ConnectionPid, get_sslsocket).
+
+handshake_complete(ConnectionPid, Node, DHandle) ->
+ call(ConnectionPid, {handshake_complete, Node, DHandle}).
+
%%--------------------------------------------------------------------
-spec prf(pid(), binary() | 'master_secret', binary(),
[binary() | ssl:prf_random()], non_neg_integer()) ->
@@ -359,6 +368,12 @@ init({call, From}, {start, {Opts, EmOpts}, Timeout},
socket_options = SockOpts} = State0, Connection) ->
try
SslOpts = ssl:handle_options(Opts, OrigSSLOptions),
+ case SslOpts of
+ #ssl_options{erl_dist = true} ->
+ process_flag(priority, max);
+ _ ->
+ ok
+ end,
State = ssl_config(SslOpts, Role, State0),
init({call, From}, {start, Timeout},
State#state{ssl_options = SslOpts, socket_options = new_emulated(EmOpts, SockOpts)}, Connection)
@@ -518,7 +533,7 @@ certify(internal, #server_key_exchange{exchange_keys = Keys},
when Alg == dhe_dss; Alg == dhe_rsa;
Alg == ecdhe_rsa; Alg == ecdhe_ecdsa;
Alg == dh_anon; Alg == ecdh_anon;
- Alg == psk; Alg == dhe_psk; Alg == rsa_psk;
+ Alg == psk; Alg == dhe_psk; Alg == ecdhe_psk; Alg == rsa_psk;
Alg == srp_dss; Alg == srp_rsa; Alg == srp_anon ->
Params = ssl_handshake:decode_server_key(Keys, Alg, ssl:tls_version(Version)),
@@ -543,6 +558,15 @@ certify(internal, #server_key_exchange{exchange_keys = Keys},
end
end;
+certify(internal, #certificate_request{},
+ #state{role = client, negotiated_version = Version,
+ key_algorithm = Alg} = State, _)
+ when Alg == dh_anon; Alg == ecdh_anon;
+ Alg == psk; Alg == dhe_psk; Alg == ecdhe_psk; Alg == rsa_psk;
+ Alg == srp_dss; Alg == srp_rsa; Alg == srp_anon ->
+ handle_own_alert(?ALERT_REC(?FATAL, ?HANDSHAKE_FAILURE),
+ Version, certify, State);
+
certify(internal, #certificate_request{} = CertRequest,
#state{session = #session{own_certificate = Cert},
role = client,
@@ -741,7 +765,7 @@ cipher(Type, Msg, State, Connection) ->
#state{}, tls_connection | dtls_connection) ->
gen_statem:state_function_result().
%%--------------------------------------------------------------------
-connection({call, From}, {application_data, Data},
+connection({call, {FromPid, _} = From}, {application_data, Data},
#state{protocol_cb = Connection} = State, Connection) ->
%% We should look into having a worker process to do this to
%% parallize send and receive decoding and not block the receiver
@@ -749,7 +773,13 @@ connection({call, From}, {application_data, Data},
try
write_application_data(Data, From, State)
catch throw:Error ->
- hibernate_after(?FUNCTION_NAME, State, [{reply, From, Error}])
+ case self() of
+ FromPid ->
+ {stop, {shutdown, Error}};
+ _ ->
+ hibernate_after(
+ ?FUNCTION_NAME, State, [{reply, From, Error}])
+ end
end;
connection({call, RecvFrom}, {recv, N, Timeout},
#state{protocol_cb = Connection, socket_options =
@@ -777,8 +807,50 @@ connection({call, From}, negotiated_protocol,
#state{negotiated_protocol = SelectedProtocol} = State, _) ->
hibernate_after(?FUNCTION_NAME, State,
[{reply, From, {ok, SelectedProtocol}}]);
+connection(
+ {call, From}, {handshake_complete, _Node, DHandle},
+ #state{
+ ssl_options = #ssl_options{erl_dist = true},
+ socket_options = SockOpts,
+ protocol_specific = ProtocolSpecific} = State,
+ Connection) ->
+ %% From now on we execute on normal priority
+ process_flag(priority, normal),
+ try erlang:dist_ctrl_get_data_notification(DHandle) of
+ _ ->
+ NewState =
+ State#state{
+ socket_options =
+ SockOpts#socket_options{active = true},
+ protocol_specific =
+ ProtocolSpecific#{d_handle => DHandle}},
+ {Record, NewerState} = Connection:next_record_if_active(NewState),
+ Connection:next_event(connection, Record, NewerState, [{reply, From, ok}])
+ catch _:Reason ->
+ death_row(State, Reason)
+ end;
connection({call, From}, Msg, State, Connection) ->
handle_call(Msg, From, ?FUNCTION_NAME, State, Connection);
+connection(
+ info, dist_data = Msg,
+ #state{
+ ssl_options = #ssl_options{erl_dist = true},
+ protocol_specific = #{d_handle := DHandle}} = State,
+ _) ->
+ eat_msgs(Msg),
+ try send_dist_data(?FUNCTION_NAME, State, DHandle, [])
+ catch _:Reason ->
+ death_row(State, Reason)
+ end;
+connection(
+ info, tick = Msg,
+ #state{
+ ssl_options = #ssl_options{erl_dist = true},
+ protocol_specific = #{d_handle := _}},
+ _) ->
+ eat_msgs(Msg),
+ {keep_state_and_data,
+ [{next_event, {call, {self(), undefined}}, {application_data, <<>>}}]};
connection(info, Msg, State, _) ->
handle_info(Msg, ?FUNCTION_NAME, State);
connection(internal, {recv, _}, State, Connection) ->
@@ -787,6 +859,30 @@ connection(Type, Msg, State, Connection) ->
handle_common_event(Type, Msg, ?FUNCTION_NAME, State, Connection).
%%--------------------------------------------------------------------
+-spec death_row(gen_statem:event_type(), term(),
+ #state{}, tls_connection | dtls_connection) ->
+ gen_statem:state_function_result().
+%%--------------------------------------------------------------------
+%% We just wait for the owner to die which triggers the monitor,
+%% or the socket may die too
+death_row(
+ info, {'DOWN', MonitorRef, _, _, Reason},
+ #state{user_application={MonitorRef,_Pid} = State},
+ _) ->
+ {stop, {shutdown, Reason}, State};
+death_row(
+ info, {'EXIT', Socket, Reason}, #state{socket = Socket} = State, _) ->
+ {stop, {shutdown, Reason}, State};
+death_row(state_timeout, Reason, _State, _Connection) ->
+ {stop, {shutdown,Reason}};
+death_row(_Type, _Msg, State, _Connection) ->
+ {keep_state, State, [postpone]}.
+
+%% State entry function
+death_row(State, Reason) ->
+ {next_state, death_row, State, [{state_timeout, 5000, Reason}]}.
+
+%%--------------------------------------------------------------------
-spec downgrade(gen_statem:event_type(), term(),
#state{}, tls_connection | dtls_connection) ->
gen_statem:state_function_result().
@@ -797,10 +893,10 @@ downgrade(internal, #alert{description = ?CLOSE_NOTIFY},
tls_socket:setopts(Transport, Socket, [{active, false}, {packet, 0}, {mode, binary}]),
Transport:controlling_process(Socket, Pid),
gen_statem:reply(From, {ok, Socket}),
- {stop, normal, State};
+ stop_normal(State);
downgrade(timeout, downgrade, #state{downgrade = {_, From}} = State, _) ->
gen_statem:reply(From, {error, timeout}),
- {stop, normal, State};
+ stop_normal(State);
downgrade(Type, Event, State, Connection) ->
handle_common_event(Type, Event, ?FUNCTION_NAME, State, Connection).
@@ -870,7 +966,7 @@ handle_call({shutdown, How0}, From, _,
#state{transport_cb = Transport,
negotiated_version = Version,
connection_states = ConnectionStates,
- socket = Socket}, Connection) ->
+ socket = Socket} = State, Connection) ->
case How0 of
How when How == write; How == both ->
Alert = ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
@@ -886,7 +982,7 @@ handle_call({shutdown, How0}, From, _,
{keep_state_and_data, [{reply, From, ok}]};
Error ->
gen_statem:reply(From, {error, Error}),
- {stop, normal}
+ stop_normal(State)
end;
handle_call({recv, _N, _Timeout}, From, _,
#state{socket_options =
@@ -921,6 +1017,15 @@ handle_call({set_opts, Opts0}, From, StateName,
handle_call(renegotiate, From, StateName, _, _) when StateName =/= connection ->
{keep_state_and_data, [{reply, From, {error, already_renegotiating}}]};
+
+handle_call(
+ get_sslsocket, From, _StateName,
+ #state{transport_cb = Transport, socket = Socket, tracker = Tracker},
+ Connection) ->
+ SslSocket =
+ Connection:socket(self(), Transport, Socket, Connection, Tracker),
+ {keep_state_and_data, [{reply, From, SslSocket}]};
+
handle_call({prf, Secret, Label, Seed, WantedLength}, From, _,
#state{connection_states = ConnectionStates,
negotiated_version = Version}, _) ->
@@ -957,18 +1062,19 @@ handle_info({ErrorTag, Socket, econnaborted}, StateName,
tracker = Tracker} = State) when StateName =/= connection ->
alert_user(Transport, Tracker,Socket,
StartFrom, ?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), Role, Connection),
- {stop, normal, State};
+ stop_normal(State);
handle_info({ErrorTag, Socket, Reason}, StateName, #state{socket = Socket,
error_tag = ErrorTag} = State) ->
Report = io_lib:format("SSL: Socket error: ~p ~n", [Reason]),
- error_logger:info_report(Report),
+ error_logger:error_report(Report),
handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State),
- {stop, normal, State};
+ stop_normal(State);
-handle_info({'DOWN', MonitorRef, _, _, _}, _,
- State = #state{user_application={MonitorRef,_Pid}}) ->
- {stop, normal, State};
+handle_info(
+ {'DOWN', MonitorRef, _, _, _}, _,
+ #state{user_application={MonitorRef,_Pid}} = State) ->
+ stop_normal(State);
%%% So that terminate will be run when supervisor issues shutdown
handle_info({'EXIT', _Sup, shutdown}, _StateName, State) ->
@@ -976,6 +1082,8 @@ handle_info({'EXIT', _Sup, shutdown}, _StateName, State) ->
handle_info({'EXIT', Socket, normal}, _StateName, #state{socket = Socket} = State) ->
%% Handle as transport close"
{stop, {shutdown, transport_closed}, State};
+handle_info({'EXIT', Socket, Reason}, _StateName, #state{socket = Socket} = State) ->
+ {stop, {shutdown, Reason}, State};
handle_info(allow_renegotiate, StateName, State) ->
{next_state, StateName, State#state{allow_renegotiate = true}};
@@ -999,6 +1107,38 @@ handle_info(Msg, StateName, #state{socket = Socket, error_tag = Tag} = State) ->
error_logger:info_report(Report),
{next_state, StateName, State}.
+
+
+send_dist_data(StateName, State, DHandle, Acc) ->
+ case erlang:dist_ctrl_get_data(DHandle) of
+ none ->
+ erlang:dist_ctrl_get_data_notification(DHandle),
+ hibernate_after(StateName, State, lists:reverse(Acc));
+ Data ->
+ send_dist_data(
+ StateName, State, DHandle,
+ [{next_event, {call, {self(), undefined}}, {application_data, Data}}
+ |Acc])
+ end.
+
+%% Overload mitigation
+eat_msgs(Msg) ->
+ receive Msg -> eat_msgs(Msg)
+ after 0 -> ok
+ end.
+
+%% When running with erl_dist the stop reason 'normal'
+%% would be too silent and prevent cleanup
+stop_normal(State) ->
+ Reason =
+ case State of
+ #state{ssl_options = #ssl_options{erl_dist = true}} ->
+ {shutdown, normal};
+ _ ->
+ normal
+ end,
+ {stop, Reason, State}.
+
%%--------------------------------------------------------------------
%% gen_statem callbacks
%%--------------------------------------------------------------------
@@ -1073,7 +1213,7 @@ format_status(terminate, [_, StateName, State]) ->
%%--------------------------------------------------------------------
%%%
%%--------------------------------------------------------------------
-write_application_data(Data0, From,
+write_application_data(Data0, {FromPid, _} = From,
#state{socket = Socket,
negotiated_version = Version,
protocol_cb = Connection,
@@ -1088,10 +1228,19 @@ write_application_data(Data0, From,
Connection:renegotiate(State#state{renegotiation = {true, internal}},
[{next_event, {call, From}, {application_data, Data0}}]);
false ->
- {Msgs, ConnectionStates} = Connection:encode_data(Data, Version, ConnectionStates0),
- Result = Connection:send(Transport, Socket, Msgs),
- ssl_connection:hibernate_after(connection, State#state{connection_states = ConnectionStates},
- [{reply, From, Result}])
+ {Msgs, ConnectionStates} =
+ Connection:encode_data(Data, Version, ConnectionStates0),
+ NewState = State#state{connection_states = ConnectionStates},
+ case Connection:send(Transport, Socket, Msgs) of
+ ok when FromPid =:= self() ->
+ hibernate_after(connection, NewState, []);
+ Error when FromPid =:= self() ->
+ {stop, {shutdown, Error}, NewState};
+ ok ->
+ hibernate_after(connection, NewState, [{reply, From, ok}]);
+ Result ->
+ hibernate_after(connection, NewState, [{reply, From, Result}])
+ end
end.
read_application_data(Data, #state{user_application = {_Mon, Pid},
@@ -1111,30 +1260,57 @@ read_application_data(Data, #state{user_application = {_Mon, Pid},
end,
case get_data(SOpts, BytesToRead, Buffer1) of
{ok, ClientData, Buffer} -> % Send data
- SocketOpt = deliver_app_data(Transport, Socket, SOpts,
- ClientData, Pid, RecvFrom, Tracker, Connection),
- cancel_timer(Timer),
- State = State0#state{user_data_buffer = Buffer,
- start_or_recv_from = undefined,
- timer = undefined,
- bytes_to_read = undefined,
- socket_options = SocketOpt
- },
- if
- SocketOpt#socket_options.active =:= false; Buffer =:= <<>> ->
- %% Passive mode, wait for active once or recv
- %% Active and empty, get more data
- Connection:next_record_if_active(State);
- true -> %% We have more data
- read_application_data(<<>>, State)
- end;
+ case State0 of
+ #state{
+ ssl_options = #ssl_options{erl_dist = true},
+ protocol_specific = #{d_handle := DHandle}} ->
+ State =
+ State0#state{
+ user_data_buffer = Buffer,
+ bytes_to_read = undefined},
+ try erlang:dist_ctrl_put_data(DHandle, ClientData) of
+ _
+ when SOpts#socket_options.active =:= false;
+ Buffer =:= <<>> ->
+ %% Passive mode, wait for active once or recv
+ %% Active and empty, get more data
+ Connection:next_record_if_active(State);
+ _ -> %% We have more data
+ read_application_data(<<>>, State)
+ catch _:Reason ->
+ death_row(State, Reason)
+ end;
+ _ ->
+ SocketOpt =
+ deliver_app_data(
+ Transport, Socket, SOpts,
+ ClientData, Pid, RecvFrom, Tracker, Connection),
+ cancel_timer(Timer),
+ State =
+ State0#state{
+ user_data_buffer = Buffer,
+ start_or_recv_from = undefined,
+ timer = undefined,
+ bytes_to_read = undefined,
+ socket_options = SocketOpt
+ },
+ if
+ SocketOpt#socket_options.active =:= false;
+ Buffer =:= <<>> ->
+ %% Passive mode, wait for active once or recv
+ %% Active and empty, get more data
+ Connection:next_record_if_active(State);
+ true -> %% We have more data
+ read_application_data(<<>>, State)
+ end
+ end;
{more, Buffer} -> % no reply, we need more data
Connection:next_record(State0#state{user_data_buffer = Buffer});
{passive, Buffer} ->
Connection:next_record_if_active(State0#state{user_data_buffer = Buffer});
{error,_Reason} -> %% Invalid packet in packet mode
deliver_packet_error(Transport, Socket, SOpts, Buffer1, Pid, RecvFrom, Tracker, Connection),
- {stop, normal, State0}
+ stop_normal(State0)
end.
%%--------------------------------------------------------------------
%%%
@@ -1144,12 +1320,12 @@ handle_alert(#alert{level = ?FATAL} = Alert, StateName,
protocol_cb = Connection,
ssl_options = SslOpts, start_or_recv_from = From, host = Host,
port = Port, session = Session, user_application = {_Mon, Pid},
- role = Role, socket_options = Opts, tracker = Tracker}) ->
+ role = Role, socket_options = Opts, tracker = Tracker} = State) ->
invalidate_session(Role, Host, Port, Session),
log_alert(SslOpts#ssl_options.log_alert, Role, Connection:protocol_name(),
StateName, Alert#alert{role = opposite_role(Role)}),
alert_user(Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, Role, Connection),
- {stop, normal};
+ stop_normal(State);
handle_alert(#alert{level = ?WARNING, description = ?CLOSE_NOTIFY} = Alert,
StateName, State) ->
@@ -1402,6 +1578,16 @@ certify_client_key_exchange(#client_dhe_psk_identity{} = ClientKey,
PremasterSecret =
ssl_handshake:premaster_secret(ClientKey, ServerDhPrivateKey, Params, PSKLookup),
calculate_master_secret(PremasterSecret, State0, Connection, certify, cipher);
+
+certify_client_key_exchange(#client_ecdhe_psk_identity{} = ClientKey,
+ #state{diffie_hellman_keys = ServerEcDhPrivateKey,
+ ssl_options =
+ #ssl_options{user_lookup_fun = PSKLookup}} = State,
+ Connection) ->
+ PremasterSecret =
+ ssl_handshake:premaster_secret(ClientKey, ServerEcDhPrivateKey, PSKLookup),
+ calculate_master_secret(PremasterSecret, State, Connection, certify, cipher);
+
certify_client_key_exchange(#client_rsa_psk_identity{} = ClientKey,
#state{private_key = Key,
ssl_options =
@@ -1421,6 +1607,7 @@ certify_server(#state{key_algorithm = Algo} = State, _) when Algo == dh_anon;
Algo == ecdh_anon;
Algo == psk;
Algo == dhe_psk;
+ Algo == ecdhe_psk;
Algo == srp_anon ->
State;
@@ -1527,6 +1714,28 @@ key_exchange(#state{role = server, key_algorithm = dhe_psk,
State = Connection:queue_handshake(Msg, State0),
State#state{diffie_hellman_keys = DHKeys};
+key_exchange(#state{role = server, key_algorithm = ecdhe_psk,
+ ssl_options = #ssl_options{psk_identity = PskIdentityHint},
+ hashsign_algorithm = HashSignAlgo,
+ private_key = PrivateKey,
+ session = #session{ecc = ECCCurve},
+ connection_states = ConnectionStates0,
+ negotiated_version = Version
+ } = State0, Connection) ->
+ ECDHKeys = public_key:generate_key(ECCCurve),
+ #{security_parameters := SecParams} =
+ ssl_record:pending_connection_state(ConnectionStates0, read),
+ #security_parameters{client_random = ClientRandom,
+ server_random = ServerRandom} = SecParams,
+ Msg = ssl_handshake:key_exchange(server, ssl:tls_version(Version),
+ {ecdhe_psk,
+ PskIdentityHint, ECDHKeys,
+ HashSignAlgo, ClientRandom,
+ ServerRandom,
+ PrivateKey}),
+ State = Connection:queue_handshake(Msg, State0),
+ State#state{diffie_hellman_keys = ECDHKeys};
+
key_exchange(#state{role = server, key_algorithm = rsa_psk,
ssl_options = #ssl_options{psk_identity = undefined}} = State, _) ->
State;
@@ -1625,6 +1834,17 @@ key_exchange(#state{role = client,
{dhe_psk,
SslOpts#ssl_options.psk_identity, DhPubKey}),
Connection:queue_handshake(Msg, State0);
+
+key_exchange(#state{role = client,
+ ssl_options = SslOpts,
+ key_algorithm = ecdhe_psk,
+ negotiated_version = Version,
+ diffie_hellman_keys = ECDHKeys} = State0, Connection) ->
+ Msg = ssl_handshake:key_exchange(client, ssl:tls_version(Version),
+ {ecdhe_psk,
+ SslOpts#ssl_options.psk_identity, ECDHKeys}),
+ Connection:queue_handshake(Msg, State0);
+
key_exchange(#state{role = client,
ssl_options = SslOpts,
key_algorithm = rsa_psk,
@@ -1680,6 +1900,12 @@ rsa_psk_key_exchange(Version, PskIdentity, PremasterSecret,
rsa_psk_key_exchange(_, _, _, _) ->
throw (?ALERT_REC(?FATAL,?HANDSHAKE_FAILURE, pub_key_is_not_rsa)).
+request_client_cert(#state{key_algorithm = Alg} = State, _)
+ when Alg == dh_anon; Alg == ecdh_anon;
+ Alg == psk; Alg == dhe_psk; Alg == ecdhe_psk; Alg == rsa_psk;
+ Alg == srp_dss; Alg == srp_rsa; Alg == srp_anon ->
+ State;
+
request_client_cert(#state{ssl_options = #ssl_options{verify = verify_peer,
signature_algs = SupportedHashSigns},
connection_states = ConnectionStates0,
@@ -1801,6 +2027,18 @@ calculate_secret(#server_dhe_psk_params{
calculate_master_secret(PremasterSecret, State#state{diffie_hellman_keys = Keys},
Connection, certify, certify);
+calculate_secret(#server_ecdhe_psk_params{
+ dh_params = #server_ecdh_params{curve = ECCurve}} = ServerKey,
+ #state{ssl_options = #ssl_options{user_lookup_fun = PSKLookup}} =
+ State=#state{session=Session}, Connection) ->
+ ECDHKeys = public_key:generate_key(ECCurve),
+
+ PremasterSecret = ssl_handshake:premaster_secret(ServerKey, ECDHKeys, PSKLookup),
+ calculate_master_secret(PremasterSecret,
+ State#state{diffie_hellman_keys = ECDHKeys,
+ session = Session#session{ecc = ECCurve}},
+ Connection, certify, certify);
+
calculate_secret(#server_srp_params{srp_n = Prime, srp_g = Generator} = ServerKey,
#state{ssl_options = #ssl_options{srp_identity = SRPId}} = State,
Connection) ->
@@ -1885,6 +2123,7 @@ is_anonymous(Algo) when Algo == dh_anon;
Algo == ecdh_anon;
Algo == psk;
Algo == dhe_psk;
+ Algo == ecdhe_psk;
Algo == rsa_psk;
Algo == srp_anon ->
true;
diff --git a/lib/ssl/src/ssl_dist_sup.erl b/lib/ssl/src/ssl_dist_sup.erl
index 690b896919..c241a9bced 100644
--- a/lib/ssl/src/ssl_dist_sup.erl
+++ b/lib/ssl/src/ssl_dist_sup.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2011-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2011-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -46,8 +46,7 @@ start_link() ->
init([]) ->
AdminSup = ssl_admin_child_spec(),
ConnectionSup = ssl_connection_sup(),
- ProxyServer = proxy_server_child_spec(),
- {ok, {{one_for_all, 10, 3600}, [AdminSup, ProxyServer, ConnectionSup]}}.
+ {ok, {{one_for_all, 10, 3600}, [AdminSup, ConnectionSup]}}.
%%--------------------------------------------------------------------
%%% Internal functions
@@ -69,12 +68,3 @@ ssl_connection_sup() ->
Modules = [ssl_connection_sup],
Type = supervisor,
{Name, StartFunc, Restart, Shutdown, Type, Modules}.
-
-proxy_server_child_spec() ->
- Name = ssl_tls_dist_proxy,
- StartFunc = {ssl_tls_dist_proxy, start_link, []},
- Restart = permanent,
- Shutdown = 4000,
- Modules = [ssl_tls_dist_proxy],
- Type = worker,
- {Name, StartFunc, Restart, Shutdown, Type, Modules}.
diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl
index b47a11dc0d..8681765284 100644
--- a/lib/ssl/src/ssl_handshake.erl
+++ b/lib/ssl/src/ssl_handshake.erl
@@ -227,6 +227,7 @@ certificate_request(CipherSuite, CertDbHandle, CertDbRef, HashSigns, Version) ->
{ecdh, #'ECPrivateKey'{}} |
{psk, binary()} |
{dhe_psk, binary(), binary()} |
+ {ecdhe_psk, binary(), #'ECPrivateKey'{}} |
{srp, {binary(), binary()}, #srp_user{}, {HashAlgo::atom(), SignAlgo::atom()},
binary(), binary(), public_key:private_key()}) ->
#client_key_exchange{} | #server_key_exchange{}.
@@ -264,6 +265,13 @@ key_exchange(client, _Version, {dhe_psk, Identity, PublicKey}) ->
dh_public = PublicKey}
};
+key_exchange(client, _Version, {ecdhe_psk, Identity, #'ECPrivateKey'{publicKey = ECPublicKey}}) ->
+ #client_key_exchange{
+ exchange_keys = #client_ecdhe_psk_identity{
+ identity = Identity,
+ dh_public = ECPublicKey}
+ };
+
key_exchange(client, _Version, {psk_premaster_secret, PskIdentity, Secret, {_, PublicKey, _}}) ->
EncPremasterSecret =
encrypted_premaster_secret(Secret, PublicKey),
@@ -310,6 +318,16 @@ key_exchange(server, Version, {dhe_psk, PskIdentityHint, {PublicKey, _},
enc_server_key_exchange(Version, ServerEDHPSKParams,
HashSign, ClientRandom, ServerRandom, PrivateKey);
+key_exchange(server, Version, {ecdhe_psk, PskIdentityHint,
+ #'ECPrivateKey'{publicKey = ECPublicKey,
+ parameters = ECCurve},
+ HashSign, ClientRandom, ServerRandom, PrivateKey}) ->
+ ServerECDHEPSKParams = #server_ecdhe_psk_params{
+ hint = PskIdentityHint,
+ dh_params = #server_ecdh_params{curve = ECCurve, public = ECPublicKey}},
+ enc_server_key_exchange(Version, ServerECDHEPSKParams, HashSign,
+ ClientRandom, ServerRandom, PrivateKey);
+
key_exchange(server, Version, {srp, {PublicKey, _},
#srp_user{generator = Generator, prime = Prime,
salt = Salt},
@@ -532,14 +550,31 @@ premaster_secret(#server_dhe_psk_params{
LookupFun) ->
PremasterSecret = premaster_secret(PublicDhKey, PrivateDhKey, Params),
psk_secret(IdentityHint, LookupFun, PremasterSecret);
+
+premaster_secret(#server_ecdhe_psk_params{
+ hint = IdentityHint,
+ dh_params = #server_ecdh_params{
+ public = ECServerPubKey}},
+ PrivateEcDhKey,
+ LookupFun) ->
+ PremasterSecret = premaster_secret(#'ECPoint'{point = ECServerPubKey}, PrivateEcDhKey),
+ psk_secret(IdentityHint, LookupFun, PremasterSecret);
+
premaster_secret({rsa_psk, PSKIdentity}, PSKLookup, RSAPremasterSecret) ->
- psk_secret(PSKIdentity, PSKLookup, RSAPremasterSecret).
+ psk_secret(PSKIdentity, PSKLookup, RSAPremasterSecret);
+
+premaster_secret(#client_ecdhe_psk_identity{
+ identity = PSKIdentity,
+ dh_public = PublicEcDhPoint}, PrivateEcDhKey, PSKLookup) ->
+ PremasterSecret = premaster_secret(#'ECPoint'{point = PublicEcDhPoint}, PrivateEcDhKey),
+ psk_secret(PSKIdentity, PSKLookup, PremasterSecret).
premaster_secret(#client_dhe_psk_identity{
identity = PSKIdentity,
dh_public = PublicDhKey}, PrivateKey, #'DHParameter'{} = Params, PSKLookup) ->
PremasterSecret = premaster_secret(PublicDhKey, PrivateKey, Params),
psk_secret(PSKIdentity, PSKLookup, PremasterSecret).
+
premaster_secret(#client_psk_identity{identity = PSKIdentity}, PSKLookup) ->
psk_secret(PSKIdentity, PSKLookup);
premaster_secret({psk, PSKIdentity}, PSKLookup) ->
@@ -887,6 +922,7 @@ enc_server_key_exchange(Version, Params, {HashAlgo, SignAlgo},
| #client_ec_diffie_hellman_public{}
| #client_psk_identity{}
| #client_dhe_psk_identity{}
+ | #client_ecdhe_psk_identity{}
| #client_rsa_psk_identity{}
| #client_srp_public{}.
%%
@@ -1048,6 +1084,7 @@ dec_server_key(<<?UINT16(Len), PskIdentityHint:Len/binary, _/binary>> = KeyStruc
params_bin = BinMsg,
hashsign = HashSign,
signature = Signature};
+
dec_server_key(<<?UINT16(Len), IdentityHint:Len/binary,
?UINT16(PLen), P:PLen/binary,
?UINT16(GLen), G:GLen/binary,
@@ -1062,6 +1099,22 @@ dec_server_key(<<?UINT16(Len), IdentityHint:Len/binary,
params_bin = BinMsg,
hashsign = HashSign,
signature = Signature};
+dec_server_key(<<?UINT16(Len), IdentityHint:Len/binary,
+ ?BYTE(?NAMED_CURVE), ?UINT16(CurveID),
+ ?BYTE(PointLen), ECPoint:PointLen/binary,
+ _/binary>> = KeyStruct,
+ ?KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK, Version) ->
+ DHParams = #server_ecdh_params{
+ curve = {namedCurve, tls_v1:enum_to_oid(CurveID)},
+ public = ECPoint},
+ Params = #server_ecdhe_psk_params{
+ hint = IdentityHint,
+ dh_params = DHParams},
+ {BinMsg, HashSign, Signature} = dec_server_key_params(Len + 2 + PointLen + 4, KeyStruct, Version),
+ #server_key_params{params = Params,
+ params_bin = BinMsg,
+ hashsign = HashSign,
+ signature = Signature};
dec_server_key(<<?UINT16(NLen), N:NLen/binary,
?UINT16(GLen), G:GLen/binary,
?BYTE(SLen), S:SLen/binary,
@@ -1132,7 +1185,8 @@ filter_hashsigns([Suite | Suites], [{KeyExchange,_,_,_} | Algos], HashSigns, Acc
KeyExchange == ecdh_anon;
KeyExchange == srp_anon;
KeyExchange == psk;
- KeyExchange == dhe_psk ->
+ KeyExchange == dhe_psk;
+ KeyExchange == ecdhe_psk ->
%% In this case hashsigns is not used as the kexchange is anonaymous
filter_hashsigns(Suites, Algos, HashSigns, [Suite| Acc]).
@@ -1496,6 +1550,8 @@ advertises_ec_ciphers([{ecdhe_rsa, _,_,_} | _]) ->
true;
advertises_ec_ciphers([{ecdh_anon, _,_,_} | _]) ->
true;
+advertises_ec_ciphers([{ecdhe_psk, _,_,_} | _]) ->
+ true;
advertises_ec_ciphers([_| Rest]) ->
advertises_ec_ciphers(Rest).
@@ -1798,6 +1854,18 @@ encode_server_key(#server_dhe_psk_params{
YLen = byte_size(Y),
<<?UINT16(Len), PskIdentityHint/binary,
?UINT16(PLen), P/binary, ?UINT16(GLen), G/binary, ?UINT16(YLen), Y/binary>>;
+encode_server_key(Params = #server_ecdhe_psk_params{hint = undefined}) ->
+ encode_server_key(Params#server_ecdhe_psk_params{hint = <<>>});
+encode_server_key(#server_ecdhe_psk_params{
+ hint = PskIdentityHint,
+ dh_params = #server_ecdh_params{
+ curve = {namedCurve, ECCurve}, public = ECPubKey}}) ->
+ %%TODO: support arbitrary keys
+ Len = byte_size(PskIdentityHint),
+ KLen = size(ECPubKey),
+ <<?UINT16(Len), PskIdentityHint/binary,
+ ?BYTE(?NAMED_CURVE), ?UINT16((tls_v1:oid_to_enum(ECCurve))),
+ ?BYTE(KLen), ECPubKey/binary>>;
encode_server_key(#server_srp_params{srp_n = N, srp_g = G, srp_s = S, srp_b = B}) ->
NLen = byte_size(N),
GLen = byte_size(G),
@@ -1830,6 +1898,12 @@ encode_client_key(#client_dhe_psk_identity{identity = Id, dh_public = DHPublic},
Len = byte_size(Id),
DHLen = byte_size(DHPublic),
<<?UINT16(Len), Id/binary, ?UINT16(DHLen), DHPublic/binary>>;
+encode_client_key(Identity = #client_ecdhe_psk_identity{identity = undefined}, Version) ->
+ encode_client_key(Identity#client_ecdhe_psk_identity{identity = <<"psk_identity">>}, Version);
+encode_client_key(#client_ecdhe_psk_identity{identity = Id, dh_public = DHPublic}, _) ->
+ Len = byte_size(Id),
+ DHLen = byte_size(DHPublic),
+ <<?UINT16(Len), Id/binary, ?BYTE(DHLen), DHPublic/binary>>;
encode_client_key(Identity = #client_rsa_psk_identity{identity = undefined}, Version) ->
encode_client_key(Identity#client_rsa_psk_identity{identity = <<"psk_identity">>}, Version);
encode_client_key(#client_rsa_psk_identity{identity = Id, exchange_keys = ExchangeKeys}, Version) ->
@@ -1881,6 +1955,10 @@ dec_client_key(<<?UINT16(Len), Id:Len/binary,
?UINT16(DH_YLen), DH_Y:DH_YLen/binary>>,
?KEY_EXCHANGE_DHE_PSK, _) ->
#client_dhe_psk_identity{identity = Id, dh_public = DH_Y};
+dec_client_key(<<?UINT16(Len), Id:Len/binary,
+ ?BYTE(DH_YLen), DH_Y:DH_YLen/binary>>,
+ ?KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK, _) ->
+ #client_ecdhe_psk_identity{identity = Id, dh_public = DH_Y};
dec_client_key(<<?UINT16(Len), Id:Len/binary, PKEPMS/binary>>,
?KEY_EXCHANGE_RSA_PSK, {3, 0}) ->
#client_rsa_psk_identity{identity = Id,
@@ -2058,6 +2136,8 @@ key_exchange_alg(psk) ->
?KEY_EXCHANGE_PSK;
key_exchange_alg(dhe_psk) ->
?KEY_EXCHANGE_DHE_PSK;
+key_exchange_alg(ecdhe_psk) ->
+ ?KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK;
key_exchange_alg(rsa_psk) ->
?KEY_EXCHANGE_RSA_PSK;
key_exchange_alg(Alg)
@@ -2316,6 +2396,7 @@ is_acceptable_hash_sign({_, ecdsa} = Algos, ecdsa, ecdsa, ecdhe_ecdsa, Supported
is_acceptable_hash_sign(_, _, _, KeyExAlgo, _) when
KeyExAlgo == psk;
KeyExAlgo == dhe_psk;
+ KeyExAlgo == ecdhe_psk;
KeyExAlgo == srp_anon;
KeyExAlgo == dh_anon;
KeyExAlgo == ecdhe_anon
diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl
index 324b7dbde3..a191fcf766 100644
--- a/lib/ssl/src/ssl_handshake.hrl
+++ b/lib/ssl/src/ssl_handshake.hrl
@@ -133,6 +133,7 @@
-define(KEY_EXCHANGE_DIFFIE_HELLMAN, 1).
-define(KEY_EXCHANGE_EC_DIFFIE_HELLMAN, 6).
-define(KEY_EXCHANGE_PSK, 2).
+-define(KEY_EXCHANGE_EC_DIFFIE_HELLMAN_PSK, 7).
-define(KEY_EXCHANGE_DHE_PSK, 3).
-define(KEY_EXCHANGE_RSA_PSK, 4).
-define(KEY_EXCHANGE_SRP, 5).
@@ -162,6 +163,11 @@
dh_params
}).
+-record(server_ecdhe_psk_params, {
+ hint,
+ dh_params
+ }).
+
-record(server_srp_params, {
srp_n, %% opaque srp_N<1..2^16-1>
srp_g, %% opaque srp_g<1..2^16-1>
@@ -254,6 +260,11 @@
dh_public
}).
+-record(client_ecdhe_psk_identity, {
+ identity,
+ dh_public
+ }).
+
-record(client_rsa_psk_identity, {
identity,
exchange_keys
diff --git a/lib/ssl/src/ssl_tls_dist_proxy.erl b/lib/ssl/src/ssl_tls_dist_proxy.erl
deleted file mode 100644
index 08947f24dd..0000000000
--- a/lib/ssl/src/ssl_tls_dist_proxy.erl
+++ /dev/null
@@ -1,479 +0,0 @@
-%%
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 2011-2016. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%
-%% %CopyrightEnd%
-%%
--module(ssl_tls_dist_proxy).
-
-
--export([listen/2, accept/2, connect/3, get_tcp_address/1]).
--export([init/1, start_link/0, handle_call/3, handle_cast/2, handle_info/2,
- terminate/2, code_change/3, ssl_options/2]).
-
--include_lib("kernel/include/net_address.hrl").
-
--record(state,
- {listen,
- accept_loop
- }).
-
--define(PPRE, 4).
--define(PPOST, 4).
-
-
-%%====================================================================
-%% Internal application API
-%%====================================================================
-
-listen(Driver, Name) ->
- gen_server:call(?MODULE, {listen, Driver, Name}, infinity).
-
-accept(Driver, Listen) ->
- gen_server:call(?MODULE, {accept, Driver, Listen}, infinity).
-
-connect(Driver, Ip, Port) ->
- gen_server:call(?MODULE, {connect, Driver, Ip, Port}, infinity).
-
-
-do_listen(Options) ->
- {First,Last} = case application:get_env(kernel,inet_dist_listen_min) of
- {ok,N} when is_integer(N) ->
- case application:get_env(kernel,
- inet_dist_listen_max) of
- {ok,M} when is_integer(M) ->
- {N,M};
- _ ->
- {N,N}
- end;
- _ ->
- {0,0}
- end,
- do_listen(First, Last, listen_options([{backlog,128}|Options])).
-
-do_listen(First,Last,_) when First > Last ->
- {error,eaddrinuse};
-do_listen(First,Last,Options) ->
- case gen_tcp:listen(First, Options) of
- {error, eaddrinuse} ->
- do_listen(First+1,Last,Options);
- Other ->
- Other
- end.
-
-listen_options(Opts0) ->
- Opts1 =
- case application:get_env(kernel, inet_dist_use_interface) of
- {ok, Ip} ->
- [{ip, Ip} | Opts0];
- _ ->
- Opts0
- end,
- case application:get_env(kernel, inet_dist_listen_options) of
- {ok,ListenOpts} ->
- ListenOpts ++ Opts1;
- _ ->
- Opts1
- end.
-
-connect_options(Opts) ->
- case application:get_env(kernel, inet_dist_connect_options) of
- {ok,ConnectOpts} ->
- lists:ukeysort(1, ConnectOpts ++ Opts);
- _ ->
- Opts
- end.
-
-%%====================================================================
-%% gen_server callbacks
-%%====================================================================
-
-start_link() ->
- gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
-init([]) ->
- process_flag(priority, max),
- {ok, #state{}}.
-
-handle_call({listen, Driver, Name}, _From, State) ->
- case gen_tcp:listen(0, [{active, false}, {packet,?PPRE}, {ip, loopback}]) of
- {ok, Socket} ->
- {ok, World} = do_listen([{active, false}, binary, {packet,?PPRE}, {reuseaddr, true},
- Driver:family()]),
- {ok, TcpAddress} = get_tcp_address(Socket),
- {ok, WorldTcpAddress} = get_tcp_address(World),
- {_,Port} = WorldTcpAddress#net_address.address,
- ErlEpmd = net_kernel:epmd_module(),
- case ErlEpmd:register_node(Name, Port, Driver) of
- {ok, Creation} ->
- {reply, {ok, {Socket, TcpAddress, Creation}},
- State#state{listen={Socket, World}}};
- {error, _} = Error ->
- {reply, Error, State}
- end;
- Error ->
- {reply, Error, State}
- end;
-
-handle_call({accept, _Driver, Listen}, {From, _}, State = #state{listen={_, World}}) ->
- Self = self(),
- ErtsPid = spawn_link(fun() -> accept_loop(Self, erts, Listen, From) end),
- WorldPid = spawn_link(fun() -> accept_loop(Self, world, World, Listen) end),
- {reply, ErtsPid, State#state{accept_loop={ErtsPid, WorldPid}}};
-
-handle_call({connect, Driver, Ip, Port}, {From, _}, State) ->
- Me = self(),
- Pid = spawn_link(fun() -> setup_proxy(Driver, Ip, Port, Me) end),
- receive
- {Pid, go_ahead, LPort} ->
- Res = {ok, Socket} = try_connect(LPort),
- case gen_tcp:controlling_process(Socket, From) of
- {error, badarg} = Error -> {reply, Error, State}; % From is dead anyway.
- ok ->
- flush_old_controller(From, Socket),
- {reply, Res, State}
- end;
- {Pid, Error} ->
- {reply, Error, State}
- end;
-
-handle_call(_What, _From, State) ->
- {reply, ok, State}.
-
-handle_cast(_What, State) ->
- {noreply, State}.
-
-handle_info(_What, State) ->
- {noreply, State}.
-
-terminate(_Reason, _St) ->
- ok.
-
-code_change(_OldVsn, St, _Extra) ->
- {ok, St}.
-
-%%--------------------------------------------------------------------
-%%% Internal functions
-%%--------------------------------------------------------------------
-get_tcp_address(Socket) ->
- case inet:sockname(Socket) of
- {ok, Address} ->
- {ok, Host} = inet:gethostname(),
- NetAddress = #net_address{
- address = Address,
- host = Host,
- protocol = proxy,
- family = inet
- },
- {ok, NetAddress};
- {error, _} = Error -> Error
- end.
-
-accept_loop(Proxy, erts = Type, Listen, Extra) ->
- process_flag(priority, max),
- case gen_tcp:accept(Listen) of
- {ok, Socket} ->
- Extra ! {accept,self(),Socket,inet,proxy},
- receive
- {_Kernel, controller, Pid} ->
- inet:setopts(Socket, [nodelay()]),
- ok = gen_tcp:controlling_process(Socket, Pid),
- flush_old_controller(Pid, Socket),
- Pid ! {self(), controller};
- {_Kernel, unsupported_protocol} ->
- exit(unsupported_protocol)
- end;
- {error, closed} ->
- %% The listening socket is closed: the proxy process is
- %% shutting down. Exit normally, to avoid generating a
- %% spurious error report.
- exit(normal);
- Error ->
- exit(Error)
- end,
- accept_loop(Proxy, Type, Listen, Extra);
-
-accept_loop(Proxy, world = Type, Listen, Extra) ->
- process_flag(priority, max),
- case gen_tcp:accept(Listen) of
- {ok, Socket} ->
- Opts = get_ssl_options(server),
- wait_for_code_server(),
- case ssl:ssl_accept(Socket, Opts) of
- {ok, SslSocket} ->
- PairHandler =
- spawn_link(fun() ->
- setup_connection(SslSocket, Extra)
- end),
- ok = ssl:controlling_process(SslSocket, PairHandler),
- flush_old_controller(PairHandler, SslSocket);
- {error, {options, _}} = Error ->
- %% Bad options: that's probably our fault. Let's log that.
- error_logger:error_msg("Cannot accept TLS distribution connection: ~s~n",
- [ssl:format_error(Error)]),
- gen_tcp:close(Socket);
- _ ->
- gen_tcp:close(Socket)
- end;
- Error ->
- exit(Error)
- end,
- accept_loop(Proxy, Type, Listen, Extra).
-
-wait_for_code_server() ->
- %% This is an ugly hack. Upgrading a socket to TLS requires the
- %% crypto module to be loaded. Loading the crypto module triggers
- %% its on_load function, which calls code:priv_dir/1 to find the
- %% directory where its NIF library is. However, distribution is
- %% started earlier than the code server, so the code server is not
- %% necessarily started yet, and code:priv_dir/1 might fail because
- %% of that, if we receive an incoming connection on the
- %% distribution port early enough.
- %%
- %% If the on_load function of a module fails, the module is
- %% unloaded, and the function call that triggered loading it fails
- %% with 'undef', which is rather confusing.
- %%
- %% Thus, the ssl_tls_dist_proxy process will terminate, and be
- %% restarted by ssl_dist_sup. However, it won't have any memory
- %% of being asked by net_kernel to listen for incoming
- %% connections. Hence, the node will believe that it's open for
- %% distribution, but it actually isn't.
- %%
- %% So let's avoid that by waiting for the code server to start.
- case whereis(code_server) of
- undefined ->
- timer:sleep(10),
- wait_for_code_server();
- Pid when is_pid(Pid) ->
- ok
- end.
-
-try_connect(Port) ->
- case gen_tcp:connect({127,0,0,1}, Port, [{active, false}, {packet,?PPRE}, nodelay()]) of
- R = {ok, _S} ->
- R;
- {error, _R} ->
- try_connect(Port)
- end.
-
-setup_proxy(Driver, Ip, Port, Parent) ->
- process_flag(trap_exit, true),
- Opts = connect_options(get_ssl_options(client)),
- case ssl:connect(Ip, Port, [{active, true}, binary, {packet,?PPRE}, nodelay(),
- Driver:family()] ++ Opts) of
- {ok, World} ->
- {ok, ErtsL} = gen_tcp:listen(0, [{active, true}, {ip, loopback}, binary, {packet,?PPRE}]),
- {ok, #net_address{address={_,LPort}}} = get_tcp_address(ErtsL),
- Parent ! {self(), go_ahead, LPort},
- case gen_tcp:accept(ErtsL) of
- {ok, Erts} ->
- %% gen_tcp:close(ErtsL),
- loop_conn_setup(World, Erts);
- Err ->
- Parent ! {self(), Err}
- end;
- {error, {options, _}} = Err ->
- %% Bad options: that's probably our fault. Let's log that.
- error_logger:error_msg("Cannot open TLS distribution connection: ~s~n",
- [ssl:format_error(Err)]),
- Parent ! {self(), Err};
- Err ->
- Parent ! {self(), Err}
- end.
-
-
-%% we may not always want the nodelay behaviour
-%% %% for performance reasons
-
-nodelay() ->
- case application:get_env(kernel, dist_nodelay) of
- undefined ->
- {nodelay, true};
- {ok, true} ->
- {nodelay, true};
- {ok, false} ->
- {nodelay, false};
- _ ->
- {nodelay, true}
- end.
-
-setup_connection(World, ErtsListen) ->
- process_flag(trap_exit, true),
- {ok, TcpAddress} = get_tcp_address(ErtsListen),
- {_Addr,Port} = TcpAddress#net_address.address,
- {ok, Erts} = gen_tcp:connect({127,0,0,1}, Port, [{active, true}, binary, {packet,?PPRE}, nodelay()]),
- ssl:setopts(World, [{active,true}, {packet,?PPRE}, nodelay()]),
- loop_conn_setup(World, Erts).
-
-loop_conn_setup(World, Erts) ->
- receive
- {ssl, World, Data = <<$a, _/binary>>} ->
- gen_tcp:send(Erts, Data),
- ssl:setopts(World, [{packet,?PPOST}, nodelay()]),
- inet:setopts(Erts, [{packet,?PPOST}, nodelay()]),
- loop_conn(World, Erts);
- {tcp, Erts, Data = <<$a, _/binary>>} ->
- ssl:send(World, Data),
- ssl:setopts(World, [{packet,?PPOST}, nodelay()]),
- inet:setopts(Erts, [{packet,?PPOST}, nodelay()]),
- loop_conn(World, Erts);
- {ssl, World, Data = <<_, _/binary>>} ->
- gen_tcp:send(Erts, Data),
- loop_conn_setup(World, Erts);
- {tcp, Erts, Data = <<_, _/binary>>} ->
- ssl:send(World, Data),
- loop_conn_setup(World, Erts);
- {ssl, World, Data} ->
- gen_tcp:send(Erts, Data),
- loop_conn_setup(World, Erts);
- {tcp, Erts, Data} ->
- ssl:send(World, Data),
- loop_conn_setup(World, Erts);
- {tcp_closed, Erts} ->
- ssl:close(World);
- {ssl_closed, World} ->
- gen_tcp:close(Erts);
- {ssl_error, World, _} ->
-
- ssl:close(World)
- end.
-
-loop_conn(World, Erts) ->
- receive
- {ssl, World, Data} ->
- gen_tcp:send(Erts, Data),
- loop_conn(World, Erts);
- {tcp, Erts, Data} ->
- ssl:send(World, Data),
- loop_conn(World, Erts);
- {tcp_closed, Erts} ->
- ssl:close(World);
- {ssl_closed, World} ->
- gen_tcp:close(Erts);
- {ssl_error, World, _} ->
- ssl:close(World)
- end.
-
-get_ssl_options(Type) ->
- case init:get_argument(ssl_dist_opt) of
- {ok, Args} ->
- [{erl_dist, true} | ssl_options(Type, lists:append(Args))];
- _ ->
- [{erl_dist, true}]
- end.
-
-ssl_options(_,[]) ->
- [];
-ssl_options(server, ["client_" ++ _, _Value |T]) ->
- ssl_options(server,T);
-ssl_options(client, ["server_" ++ _, _Value|T]) ->
- ssl_options(client,T);
-ssl_options(server, ["server_certfile", Value|T]) ->
- [{certfile, Value} | ssl_options(server,T)];
-ssl_options(client, ["client_certfile", Value | T]) ->
- [{certfile, Value} | ssl_options(client,T)];
-ssl_options(server, ["server_cacertfile", Value|T]) ->
- [{cacertfile, Value} | ssl_options(server,T)];
-ssl_options(client, ["client_cacertfile", Value|T]) ->
- [{cacertfile, Value} | ssl_options(client,T)];
-ssl_options(server, ["server_keyfile", Value|T]) ->
- [{keyfile, Value} | ssl_options(server,T)];
-ssl_options(client, ["client_keyfile", Value|T]) ->
- [{keyfile, Value} | ssl_options(client,T)];
-ssl_options(server, ["server_password", Value|T]) ->
- [{password, Value} | ssl_options(server,T)];
-ssl_options(client, ["client_password", Value|T]) ->
- [{password, Value} | ssl_options(client,T)];
-ssl_options(server, ["server_verify", Value|T]) ->
- [{verify, atomize(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_verify", Value|T]) ->
- [{verify, atomize(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_verify_fun", Value|T]) ->
- [{verify_fun, verify_fun(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_verify_fun", Value|T]) ->
- [{verify_fun, verify_fun(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_crl_check", Value|T]) ->
- [{crl_check, atomize(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_crl_check", Value|T]) ->
- [{crl_check, atomize(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_crl_cache", Value|T]) ->
- [{crl_cache, termify(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_crl_cache", Value|T]) ->
- [{crl_cache, termify(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_reuse_sessions", Value|T]) ->
- [{reuse_sessions, atomize(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_reuse_sessions", Value|T]) ->
- [{reuse_sessions, atomize(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_secure_renegotiate", Value|T]) ->
- [{secure_renegotiate, atomize(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_secure_renegotiate", Value|T]) ->
- [{secure_renegotiate, atomize(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_depth", Value|T]) ->
- [{depth, list_to_integer(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_depth", Value|T]) ->
- [{depth, list_to_integer(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_hibernate_after", Value|T]) ->
- [{hibernate_after, list_to_integer(Value)} | ssl_options(server,T)];
-ssl_options(client, ["client_hibernate_after", Value|T]) ->
- [{hibernate_after, list_to_integer(Value)} | ssl_options(client,T)];
-ssl_options(server, ["server_ciphers", Value|T]) ->
- [{ciphers, Value} | ssl_options(server,T)];
-ssl_options(client, ["client_ciphers", Value|T]) ->
- [{ciphers, Value} | ssl_options(client,T)];
-ssl_options(server, ["server_dhfile", Value|T]) ->
- [{dhfile, Value} | ssl_options(server,T)];
-ssl_options(server, ["server_fail_if_no_peer_cert", Value|T]) ->
- [{fail_if_no_peer_cert, atomize(Value)} | ssl_options(server,T)];
-ssl_options(Type, Opts) ->
- error(malformed_ssl_dist_opt, [Type, Opts]).
-
-atomize(List) when is_list(List) ->
- list_to_atom(List);
-atomize(Atom) when is_atom(Atom) ->
- Atom.
-
-termify(String) when is_list(String) ->
- {ok, Tokens, _} = erl_scan:string(String ++ "."),
- {ok, Term} = erl_parse:parse_term(Tokens),
- Term.
-
-verify_fun(Value) ->
- case termify(Value) of
- {Mod, Func, State} when is_atom(Mod), is_atom(Func) ->
- Fun = fun Mod:Func/3,
- {Fun, State};
- _ ->
- error(malformed_ssl_dist_opt, [Value])
- end.
-
-flush_old_controller(Pid, Socket) ->
- receive
- {tcp, Socket, Data} ->
- Pid ! {tcp, Socket, Data},
- flush_old_controller(Pid, Socket);
- {tcp_closed, Socket} ->
- Pid ! {tcp_closed, Socket},
- flush_old_controller(Pid, Socket);
- {ssl, Socket, Data} ->
- Pid ! {ssl, Socket, Data},
- flush_old_controller(Pid, Socket);
- {ssl_closed, Socket} ->
- Pid ! {ssl_closed, Socket},
- flush_old_controller(Pid, Socket)
- after 0 ->
- ok
- end.
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index ccda58e0a9..62452808ae 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -65,7 +65,7 @@
%% gen_statem state functions
-export([init/3, error/3, downgrade/3, %% Initiation and take down states
hello/3, certify/3, cipher/3, abbreviated/3, %% Handshake states
- connection/3]).
+ connection/3, death_row/3]).
%% gen_statem callbacks
-export([callback_mode/0, terminate/3, code_change/4, format_status/2]).
@@ -381,6 +381,13 @@ connection(Type, Event, State) ->
ssl_connection:?FUNCTION_NAME(Type, Event, State, ?MODULE).
%%--------------------------------------------------------------------
+-spec death_row(gen_statem:event_type(), term(), #state{}) ->
+ gen_statem:state_function_result().
+%%--------------------------------------------------------------------
+death_row(Type, Event, State) ->
+ ssl_connection:death_row(Type, Event, State, ?MODULE).
+
+%%--------------------------------------------------------------------
-spec downgrade(gen_statem:event_type(), term(), #state{}) ->
gen_statem:state_function_result().
%%--------------------------------------------------------------------
@@ -437,7 +444,7 @@ handle_info({CloseTag, Socket}, StateName,
next_event(StateName, no_record, State)
end;
handle_info(Msg, StateName, State) ->
- ssl_connection:handle_info(Msg, StateName, State).
+ ssl_connection:StateName(info, Msg, State, ?MODULE).
handle_common_event(internal, #alert{} = Alert, StateName,
#state{negotiated_version = Version} = State) ->
diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl
index 13265debb1..4e7252f469 100644
--- a/lib/ssl/test/ssl_test_lib.erl
+++ b/lib/ssl/test/ssl_test_lib.erl
@@ -1533,10 +1533,14 @@ is_psk_anon_suite({psk, _,_}) ->
true;
is_psk_anon_suite({dhe_psk,_,_}) ->
true;
+is_psk_anon_suite({ecdhe_psk,_,_}) ->
+ true;
is_psk_anon_suite({psk, _,_,_}) ->
true;
is_psk_anon_suite({dhe_psk, _,_,_}) ->
true;
+is_psk_anon_suite({ecdhe_psk, _,_,_}) ->
+ true;
is_psk_anon_suite(_) ->
false.
diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile
index 93eac8220d..aeed79408b 100644
--- a/lib/stdlib/doc/src/Makefile
+++ b/lib/stdlib/doc/src/Makefile
@@ -98,6 +98,7 @@ XML_REF3_FILES = \
sys.xml \
timer.xml \
unicode.xml \
+ uri_string.xml \
win32reg.xml \
zip.xml
diff --git a/lib/stdlib/doc/src/c.xml b/lib/stdlib/doc/src/c.xml
index 7666699183..697e1715e7 100644
--- a/lib/stdlib/doc/src/c.xml
+++ b/lib/stdlib/doc/src/c.xml
@@ -94,6 +94,15 @@
</func>
<func>
+ <name name="erlangrc" arity="1"/>
+ <fsummary>Load an erlang resource file.</fsummary>
+ <desc>
+ <p>Search <c>PathList</c> and load <c>.erlang</c> resource file if
+ found.</p>
+ </desc>
+ </func>
+
+ <func>
<name name="flush" arity="0"/>
<fsummary>Flush any messages sent to the shell.</fsummary>
<desc>
diff --git a/lib/stdlib/doc/src/filelib.xml b/lib/stdlib/doc/src/filelib.xml
index 80c4acffdb..5e631aac21 100644
--- a/lib/stdlib/doc/src/filelib.xml
+++ b/lib/stdlib/doc/src/filelib.xml
@@ -45,6 +45,30 @@
<p>For more information about raw filenames, see the
<seealso marker="kernel:file"><c>file</c></seealso> module.</p>
+
+ <note>
+ <p>
+ Functionality in this module generally assumes valid input and
+ does not necessarily fail on input that does not use a valid
+ encoding, but may instead very likely produce invalid output.
+ </p>
+ <p>
+ File operations used to accept filenames containing
+ null characters (integer value zero). This caused
+ the name to be truncated and in some cases arguments
+ to primitive operations to be mixed up. Filenames
+ containing null characters inside the filename
+ are now <em>rejected</em> and will cause primitive
+ file operations to fail.
+ </p>
+ </note>
+ <warning><p>
+ Currently null characters at the end of the filename
+ will be accepted by primitive file operations. Such
+ filenames are however still documented as invalid. The
+ implementation will also change in the future and
+ reject such filenames.
+ </p></warning>
</description>
<datatypes>
@@ -193,6 +217,11 @@
<p>Other characters represent themselves. Only filenames that
have exactly the same character in the same position match.
Matching is case-sensitive, for example, "a" does not match "A".</p>
+ <p>Directory separators must always be written as <c>/</c>, even on
+ Windows.</p>
+ <p>A character preceded by <c>\</c> loses its special meaning. Note
+ that <c>\</c> must be written as <c>\\</c> in a string literal.
+ For example, "\\?*" will match any filename starting with <c>?</c>.</p>
<p>Notice that multiple "*" characters are allowed
(as in Unix wildcards, but opposed to Windows/DOS wildcards).</p>
<p><em>Examples:</em></p>
diff --git a/lib/stdlib/doc/src/filename.xml b/lib/stdlib/doc/src/filename.xml
index 14fd5ef787..d2608ad542 100644
--- a/lib/stdlib/doc/src/filename.xml
+++ b/lib/stdlib/doc/src/filename.xml
@@ -46,7 +46,10 @@
filename by removing redundant directory separators, use
<seealso marker="#join/1"><c>join/1</c></seealso>.</p>
- <p>The module supports raw filenames in the way that if a binary is
+ <p>
+ The module supports
+ <seealso marker="unicode_usage#notes-about-raw-filenames">raw
+ filenames</seealso> in the way that if a binary is
present, or the filename cannot be interpreted according to the return
value of <seealso marker="kernel:file#native_name_encoding/0">
<c>file:native_name_encoding/0</c></seealso>, a raw filename is also
@@ -56,6 +59,30 @@
(the join operation is performed of course). For more information
about raw filenames, see the
<seealso marker="kernel:file"><c>file</c></seealso> module.</p>
+
+ <note>
+ <p>
+ Functionality in this module generally assumes valid input and
+ does not necessarily fail on input that does not use a valid
+ encoding, but may instead very likely produce invalid output.
+ </p>
+ <p>
+ File operations used to accept filenames containing
+ null characters (integer value zero). This caused
+ the name to be truncated and in some cases arguments
+ to primitive operations to be mixed up. Filenames
+ containing null characters inside the filename
+ are now <em>rejected</em> and will cause primitive
+ file operations to fail.
+ </p>
+ </note>
+ <warning><p>
+ Currently null characters at the end of the filename
+ will be accepted by primitive file operations. Such
+ filenames are however still documented as invalid. The
+ implementation will also change in the future and
+ reject such filenames.
+ </p></warning>
</description>
<datatypes>
<datatype>
@@ -555,6 +582,7 @@ unsafe</pre>
["a:/","msdev","include"]</pre>
</desc>
</func>
+
</funcs>
</erlref>
diff --git a/lib/stdlib/doc/src/gen_server.xml b/lib/stdlib/doc/src/gen_server.xml
index 7d137fc772..da74e793e6 100644
--- a/lib/stdlib/doc/src/gen_server.xml
+++ b/lib/stdlib/doc/src/gen_server.xml
@@ -60,6 +60,8 @@ gen_server:abcast -----> Module:handle_cast/2
- -----> Module:handle_info/2
+- -----> Module:handle_continue/2
+
- -----> Module:terminate/2
- -----> Module:code_change/3</pre>
@@ -88,6 +90,13 @@ gen_server:abcast -----> Module:handle_cast/2
implies at least two garbage collections (when hibernating and
shortly after waking up) and is not something you want to do
between each call to a busy server.</p>
+
+ <p>If the <c>gen_server</c> process needs to perform an action
+ immediately after initialization or to break the execution of a
+ callback into multiple steps, it can return <c>{continue,Continue}</c>
+ in place of the time-out or hibernation value, which will immediately
+ invoke the <c>handle_continue/2</c> callback.</p>
+
</description>
<funcs>
@@ -610,12 +619,15 @@ gen_server:abcast -----> Module:handle_cast/2
<v>State = term()</v>
<v>Result = {reply,Reply,NewState} | {reply,Reply,NewState,Timeout}</v>
<v>&nbsp;&nbsp;| {reply,Reply,NewState,hibernate}</v>
+ <v>&nbsp;&nbsp;| {reply,Reply,NewState,{continue,Continue}}</v>
<v>&nbsp;&nbsp;| {noreply,NewState} | {noreply,NewState,Timeout}</v>
<v>&nbsp;&nbsp;| {noreply,NewState,hibernate}</v>
+ <v>&nbsp;&nbsp;| {noreply,NewState,{continue,Continue}}</v>
<v>&nbsp;&nbsp;| {stop,Reason,Reply,NewState} | {stop,Reason,NewState}</v>
<v>&nbsp;Reply = term()</v>
<v>&nbsp;NewState = term()</v>
<v>&nbsp;Timeout = int()>=0 | infinity</v>
+ <v>&nbsp;Continue = term()</v>
<v>&nbsp;Reason = term()</v>
</type>
<desc>
@@ -673,9 +685,11 @@ gen_server:abcast -----> Module:handle_cast/2
<v>State = term()</v>
<v>Result = {noreply,NewState} | {noreply,NewState,Timeout}</v>
<v>&nbsp;&nbsp;| {noreply,NewState,hibernate}</v>
+ <v>&nbsp;&nbsp;| {noreply,NewState,{continue,Continue}}</v>
<v>&nbsp;&nbsp;| {stop,Reason,NewState}</v>
<v>&nbsp;NewState = term()</v>
<v>&nbsp;Timeout = int()>=0 | infinity</v>
+ <v>&nbsp;Continue = term()</v>
<v>&nbsp;Reason = term()</v>
</type>
<desc>
@@ -690,6 +704,41 @@ gen_server:abcast -----> Module:handle_cast/2
</func>
<func>
+ <name>Module:handle_continue(Continue, State) -> Result</name>
+ <fsummary>Handle a continue instruction.</fsummary>
+ <type>
+ <v>Continue = term()</v>
+ <v>State = term()</v>
+ <v>Result = {noreply,NewState} | {noreply,NewState,Timeout}</v>
+ <v>&nbsp;&nbsp;| {noreply,NewState,hibernate}</v>
+ <v>&nbsp;&nbsp;| {noreply,NewState,{continue,Continue}}</v>
+ <v>&nbsp;&nbsp;| {stop,Reason,NewState}</v>
+ <v>&nbsp;NewState = term()</v>
+ <v>&nbsp;Timeout = int()>=0 | infinity</v>
+ <v>&nbsp;Continue = term()</v>
+ <v>&nbsp;Reason = normal | term()</v>
+ </type>
+ <desc>
+ <note>
+ <p>This callback is optional, so callback modules need to
+ export it only if they return <c>{continue,Continue}</c>
+ from another callback. If continue is used and the callback
+ is not implemented, the process will exit with <c>undef</c>
+ error.</p>
+ </note>
+ <p>This function is called by a <c>gen_server</c> process whenever
+ a previous callback returns <c>{continue, Continue}</c>.
+ <c>handle_continue/2</c> is invoked immediately after the previous
+ callback, which makes it useful for performing work after
+ initialization or for splitting the work in a callback in
+ multiple steps, updating the process state along the way.</p>
+ <p>For a description of the other arguments and possible return values,
+ see <seealso marker="#Module:handle_call/3">
+ <c>Module:handle_call/3</c></seealso>.</p>
+ </desc>
+ </func>
+
+ <func>
<name>Module:handle_info(Info, State) -> Result</name>
<fsummary>Handle an incoming message.</fsummary>
<type>
@@ -697,6 +746,7 @@ gen_server:abcast -----> Module:handle_cast/2
<v>State = term()</v>
<v>Result = {noreply,NewState} | {noreply,NewState,Timeout}</v>
<v>&nbsp;&nbsp;| {noreply,NewState,hibernate}</v>
+ <v>&nbsp;&nbsp;| {noreply,NewState,{continue,Continue}}</v>
<v>&nbsp;&nbsp;| {stop,Reason,NewState}</v>
<v>&nbsp;NewState = term()</v>
<v>&nbsp;Timeout = int()>=0 | infinity</v>
@@ -726,7 +776,7 @@ gen_server:abcast -----> Module:handle_cast/2
<type>
<v>Args = term()</v>
<v>Result = {ok,State} | {ok,State,Timeout} | {ok,State,hibernate}</v>
- <v>&nbsp;| {stop,Reason} | ignore</v>
+ <v>&nbsp;| {ok,State,{continue,Continue}} | {stop,Reason} | ignore</v>
<v>&nbsp;State = term()</v>
<v>&nbsp;Timeout = int()>=0 | infinity</v>
<v>&nbsp;Reason = term()</v>
diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml
index a7caa71dcb..4a824f073e 100644
--- a/lib/stdlib/doc/src/gen_statem.xml
+++ b/lib/stdlib/doc/src/gen_statem.xml
@@ -1329,7 +1329,7 @@ handle_event(_, _, State, Data) ->
<c><anno>T</anno></c> is the time-out time.
<c>{clean_timeout,<anno>T</anno>}</c> works like
just <c>T</c> described in the note above
- and uses a proxy process for <c>T &lt; infinity</c>,
+ and uses a proxy process
while <c>{dirty_timeout,<anno>T</anno>}</c>
bypasses the proxy process which is more lightweight.
</p>
@@ -1339,8 +1339,12 @@ handle_event(_, _, State, Data) ->
with <c>{dirty_timeout,<anno>T</anno>}</c>
to avoid that the calling process dies when the call
times out, you will have to be prepared to handle
- a late reply.
- So why not just let the calling process die?
+ a late reply. Note that there is an odd chance
+ to get a late reply even with
+ <c>{dirty_timeout,infinity}</c> or <c>infinity</c>
+ for example in the event of network problems.
+ So why not just let the calling process die
+ by not catching the exception?
</p>
</note>
<p>
@@ -1851,7 +1855,7 @@ handle_event(_, _, State, Data) ->
</p>
<note>
<p>
- Note that if the <c>gen_statem</c> is started trough
+ Note that if the <c>gen_statem</c> is started through
<seealso marker="proc_lib"><c>proc_lib</c></seealso>
and
<seealso marker="#enter_loop/4"><c>enter_loop/4-6</c></seealso>,
diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml
index 89fb858823..21f680a0ee 100644
--- a/lib/stdlib/doc/src/rand.xml
+++ b/lib/stdlib/doc/src/rand.xml
@@ -133,8 +133,9 @@
variable <c>rand_seed</c> to remember the current state.</p>
<p>If a process calls
- <seealso marker="#uniform-0"><c>uniform/0</c></seealso> or
- <seealso marker="#uniform-1"><c>uniform/1</c></seealso> without
+ <seealso marker="#uniform-0"><c>uniform/0</c></seealso>,
+ <seealso marker="#uniform-1"><c>uniform/1</c></seealso> or
+ <seealso marker="#uniform_real-0"><c>uniform_real/0</c></seealso> without
setting a seed first, <seealso marker="#seed-1"><c>seed/1</c></seealso>
is called automatically with the default algorithm and creates a
non-constant seed.</p>
@@ -168,10 +169,17 @@ R3 = rand:uniform(),</pre>
S0 = rand:seed_s(exrop),
{R4, S1} = rand:uniform_s(S0),</pre>
+ <p>Textbook basic form Box-Muller standard normal deviate</p>
+
+ <pre>
+R5 = rand:uniform_real(),
+R6 = rand:uniform(),
+SND0 = math:sqrt(-2 * math:log(R5)) * math:cos(math:pi() * R6)</pre>
+
<p>Create a standard normal deviate:</p>
<pre>
-{SND0, S2} = rand:normal_s(S1),</pre>
+{SND1, S2} = rand:normal_s(S1),</pre>
<p>Create a normal deviate with mean -3 and variance 0.5:</p>
@@ -414,7 +422,8 @@ tests. We suggest to use a sign test to extract a random Boolean value.</pre>
This function may return exactly <c>0.0</c> which can be
fatal for certain applications. If that is undesired
you can use <c>(1.0 - rand:uniform())</c> to get the
- interval <c>0.0 &lt; <anno>X</anno> =&lt; 1.0</c>.
+ interval <c>0.0 &lt; <anno>X</anno> =&lt; 1.0</c>, or instead use
+ <seealso marker="#uniform_real-0"><c>uniform_real/0</c></seealso>.
</p>
<p>
If neither endpoint is desired you can test and re-try
@@ -432,6 +441,42 @@ end.</pre>
</func>
<func>
+ <name name="uniform_real" arity="0"/>
+ <fsummary>Return a random float.</fsummary>
+ <desc><marker id="uniform_real-0"/>
+ <p>
+ Returns a random float
+ uniformly distributed in the value range
+ <c>DBL_MIN =&lt; <anno>X</anno> &lt; 1.0</c>
+ and updates the state in the process dictionary.
+ </p>
+ <p>
+ Conceptually, a random real number <c>R</c> is generated
+ from the interval <c>0 =&lt; R &lt; 1</c> and then the
+ closest rounded down normalized number
+ in the IEEE 754 Double precision format
+ is returned.
+ </p>
+ <note>
+ <p>
+ The generated numbers from this function has got better
+ granularity for small numbers than the regular
+ <seealso marker="#uniform-0"><c>uniform/0</c></seealso>
+ because all bits in the mantissa are random.
+ This property, in combination with the fact that exactly zero
+ is never returned is useful for algoritms doing for example
+ <c>1.0 / <anno>X</anno></c> or <c>math:log(<anno>X</anno>)</c>.
+ </p>
+ </note>
+ <p>
+ See
+ <seealso marker="#uniform_real_s-1"><c>uniform_real_s/1</c></seealso>
+ for more explanation.
+ </p>
+ </desc>
+ </func>
+
+ <func>
<name name="uniform" arity="1"/>
<fsummary>Return a random integer.</fsummary>
<desc><marker id="uniform-1"/>
@@ -460,7 +505,8 @@ end.</pre>
This function may return exactly <c>0.0</c> which can be
fatal for certain applications. If that is undesired
you can use <c>(1.0 - rand:uniform(State))</c> to get the
- interval <c>0.0 &lt; <anno>X</anno> =&lt; 1.0</c>.
+ interval <c>0.0 &lt; <anno>X</anno> =&lt; 1.0</c>, or instead use
+ <seealso marker="#uniform_real_s-1"><c>uniform_real_s/1</c></seealso>.
</p>
<p>
If neither endpoint is desired you can test and re-try
@@ -478,6 +524,68 @@ end.</pre>
</func>
<func>
+ <name name="uniform_real_s" arity="1"/>
+ <fsummary>Return a random float.</fsummary>
+ <desc>
+ <p>
+ Returns, for a specified state, a random float
+ uniformly distributed in the value range
+ <c>DBL_MIN =&lt; <anno>X</anno> &lt; 1.0</c>
+ and updates the state in the process dictionary.
+ </p>
+ <p>
+ Conceptually, a random real number <c>R</c> is generated
+ from the interval <c>0 =&lt; R &lt; 1</c> and then the
+ closest rounded down normalized number
+ in the IEEE 754 Double precision format
+ is returned.
+ </p>
+ <note>
+ <p>
+ The generated numbers from this function has got better
+ granularity for small numbers than the regular
+ <seealso marker="#uniform_s-1"><c>uniform_s/1</c></seealso>
+ because all bits in the mantissa are random.
+ This property, in combination with the fact that exactly zero
+ is never returned is useful for algoritms doing for example
+ <c>1.0 / <anno>X</anno></c> or <c>math:log(<anno>X</anno>)</c>.
+ </p>
+ </note>
+ <p>
+ The concept implicates that the probability to get
+ exactly zero is extremely low; so low that this function
+ is in fact guaranteed to never return zero. The smallest
+ number that it might return is <c>DBL_MIN</c>, which is
+ 2.0^(-1022).
+ </p>
+ <p>
+ The value range stated at the top of this function
+ description is technically correct, but
+ <c>0.0 =&lt; <anno>X</anno> &lt; 1.0</c>
+ is a better description of the generated numbers'
+ statistical distribution. Except that exactly 0.0
+ is never returned, which is not possible to observe
+ statistically.
+ </p>
+ <p>
+ For example; for all sub ranges
+ <c>N*2.0^(-53) =&lt; X &lt; (N+1)*2.0^(-53)</c>
+ where
+ <c>0 =&lt; integer(N) &lt; 2.0^53</c>
+ the probability is the same.
+ Compare that with the form of the numbers generated by
+ <seealso marker="#uniform_s-1"><c>uniform_s/1</c></seealso>.
+ </p>
+ <p>
+ Having to generate extra random bits for
+ small numbers costs a little performance.
+ This function is about 20% slower than the regular
+ <seealso marker="#uniform_s-1"><c>uniform_s/1</c></seealso>
+ </p>
+ </desc>
+ </func>
+
+ <func>
<name name="uniform_s" arity="2"/>
<fsummary>Return a random integer.</fsummary>
<desc>
diff --git a/lib/stdlib/doc/src/ref_man.xml b/lib/stdlib/doc/src/ref_man.xml
index 878a3babc5..68bfddbc71 100644
--- a/lib/stdlib/doc/src/ref_man.xml
+++ b/lib/stdlib/doc/src/ref_man.xml
@@ -93,6 +93,7 @@
<xi:include href="sys.xml"/>
<xi:include href="timer.xml"/>
<xi:include href="unicode.xml"/>
+ <xi:include href="uri_string.xml"/>
<xi:include href="win32reg.xml"/>
<xi:include href="zip.xml"/>
</application>
diff --git a/lib/stdlib/doc/src/specs.xml b/lib/stdlib/doc/src/specs.xml
index 45b207b13d..d559adf9b6 100644
--- a/lib/stdlib/doc/src/specs.xml
+++ b/lib/stdlib/doc/src/specs.xml
@@ -60,6 +60,7 @@
<xi:include href="../specs/specs_sys.xml"/>
<xi:include href="../specs/specs_timer.xml"/>
<xi:include href="../specs/specs_unicode.xml"/>
+ <xi:include href="../specs/specs_uri_string.xml"/>
<xi:include href="../specs/specs_win32reg.xml"/>
<xi:include href="../specs/specs_zip.xml"/>
</specs>
diff --git a/lib/stdlib/doc/src/string.xml b/lib/stdlib/doc/src/string.xml
index 9d5edd9ecf..130fc74a28 100644
--- a/lib/stdlib/doc/src/string.xml
+++ b/lib/stdlib/doc/src/string.xml
@@ -109,10 +109,8 @@
<p>This module has been reworked in Erlang/OTP 20 to
handle <seealso marker="unicode#type-chardata">
<c>unicode:chardata()</c></seealso> and operate on grapheme
- clusters. The <seealso marker="#oldapi"> <c>old
- functions</c></seealso> that only work on Latin-1 lists as input
- are still available but should not be
- used. They will be deprecated in Erlang/OTP 21.
+ clusters. The <c>old functions</c> that only work on Latin-1 lists as input
+ are kept for backwards compatibility reasons but should not be used.
</p>
</description>
@@ -594,7 +592,7 @@ ÖÄÅ</pre>
or <c>both</c>, indicates from which direction characters
are to be removed.
</p>
- <p> Default <c><anno>Characters</anno></c> are the set of
+ <p> Default <c><anno>Characters</anno></c> is the set of
nonbreakable whitespace codepoints, defined as
Pattern_White_Space in
<url href="http://unicode.org/reports/tr31/">Unicode Standard Annex #31</url>.
@@ -631,393 +629,5 @@ ÖÄÅ</pre>
</func>
</funcs>
-
- <section>
- <marker id="oldapi"/>
- <title>Obsolete API functions</title>
- <p>Here follows the function of the old API.
- These functions only work on a list of Latin-1 characters.
- </p>
- <note><p>
- The functions are kept for backward compatibility, but are
- not recommended.
- They will be deprecated in Erlang/OTP 21.
- </p>
- <p>Any undocumented functions in <c>string</c> are not to be used.</p>
- </note>
- </section>
-
- <funcs>
- <func>
- <name name="centre" arity="2"/>
- <name name="centre" arity="3"/>
- <fsummary>Center a string.</fsummary>
- <desc>
- <p>Returns a string, where <c><anno>String</anno></c> is centered in the
- string and surrounded by blanks or <c><anno>Character</anno></c>.
- The resulting string has length <c><anno>Number</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#pad/3"><c>pad/3</c></seealso>.
- </p>
- </desc>
- </func>
-
- <func>
- <name name="chars" arity="2"/>
- <name name="chars" arity="3"/>
- <fsummary>Return a string consisting of numbers of characters.</fsummary>
- <desc>
- <p>Returns a string consisting of <c><anno>Number</anno></c> characters
- <c><anno>Character</anno></c>. Optionally, the string can end with
- string <c><anno>Tail</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="lists#duplicate/2"><c>lists:duplicate/2</c></seealso>.</p>
- </desc>
- </func>
-
- <func>
- <name name="chr" arity="2"/>
- <fsummary>Return the index of the first occurrence of
- a character in a string.</fsummary>
- <desc>
- <p>Returns the index of the first occurrence of
- <c><anno>Character</anno></c> in <c><anno>String</anno></c>. Returns
- <c>0</c> if <c><anno>Character</anno></c> does not occur.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#find/2"><c>find/2</c></seealso>.</p>
- </desc>
- </func>
-
- <func>
- <name name="concat" arity="2"/>
- <fsummary>Concatenate two strings.</fsummary>
- <desc>
- <p>Concatenates <c><anno>String1</anno></c> and
- <c><anno>String2</anno></c> to form a new string
- <c><anno>String3</anno></c>, which is returned.</p>
- <p>
- This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use <c>[<anno>String1</anno>, <anno>String2</anno>]</c> as
- <c>Data</c> argument, and call
- <seealso marker="unicode#characters_to_list/2">
- <c>unicode:characters_to_list/2</c></seealso> or
- <seealso marker="unicode#characters_to_binary/2">
- <c>unicode:characters_to_binary/2</c></seealso>
- to flatten the output.
- </p>
- </desc>
- </func>
-
- <func>
- <name name="copies" arity="2"/>
- <fsummary>Copy a string.</fsummary>
- <desc>
- <p>Returns a string containing <c><anno>String</anno></c> repeated
- <c><anno>Number</anno></c> times.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="lists#duplicate/2"><c>lists:duplicate/2</c></seealso>.</p>
- </desc>
- </func>
-
- <func>
- <name name="cspan" arity="2"/>
- <fsummary>Span characters at start of a string.</fsummary>
- <desc>
- <p>Returns the length of the maximum initial segment of
- <c><anno>String</anno></c>, which consists entirely of characters
- not from <c><anno>Chars</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#take/3"><c>take/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:cspan("\t abcdef", " \t").
-0</code>
- </desc>
- </func>
-
- <func>
- <name name="join" arity="2"/>
- <fsummary>Join a list of strings with separator.</fsummary>
- <desc>
- <p>Returns a string with the elements of <c><anno>StringList</anno></c>
- separated by the string in <c><anno>Separator</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="lists#join/2"><c>lists:join/2</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> join(["one", "two", "three"], ", ").
-"one, two, three"</code>
- </desc>
- </func>
-
- <func>
- <name name="left" arity="2"/>
- <name name="left" arity="3"/>
- <fsummary>Adjust left end of a string.</fsummary>
- <desc>
- <p>Returns <c><anno>String</anno></c> with the length adjusted in
- accordance with <c><anno>Number</anno></c>. The left margin is
- fixed. If <c>length(<anno>String</anno>)</c> &lt;
- <c><anno>Number</anno></c>, then <c><anno>String</anno></c> is padded
- with blanks or <c><anno>Character</anno></c>s.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#pad/2"><c>pad/2</c></seealso> or
- <seealso marker="#pad/3"><c>pad/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:left("Hello",10,$.).
-"Hello....."</code>
- </desc>
- </func>
-
- <func>
- <name name="len" arity="1"/>
- <fsummary>Return the length of a string.</fsummary>
- <desc>
- <p>Returns the number of characters in <c><anno>String</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#length/1"><c>length/1</c></seealso>.</p>
- </desc>
- </func>
-
- <func>
- <name name="rchr" arity="2"/>
- <fsummary>Return the index of the last occurrence of
- a character in a string.</fsummary>
- <desc>
- <p>Returns the index of the last occurrence of
- <c><anno>Character</anno></c> in <c><anno>String</anno></c>. Returns
- <c>0</c> if <c><anno>Character</anno></c> does not occur.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#find/3"><c>find/3</c></seealso>.</p>
- </desc>
- </func>
-
- <func>
- <name name="right" arity="2"/>
- <name name="right" arity="3"/>
- <fsummary>Adjust right end of a string.</fsummary>
- <desc>
- <p>Returns <c><anno>String</anno></c> with the length adjusted in
- accordance with <c><anno>Number</anno></c>. The right margin is
- fixed. If the length of <c>(<anno>String</anno>)</c> &lt;
- <c><anno>Number</anno></c>, then <c><anno>String</anno></c> is padded
- with blanks or <c><anno>Character</anno></c>s.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#pad/3"><c>pad/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:right("Hello", 10, $.).
-".....Hello"</code>
- </desc>
- </func>
-
- <func>
- <name name="rstr" arity="2"/>
- <fsummary>Find the index of a substring.</fsummary>
- <desc>
- <p>Returns the position where the last occurrence of
- <c><anno>SubString</anno></c> begins in <c><anno>String</anno></c>.
- Returns <c>0</c> if <c><anno>SubString</anno></c>
- does not exist in <c><anno>String</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#find/3"><c>find/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:rstr(" Hello Hello World World ", "Hello World").
-8</code>
- </desc>
- </func>
-
- <func>
- <name name="span" arity="2"/>
- <fsummary>Span characters at start of a string.</fsummary>
- <desc>
- <p>Returns the length of the maximum initial segment of
- <c><anno>String</anno></c>, which consists entirely of characters
- from <c><anno>Chars</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#take/2"><c>take/2</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:span("\t abcdef", " \t").
-5</code>
- </desc>
- </func>
-
- <func>
- <name name="str" arity="2"/>
- <fsummary>Find the index of a substring.</fsummary>
- <desc>
- <p>Returns the position where the first occurrence of
- <c><anno>SubString</anno></c> begins in <c><anno>String</anno></c>.
- Returns <c>0</c> if <c><anno>SubString</anno></c>
- does not exist in <c><anno>String</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#find/2"><c>find/2</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:str(" Hello Hello World World ", "Hello World").
-8</code>
- </desc>
- </func>
-
- <func>
- <name name="strip" arity="1"/>
- <name name="strip" arity="2"/>
- <name name="strip" arity="3"/>
- <fsummary>Strip leading or trailing characters.</fsummary>
- <desc>
- <p>Returns a string, where leading or trailing, or both, blanks or a
- number of <c><anno>Character</anno></c> have been removed.
- <c><anno>Direction</anno></c>, which can be <c>left</c>, <c>right</c>,
- or <c>both</c>, indicates from which direction blanks are to be
- removed. <c>strip/1</c> is equivalent to
- <c>strip(String, both)</c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#trim/3"><c>trim/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:strip("...Hello.....", both, $.).
-"Hello"</code>
- </desc>
- </func>
-
- <func>
- <name name="sub_string" arity="2"/>
- <name name="sub_string" arity="3"/>
- <fsummary>Extract a substring.</fsummary>
- <desc>
- <p>Returns a substring of <c><anno>String</anno></c>, starting at
- position <c><anno>Start</anno></c> to the end of the string, or to
- and including position <c><anno>Stop</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#slice/3"><c>slice/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-sub_string("Hello World", 4, 8).
-"lo Wo"</code>
- </desc>
- </func>
-
- <func>
- <name name="substr" arity="2"/>
- <name name="substr" arity="3"/>
- <fsummary>Return a substring of a string.</fsummary>
- <desc>
- <p>Returns a substring of <c><anno>String</anno></c>, starting at
- position <c><anno>Start</anno></c>, and ending at the end of the
- string or at length <c><anno>Length</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#slice/3"><c>slice/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> substr("Hello World", 4, 5).
-"lo Wo"</code>
- </desc>
- </func>
-
- <func>
- <name name="sub_word" arity="2"/>
- <name name="sub_word" arity="3"/>
- <fsummary>Extract subword.</fsummary>
- <desc>
- <p>Returns the word in position <c><anno>Number</anno></c> of
- <c><anno>String</anno></c>. Words are separated by blanks or
- <c><anno>Character</anno></c>s.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#nth_lexeme/3"><c>nth_lexeme/3</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> string:sub_word(" Hello old boy !",3,$o).
-"ld b"</code>
- </desc>
- </func>
-
- <func>
- <name name="to_lower" arity="1" clause_i="1"/>
- <name name="to_lower" arity="1" clause_i="2"/>
- <name name="to_upper" arity="1" clause_i="1"/>
- <name name="to_upper" arity="1" clause_i="2"/>
- <fsummary>Convert case of string (ISO/IEC 8859-1).</fsummary>
- <type variable="String" name_i="1"/>
- <type variable="Result" name_i="1"/>
- <type variable="Char"/>
- <type variable="CharResult"/>
- <desc>
- <p>The specified string or character is case-converted. Notice that
- the supported character set is ISO/IEC 8859-1 (also called Latin 1);
- all values outside this set are unchanged</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso> use
- <seealso marker="#lowercase/1"><c>lowercase/1</c></seealso>,
- <seealso marker="#uppercase/1"><c>uppercase/1</c></seealso>,
- <seealso marker="#titlecase/1"><c>titlecase/1</c></seealso> or
- <seealso marker="#casefold/1"><c>casefold/1</c></seealso>.</p>
- </desc>
- </func>
-
- <func>
- <name name="tokens" arity="2"/>
- <fsummary>Split string into tokens.</fsummary>
- <desc>
- <p>Returns a list of tokens in <c><anno>String</anno></c>, separated
- by the characters in <c><anno>SeparatorList</anno></c>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> tokens("abc defxxghix jkl", "x ").
-["abc", "def", "ghi", "jkl"]</code>
- <p>Notice that, as shown in this example, two or more
- adjacent separator characters in <c><anno>String</anno></c>
- are treated as one. That is, there are no empty
- strings in the resulting list of tokens.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#lexemes/2"><c>lexemes/2</c></seealso>.</p>
- </desc>
- </func>
-
- <func>
- <name name="words" arity="1"/>
- <name name="words" arity="2"/>
- <fsummary>Count blank separated words.</fsummary>
- <desc>
- <p>Returns the number of words in <c><anno>String</anno></c>, separated
- by blanks or <c><anno>Character</anno></c>.</p>
- <p>This function is <seealso marker="#oldapi">obsolete</seealso>.
- Use
- <seealso marker="#lexemes/2"><c>lexemes/2</c></seealso>.</p>
- <p><em>Example:</em></p>
- <code type="none">
-> words(" Hello old boy!", $o).
-4</code>
- </desc>
- </func>
- </funcs>
-
- <section>
- <title>Notes</title>
- <p>Some of the general string functions can seem to overlap each
- other. The reason is that this string package is the
- combination of two earlier packages and all functions of
- both packages have been retained.</p>
- </section>
-
</erlref>
diff --git a/lib/stdlib/doc/src/unicode_usage.xml b/lib/stdlib/doc/src/unicode_usage.xml
index 26dc46719e..789e063c12 100644
--- a/lib/stdlib/doc/src/unicode_usage.xml
+++ b/lib/stdlib/doc/src/unicode_usage.xml
@@ -719,8 +719,8 @@ Eshell V5.10.1 (abort with ^G)
</section>
<section>
- <title>Unicode Filenames</title>
<marker id="unicode_file_names"/>
+ <title>Unicode Filenames</title>
<p>Most modern operating systems support Unicode filenames in some way.
There are many different ways to do this and Erlang by default treats the
different approaches differently:</p>
@@ -855,8 +855,12 @@ Eshell V5.10.1 (abort with ^G)
</note>
<section>
- <title>Notes About Raw Filenames</title>
<marker id="notes-about-raw-filenames"/>
+ <title>Notes About Raw Filenames</title>
+ <note><p>
+ Note that raw filenames <em>not</em> necessarily are encoded the
+ same way as on the OS level.
+ </p></note>
<p>Raw filenames were introduced together with Unicode filename support
in ERTS 5.8.2 (Erlang/OTP R14B01). The reason &quot;raw
filenames&quot; were introduced in the system was
diff --git a/lib/stdlib/doc/src/uri_string.xml b/lib/stdlib/doc/src/uri_string.xml
new file mode 100644
index 0000000000..9ace2b0a05
--- /dev/null
+++ b/lib/stdlib/doc/src/uri_string.xml
@@ -0,0 +1,230 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE erlref SYSTEM "erlref.dtd">
+
+<erlref>
+ <header>
+ <copyright>
+ <year>2017</year><year>2017</year>
+ <holder>Ericsson AB. All Rights Reserved.</holder>
+ </copyright>
+ <legalnotice>
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ </legalnotice>
+
+ <title>uri_string</title>
+ <prepared>Péter Dimitrov</prepared>
+ <docno>1</docno>
+ <date>2017-10-24</date>
+ <rev>A</rev>
+ </header>
+ <module>uri_string</module>
+ <modulesummary>URI processing functions.</modulesummary>
+ <description>
+ <p>This module contains functions for parsing and handling URIs
+ (<url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>).
+ </p>
+ <p>A URI is an identifier consisting of a sequence of characters matching the syntax
+ rule named <em>URI</em> in <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>.
+ </p>
+ <p> The generic URI syntax consists of a hierarchical sequence of components referred
+ to as the scheme, authority, path, query, and fragment:</p>
+ <pre>
+ URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
+ hier-part = "//" authority path-abempty
+ / path-absolute
+ / path-rootless
+ / path-empty
+ scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+ authority = [ userinfo "@" ] host [ ":" port ]
+ userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
+
+ reserved = gen-delims / sub-delims
+ gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+ sub-delims = "!" / "$" / "&amp;" / "'" / "(" / ")"
+ / "*" / "+" / "," / ";" / "="
+
+ unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
+ </pre><br></br>
+ <p>The interpretation of a URI depends only on the characters used and not on how those
+ characters are represented in a network protocol.</p>
+ <p>The functions implemented by this module cover the following use cases:</p>
+ <list type="bulleted">
+ <item>Parsing URIs into its components and returing a map<br></br>
+ <seealso marker="#parse/1"><c>parse/1</c></seealso>
+ </item>
+ <item>Recomposing a map of URI components into a URI string<br></br>
+ <seealso marker="#recompose/1"><c>recompose/1</c></seealso>
+ </item>
+ <item>Changing inbound binary and percent-encoding of URIs<br></br>
+ <seealso marker="#transcode/2"><c>transcode/2</c></seealso>
+ </item>
+ <item>Transforming URIs into a normalized form<br></br>
+ <seealso marker="#normalize/1"><c>normalize/1</c></seealso>
+ </item>
+ </list>
+ <p>There are four different encodings present during the handling of URIs:</p>
+ <list type="bulleted">
+ <item>Inbound binary encoding in binaries</item>
+ <item>Inbound percent-encoding in lists and binaries</item>
+ <item>Outbound binary encoding in binaries</item>
+ <item>Outbound percent-encoding in lists and binaries</item>
+ </list>
+ <p>Functions with <c>uri_string()</c> argument accept lists, binaries and
+ mixed lists (lists with binary elements) as input type. All of the functions but
+ <c>transcode/2</c> expects input as lists of unicode codepoints, UTF-8 encoded binaries
+ and UTF-8 percent-encoded URI parts ("%C3%B6" corresponds to the unicode character "ö").</p>
+ <p>Unless otherwise specified the return value type and encoding are the same as the input
+ type and encoding. That is, binary input returns binary output, list input returns a list
+ output but mixed input returns list output.</p>
+ <p>In case of lists there is only percent-encoding. In binaries, however, both binary encoding
+ and percent-encoding shall be considered. <c>transcode/2</c> provides the means to convert
+ between the supported encodings, it takes a <c>uri_string()</c> and a list of options
+ specifying inbound and outbound encodings.</p>
+ <p><url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url> does not mandate any specific
+ character encoding and it is usually defined by the protocol or surrounding text. This library
+ takes the same assumption, binary and percent-encoding are handled as one configuration unit,
+ they cannot be set to different values.</p>
+ </description>
+
+ <datatypes>
+ <datatype>
+ <name name="error"/>
+ <desc>
+ <p>Error tuple indicating the type of error. Possible values of the second component:</p>
+ <list type="bulleted">
+ <item><c>invalid_input</c></item>
+ <item><c>invalid_map</c></item>
+ <item><c>invalid_percent_encoding</c></item>
+ <item><c>invalid_scheme</c></item>
+ <item><c>invalid_uri</c></item>
+ <item><c>invalid_utf8</c></item>
+ </list>
+ <p>The third component is a term providing additional information about the
+ cause of the error.</p>
+ </desc>
+ </datatype>
+ <datatype>
+ <name name="uri_map"/>
+ <desc>
+ <p>Map holding the main components of a URI.</p>
+ </desc>
+ </datatype>
+ <datatype>
+ <name name="uri_string"/>
+ <desc>
+ <p>List of unicode codepoints, a UTF-8 encoded binary, or a mix of the two,
+ representing an <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>
+ compliant URI (<em>percent-encoded form</em>).
+ A URI is a sequence of characters from a very limited set: the letters of
+ the basic Latin alphabet, digits, and a few special characters.</p>
+ </desc>
+ </datatype>
+ </datatypes>
+
+ <funcs>
+
+ <func>
+ <name name="normalize" arity="1"/>
+ <fsummary>Syntax-based normalization.</fsummary>
+ <desc>
+ <p>Transforms <c><anno>URIString</anno></c> into a normalized form
+ using Syntax-Based Normalization as defined by
+ <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>.</p>
+ <p>This function implements case normalization, percent-encoding
+ normalization, path segment normalization and scheme based normalization
+ for HTTP(S) with basic support for FTP, SSH, SFTP and TFTP.</p>
+ <p><em>Example:</em></p>
+ <pre>
+1> <input>uri_string:normalize("/a/b/c/./../../g").</input>
+"/a/g"
+2> <![CDATA[uri_string:normalize(<<"mid/content=5/../6">>).]]>
+<![CDATA[<<"mid/6">>]]>
+3> uri_string:normalize("http://localhost:80").
+"https://localhost/"
+ </pre>
+ </desc>
+ </func>
+
+ <func>
+ <name name="parse" arity="1"/>
+ <fsummary>Parse URI into a map.</fsummary>
+ <desc>
+ <p>Parses an <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>
+ compliant <c>uri_string()</c> into a <c>uri_map()</c>, that holds the parsed
+ components of the <c>URI</c>.
+ If parsing fails, an error tuple is returned.</p>
+ <p>See also the opposite operation <seealso marker="#recompose/1">
+ <c>recompose/1</c></seealso>.</p>
+ <p><em>Example:</em></p>
+ <pre>
+1> <input>uri_string:parse("foo://[email protected]:8042/over/there?name=ferret#nose").</input>
+#{fragment => "nose",host => "example.com",
+ path => "/over/there",port => 8042,query => "name=ferret",
+ scheme => foo,userinfo => "user"}
+2> <![CDATA[uri_string:parse(<<"foo://[email protected]:8042/over/there?name=ferret">>).]]>
+<![CDATA[#{host => <<"example.com">>,path => <<"/over/there">>,
+ port => 8042,query => <<"name=ferret">>,scheme => <<"foo">>,
+ userinfo => <<"user">>}]]>
+ </pre>
+ </desc>
+ </func>
+
+ <func>
+ <name name="recompose" arity="1"/>
+ <fsummary>Recompose URI.</fsummary>
+ <desc>
+ <p>Creates an <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url> compliant
+ <c><anno>URIString</anno></c> (percent-encoded), based on the components of
+ <c><anno>URIMap</anno></c>.
+ If the <c><anno>URIMap</anno></c> is invalid, an error tuple is returned.</p>
+ <p>See also the opposite operation <seealso marker="#parse/1">
+ <c>parse/1</c></seealso>.</p>
+ <p><em>Example:</em></p>
+ <pre>
+1> <input>URIMap = #{fragment => "nose", host => "example.com", path => "/over/there",</input>
+1> port => 8042, query => "name=ferret", scheme => "foo", userinfo => "user"}.
+#{fragment => "top",host => "example.com",
+ path => "/over/there",port => 8042,query => "?name=ferret",
+ scheme => foo,userinfo => "user"}
+
+2> <input>uri_string:recompose(URIMap).</input>
+"foo://example.com:8042/over/there?name=ferret#nose"</pre>
+ </desc>
+ </func>
+
+ <func>
+ <name name="transcode" arity="2"/>
+ <fsummary>Transcode URI.</fsummary>
+ <desc>
+ <p>Transcodes an <url href="https://www.ietf.org/rfc/rfc3986.txt">RFC 3986</url>
+ compliant <c><anno>URIString</anno></c>,
+ where <c><anno>Options</anno></c> is a list of tagged tuples, specifying the inbound
+ (<c>in_encoding</c>) and outbound (<c>out_encoding</c>) encodings. <c>in_encoding</c>
+ and <c>out_encoding</c> specifies both binary encoding and percent-encoding for the
+ input and output data. Mixed encoding, where binary encoding is not the same as
+ percent-encoding, is not supported.
+ If an argument is invalid, an error tuple is returned.</p>
+ <p><em>Example:</em></p>
+ <pre>
+1> <input><![CDATA[uri_string:transcode(<<"foo%00%00%00%F6bar"/utf32>>,]]></input>
+1> [{in_encoding, utf32},{out_encoding, utf8}]).
+<![CDATA[<<"foo%C3%B6bar"/utf8>>]]>
+2> uri_string:transcode("foo%F6bar", [{in_encoding, latin1},
+2> {out_encoding, utf8}]).
+"foo%C3%B6bar"
+ </pre>
+ </desc>
+ </func>
+
+ </funcs>
+</erlref>
diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile
index bf836203ec..8b156929d7 100644
--- a/lib/stdlib/src/Makefile
+++ b/lib/stdlib/src/Makefile
@@ -121,6 +121,7 @@ MODULES= \
timer \
unicode \
unicode_util \
+ uri_string \
win32reg \
zip
diff --git a/lib/stdlib/src/base64.erl b/lib/stdlib/src/base64.erl
index 5885745fb1..c8cf6fdffe 100644
--- a/lib/stdlib/src/base64.erl
+++ b/lib/stdlib/src/base64.erl
@@ -113,9 +113,9 @@ encode_binary(Bin) ->
Data :: ascii_binary().
decode(Bin) when is_binary(Bin) ->
- decode_binary(<<>>, Bin);
+ decode_binary(Bin, <<>>);
decode(List) when is_list(List) ->
- list_to_binary(decode_l(List)).
+ decode_list(List, <<>>).
-spec mime_decode(Base64) -> Data when
Base64 :: ascii_string() | ascii_binary(),
@@ -186,31 +186,41 @@ mime_decode_to_string(List) when is_list(List) ->
bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,
bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad}).
-decode_binary(Result0, <<C:8,T0/bits>>) ->
- case element(C, ?DECODE_MAP) of
- bad ->
- erlang:error({badarg,C});
- ws ->
- decode_binary(Result0, T0);
- eq ->
- case strip_ws(T0) of
- <<$=:8,T/binary>> ->
- <<>> = strip_ws(T),
- Split = byte_size(Result0) - 1,
- <<Result:Split/bytes,_:4>> = Result0,
- Result;
- T ->
- <<>> = strip_ws(T),
- Split = byte_size(Result0) - 1,
- <<Result:Split/bytes,_:2>> = Result0,
- Result
- end;
- Bits ->
- decode_binary(<<Result0/bits,Bits:6>>, T0)
+decode_binary(<<C1:8, Cs/bits>>, A) ->
+ case element(C1, ?DECODE_MAP) of
+ ws -> decode_binary(Cs, A);
+ B1 -> decode_binary(Cs, A, B1)
end;
-decode_binary(Result, <<>>) ->
- true = is_binary(Result),
- Result.
+decode_binary(<<>>, A) ->
+ A.
+
+decode_binary(<<C2:8, Cs/bits>>, A, B1) ->
+ case element(C2, ?DECODE_MAP) of
+ ws -> decode_binary(Cs, A, B1);
+ B2 -> decode_binary(Cs, A, B1, B2)
+ end.
+
+decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) ->
+ case element(C3, ?DECODE_MAP) of
+ ws -> decode_binary(Cs, A, B1, B2);
+ B3 -> decode_binary(Cs, A, B1, B2, B3)
+ end.
+
+decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) ->
+ case element(C4, ?DECODE_MAP) of
+ ws -> decode_binary(Cs, A, B1, B2, B3);
+ eq when B3 =:= eq -> only_ws_binary(Cs, <<A/binary,B1:6,(B2 bsr 4):2>>);
+ eq -> only_ws_binary(Cs, <<A/binary,B1:6,B2:6,(B3 bsr 2):4>>);
+ B4 -> decode_binary(Cs, <<A/binary,B1:6,B2:6,B3:6,B4:6>>)
+ end.
+
+only_ws_binary(<<>>, A) ->
+ A;
+only_ws_binary(<<C:8, Cs/bits>>, A) ->
+ case element(C, ?DECODE_MAP) of
+ ws -> only_ws_binary(Cs, A);
+ _ -> erlang:error(function_clause)
+ end.
%% Skipping pad character if not at end of string. Also liberal about
%% excess padding and skipping of other illegal (non-base64 alphabet)
@@ -262,6 +272,42 @@ mime_decode_binary_after_eq(Result0, <<>>, Eq) ->
Result
end.
+decode_list([C1 | Cs], A) ->
+ case element(C1, ?DECODE_MAP) of
+ ws -> decode_list(Cs, A);
+ B1 -> decode_list(Cs, A, B1)
+ end;
+decode_list([], A) ->
+ A.
+
+decode_list([C2 | Cs], A, B1) ->
+ case element(C2, ?DECODE_MAP) of
+ ws -> decode_list(Cs, A, B1);
+ B2 -> decode_list(Cs, A, B1, B2)
+ end.
+
+decode_list([C3 | Cs], A, B1, B2) ->
+ case element(C3, ?DECODE_MAP) of
+ ws -> decode_list(Cs, A, B1, B2);
+ B3 -> decode_list(Cs, A, B1, B2, B3)
+ end.
+
+decode_list([C4 | Cs], A, B1, B2, B3) ->
+ case element(C4, ?DECODE_MAP) of
+ ws -> decode_list(Cs, A, B1, B2, B3);
+ eq when B3 =:= eq -> only_ws(Cs, <<A/binary,B1:6,(B2 bsr 4):2>>);
+ eq -> only_ws(Cs, <<A/binary,B1:6,B2:6,(B3 bsr 2):4>>);
+ B4 -> decode_list(Cs, <<A/binary,B1:6,B2:6,B3:6,B4:6>>)
+ end.
+
+only_ws([], A) ->
+ A;
+only_ws([C | Cs], A) ->
+ case element(C, ?DECODE_MAP) of
+ ws -> only_ws(Cs, A);
+ _ -> erlang:error(function_clause)
+ end.
+
decode([], A) -> A;
decode([$=,$=,C2,C1|Cs], A) ->
Bits2x6 = (b64d(C1) bsl 18) bor (b64d(C2) bsl 12),
@@ -292,16 +338,6 @@ strip_spaces([$\r|Cs], A) -> strip_spaces(Cs, A);
strip_spaces([$\n|Cs], A) -> strip_spaces(Cs, A);
strip_spaces([C|Cs], A) -> strip_spaces(Cs, [C | A]).
-strip_ws(<<$\t,T/binary>>) ->
- strip_ws(T);
-strip_ws(<<$\n,T/binary>>) ->
- strip_ws(T);
-strip_ws(<<$\r,T/binary>>) ->
- strip_ws(T);
-strip_ws(<<$\s,T/binary>>) ->
- strip_ws(T);
-strip_ws(T) -> T.
-
%% Skipping pad character if not at end of string. Also liberal about
%% excess padding and skipping of other illegal (non-base64 alphabet)
%% characters. See section 3.3 of RFC4648
diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl
index c04a201ce1..9a447af5b7 100644
--- a/lib/stdlib/src/c.erl
+++ b/lib/stdlib/src/c.erl
@@ -668,19 +668,23 @@ lm() ->
[l(M) || M <- mm()].
%% erlangrc(Home)
-%% Try to run a ".erlang" file, first in the current directory
-%% else in home directory.
+%% Try to run a ".erlang" file in home directory.
+
+-spec erlangrc() -> {ok, file:filename()} | {error, term()}.
erlangrc() ->
case init:get_argument(home) of
{ok,[[Home]]} ->
erlangrc([Home]);
_ ->
- f_p_e(["."], ".erlang")
+ {error, enoent}
end.
-erlangrc([Home]) ->
- f_p_e([".",Home], ".erlang").
+-spec erlangrc(PathList) -> {ok, file:filename()} | {error, term()}
+ when PathList :: [Dir :: file:name()].
+
+erlangrc([Home|_]=Paths) when is_list(Home) ->
+ f_p_e(Paths, ".erlang").
error(Fmt, Args) ->
error_logger:error_msg(Fmt, Args).
@@ -692,11 +696,11 @@ f_p_e(P, F) ->
{error, E={Line, _Mod, _Term}} ->
error("file:path_eval(~tp,~tp): error on line ~p: ~ts~n",
[P, F, Line, file:format_error(E)]),
- ok;
+ {error, E};
{error, E} ->
error("file:path_eval(~tp,~tp): ~ts~n",
[P, F, file:format_error(E)]),
- ok;
+ {error, E};
Other ->
Other
end.
diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl
index 31d0d499e3..00e6a10d8a 100644
--- a/lib/stdlib/src/epp.erl
+++ b/lib/stdlib/src/epp.erl
@@ -479,7 +479,7 @@ com_enc(_B, _Fun, _N, L, Ps) ->
com_enc_end([L | Ps]).
com_enc_end(Ps0) ->
- Ps = lists:reverse([lists:reverse(string:to_lower(P)) || P <- Ps0]),
+ Ps = lists:reverse([lists:reverse(lowercase(P)) || P <- Ps0]),
com_encoding(Ps).
com_encoding(["latin","1"|_]) ->
@@ -489,6 +489,9 @@ com_encoding(["utf","8"|_]) ->
com_encoding(_) ->
throw(no). % Don't try any further
+lowercase(S) ->
+ unicode:characters_to_list(string:lowercase(S)).
+
normalize_typed_record_fields([]) ->
{typed, []};
normalize_typed_record_fields(Fields) ->
diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl
index 9cd4727dc3..f58cb35cea 100644
--- a/lib/stdlib/src/erl_lint.erl
+++ b/lib/stdlib/src/erl_lint.erl
@@ -3910,10 +3910,9 @@ check_format_string(Fmt) ->
extract_sequences(Fmt, []).
extract_sequences(Fmt, Need0) ->
- case string:chr(Fmt, $~) of
- 0 -> {ok,lists:reverse(Need0)}; %That's it
- Pos ->
- Fmt1 = string:substr(Fmt, Pos+1), %Skip ~
+ case string:find(Fmt, [$~]) of
+ nomatch -> {ok,lists:reverse(Need0)}; %That's it
+ [$~|Fmt1] ->
case extract_sequence(1, Fmt1, Need0) of
{ok,Need1,Rest} -> extract_sequences(Rest, Need1);
Error -> Error
diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl
index ee5e7a11bf..367dbefb82 100644
--- a/lib/stdlib/src/erl_pp.erl
+++ b/lib/stdlib/src/erl_pp.erl
@@ -237,13 +237,20 @@ lform({attribute,Line,Name,Arg}, Opts) ->
lform({function,Line,Name,Arity,Clauses}, Opts) ->
lfunction({function,Line,Name,Arity,Clauses}, Opts);
%% These are specials to make it easier for the compiler.
-lform({error,E}, _Opts) ->
- leaf(format("~p\n", [{error,E}]));
-lform({warning,W}, _Opts) ->
- leaf(format("~p\n", [{warning,W}]));
+lform({error,_}=E, Opts) ->
+ message(E, Opts);
+lform({warning,_}=W, Opts) ->
+ message(W, Opts);
lform({eof,_Line}, _Opts) ->
$\n.
+message(M, #options{encoding = Encoding}) ->
+ F = case Encoding of
+ latin1 -> "~p\n";
+ unicode -> "~tp\n"
+ end,
+ leaf(format(F, [M])).
+
lattribute({attribute,_Line,type,Type}, Opts) ->
[typeattr(type, Type, Opts),leaf(".\n")];
lattribute({attribute,_Line,opaque,Type}, Opts) ->
@@ -598,8 +605,6 @@ lexpr({'fun',_,{clauses,Cs},Extra}, _Prec, Opts) ->
lexpr({named_fun,_,Name,Cs,Extra}, _Prec, Opts) ->
{force_nl,fun_info(Extra),
{list,[{first,['fun', " "],fun_clauses(Cs, Opts, {named, Name})},'end']}};
-lexpr({'query',_,Lc}, _Prec, Opts) ->
- {list,[{step,leaf("query"),lexpr(Lc, 0, Opts)},'end']};
lexpr({call,_,{remote,_,{atom,_,M},{atom,_,F}=N}=Name,Args}, Prec, Opts) ->
case erl_internal:bif(M, F, length(Args)) of
true ->
@@ -904,7 +909,7 @@ maybe_paren(_P, _Prec, Expr) ->
Expr.
leaf(S) ->
- {leaf,chars_size(S),S}.
+ {leaf,string:length(S),S}.
%%% Do the formatting. Currently nothing fancy. Could probably have
%%% done it in one single pass.
@@ -964,7 +969,7 @@ f({seq,Before,After,Sep,LItems}, I0, ST, WT, PP) ->
Sizes = BSizeL ++ SizeL,
NSepChars = if
is_list(Sep), Sep =/= [] ->
- erlang:max(0, length(CharsL)-1);
+ erlang:max(0, length(CharsL)-1); % not string:length
true ->
0
end,
@@ -1120,7 +1125,7 @@ incr(I, Incr) ->
I+Incr.
indentation(E, I) when I < 0 ->
- chars_size(E);
+ string:length(E);
indentation(E, I0) ->
I = io_lib_format:indentation(E, I0),
case has_nl(E) of
@@ -1157,19 +1162,19 @@ write_a_string(S, I, PP) ->
write_a_string([], _N, _Len, _PP) ->
[];
write_a_string(S, N, Len, PP) ->
- SS = string:sub_string(S, 1, N),
+ SS = string:slice(S, 0, N),
Sl = write_string(SS, PP),
- case (chars_size(Sl) > Len) and (N > ?MIN_SUBSTRING) of
+ case (string:length(Sl) > Len) and (N > ?MIN_SUBSTRING) of
true ->
write_a_string(S, N-1, Len, PP);
false ->
[flat_leaf(Sl) |
- write_a_string(lists:nthtail(length(SS), S), Len, Len, PP)]
+ write_a_string(string:slice(S, string:length(SS)), Len, Len, PP)]
end.
flat_leaf(S) ->
L = lists:flatten(S),
- {leaf,length(L),L}.
+ {leaf,string:length(L),L}.
write_value(V, PP) ->
(PP#pp.value_fun)(V).
@@ -1190,15 +1195,6 @@ write_char(C, PP) ->
a0() ->
erl_anno:new(0).
-chars_size([C | Es]) when is_integer(C) ->
- 1 + chars_size(Es);
-chars_size([E | Es]) ->
- chars_size(E) + chars_size(Es);
-chars_size([]) ->
- 0;
-chars_size(B) when is_binary(B) ->
- byte_size(B).
-
-define(N_SPACES, 30).
spacetab() ->
diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl
index 47223b129c..4774c4bf19 100644
--- a/lib/stdlib/src/erl_scan.erl
+++ b/lib/stdlib/src/erl_scan.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1996-2015. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@@ -752,7 +752,7 @@ scan_string(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) ->
{char_error,Ncs,Error,Nline,Ncol,EndCol} ->
scan_error(Error, Nline, Ncol, Nline, EndCol, Ncs);
{error,Nline,Ncol,Nwcs,Ncs} ->
- Estr = string:substr(Nwcs, 1, 16), % Expanded escape chars.
+ Estr = string:slice(Nwcs, 0, 16), % Expanded escape chars.
scan_error({string,$\",Estr}, Line0, Col0, Nline, Ncol, Ncs); %"
{Ncs,Nline,Ncol,Nstr,Nwcs} ->
Anno = anno(Line0, Col0, St, Nstr),
@@ -767,7 +767,7 @@ scan_qatom(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) ->
{char_error,Ncs,Error,Nline,Ncol,EndCol} ->
scan_error(Error, Nline, Ncol, Nline, EndCol, Ncs);
{error,Nline,Ncol,Nwcs,Ncs} ->
- Estr = string:substr(Nwcs, 1, 16), % Expanded escape chars.
+ Estr = string:slice(Nwcs, 0, 16), % Expanded escape chars.
scan_error({string,$\',Estr}, Line0, Col0, Nline, Ncol, Ncs); %'
{Ncs,Nline,Ncol,Nstr,Nwcs} ->
case catch list_to_atom(Nwcs) of
diff --git a/lib/stdlib/src/escript.erl b/lib/stdlib/src/escript.erl
index 2b9d8ff65b..132f8efbbe 100644
--- a/lib/stdlib/src/escript.erl
+++ b/lib/stdlib/src/escript.erl
@@ -224,8 +224,8 @@ return_sections(S, Bin) ->
normalize_section(Name, undefined) ->
{Name, undefined};
normalize_section(shebang, "#!" ++ Chars) ->
- Chopped = string:strip(Chars, right, $\n),
- Stripped = string:strip(Chopped, both),
+ Chopped = string:trim(Chars, trailing, "$\n"),
+ Stripped = string:trim(Chopped, both),
if
Stripped =:= ?SHEBANG ->
{shebang, default};
@@ -233,8 +233,8 @@ normalize_section(shebang, "#!" ++ Chars) ->
{shebang, Stripped}
end;
normalize_section(comment, Chars) ->
- Chopped = string:strip(Chars, right, $\n),
- Stripped = string:strip(string:strip(Chopped, left, $%), both),
+ Chopped = string:trim(Chars, trailing, "$\n"),
+ Stripped = string:trim(string:trim(Chopped, leading, "$%"), both),
if
Stripped =:= ?COMMENT ->
{comment, default};
@@ -242,8 +242,8 @@ normalize_section(comment, Chars) ->
{comment, Stripped}
end;
normalize_section(emu_args, "%%!" ++ Chars) ->
- Chopped = string:strip(Chars, right, $\n),
- Stripped = string:strip(Chopped, both),
+ Chopped = string:trim(Chars, trailing, "$\n"),
+ Stripped = string:trim(Chopped, both),
{emu_args, Stripped};
normalize_section(Name, Chars) ->
{Name, Chars}.
diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl
index 1db004c91e..b6548626f3 100644
--- a/lib/stdlib/src/ets.erl
+++ b/lib/stdlib/src/ets.erl
@@ -1719,7 +1719,7 @@ get_line(P, Default) ->
line_string(Binary) when is_binary(Binary) -> unicode:characters_to_list(Binary);
line_string(Other) -> Other.
-nonl(S) -> string:strip(S, right, $\n).
+nonl(S) -> string:trim(S, trailing, "$\n").
print_number(Tab, Key, Num) ->
Os = ets:lookup(Tab, Key),
@@ -1748,7 +1748,7 @@ do_display_item(_Height, Width, I, Opos) ->
L = to_string(I),
L2 = if
length(L) > Width - 8 ->
- string:substr(L, 1, Width-13) ++ " ...";
+ string:slice(L, 0, Width-13) ++ " ...";
true ->
L
end,
diff --git a/lib/stdlib/src/filelib.erl b/lib/stdlib/src/filelib.erl
index d7c313f214..0f90b3fc33 100644
--- a/lib/stdlib/src/filelib.erl
+++ b/lib/stdlib/src/filelib.erl
@@ -365,11 +365,18 @@ do_list_dir(Dir, Mod) -> eval_list_dir(Dir, Mod).
%%% Compiling a wildcard.
+
+%% Define characters used for escaping a \.
+-define(ESCAPE_PREFIX, $@).
+-define(ESCAPE_CHARACTER, [?ESCAPE_PREFIX,$e]).
+-define(ESCAPED_ESCAPE_PREFIX, [?ESCAPE_PREFIX,?ESCAPE_PREFIX]).
+
%% Only for debugging.
compile_wildcard(Pattern) when is_list(Pattern) ->
{compiled_wildcard,?HANDLE_ERROR(compile_wildcard(Pattern, "."))}.
-compile_wildcard(Pattern, Cwd0) ->
+compile_wildcard(Pattern0, Cwd0) ->
+ Pattern = convert_escapes(Pattern0),
[Root|Rest] = filename:split(Pattern),
case filename:pathtype(Root) of
relative ->
@@ -409,7 +416,8 @@ compile_join({cwd,Cwd}, File0) ->
compile_join({root,PrefixLen,Root}, File) ->
{root,PrefixLen,filename:join(Root, File)}.
-compile_part(Part) ->
+compile_part(Part0) ->
+ Part = wrap_escapes(Part0),
compile_part(Part, false, []).
compile_part_to_sep(Part) ->
@@ -445,6 +453,8 @@ compile_part([${|Rest], Upto, Result) ->
error ->
compile_part(Rest, Upto, [${|Result])
end;
+compile_part([{escaped,X}|Rest], Upto, Result) ->
+ compile_part(Rest, Upto, [X|Result]);
compile_part([X|Rest], Upto, Result) ->
compile_part(Rest, Upto, [X|Result]);
compile_part([], _Upto, Result) ->
@@ -461,6 +471,8 @@ compile_charset1([Lower, $-, Upper|Rest], Ordset) when Lower =< Upper ->
compile_charset1(Rest, compile_range(Lower, Upper, Ordset));
compile_charset1([$]|Rest], Ordset) ->
{ok, {one_of, gb_sets:from_ordset(Ordset)}, Rest};
+compile_charset1([{escaped,X}|Rest], Ordset) ->
+ compile_charset1(Rest, ordsets:add_element(X, Ordset));
compile_charset1([X|Rest], Ordset) ->
compile_charset1(Rest, ordsets:add_element(X, Ordset));
compile_charset1([], _Ordset) ->
@@ -486,6 +498,32 @@ compile_alt(Pattern, Result) ->
error
end.
+%% Convert backslashes to an illegal Unicode character to
+%% protect in from filename:split/1.
+
+convert_escapes([?ESCAPE_PREFIX|T]) ->
+ ?ESCAPED_ESCAPE_PREFIX ++ convert_escapes(T);
+convert_escapes([$\\|T]) ->
+ ?ESCAPE_CHARACTER ++ convert_escapes(T);
+convert_escapes([H|T]) ->
+ [H|convert_escapes(T)];
+convert_escapes([]) ->
+ [].
+
+%% Wrap each escape in a tuple to remove the special meaning for
+%% the character that follows.
+
+wrap_escapes(?ESCAPED_ESCAPE_PREFIX ++ T) ->
+ [?ESCAPE_PREFIX|wrap_escapes(T)];
+wrap_escapes(?ESCAPE_CHARACTER ++ [C|T]) ->
+ [{escaped,C}|wrap_escapes(T)];
+wrap_escapes(?ESCAPE_CHARACTER) ->
+ [];
+wrap_escapes([H|T]) ->
+ [H|wrap_escapes(T)];
+wrap_escapes([]) ->
+ [].
+
badpattern(Reason) ->
error({badpattern,Reason}).
diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl
index ee807dfd09..a322bd002d 100644
--- a/lib/stdlib/src/filename.erl
+++ b/lib/stdlib/src/filename.erl
@@ -34,6 +34,38 @@
%% we flatten the arguments immediately on function entry as that makes
%% it easier to ensure that the code works.
+%%
+%% *** Requirements on Raw Filename Format ***
+%%
+%% These requirements are due to the 'filename' module
+%% in stdlib. This since it is documented that it
+%% should be able to operate on raw filenames as well
+%% as ordinary filenames.
+%%
+%% A raw filename *must* be a byte sequence where:
+%% 1. Codepoints 0-127 (7-bit ascii) *must* be encoded
+%% as a byte with the corresponding value. That is,
+%% the most significant bit in the byte encoding the
+%% codepoint is never set.
+%% 2. Codepoints greater than 127 *must* be encoded
+%% with the most significant bit set in *every* byte
+%% encoding it.
+%%
+%% Latin1 and UTF-8 meet these requirements while
+%% UTF-16 and UTF-32 don't.
+%%
+%% On Windows filenames are natively stored as malformed
+%% UTF-16LE (lonely surrogates may appear). A more correct
+%% description than UTF-16 would be an array of 16-bit
+%% words... In order to meet the requirements of the
+%% raw file format we convert the malformed UTF-16LE to
+%% malformed UTF-8 which meet the requirements.
+%%
+%% Note that these requirements are today only OTP
+%% internal (erts-stdlib internal) requirements that
+%% could be changed.
+%%
+
-export([absname/1, absname/2, absname_join/2,
basename/1, basename/2, dirname/1,
extension/1, join/1, join/2, pathtype/1,
@@ -41,6 +73,7 @@
safe_relative_path/1]).
-export([find_src/1, find_src/2]). % deprecated
-export([basedir/2, basedir/3]).
+-export([validate/1]).
%% Undocumented and unsupported exports.
-export([append/2]).
@@ -1053,10 +1086,10 @@ basedir_linux(Type) ->
user_log -> getenv("XDG_CACHE_HOME", ?basedir_linux_user_log, true);
site_data ->
Base = getenv("XDG_DATA_DIRS",?basedir_linux_site_data,false),
- string:tokens(Base,":");
+ string:lexemes(Base, ":");
site_config ->
Base = getenv("XDG_CONFIG_DIRS",?basedir_linux_site_config,false),
- string:tokens(Base,":")
+ string:lexemes(Base, ":")
end.
-define(basedir_darwin_user_data, "Library/Application Support").
@@ -1152,3 +1185,72 @@ basedir_os_type() ->
{win32,_} -> windows;
_ -> linux
end.
+
+%%
+%% validate/1
+%%
+
+-spec validate(FileName) -> boolean() when
+ FileName :: file:name_all().
+
+validate(FileName) when is_binary(FileName) ->
+ %% Raw filename...
+ validate_bin(FileName);
+validate(FileName) when is_list(FileName);
+ is_atom(FileName) ->
+ validate_list(FileName,
+ file:native_name_encoding(),
+ os:type()).
+
+validate_list(FileName, Enc, Os) ->
+ try
+ true = validate_list(FileName, Enc, Os, 0) > 0
+ catch
+ _ : _ -> false
+ end.
+
+validate_list([], _Enc, _Os, Chars) ->
+ Chars;
+validate_list(C, Enc, Os, Chars) when is_integer(C) ->
+ validate_char(C, Enc, Os),
+ Chars+1;
+validate_list(A, Enc, Os, Chars) when is_atom(A) ->
+ validate_list(atom_to_list(A), Enc, Os, Chars);
+validate_list([H|T], Enc, Os, Chars) ->
+ NewChars = validate_list(H, Enc, Os, Chars),
+ validate_list(T, Enc, Os, NewChars).
+
+%% C is always an integer...
+% validate_char(C, _, _) when not is_integer(C) ->
+% throw(invalid);
+validate_char(C, _, _) when C < 1 ->
+ throw(invalid); %% No negative or null characters...
+validate_char(C, latin1, _) when C > 255 ->
+ throw(invalid);
+validate_char(C, utf8, _) when C >= 16#110000 ->
+ throw(invalid);
+validate_char(C, utf8, {win32, _}) when C > 16#ffff ->
+ throw(invalid); %% invalid win wchar...
+validate_char(_C, utf8, {win32, _}) ->
+ ok; %% Range below is accepted on windows...
+validate_char(C, utf8, _) when 16#D800 =< C, C =< 16#DFFF ->
+ throw(invalid); %% invalid unicode range...
+validate_char(_, _, _) ->
+ ok.
+
+validate_bin(Bin) ->
+ %% Raw filename. That is, we do not interpret
+ %% the encoding, but we still do not accept
+ %% null characters...
+ try
+ true = validate_bin(Bin, 0) > 0
+ catch
+ _ : _ -> false
+ end.
+
+validate_bin(<<>>, Bs) ->
+ Bs;
+validate_bin(<<0, _Rest/binary>>, _Bs) ->
+ throw(invalid); %% No null characters allowed...
+validate_bin(<<_B, Rest/binary>>, Bs) ->
+ validate_bin(Rest, Bs+1).
diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl
index 7daa7a9fe4..ac172325b5 100644
--- a/lib/stdlib/src/gen_server.erl
+++ b/lib/stdlib/src/gen_server.erl
@@ -116,23 +116,27 @@
%%%=========================================================================
-callback init(Args :: term()) ->
- {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate} |
+ {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate | {continue, term()}} |
{stop, Reason :: term()} | ignore.
-callback handle_call(Request :: term(), From :: {pid(), Tag :: term()},
State :: term()) ->
{reply, Reply :: term(), NewState :: term()} |
- {reply, Reply :: term(), NewState :: term(), timeout() | hibernate} |
+ {reply, Reply :: term(), NewState :: term(), timeout() | hibernate | {continue, term()}} |
{noreply, NewState :: term()} |
- {noreply, NewState :: term(), timeout() | hibernate} |
+ {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} |
{stop, Reason :: term(), Reply :: term(), NewState :: term()} |
{stop, Reason :: term(), NewState :: term()}.
-callback handle_cast(Request :: term(), State :: term()) ->
{noreply, NewState :: term()} |
- {noreply, NewState :: term(), timeout() | hibernate} |
+ {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} |
{stop, Reason :: term(), NewState :: term()}.
-callback handle_info(Info :: timeout | term(), State :: term()) ->
{noreply, NewState :: term()} |
- {noreply, NewState :: term(), timeout() | hibernate} |
+ {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} |
+ {stop, Reason :: term(), NewState :: term()}.
+-callback handle_continue(Info :: term(), State :: term()) ->
+ {noreply, NewState :: term()} |
+ {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} |
{stop, Reason :: term(), NewState :: term()}.
-callback terminate(Reason :: (normal | shutdown | {shutdown, term()} |
term()),
@@ -149,7 +153,7 @@
Status :: term().
-optional_callbacks(
- [handle_info/2, terminate/2, code_change/3, format_status/2]).
+ [handle_info/2, handle_continue/2, terminate/2, code_change/3, format_status/2]).
%%% -----------------------------------------------------------------
%%% Starts a generic server.
@@ -309,7 +313,7 @@ enter_loop(Mod, Options, State, ServerName, Timeout) ->
Name = gen:get_proc_name(ServerName),
Parent = gen:get_parent(),
Debug = gen:debug_options(Name, Options),
- HibernateAfterTimeout = gen:hibernate_after(Options),
+ HibernateAfterTimeout = gen:hibernate_after(Options),
loop(Parent, Name, State, Mod, Timeout, HibernateAfterTimeout, Debug).
%%%========================================================================
@@ -374,6 +378,19 @@ init_it(Mod, Args) ->
%%% ---------------------------------------------------
%%% The MAIN loop.
%%% ---------------------------------------------------
+
+loop(Parent, Name, State, Mod, {continue, Continue} = Msg, HibernateAfterTimeout, Debug) ->
+ Reply = try_dispatch(Mod, handle_continue, Continue, State),
+ case Debug of
+ [] ->
+ handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod,
+ HibernateAfterTimeout, State);
+ _ ->
+ Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, Msg),
+ handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod,
+ HibernateAfterTimeout, State, Debug1)
+ end;
+
loop(Parent, Name, State, Mod, hibernate, HibernateAfterTimeout, Debug) ->
proc_lib:hibernate(?MODULE,wake_hib,[Parent, Name, State, Mod, HibernateAfterTimeout, Debug]);
diff --git a/lib/stdlib/src/gen_statem.erl b/lib/stdlib/src/gen_statem.erl
index 1110d18af6..cd6312855d 100644
--- a/lib/stdlib/src/gen_statem.erl
+++ b/lib/stdlib/src/gen_statem.erl
@@ -296,7 +296,7 @@
(Reason :: term()).
%% Format the callback module state in some sensible that is
-%% often condensed way. For StatusOption =:= 'normal' the perferred
+%% often condensed way. For StatusOption =:= 'normal' the preferred
%% return term is [{data,[{"State",FormattedState}]}], and for
%% StatusOption =:= 'terminate' it is just FormattedState.
-callback format_status(
@@ -510,8 +510,6 @@ call(ServerRef, Request, Timeout) ->
parse_timeout(Timeout) ->
case Timeout of
- {clean_timeout,infinity} ->
- {dirty_timeout,infinity};
{clean_timeout,_} ->
Timeout;
{dirty_timeout,_} ->
diff --git a/lib/stdlib/src/io_lib_format.erl b/lib/stdlib/src/io_lib_format.erl
index 4b2d15c8b3..e345810ca0 100644
--- a/lib/stdlib/src/io_lib_format.erl
+++ b/lib/stdlib/src/io_lib_format.erl
@@ -380,7 +380,7 @@ float_e(_Fl, {Ds,E}, P) ->
{Fs,false} -> [Fs|float_exp(E-1)]
end.
-%% float_man([Digit], Icount, Dcount) -> {[Chars],CarryFlag}.
+%% float_man([Digit], Icount, Dcount) -> {[Char],CarryFlag}.
%% Generate the characters in the mantissa from the digits with Icount
%% characters before the '.' and Dcount decimals. Handle carry and let
%% caller decide what to do at top.
@@ -395,7 +395,7 @@ float_man([D|Ds], I, Dc) ->
{Cs,false} -> {[D|Cs],false}
end;
float_man([], I, Dc) -> %Pad with 0's
- {string:chars($0, I, [$.|string:chars($0, Dc)]),false}.
+ {lists:duplicate(I, $0) ++ [$.|lists:duplicate(Dc, $0)],false}.
float_man([D|_], 0) when D >= $5 -> {[],true};
float_man([_|_], 0) -> {[],false};
@@ -405,7 +405,7 @@ float_man([D|Ds], Dc) ->
{Cs,true} -> {[D+1|Cs],false};
{Cs,false} -> {[D|Cs],false}
end;
-float_man([], Dc) -> {string:chars($0, Dc),false}. %Pad with 0's
+float_man([], Dc) -> {lists:duplicate(Dc, $0),false}. %Pad with 0's
%% float_exp(Exponent) -> [Char].
%% Generate the exponent of a floating point number. Always include sign.
@@ -429,7 +429,7 @@ fwrite_f(Fl, F, Adj, P, Pad) when P >= 1 ->
float_f(Fl, Fd, P) when Fl < 0.0 ->
[$-|float_f(-Fl, Fd, P)];
float_f(Fl, {Ds,E}, P) when E =< 0 ->
- float_f(Fl, {string:chars($0, -E+1, Ds),1}, P); %Prepend enough 0's
+ float_f(Fl, {lists:duplicate(-E+1, $0)++Ds,1}, P); %Prepend enough 0's
float_f(_Fl, {Ds,E}, P) ->
case float_man(Ds, E, P) of
{Fs,true} -> "1" ++ Fs; %Handle carry
@@ -751,7 +751,7 @@ adjust(Data, Pad, right) -> [Pad|Data].
flat_trunc(List, N) when is_integer(N), N >= 0 ->
string:slice(List, 0, N).
-%% A deep version of string:chars/2,3
+%% A deep version of lists:duplicate/2
chars(_C, 0) ->
[];
diff --git a/lib/stdlib/src/lib.erl b/lib/stdlib/src/lib.erl
index c6eb0d7915..a7980cc294 100644
--- a/lib/stdlib/src/lib.erl
+++ b/lib/stdlib/src/lib.erl
@@ -646,7 +646,7 @@ pp_arguments(PF, As, I, Enc) ->
Ll = length(L),
A = list_to_atom(lists:duplicate(Ll, $a)),
S0 = unicode:characters_to_list(PF([A | T], I+1), Enc),
- brackets_to_parens([$[,L,string:sub_string(S0, 2+Ll)], Enc);
+ brackets_to_parens([$[,L,string:slice(S0, 1+Ll)], Enc);
_ ->
brackets_to_parens(PF(As, I+1), Enc)
end.
diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl
index 5b488cc677..122b476ddb 100644
--- a/lib/stdlib/src/otp_internal.erl
+++ b/lib/stdlib/src/otp_internal.erl
@@ -609,6 +609,52 @@ obsolete_1(filename, find_src, 2) ->
obsolete_1(erlang, hash, 2) ->
{removed, {erlang, phash2, 2}, "20.0"};
+%% Added in OTP-21
+obsolete_1(string, len, 1) ->
+ {deprecated, "deprecated; use string:length/3 instead"};
+obsolete_1(string, concat, 2) ->
+ {deprecated, "deprecated; use [Str1,Str2] instead"};
+obsolete_1(string, str, 2) ->
+ {deprecated, "deprecated; use string:find/2 instead"};
+obsolete_1(string, rstr, 2) ->
+ {deprecated, "deprecated; use string:find/3 instead"};
+obsolete_1(string, chr, 2) ->
+ {deprecated, "deprecated; use string:find/2 instead"};
+obsolete_1(string, rchr, 2) ->
+ {deprecated, "deprecated; use string:find/3 instead"};
+obsolete_1(string, span, 2) ->
+ {deprecated, "deprecated; use string:take/2 instead"};
+obsolete_1(string, cspan, 2) ->
+ {deprecated, "deprecated; use string:take/3 instead"};
+obsolete_1(string, substr, _) ->
+ {deprecated, "deprecated; use string:slice/3 instead"};
+obsolete_1(string, tokens, 2) ->
+ {deprecated, "deprecated; use string:lexemes/2 instead"};
+obsolete_1(string, chars, _) ->
+ {deprecated, "deprecated; use lists:duplicate/2 instead"};
+obsolete_1(string, copies, _) ->
+ {deprecated, "deprecated; use lists:duplicate/2 instead"};
+obsolete_1(string, words, _) ->
+ {deprecated, "deprecated; use string:lexemes/2 instead"};
+obsolete_1(string, strip, _) ->
+ {deprecated, "deprecated; use string:trim/3 instead"};
+obsolete_1(string, sub_word, _) ->
+ {deprecated, "deprecated; use string:nth_lexeme/3 instead"};
+obsolete_1(string, sub_string, _) ->
+ {deprecated, "deprecated; use string:slice/3 instead"};
+obsolete_1(string, left, _) ->
+ {deprecated, "deprecated; use string:pad/3 instead"};
+obsolete_1(string, right, _) ->
+ {deprecated, "deprecated; use string:pad/3 instead"};
+obsolete_1(string, centre, _) ->
+ {deprecated, "deprecated; use string:pad/3 instead"};
+obsolete_1(string, join, _) ->
+ {deprecated, "deprecated; use lists:join/2 instead"};
+obsolete_1(string, to_upper, _) ->
+ {deprecated, "deprecated; use string:uppercase/1 or string:titlecase/1 instead"};
+obsolete_1(string, to_lower, _) ->
+ {deprecated, "deprecated; use string:lowercase/1 or string:casefold/1 instead"};
+
%% not obsolete
obsolete_1(_, _, _) ->
diff --git a/lib/stdlib/src/pool.erl b/lib/stdlib/src/pool.erl
index 05950a1d7c..b12ff205b1 100644
--- a/lib/stdlib/src/pool.erl
+++ b/lib/stdlib/src/pool.erl
@@ -25,7 +25,7 @@
%% with the least load !!!!
%% This function is callable from any node including the master
%% That is part of the pool
-%% nodes are scheduled on a per usgae basis and per load basis,
+%% nodes are scheduled on a per usage basis and per load basis,
%% Whenever we use a node, we put at the end of the queue, and whenever
%% a node report a change in load, we insert it accordingly
@@ -197,7 +197,7 @@ pure_insert({Load,Node},[{L,N}|Tail]) when Load < L ->
pure_insert(L,[H|T]) -> [H|pure_insert(L,T)].
%% Really should not measure the contributions from
-%% the back ground processes here .... which we do :-(
+%% the background processes here .... which we do :-(
%% We don't have to monitor the master, since we're slaves anyway
statistic_collector() ->
@@ -213,7 +213,7 @@ statistic_collector(I) ->
stat_loop(M, 999999)
end.
-%% Do not tell the master about our load if it has not changed
+%% Do not tell the master about our load if it has not changed
stat_loop(M, Old) ->
sleep(2000),
diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl
index 7a8a5e6d4a..362e98006e 100644
--- a/lib/stdlib/src/rand.erl
+++ b/lib/stdlib/src/rand.erl
@@ -21,8 +21,8 @@
%% Multiple PRNG module for Erlang/OTP
%% Copyright (c) 2015-2016 Kenji Rikitake
%%
-%% exrop (xoroshiro116+) added and statistical distribution
-%% improvements by the Erlang/OTP team 2017
+%% exrop (xoroshiro116+) added, statistical distribution
+%% improvements and uniform_real added by the Erlang/OTP team 2017
%% =====================================================================
-module(rand).
@@ -30,10 +30,14 @@
-export([seed_s/1, seed_s/2, seed/1, seed/2,
export_seed/0, export_seed_s/1,
uniform/0, uniform/1, uniform_s/1, uniform_s/2,
+ uniform_real/0, uniform_real_s/1,
jump/0, jump/1,
normal/0, normal/2, normal_s/1, normal_s/3
]).
+%% Debug
+-export([make_float/3, float2str/1, bc64/1]).
+
-compile({inline, [exs64_next/1, exsplus_next/1,
exs1024_next/1, exs1024_calc/2,
exrop_next/1, exrop_next_s/2,
@@ -60,6 +64,10 @@
%% N i evaluated 3 times
(?BSL((Bits), (X), (N)) bor ((X) bsr ((Bits)-(N))))).
+-define(
+ BC(V, N),
+ bc((V), ?BIT((N) - 1), N)).
+
%%-define(TWO_POW_MINUS53, (math:pow(2, -53))).
-define(TWO_POW_MINUS53, 1.11022302462515657e-16).
@@ -84,14 +92,21 @@
%% The 'bits' field indicates how many bits the integer
%% returned from 'next' has got, i.e 'next' shall return
%% an random integer in the range 0..(2^Bits - 1).
-%% At least 53 bits is required for the floating point
-%% producing fallbacks. This field is only used when
-%% the 'uniform' or 'uniform_n' fields are not defined.
+%% At least 55 bits is required for the floating point
+%% producing fallbacks, but 56 bits would be more future proof.
%%
%% The fields 'next', 'uniform' and 'uniform_n'
-%% implement the algorithm. If 'uniform' or 'uinform_n'
+%% implement the algorithm. If 'uniform' or 'uniform_n'
%% is not present there is a fallback using 'next' and either
-%% 'bits' or the deprecated 'max'.
+%% 'bits' or the deprecated 'max'. The 'next' function
+%% must generate a word with at least 56 good random bits.
+%%
+%% The 'weak_low_bits' field indicate how many bits are of
+%% lesser quality and they will not be used by the floating point
+%% producing functions, nor by the range producing functions
+%% when more bits are needed, to avoid weak bits in the middle
+%% of the generated bits. The lowest bits from the range
+%% functions still have the generator's quality.
%%
-type alg_handler() ::
#{type := alg(),
@@ -148,11 +163,7 @@
%% For ranges larger than the algorithm bit size
uniform_range(Range, #{next:=Next, bits:=Bits} = Alg, R, V) ->
- WeakLowBits =
- case Alg of
- #{weak_low_bits:=WLB} -> WLB;
- #{} -> 0
- end,
+ WeakLowBits = maps:get(weak_low_bits, Alg, 0),
%% Maybe waste the lowest bit(s) when shifting in new bits
Shift = Bits - WeakLowBits,
ShiftMask = bnot ?MASK(WeakLowBits),
@@ -297,7 +308,7 @@ uniform_s({#{bits:=Bits, next:=Next} = Alg, R0}) ->
{(V bsr (Bits - 53)) * ?TWO_POW_MINUS53, {Alg, R1}};
uniform_s({#{max:=Max, next:=Next} = Alg, R0}) ->
{V, R1} = Next(R0),
- %% Old broken algorithm with non-uniform density
+ %% Old algorithm with non-uniform density
{V / (Max + 1), {Alg, R1}}.
@@ -317,7 +328,7 @@ uniform_s(N, {#{bits:=Bits, next:=Next} = Alg, R0})
?uniform_range(N, Alg, R1, V, MaxMinusN, I);
uniform_s(N, {#{max:=Max, next:=Next} = Alg, R0})
when is_integer(N), 1 =< N ->
- %% Old broken algorithm with skewed probability
+ %% Old algorithm with skewed probability
%% and gap in ranges > Max
{V, R1} = Next(R0),
if
@@ -328,6 +339,189 @@ uniform_s(N, {#{max:=Max, next:=Next} = Alg, R0})
{trunc(F * N) + 1, {Alg, R1}}
end.
+%% uniform_real/0: returns a random float X where 0.0 < X =< 1.0,
+%% updating the state in the process dictionary.
+
+-spec uniform_real() -> X :: float().
+uniform_real() ->
+ {X, Seed} = uniform_real_s(seed_get()),
+ _ = seed_put(Seed),
+ X.
+
+%% uniform_real_s/1: given a state, uniform_s/1
+%% returns a random float X where 0.0 < X =< 1.0,
+%% and a new state.
+%%
+%% This function does not use the same form of uniformity
+%% as the uniform_s/1 function.
+%%
+%% Instead, this function does not generate numbers with equal
+%% distance in the interval, but rather tries to keep all mantissa
+%% bits random also for small numbers, meaning that the distance
+%% between possible numbers decreases when the numbers
+%% approaches 0.0, as does the possibility for a particular
+%% number. Hence uniformity is preserved.
+%%
+%% To generate 56 bits at the time instead of 53 is actually
+%% a speed optimization since the probability to have to
+%% generate a second word decreases by 1/2 for every extra bit.
+%%
+%% This function generates normalized numbers, so the smallest number
+%% that can be generated is 2^-1022 with the distance 2^-1074
+%% to the next to smallest number, compared to 2^-53 for uniform_s/1.
+%%
+%% This concept of uniformity should work better for applications
+%% where you need to calculate 1.0/X or math:log(X) since those
+%% operations benefits from larger precision approaching 0.0,
+%% and that this function does not return 0.0 nor denormalized
+%% numbers very close to 0.0. The log() operation in The Box-Muller
+%% transformation for normal distribution is an example of this.
+%%
+%%-define(TWO_POW_MINUS55, (math:pow(2, -55))).
+%%-define(TWO_POW_MINUS110, (math:pow(2, -110))).
+%%-define(TWO_POW_MINUS55, 2.7755575615628914e-17).
+%%-define(TWO_POW_MINUS110, 7.7037197775489436e-34).
+%%
+-spec uniform_real_s(State :: state()) -> {X :: float(), NewState :: state()}.
+uniform_real_s({#{bits:=Bits, next:=Next} = Alg, R0}) ->
+ %% Generate a 56 bit number without using the weak low bits.
+ %%
+ %% Be sure to use only 53 bits when multiplying with
+ %% math:pow(2.0, -N) to avoid rounding which would make
+ %% "even" floats more probable than "odd".
+ %%
+ {V1, R1} = Next(R0),
+ M1 = V1 bsr (Bits - 56),
+ if
+ ?BIT(55) =< M1 ->
+ %% We have 56 bits - waste 3
+ {(M1 bsr 3) * math:pow(2.0, -53), {Alg, R1}};
+ ?BIT(54) =< M1 ->
+ %% We have 55 bits - waste 2
+ {(M1 bsr 2) * math:pow(2.0, -54), {Alg, R1}};
+ ?BIT(53) =< M1 ->
+ %% We have 54 bits - waste 1
+ {(M1 bsr 1) * math:pow(2.0, -55), {Alg, R1}};
+ ?BIT(52) =< M1 ->
+ %% We have 53 bits - use all
+ {M1 * math:pow(2.0, -56), {Alg, R1}};
+ true ->
+ %% Need more bits
+ {V2, R2} = Next(R1),
+ uniform_real_s(Alg, Next, M1, -56, R2, V2, Bits)
+ end;
+uniform_real_s({#{max:=_, next:=Next} = Alg, R0}) ->
+ %% Generate a 56 bit number.
+ %% Ignore the weak low bits for these old algorithms,
+ %% just produce something reasonable.
+ %%
+ %% Be sure to use only 53 bits when multiplying with
+ %% math:pow(2.0, -N) to avoid rounding which would make
+ %% "even" floats more probable than "odd".
+ %%
+ {V1, R1} = Next(R0),
+ M1 = ?MASK(56, V1),
+ if
+ ?BIT(55) =< M1 ->
+ %% We have 56 bits - waste 3
+ {(M1 bsr 3) * math:pow(2.0, -53), {Alg, R1}};
+ ?BIT(54) =< M1 ->
+ %% We have 55 bits - waste 2
+ {(M1 bsr 2) * math:pow(2.0, -54), {Alg, R1}};
+ ?BIT(53) =< M1 ->
+ %% We have 54 bits - waste 1
+ {(M1 bsr 1) * math:pow(2.0, -55), {Alg, R1}};
+ ?BIT(52) =< M1 ->
+ %% We have 53 bits - use all
+ {M1 * math:pow(2.0, -56), {Alg, R1}};
+ true ->
+ %% Need more bits
+ {V2, R2} = Next(R1),
+ uniform_real_s(Alg, Next, M1, -56, R2, V2, 56)
+ end.
+
+uniform_real_s(Alg, _Next, M0, -1064, R1, V1, Bits) -> % 19*56
+ %% This is a very theoretical bottom case.
+ %% The odds of getting here is about 2^-1008,
+ %% through a white box test case, or thanks to
+ %% a malfunctioning PRNG producing 18 56-bit zeros in a row.
+ %%
+ %% Fill up to 53 bits, we have at most 52
+ B0 = (53 - ?BC(M0, 52)), % Missing bits
+ {(((M0 bsl B0) bor (V1 bsr (Bits - B0))) * math:pow(2.0, -1064 - B0)),
+ {Alg, R1}};
+uniform_real_s(Alg, Next, M0, BitNo, R1, V1, Bits) ->
+ if
+ %% Optimize the most probable.
+ %% Fill up to 53 bits.
+ ?BIT(51) =< M0 ->
+ %% We have 52 bits in M0 - need 1
+ {(((M0 bsl 1) bor (V1 bsr (Bits - 1)))
+ * math:pow(2.0, BitNo - 1)),
+ {Alg, R1}};
+ ?BIT(50) =< M0 ->
+ %% We have 51 bits in M0 - need 2
+ {(((M0 bsl 2) bor (V1 bsr (Bits - 2)))
+ * math:pow(2.0, BitNo - 2)),
+ {Alg, R1}};
+ ?BIT(49) =< M0 ->
+ %% We have 50 bits in M0 - need 3
+ {(((M0 bsl 3) bor (V1 bsr (Bits - 3)))
+ * math:pow(2.0, BitNo - 3)),
+ {Alg, R1}};
+ M0 == 0 ->
+ M1 = V1 bsr (Bits - 56),
+ if
+ ?BIT(55) =< M1 ->
+ %% We have 56 bits - waste 3
+ {(M1 bsr 3) * math:pow(2.0, BitNo - 53), {Alg, R1}};
+ ?BIT(54) =< M1 ->
+ %% We have 55 bits - waste 2
+ {(M1 bsr 2) * math:pow(2.0, BitNo - 54), {Alg, R1}};
+ ?BIT(53) =< M1 ->
+ %% We have 54 bits - waste 1
+ {(M1 bsr 1) * math:pow(2.0, BitNo - 55), {Alg, R1}};
+ ?BIT(52) =< M1 ->
+ %% We have 53 bits - use all
+ {M1 * math:pow(2.0, BitNo - 56), {Alg, R1}};
+ BitNo =:= -1008 ->
+ %% Endgame
+ %% For the last round we can not have 14 zeros or more
+ %% at the top of M1 because then we will underflow,
+ %% so we need at least 43 bits
+ if
+ ?BIT(42) =< M1 ->
+ %% We have 43 bits - get the last bits
+ uniform_real_s(Alg, Next, M1, BitNo - 56, R1);
+ true ->
+ %% Would underflow 2^-1022 - start all over
+ %%
+ %% We could just crash here since the odds for
+ %% the PRNG being broken is much higher than
+ %% for a good PRNG generating this many zeros
+ %% in a row. Maybe we should write an error
+ %% report or call this a system limit...?
+ uniform_real_s({Alg, R1})
+ end;
+ true ->
+ %% Need more bits
+ uniform_real_s(Alg, Next, M1, BitNo - 56, R1)
+ end;
+ true ->
+ %% Fill up to 53 bits
+ B0 = 53 - ?BC(M0, 49), % Number of bits we need to append
+ {(((M0 bsl B0) bor (V1 bsr (Bits - B0)))
+ * math:pow(2.0, BitNo - B0)),
+ {Alg, R1}}
+ end.
+%%
+uniform_real_s(#{bits:=Bits} = Alg, Next, M0, BitNo, R0) ->
+ {V1, R1} = Next(R0),
+ uniform_real_s(Alg, Next, M0, BitNo, R1, V1, Bits);
+uniform_real_s(#{max:=_} = Alg, Next, M0, BitNo, R0) ->
+ {V1, R1} = Next(R0),
+ uniform_real_s(Alg, Next, M0, BitNo, R1, ?MASK(56, V1), 56).
+
%% jump/1: given a state, jump/1
%% returns a new state which is equivalent to that
%% after a large number of call defined for each algorithm.
@@ -1025,3 +1219,42 @@ normal_fi(Indx) ->
1.0214971439701471e-02,8.6165827693987316e-03,7.0508754713732268e-03,
5.5224032992509968e-03,4.0379725933630305e-03,2.6090727461021627e-03,
1.2602859304985975e-03}).
+
+%%%bitcount64(0) -> 0;
+%%%bitcount64(V) -> 1 + bitcount(V, 64).
+%%%
+%%%-define(
+%%% BITCOUNT(V, N),
+%%% bitcount(V, N) ->
+%%% if
+%%% (1 bsl ((N) bsr 1)) =< (V) ->
+%%% ((N) bsr 1) + bitcount((V) bsr ((N) bsr 1), ((N) bsr 1));
+%%% true ->
+%%% bitcount((V), ((N) bsr 1))
+%%% end).
+%%%?BITCOUNT(V, 64);
+%%%?BITCOUNT(V, 32);
+%%%?BITCOUNT(V, 16);
+%%%?BITCOUNT(V, 8);
+%%%?BITCOUNT(V, 4);
+%%%?BITCOUNT(V, 2);
+%%%bitcount(_, 1) -> 0.
+
+bc64(V) -> ?BC(V, 64).
+
+%% Linear from high bit - higher probability first gives faster execution
+bc(V, B, N) when B =< V -> N;
+bc(V, B, N) -> bc(V, B bsr 1, N - 1).
+
+make_float(S, E, M) ->
+ <<F/float>> = <<S:1, E:11, M:52>>,
+ F.
+
+float2str(N) ->
+ <<S:1, E:11, M:52>> = <<(float(N))/float>>,
+ lists:flatten(
+ io_lib:format(
+ "~c~c.~13.16.0bE~b",
+ [case S of 1 -> $-; 0 -> $+ end,
+ case E of 0 -> $0; _ -> $1 end,
+ M, E - 16#3ff])).
diff --git a/lib/stdlib/src/slave.erl b/lib/stdlib/src/slave.erl
index d7cf6386f5..b3f3206d67 100644
--- a/lib/stdlib/src/slave.erl
+++ b/lib/stdlib/src/slave.erl
@@ -320,7 +320,7 @@ mk_cmd(Host, Name, Args, Waiter, Prog0) ->
%% emulator and flags as the test node. The return from lib:progname()
%% could then typically be '/<full_path_to>/cerl -gcov').
quote_progname(Progname) ->
- do_quote_progname(string:tokens(to_list(Progname)," ")).
+ do_quote_progname(string:lexemes(to_list(Progname)," ")).
do_quote_progname([Prog]) ->
"\""++Prog++"\"";
diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src
index 3c449d3cb9..5fb48acfab 100644
--- a/lib/stdlib/src/stdlib.app.src
+++ b/lib/stdlib/src/stdlib.app.src
@@ -101,13 +101,14 @@
timer,
unicode,
unicode_util,
+ uri_string,
win32reg,
zip]},
{registered,[timer_server,rsh_starter,take_over_monitor,pool_master,
dets]},
{applications, [kernel]},
{env, []},
- {runtime_dependencies, ["sasl-3.0","kernel-5.0","erts-9.0","crypto-3.3",
+ {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-10.0","crypto-3.3",
"compiler-5.0"]}
]}.
diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src
index 800c2c61f3..e4e3fb83e9 100644
--- a/lib/stdlib/src/stdlib.appup.src
+++ b/lib/stdlib/src/stdlib.appup.src
@@ -18,9 +18,7 @@
%% %CopyrightEnd%
{"%VSN%",
%% Up from - max one major revision back
- [{<<"3\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.*
- {<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.*
+ [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-20.*
%% Down to - max one major revision back
- [{<<"3\\.[0-3](\\.[0-9]+)*">>,[restart_new_emulator]}, % OTP-19.*
- {<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.*
+ [{<<"3\\.4(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-20.*
}.
diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl
index 4972da297d..5a4d2df2a6 100644
--- a/lib/stdlib/src/string.erl
+++ b/lib/stdlib/src/string.erl
@@ -87,6 +87,16 @@
%%% May be removed
-export([list_to_float/1, list_to_integer/1]).
+-deprecated([{len,1},{concat,2},
+ {str,2},{chr,2},{rchr,2},{rstr,2},
+ {span,2},{cspan,2},{substr,'_'},{tokens,2},
+ {chars,'_'},
+ {copies,2},{words,'_'},{strip,'_'},
+ {sub_word,'_'},{left,'_'},{right,'_'},
+ {sub_string,'_'},{centre,'_'},{join,2},
+ {to_upper,1}, {to_lower,1}
+ ]).
+
%% Uses bifs: string:list_to_float/1 and string:list_to_integer/1
-spec list_to_float(String) -> {Float, Rest} | {'error', Reason} when
String :: string(),
diff --git a/lib/stdlib/src/uri_string.erl b/lib/stdlib/src/uri_string.erl
new file mode 100644
index 0000000000..22212da222
--- /dev/null
+++ b/lib/stdlib/src/uri_string.erl
@@ -0,0 +1,1842 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%
+%% [RFC 3986, Chapter 2.2. Reserved Characters]
+%%
+%% reserved = gen-delims / sub-delims
+%%
+%% gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+%%
+%% sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
+%% / "*" / "+" / "," / ";" / "="
+%%
+%%
+%% [RFC 3986, Chapter 2.3. Unreserved Characters]
+%%
+%% unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
+%%
+%%
+%% [RFC 3986, Chapter 3. Syntax Components]
+%%
+%% The generic URI syntax consists of a hierarchical sequence of
+%% components referred to as the scheme, authority, path, query, and
+%% fragment.
+%%
+%% URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
+%%
+%% hier-part = "//" authority path-abempty
+%% / path-absolute
+%% / path-rootless
+%% / path-empty
+%%
+%% The scheme and path components are required, though the path may be
+%% empty (no characters). When authority is present, the path must
+%% either be empty or begin with a slash ("/") character. When
+%% authority is not present, the path cannot begin with two slash
+%% characters ("//"). These restrictions result in five different ABNF
+%% rules for a path (Section 3.3), only one of which will match any
+%% given URI reference.
+%%
+%% The following are two example URIs and their component parts:
+%%
+%% foo://example.com:8042/over/there?name=ferret#nose
+%% \_/ \______________/\_________/ \_________/ \__/
+%% | | | | |
+%% scheme authority path query fragment
+%% | _____________________|__
+%% / \ / \
+%% urn:example:animal:ferret:nose
+%%
+%%
+%% [RFC 3986, Chapter 3.1. Scheme]
+%%
+%% Each URI begins with a scheme name that refers to a specification for
+%% assigning identifiers within that scheme.
+%%
+%% scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+%%
+%%
+%% [RFC 3986, Chapter 3.2. Authority]
+%%
+%% Many URI schemes include a hierarchical element for a naming
+%% authority so that governance of the name space defined by the
+%% remainder of the URI is delegated to that authority (which may, in
+%% turn, delegate it further).
+%%
+%% authority = [ userinfo "@" ] host [ ":" port ]
+%%
+%%
+%% [RFC 3986, Chapter 3.2.1. User Information]
+%%
+%% The userinfo subcomponent may consist of a user name and, optionally,
+%% scheme-specific information about how to gain authorization to access
+%% the resource. The user information, if present, is followed by a
+%% commercial at-sign ("@") that delimits it from the host.
+%%
+%% userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
+%%
+%%
+%% [RFC 3986, Chapter 3.2.2. Host]
+%%
+%% The host subcomponent of authority is identified by an IP literal
+%% encapsulated within square brackets, an IPv4 address in dotted-
+%% decimal form, or a registered name.
+%%
+%% host = IP-literal / IPv4address / reg-name
+%%
+%% IP-literal = "[" ( IPv6address / IPvFuture ) "]"
+%%
+%% IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )
+%%
+%% IPv6address = 6( h16 ":" ) ls32
+%% / "::" 5( h16 ":" ) ls32
+%% / [ h16 ] "::" 4( h16 ":" ) ls32
+%% / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
+%% / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
+%% / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
+%% / [ *4( h16 ":" ) h16 ] "::" ls32
+%% / [ *5( h16 ":" ) h16 ] "::" h16
+%% / [ *6( h16 ":" ) h16 ] "::"
+%%
+%% ls32 = ( h16 ":" h16 ) / IPv4address
+%% ; least-significant 32 bits of address
+%%
+%% h16 = 1*4HEXDIG
+%% ; 16 bits of address represented in hexadecimal
+%%
+%% IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
+%%
+%% dec-octet = DIGIT ; 0-9
+%% / %x31-39 DIGIT ; 10-99
+%% / "1" 2DIGIT ; 100-199
+%% / "2" %x30-34 DIGIT ; 200-249
+%% / "25" %x30-35 ; 250-255
+%%
+%% reg-name = *( unreserved / pct-encoded / sub-delims )
+%%
+%%
+%% [RFC 3986, Chapter 3.2.2. Port]
+%%
+%% The port subcomponent of authority is designated by an optional port
+%% number in decimal following the host and delimited from it by a
+%% single colon (":") character.
+%%
+%% port = *DIGIT
+%%
+%%
+%% [RFC 3986, Chapter 3.3. Path]
+%%
+%% The path component contains data, usually organized in hierarchical
+%% form, that, along with data in the non-hierarchical query component
+%% (Section 3.4), serves to identify a resource within the scope of the
+%% URI's scheme and naming authority (if any). The path is terminated
+%% by the first question mark ("?") or number sign ("#") character, or
+%% by the end of the URI.
+%%
+%% path = path-abempty ; begins with "/" or is empty
+%% / path-absolute ; begins with "/" but not "//"
+%% / path-noscheme ; begins with a non-colon segment
+%% / path-rootless ; begins with a segment
+%% / path-empty ; zero characters
+%%
+%% path-abempty = *( "/" segment )
+%% path-absolute = "/" [ segment-nz *( "/" segment ) ]
+%% path-noscheme = segment-nz-nc *( "/" segment )
+%% path-rootless = segment-nz *( "/" segment )
+%% path-empty = 0<pchar>
+%% segment = *pchar
+%% segment-nz = 1*pchar
+%% segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
+%% ; non-zero-length segment without any colon ":"
+%%
+%% pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
+%%
+%%
+%% [RFC 3986, Chapter 3.4. Query]
+%%
+%% The query component contains non-hierarchical data that, along with
+%% data in the path component (Section 3.3), serves to identify a
+%% resource within the scope of the URI's scheme and naming authority
+%% (if any). The query component is indicated by the first question
+%% mark ("?") character and terminated by a number sign ("#") character
+%% or by the end of the URI.
+%%
+%% query = *( pchar / "/" / "?" )
+%%
+%%
+%% [RFC 3986, Chapter 3.5. Fragment]
+%%
+%% The fragment identifier component of a URI allows indirect
+%% identification of a secondary resource by reference to a primary
+%% resource and additional identifying information.
+%%
+%% fragment = *( pchar / "/" / "?" )
+%%
+%%
+%% [RFC 3986, Chapter 4.1. URI Reference]
+%%
+%% URI-reference is used to denote the most common usage of a resource
+%% identifier.
+%%
+%% URI-reference = URI / relative-ref
+%%
+%%
+%% [RFC 3986, Chapter 4.2. Relative Reference]
+%%
+%% A relative reference takes advantage of the hierarchical syntax
+%% (Section 1.2.3) to express a URI reference relative to the name space
+%% of another hierarchical URI.
+%%
+%% relative-ref = relative-part [ "?" query ] [ "#" fragment ]
+%%
+%% relative-part = "//" authority path-abempty
+%% / path-absolute
+%% / path-noscheme
+%% / path-empty
+%%
+%%
+%% [RFC 3986, Chapter 4.3. Absolute URI]
+%%
+%% Some protocol elements allow only the absolute form of a URI without
+%% a fragment identifier. For example, defining a base URI for later
+%% use by relative references calls for an absolute-URI syntax rule that
+%% does not allow a fragment.
+%%
+%% absolute-URI = scheme ":" hier-part [ "?" query ]
+%%
+-module(uri_string).
+
+%%-------------------------------------------------------------------------
+%% External API
+%%-------------------------------------------------------------------------
+-export([normalize/1, parse/1,
+ recompose/1, transcode/2]).
+-export_type([error/0, uri_map/0, uri_string/0]).
+
+
+%%-------------------------------------------------------------------------
+%% Internal API
+%%-------------------------------------------------------------------------
+-export([is_host/1, is_path/1]). % suppress warnings
+
+
+%%-------------------------------------------------------------------------
+%% Macros
+%%-------------------------------------------------------------------------
+-define(CHAR(Char), <<Char/utf8>>).
+-define(STRING_EMPTY, <<>>).
+-define(STRING(MatchStr), <<MatchStr/binary>>).
+-define(STRING_REST(MatchStr, Rest), <<MatchStr/utf8, Rest/binary>>).
+
+-define(DEC2HEX(X),
+ if ((X) >= 0) andalso ((X) =< 9) -> (X) + $0;
+ ((X) >= 10) andalso ((X) =< 15) -> (X) + $A - 10
+ end).
+
+-define(HEX2DEC(X),
+ if ((X) >= $0) andalso ((X) =< $9) -> (X) - $0;
+ ((X) >= $A) andalso ((X) =< $F) -> (X) - $A + 10;
+ ((X) >= $a) andalso ((X) =< $f) -> (X) - $a + 10
+ end).
+
+
+%%%=========================================================================
+%%% API
+%%%=========================================================================
+
+%%-------------------------------------------------------------------------
+%% URI compliant with RFC 3986
+%% ASCII %x21 - %x7A ("!" - "z") except
+%% %x34 " double quote
+%% %x60 < less than
+%% %x62 > greater than
+%% %x92 \ backslash
+%% %x94 ^ caret / circumflex
+%% %x96 ` grave / accent
+%%-------------------------------------------------------------------------
+-type uri_string() :: iodata().
+-type error() :: {error, atom(), term()}.
+
+
+%%-------------------------------------------------------------------------
+%% RFC 3986, Chapter 3. Syntax Components
+%%-------------------------------------------------------------------------
+-type uri_map() ::
+ #{fragment => unicode:chardata(),
+ host => unicode:chardata(),
+ path => unicode:chardata(),
+ port => non_neg_integer() | undefined,
+ query => unicode:chardata(),
+ scheme => unicode:chardata(),
+ userinfo => unicode:chardata()} | #{}.
+
+
+%%-------------------------------------------------------------------------
+%% Normalize URIs
+%%-------------------------------------------------------------------------
+-spec normalize(URIString) -> NormalizedURI when
+ URIString :: uri_string(),
+ NormalizedURI :: uri_string().
+normalize(URIString) ->
+ %% Percent-encoding normalization and case normalization for
+ %% percent-encoded triplets are achieved by running parse and
+ %% recompose on the input URI string.
+ recompose(
+ normalize_path_segment(
+ normalize_scheme_based(
+ normalize_case(
+ parse(URIString))))).
+
+
+%%-------------------------------------------------------------------------
+%% Parse URIs
+%%-------------------------------------------------------------------------
+-spec parse(URIString) -> URIMap when
+ URIString :: uri_string(),
+ URIMap :: uri_map()
+ | error().
+parse(URIString) when is_binary(URIString) ->
+ try parse_uri_reference(URIString, #{})
+ catch
+ throw:{error, Atom, RestData} -> {error, Atom, RestData}
+ end;
+parse(URIString) when is_list(URIString) ->
+ try
+ Binary = unicode:characters_to_binary(URIString),
+ Map = parse_uri_reference(Binary, #{}),
+ convert_mapfields_to_list(Map)
+ catch
+ throw:{error, Atom, RestData} -> {error, Atom, RestData}
+ end.
+
+
+%%-------------------------------------------------------------------------
+%% Recompose URIs
+%%-------------------------------------------------------------------------
+-spec recompose(URIMap) -> URIString when
+ URIMap :: uri_map(),
+ URIString :: uri_string()
+ | error().
+recompose(Map) ->
+ case is_valid_map(Map) of
+ false ->
+ {error, invalid_map, Map};
+ true ->
+ try
+ T0 = update_scheme(Map, empty),
+ T1 = update_userinfo(Map, T0),
+ T2 = update_host(Map, T1),
+ T3 = update_port(Map, T2),
+ T4 = update_path(Map, T3),
+ T5 = update_query(Map, T4),
+ update_fragment(Map, T5)
+ catch
+ throw:{error, Atom, RestData} -> {error, Atom, RestData}
+ end
+ end.
+
+
+%%-------------------------------------------------------------------------
+%% Transcode URIs
+%%-------------------------------------------------------------------------
+-spec transcode(URIString, Options) -> Result when
+ URIString :: uri_string(),
+ Options :: [{in_encoding, unicode:encoding()}|{out_encoding, unicode:encoding()}],
+ Result :: uri_string()
+ | error().
+transcode(URIString, Options) when is_binary(URIString) ->
+ try
+ InEnc = proplists:get_value(in_encoding, Options, utf8),
+ OutEnc = proplists:get_value(out_encoding, Options, utf8),
+ List = convert_to_list(URIString, InEnc),
+ Output = transcode(List, [], InEnc, OutEnc),
+ convert_to_binary(Output, utf8, OutEnc)
+ catch
+ throw:{error, Atom, RestData} -> {error, Atom, RestData}
+ end;
+transcode(URIString, Options) when is_list(URIString) ->
+ InEnc = proplists:get_value(in_encoding, Options, utf8),
+ OutEnc = proplists:get_value(out_encoding, Options, utf8),
+ Flattened = flatten_list(URIString, InEnc),
+ try transcode(Flattened, [], InEnc, OutEnc)
+ catch
+ throw:{error, Atom, RestData} -> {error, Atom, RestData}
+ end.
+
+
+%%%========================================================================
+%%% Internal functions
+%%%========================================================================
+
+%%-------------------------------------------------------------------------
+%% Converts Map fields to lists
+%%-------------------------------------------------------------------------
+convert_mapfields_to_list(Map) ->
+ Fun = fun (_, V) when is_binary(V) -> unicode:characters_to_list(V);
+ (_, V) -> V end,
+ maps:map(Fun, Map).
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 4.1. URI Reference]
+%%
+%% URI-reference is used to denote the most common usage of a resource
+%% identifier.
+%%
+%% URI-reference = URI / relative-ref
+%%-------------------------------------------------------------------------
+-spec parse_uri_reference(binary(), uri_map()) -> uri_map().
+parse_uri_reference(<<>>, _) -> #{path => <<>>};
+parse_uri_reference(URIString, URI) ->
+ try parse_scheme_start(URIString, URI)
+ catch
+ throw:{_,_,_} ->
+ parse_relative_part(URIString, URI)
+ end.
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 4.2. Relative Reference]
+%%
+%% A relative reference takes advantage of the hierarchical syntax
+%% (Section 1.2.3) to express a URI reference relative to the name space
+%% of another hierarchical URI.
+%%
+%% relative-ref = relative-part [ "?" query ] [ "#" fragment ]
+%%
+%% relative-part = "//" authority path-abempty
+%% / path-absolute
+%% / path-noscheme
+%% / path-empty
+%%-------------------------------------------------------------------------
+-spec parse_relative_part(binary(), uri_map()) -> uri_map().
+parse_relative_part(?STRING_REST("//", Rest), URI) ->
+ %% Parse userinfo - "//" is NOT part of authority
+ try parse_userinfo(Rest, URI) of
+ {T, URI1} ->
+ Userinfo = calculate_parsed_userinfo(Rest, T),
+ URI2 = maybe_add_path(URI1),
+ URI2#{userinfo => decode_userinfo(Userinfo)}
+ catch
+ throw:{_,_,_} ->
+ {T, URI1} = parse_host(Rest, URI),
+ Host = calculate_parsed_host_port(Rest, T),
+ URI2 = maybe_add_path(URI1),
+ URI2#{host => decode_host(remove_brackets(Host))}
+ end;
+parse_relative_part(?STRING_REST($/, Rest), URI) ->
+ {T, URI1} = parse_segment(Rest, URI), % path-absolute
+ Path = calculate_parsed_part(Rest, T),
+ URI1#{path => decode_path(?STRING_REST($/, Path))};
+parse_relative_part(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % path-empty ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ URI2 = maybe_add_path(URI1),
+ URI2#{query => decode_query(Query)};
+parse_relative_part(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI), % path-empty
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ URI2 = maybe_add_path(URI1),
+ URI2#{fragment => decode_fragment(Fragment)};
+parse_relative_part(?STRING_REST(Char, Rest), URI) ->
+ case is_segment_nz_nc(Char) of
+ true ->
+ {T, URI1} = parse_segment_nz_nc(Rest, URI), % path-noscheme
+ Path = calculate_parsed_part(Rest, T),
+ URI1#{path => decode_path(?STRING_REST(Char, Path))};
+ false -> throw({error,invalid_uri,[Char]})
+ end.
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 3.3. Path]
+%%
+%% The path component contains data, usually organized in hierarchical
+%% form, that, along with data in the non-hierarchical query component
+%% (Section 3.4), serves to identify a resource within the scope of the
+%% URI's scheme and naming authority (if any). The path is terminated
+%% by the first question mark ("?") or number sign ("#") character, or
+%% by the end of the URI.
+%%
+%% path = path-abempty ; begins with "/" or is empty
+%% / path-absolute ; begins with "/" but not "//"
+%% / path-noscheme ; begins with a non-colon segment
+%% / path-rootless ; begins with a segment
+%% / path-empty ; zero characters
+%%
+%% path-abempty = *( "/" segment )
+%% path-absolute = "/" [ segment-nz *( "/" segment ) ]
+%% path-noscheme = segment-nz-nc *( "/" segment )
+%% path-rootless = segment-nz *( "/" segment )
+%% path-empty = 0<pchar>
+%% segment = *pchar
+%% segment-nz = 1*pchar
+%% segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
+%% ; non-zero-length segment without any colon ":"
+%%
+%% pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
+%%-------------------------------------------------------------------------
+
+%%-------------------------------------------------------------------------
+%% path-abempty
+%%-------------------------------------------------------------------------
+-spec parse_segment(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_segment(?STRING_REST($/, Rest), URI) ->
+ parse_segment(Rest, URI); % segment
+parse_segment(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_segment(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI),
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_segment(?STRING_REST(Char, Rest), URI) ->
+ case is_pchar(Char) of
+ true -> parse_segment(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_segment(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+
+%%-------------------------------------------------------------------------
+%% path-noscheme
+%%-------------------------------------------------------------------------
+-spec parse_segment_nz_nc(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_segment_nz_nc(?STRING_REST($/, Rest), URI) ->
+ parse_segment(Rest, URI); % segment
+parse_segment_nz_nc(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_segment_nz_nc(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI),
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_segment_nz_nc(?STRING_REST(Char, Rest), URI) ->
+ case is_segment_nz_nc(Char) of
+ true -> parse_segment_nz_nc(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_segment_nz_nc(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+
+%% Check if char is pchar.
+-spec is_pchar(char()) -> boolean().
+is_pchar($%) -> true; % pct-encoded
+is_pchar($:) -> true;
+is_pchar($@) -> true;
+is_pchar(Char) -> is_unreserved(Char) orelse is_sub_delim(Char).
+
+%% Check if char is segment_nz_nc.
+-spec is_segment_nz_nc(char()) -> boolean().
+is_segment_nz_nc($%) -> true; % pct-encoded
+is_segment_nz_nc($@) -> true;
+is_segment_nz_nc(Char) -> is_unreserved(Char) orelse is_sub_delim(Char).
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 3.1. Scheme]
+%%
+%% Each URI begins with a scheme name that refers to a specification for
+%% assigning identifiers within that scheme.
+%%
+%% scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+%%-------------------------------------------------------------------------
+-spec parse_scheme_start(binary(), uri_map()) -> uri_map().
+parse_scheme_start(?STRING_REST(Char, Rest), URI) ->
+ case is_alpha(Char) of
+ true -> {T, URI1} = parse_scheme(Rest, URI),
+ Scheme = calculate_parsed_scheme(Rest, T),
+ URI2 = maybe_add_path(URI1),
+ URI2#{scheme => ?STRING_REST(Char, Scheme)};
+ false -> throw({error,invalid_uri,[Char]})
+ end.
+
+%% Add path component if it missing after parsing the URI.
+%% According to the URI specification there is always a
+%% path component in every URI-reference and it can be
+%% empty.
+maybe_add_path(Map) ->
+ case maps:is_key(path, Map) of
+ false ->
+ Map#{path => <<>>};
+ _Else ->
+ Map
+ end.
+
+
+-spec parse_scheme(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_scheme(?STRING_REST($:, Rest), URI) ->
+ {_, URI1} = parse_hier(Rest, URI),
+ {Rest, URI1};
+parse_scheme(?STRING_REST(Char, Rest), URI) ->
+ case is_scheme(Char) of
+ true -> parse_scheme(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_scheme(?STRING_EMPTY, _URI) ->
+ throw({error,invalid_uri,<<>>}).
+
+
+%% Check if char is allowed in scheme
+-spec is_scheme(char()) -> boolean().
+is_scheme($+) -> true;
+is_scheme($-) -> true;
+is_scheme($.) -> true;
+is_scheme(Char) -> is_alpha(Char) orelse is_digit(Char).
+
+
+%%-------------------------------------------------------------------------
+%% hier-part = "//" authority path-abempty
+%% / path-absolute
+%% / path-rootless
+%% / path-empty
+%%-------------------------------------------------------------------------
+-spec parse_hier(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_hier(?STRING_REST("//", Rest), URI) ->
+ % Parse userinfo - "//" is NOT part of authority
+ try parse_userinfo(Rest, URI) of
+ {T, URI1} ->
+ Userinfo = calculate_parsed_userinfo(Rest, T),
+ {Rest, URI1#{userinfo => decode_userinfo(Userinfo)}}
+ catch
+ throw:{_,_,_} ->
+ {T, URI1} = parse_host(Rest, URI),
+ Host = calculate_parsed_host_port(Rest, T),
+ {Rest, URI1#{host => decode_host(remove_brackets(Host))}}
+ end;
+parse_hier(?STRING_REST($/, Rest), URI) ->
+ {T, URI1} = parse_segment(Rest, URI), % path-absolute
+ Path = calculate_parsed_part(Rest, T),
+ {Rest, URI1#{path => decode_path(?STRING_REST($/, Path))}};
+parse_hier(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % path-empty ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_hier(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI), % path-empty
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_hier(?STRING_REST(Char, Rest), URI) -> % path-rootless
+ case is_pchar(Char) of
+ true -> % segment_nz
+ {T, URI1} = parse_segment(Rest, URI),
+ Path = calculate_parsed_part(Rest, T),
+ {Rest, URI1#{path => decode_path(?STRING_REST(Char, Path))}};
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_hier(?STRING_EMPTY, URI) ->
+ {<<>>, URI}.
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 3.2. Authority]
+%%
+%% Many URI schemes include a hierarchical element for a naming
+%% authority so that governance of the name space defined by the
+%% remainder of the URI is delegated to that authority (which may, in
+%% turn, delegate it further).
+%%
+%% The authority component is preceded by a double slash ("//") and is
+%% terminated by the next slash ("/"), question mark ("?"), or number
+%% sign ("#") character, or by the end of the URI.
+%%
+%% authority = [ userinfo "@" ] host [ ":" port ]
+%%
+%%
+%% [RFC 3986, Chapter 3.2.1. User Information]
+%%
+%% The userinfo subcomponent may consist of a user name and, optionally,
+%% scheme-specific information about how to gain authorization to access
+%% the resource. The user information, if present, is followed by a
+%% commercial at-sign ("@") that delimits it from the host.
+%%
+%% userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
+%%-------------------------------------------------------------------------
+-spec parse_userinfo(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_userinfo(?CHAR($@), URI) ->
+ {?STRING_EMPTY, URI#{host => <<>>}};
+parse_userinfo(?STRING_REST($@, Rest), URI) ->
+ {T, URI1} = parse_host(Rest, URI),
+ Host = calculate_parsed_host_port(Rest, T),
+ {Rest, URI1#{host => decode_host(remove_brackets(Host))}};
+parse_userinfo(?STRING_REST(Char, Rest), URI) ->
+ case is_userinfo(Char) of
+ true -> parse_userinfo(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_userinfo(?STRING_EMPTY, _URI) ->
+ %% URI cannot end in userinfo state
+ throw({error,invalid_uri,<<>>}).
+
+
+%% Check if char is allowed in userinfo
+-spec is_userinfo(char()) -> boolean().
+is_userinfo($%) -> true; % pct-encoded
+is_userinfo($:) -> true;
+is_userinfo(Char) -> is_unreserved(Char) orelse is_sub_delim(Char).
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 3.2.2. Host]
+%%
+%% The host subcomponent of authority is identified by an IP literal
+%% encapsulated within square brackets, an IPv4 address in dotted-
+%% decimal form, or a registered name.
+%%
+%% host = IP-literal / IPv4address / reg-name
+%%
+%% IP-literal = "[" ( IPv6address / IPvFuture ) "]"
+%%
+%% IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )
+%%
+%% IPv6address = 6( h16 ":" ) ls32
+%% / "::" 5( h16 ":" ) ls32
+%% / [ h16 ] "::" 4( h16 ":" ) ls32
+%% / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
+%% / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
+%% / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
+%% / [ *4( h16 ":" ) h16 ] "::" ls32
+%% / [ *5( h16 ":" ) h16 ] "::" h16
+%% / [ *6( h16 ":" ) h16 ] "::"
+%%
+%% ls32 = ( h16 ":" h16 ) / IPv4address
+%% ; least-significant 32 bits of address
+%%
+%% h16 = 1*4HEXDIG
+%% ; 16 bits of address represented in hexadecimal
+%%
+%% IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
+%%
+%% dec-octet = DIGIT ; 0-9
+%% / %x31-39 DIGIT ; 10-99
+%% / "1" 2DIGIT ; 100-199
+%% / "2" %x30-34 DIGIT ; 200-249
+%% / "25" %x30-35 ; 250-255
+%%
+%% reg-name = *( unreserved / pct-encoded / sub-delims )
+%%-------------------------------------------------------------------------
+-spec parse_host(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_host(?STRING_REST($:, Rest), URI) ->
+ {T, URI1} = parse_port(Rest, URI),
+ H = calculate_parsed_host_port(Rest, T),
+ Port = get_port(H),
+ {Rest, URI1#{port => Port}};
+parse_host(?STRING_REST($/, Rest), URI) ->
+ {T, URI1} = parse_segment(Rest, URI), % path-abempty
+ Path = calculate_parsed_part(Rest, T),
+ {Rest, URI1#{path => decode_path(?STRING_REST($/, Path))}};
+parse_host(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % path-empty ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_host(?STRING_REST($[, Rest), URI) ->
+ parse_ipv6_bin(Rest, [], URI);
+parse_host(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI), % path-empty
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_host(?STRING_REST(Char, Rest), URI) ->
+ case is_digit(Char) of
+ true -> parse_ipv4_bin(Rest, [Char], URI);
+ false -> parse_reg_name(?STRING_REST(Char, Rest), URI)
+ end;
+parse_host(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+
+-spec parse_reg_name(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_reg_name(?STRING_REST($:, Rest), URI) ->
+ {T, URI1} = parse_port(Rest, URI),
+ H = calculate_parsed_host_port(Rest, T),
+ Port = get_port(H),
+ {Rest, URI1#{port => Port}};
+parse_reg_name(?STRING_REST($/, Rest), URI) ->
+ {T, URI1} = parse_segment(Rest, URI), % path-abempty
+ Path = calculate_parsed_part(Rest, T),
+ {Rest, URI1#{path => decode_path(?STRING_REST($/, Path))}};
+parse_reg_name(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % path-empty ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_reg_name(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI), % path-empty
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_reg_name(?STRING_REST(Char, Rest), URI) ->
+ case is_reg_name(Char) of
+ true -> parse_reg_name(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_reg_name(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+%% Check if char is allowed in reg-name
+-spec is_reg_name(char()) -> boolean().
+is_reg_name($%) -> true;
+is_reg_name(Char) -> is_unreserved(Char) orelse is_sub_delim(Char).
+
+
+-spec parse_ipv4_bin(binary(), list(), uri_map()) -> {binary(), uri_map()}.
+parse_ipv4_bin(?STRING_REST($:, Rest), Acc, URI) ->
+ _ = validate_ipv4_address(lists:reverse(Acc)),
+ {T, URI1} = parse_port(Rest, URI),
+ H = calculate_parsed_host_port(Rest, T),
+ Port = get_port(H),
+ {Rest, URI1#{port => Port}};
+parse_ipv4_bin(?STRING_REST($/, Rest), Acc, URI) ->
+ _ = validate_ipv4_address(lists:reverse(Acc)),
+ {T, URI1} = parse_segment(Rest, URI), % path-abempty
+ Path = calculate_parsed_part(Rest, T),
+ {Rest, URI1#{path => decode_path(?STRING_REST($/, Path))}};
+parse_ipv4_bin(?STRING_REST($?, Rest), Acc, URI) ->
+ _ = validate_ipv4_address(lists:reverse(Acc)),
+ {T, URI1} = parse_query(Rest, URI), % path-empty ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_ipv4_bin(?STRING_REST($#, Rest), Acc, URI) ->
+ _ = validate_ipv4_address(lists:reverse(Acc)),
+ {T, URI1} = parse_fragment(Rest, URI), % path-empty
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_ipv4_bin(?STRING_REST(Char, Rest), Acc, URI) ->
+ case is_ipv4(Char) of
+ true -> parse_ipv4_bin(Rest, [Char|Acc], URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_ipv4_bin(?STRING_EMPTY, Acc, URI) ->
+ _ = validate_ipv4_address(lists:reverse(Acc)),
+ {?STRING_EMPTY, URI}.
+
+
+%% Check if char is allowed in IPv4 addresses
+-spec is_ipv4(char()) -> boolean().
+is_ipv4($.) -> true;
+is_ipv4(Char) -> is_digit(Char).
+
+-spec validate_ipv4_address(list()) -> list().
+validate_ipv4_address(Addr) ->
+ case inet:parse_ipv4strict_address(Addr) of
+ {ok, _} -> Addr;
+ {error, _} -> throw({error,invalid_uri,Addr})
+ end.
+
+
+-spec parse_ipv6_bin(binary(), list(), uri_map()) -> {binary(), uri_map()}.
+parse_ipv6_bin(?STRING_REST($], Rest), Acc, URI) ->
+ _ = validate_ipv6_address(lists:reverse(Acc)),
+ parse_ipv6_bin_end(Rest, URI);
+parse_ipv6_bin(?STRING_REST(Char, Rest), Acc, URI) ->
+ case is_ipv6(Char) of
+ true -> parse_ipv6_bin(Rest, [Char|Acc], URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_ipv6_bin(?STRING_EMPTY, _Acc, _URI) ->
+ throw({error,invalid_uri,<<>>}).
+
+%% Check if char is allowed in IPv6 addresses
+-spec is_ipv6(char()) -> boolean().
+is_ipv6($:) -> true;
+is_ipv6($.) -> true;
+is_ipv6(Char) -> is_hex_digit(Char).
+
+
+-spec parse_ipv6_bin_end(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_ipv6_bin_end(?STRING_REST($:, Rest), URI) ->
+ {T, URI1} = parse_port(Rest, URI),
+ H = calculate_parsed_host_port(Rest, T),
+ Port = get_port(H),
+ {Rest, URI1#{port => Port}};
+parse_ipv6_bin_end(?STRING_REST($/, Rest), URI) ->
+ {T, URI1} = parse_segment(Rest, URI), % path-abempty
+ Path = calculate_parsed_part(Rest, T),
+ {Rest, URI1#{path => decode_path(?STRING_REST($/, Path))}};
+parse_ipv6_bin_end(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % path-empty ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_ipv6_bin_end(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI), % path-empty
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_ipv6_bin_end(?STRING_REST(Char, Rest), URI) ->
+ case is_ipv6(Char) of
+ true -> parse_ipv6_bin_end(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_ipv6_bin_end(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+-spec validate_ipv6_address(list()) -> list().
+validate_ipv6_address(Addr) ->
+ case inet:parse_ipv6strict_address(Addr) of
+ {ok, _} -> Addr;
+ {error, _} -> throw({error,invalid_uri,Addr})
+ end.
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 3.2.2. Port]
+%%
+%% The port subcomponent of authority is designated by an optional port
+%% number in decimal following the host and delimited from it by a
+%% single colon (":") character.
+%%
+%% port = *DIGIT
+%%-------------------------------------------------------------------------
+-spec parse_port(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_port(?STRING_REST($/, Rest), URI) ->
+ {T, URI1} = parse_segment(Rest, URI), % path-abempty
+ Path = calculate_parsed_part(Rest, T),
+ {Rest, URI1#{path => decode_path(?STRING_REST($/, Path))}};
+parse_port(?STRING_REST($?, Rest), URI) ->
+ {T, URI1} = parse_query(Rest, URI), % path-empty ?query
+ Query = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{query => decode_query(Query)}};
+parse_port(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI), % path-empty
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_port(?STRING_REST(Char, Rest), URI) ->
+ case is_digit(Char) of
+ true -> parse_port(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_port(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 3.4. Query]
+%%
+%% The query component contains non-hierarchical data that, along with
+%% data in the path component (Section 3.3), serves to identify a
+%% resource within the scope of the URI's scheme and naming authority
+%% (if any). The query component is indicated by the first question
+%% mark ("?") character and terminated by a number sign ("#") character
+%% or by the end of the URI.
+%%
+%% query = *( pchar / "/" / "?" )
+%%-------------------------------------------------------------------------
+-spec parse_query(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_query(?STRING_REST($#, Rest), URI) ->
+ {T, URI1} = parse_fragment(Rest, URI),
+ Fragment = calculate_parsed_query_fragment(Rest, T),
+ {Rest, URI1#{fragment => decode_fragment(Fragment)}};
+parse_query(?STRING_REST(Char, Rest), URI) ->
+ case is_query(Char) of
+ true -> parse_query(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_query(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+
+%% Check if char is allowed in query
+-spec is_query(char()) -> boolean().
+is_query($/) -> true;
+is_query($?) -> true;
+is_query(Char) -> is_pchar(Char).
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 3.5. Fragment]
+%%
+%% The fragment identifier component of a URI allows indirect
+%% identification of a secondary resource by reference to a primary
+%% resource and additional identifying information.
+%%
+%% fragment = *( pchar / "/" / "?" )
+%%-------------------------------------------------------------------------
+-spec parse_fragment(binary(), uri_map()) -> {binary(), uri_map()}.
+parse_fragment(?STRING_REST(Char, Rest), URI) ->
+ case is_fragment(Char) of
+ true -> parse_fragment(Rest, URI);
+ false -> throw({error,invalid_uri,[Char]})
+ end;
+parse_fragment(?STRING_EMPTY, URI) ->
+ {?STRING_EMPTY, URI}.
+
+
+%% Check if char is allowed in fragment
+-spec is_fragment(char()) -> boolean().
+is_fragment($/) -> true;
+is_fragment($?) -> true;
+is_fragment(Char) -> is_pchar(Char).
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 2.2. Reserved Characters]
+%%
+%% reserved = gen-delims / sub-delims
+%%
+%% gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+%%
+%% sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
+%% / "*" / "+" / "," / ";" / "="
+%%
+%%-------------------------------------------------------------------------
+
+%% Check if char is sub-delim.
+-spec is_sub_delim(char()) -> boolean().
+is_sub_delim($!) -> true;
+is_sub_delim($$) -> true;
+is_sub_delim($&) -> true;
+is_sub_delim($') -> true;
+is_sub_delim($() -> true;
+is_sub_delim($)) -> true;
+
+is_sub_delim($*) -> true;
+is_sub_delim($+) -> true;
+is_sub_delim($,) -> true;
+is_sub_delim($;) -> true;
+is_sub_delim($=) -> true;
+is_sub_delim(_) -> false.
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 2.3. Unreserved Characters]
+%%
+%% unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
+%%
+%%-------------------------------------------------------------------------
+-spec is_unreserved(char()) -> boolean().
+is_unreserved($-) -> true;
+is_unreserved($.) -> true;
+is_unreserved($_) -> true;
+is_unreserved($~) -> true;
+is_unreserved(Char) -> is_alpha(Char) orelse is_digit(Char).
+
+-spec is_alpha(char()) -> boolean().
+is_alpha(C)
+ when $A =< C, C =< $Z;
+ $a =< C, C =< $z -> true;
+is_alpha(_) -> false.
+
+-spec is_digit(char()) -> boolean().
+is_digit(C)
+ when $0 =< C, C =< $9 -> true;
+is_digit(_) -> false.
+
+-spec is_hex_digit(char()) -> boolean().
+is_hex_digit(C)
+ when $0 =< C, C =< $9;$a =< C, C =< $f;$A =< C, C =< $F -> true;
+is_hex_digit(_) -> false.
+
+
+%% Remove enclosing brackets from binary
+-spec remove_brackets(binary()) -> binary().
+remove_brackets(<<$[/utf8, Rest/binary>>) ->
+ {H,T} = split_binary(Rest, byte_size(Rest) - 1),
+ case T =:= <<$]/utf8>> of
+ true -> H;
+ false -> Rest
+ end;
+remove_brackets(Addr) -> Addr.
+
+
+%%-------------------------------------------------------------------------
+%% Helper functions for calculating the parsed binary.
+%%-------------------------------------------------------------------------
+-spec calculate_parsed_scheme(binary(), binary()) -> binary().
+calculate_parsed_scheme(Input, <<>>) ->
+ strip_last_char(Input, [$:]);
+calculate_parsed_scheme(Input, Unparsed) ->
+ get_parsed_binary(Input, Unparsed).
+
+
+-spec calculate_parsed_part(binary(), binary()) -> binary().
+calculate_parsed_part(Input, <<>>) ->
+ strip_last_char(Input, [$?,$#]);
+calculate_parsed_part(Input, Unparsed) ->
+ get_parsed_binary(Input, Unparsed).
+
+
+-spec calculate_parsed_userinfo(binary(), binary()) -> binary().
+calculate_parsed_userinfo(Input, <<>>) ->
+ strip_last_char(Input, [$?,$#,$@]);
+calculate_parsed_userinfo(Input, Unparsed) ->
+ get_parsed_binary(Input, Unparsed).
+
+
+-spec calculate_parsed_host_port(binary(), binary()) -> binary().
+calculate_parsed_host_port(Input, <<>>) ->
+ strip_last_char(Input, [$:,$?,$#,$/]);
+calculate_parsed_host_port(Input, Unparsed) ->
+ get_parsed_binary(Input, Unparsed).
+
+
+calculate_parsed_query_fragment(Input, <<>>) ->
+ strip_last_char(Input, [$#]);
+calculate_parsed_query_fragment(Input, Unparsed) ->
+ get_parsed_binary(Input, Unparsed).
+
+
+get_port(<<>>) ->
+ undefined;
+get_port(B) ->
+ try binary_to_integer(B)
+ catch
+ error:badarg ->
+ throw({error, invalid_uri, B})
+ end.
+
+
+%% Strip last char if it is in list
+%%
+%% This function is optimized for speed: parse/1 is about 10% faster than
+%% with an alternative implementation based on lists and sets.
+strip_last_char(<<>>, _) -> <<>>;
+strip_last_char(Input, [C0]) ->
+ case binary:last(Input) of
+ C0 ->
+ init_binary(Input);
+ _Else ->
+ Input
+ end;
+strip_last_char(Input, [C0,C1]) ->
+ case binary:last(Input) of
+ C0 ->
+ init_binary(Input);
+ C1 ->
+ init_binary(Input);
+ _Else ->
+ Input
+ end;
+strip_last_char(Input, [C0,C1,C2]) ->
+ case binary:last(Input) of
+ C0 ->
+ init_binary(Input);
+ C1 ->
+ init_binary(Input);
+ C2 ->
+ init_binary(Input);
+ _Else ->
+ Input
+ end;
+strip_last_char(Input, [C0,C1,C2,C3]) ->
+ case binary:last(Input) of
+ C0 ->
+ init_binary(Input);
+ C1 ->
+ init_binary(Input);
+ C2 ->
+ init_binary(Input);
+ C3 ->
+ init_binary(Input);
+ _Else ->
+ Input
+ end.
+
+
+%% Get parsed binary
+get_parsed_binary(Input, Unparsed) ->
+ {First, _} = split_binary(Input, byte_size(Input) - byte_size_exl_head(Unparsed)),
+ First.
+
+
+%% Return all bytes of the binary except the last one. The binary must be non-empty.
+init_binary(B) ->
+ {Init, _} =
+ split_binary(B, byte_size(B) - 1),
+ Init.
+
+
+%% Returns the size of a binary exluding the first element.
+%% Used in calls to split_binary().
+-spec byte_size_exl_head(binary()) -> number().
+byte_size_exl_head(<<>>) -> 0;
+byte_size_exl_head(Binary) -> byte_size(Binary) + 1.
+
+
+%%-------------------------------------------------------------------------
+%% [RFC 3986, Chapter 2.1. Percent-Encoding]
+%%
+%% A percent-encoding mechanism is used to represent a data octet in a
+%% component when that octet's corresponding character is outside the
+%% allowed set or is being used as a delimiter of, or within, the
+%% component. A percent-encoded octet is encoded as a character
+%% triplet, consisting of the percent character "%" followed by the two
+%% hexadecimal digits representing that octet's numeric value. For
+%% example, "%20" is the percent-encoding for the binary octet
+%% "00100000" (ABNF: %x20), which in US-ASCII corresponds to the space
+%% character (SP). Section 2.4 describes when percent-encoding and
+%% decoding is applied.
+%%
+%% pct-encoded = "%" HEXDIG HEXDIG
+%%-------------------------------------------------------------------------
+-spec decode_userinfo(binary()) -> binary().
+decode_userinfo(Cs) ->
+ check_utf8(decode(Cs, fun is_userinfo/1, <<>>)).
+
+-spec decode_host(binary()) -> binary().
+decode_host(Cs) ->
+ check_utf8(decode(Cs, fun is_host/1, <<>>)).
+
+-spec decode_path(binary()) -> binary().
+decode_path(Cs) ->
+ check_utf8(decode(Cs, fun is_path/1, <<>>)).
+
+-spec decode_query(binary()) -> binary().
+decode_query(Cs) ->
+ check_utf8(decode(Cs, fun is_query/1, <<>>)).
+
+-spec decode_fragment(binary()) -> binary().
+decode_fragment(Cs) ->
+ check_utf8(decode(Cs, fun is_fragment/1, <<>>)).
+
+
+%% Returns Cs if it is utf8 encoded.
+check_utf8(Cs) ->
+ case unicode:characters_to_list(Cs) of
+ {incomplete,_,_} ->
+ throw({error,invalid_utf8,Cs});
+ {error,_,_} ->
+ throw({error,invalid_utf8,Cs});
+ _ -> Cs
+ end.
+
+%%-------------------------------------------------------------------------
+%% Percent-encode
+%%-------------------------------------------------------------------------
+
+%% Only validates as scheme cannot have percent-encoded characters
+-spec encode_scheme(list()|binary()) -> list() | binary().
+encode_scheme([]) ->
+ throw({error,invalid_scheme,""});
+encode_scheme(<<>>) ->
+ throw({error,invalid_scheme,<<>>});
+encode_scheme(Scheme) ->
+ case validate_scheme(Scheme) of
+ true -> Scheme;
+ false -> throw({error,invalid_scheme,Scheme})
+ end.
+
+-spec encode_userinfo(list()|binary()) -> list() | binary().
+encode_userinfo(Cs) ->
+ encode(Cs, fun is_userinfo/1).
+
+-spec encode_host(list()|binary()) -> list() | binary().
+encode_host(Cs) ->
+ case classify_host(Cs) of
+ regname -> Cs;
+ ipv4 -> Cs;
+ ipv6 -> bracket_ipv6(Cs);
+ other -> encode(Cs, fun is_reg_name/1)
+ end.
+
+-spec encode_path(list()|binary()) -> list() | binary().
+encode_path(Cs) ->
+ encode(Cs, fun is_path/1).
+
+-spec encode_query(list()|binary()) -> list() | binary().
+encode_query(Cs) ->
+ encode(Cs, fun is_query/1).
+
+-spec encode_fragment(list()|binary()) -> list() | binary().
+encode_fragment(Cs) ->
+ encode(Cs, fun is_fragment/1).
+
+%%-------------------------------------------------------------------------
+%% Helper funtions for percent-decode
+%%-------------------------------------------------------------------------
+decode(<<$%,C0,C1,Cs/binary>>, Fun, Acc) ->
+ case is_hex_digit(C0) andalso is_hex_digit(C1) of
+ true ->
+ B = ?HEX2DEC(C0)*16+?HEX2DEC(C1),
+ decode(Cs, Fun, <<Acc/binary, B>>);
+ false -> throw({error,invalid_percent_encoding,<<$%,C0,C1>>})
+ end;
+decode(<<C,Cs/binary>>, Fun, Acc) ->
+ case Fun(C) of
+ true -> decode(Cs, Fun, <<Acc/binary, C>>);
+ false -> throw({error,invalid_percent_encoding,<<C,Cs/binary>>})
+ end;
+decode(<<>>, _Fun, Acc) ->
+ Acc.
+
+%% Check if char is allowed in host
+-spec is_host(char()) -> boolean().
+is_host($:) -> true;
+is_host(Char) -> is_unreserved(Char) orelse is_sub_delim(Char).
+
+%% Check if char is allowed in path
+-spec is_path(char()) -> boolean().
+is_path($/) -> true;
+is_path(Char) -> is_pchar(Char).
+
+
+%%-------------------------------------------------------------------------
+%% Helper functions for percent-encode
+%%-------------------------------------------------------------------------
+-spec encode(list()|binary(), fun()) -> list() | binary().
+encode(Component, Fun) when is_list(Component) ->
+ B = unicode:characters_to_binary(Component),
+ unicode:characters_to_list(encode(B, Fun, <<>>));
+encode(Component, Fun) when is_binary(Component) ->
+ encode(Component, Fun, <<>>).
+%%
+encode(<<Char/utf8, Rest/binary>>, Fun, Acc) ->
+ C = encode_codepoint_binary(Char, Fun),
+ encode(Rest, Fun, <<Acc/binary,C/binary>>);
+encode(<<Char, Rest/binary>>, _Fun, _Acc) ->
+ throw({error,invalid_input,<<Char,Rest/binary>>});
+encode(<<>>, _Fun, Acc) ->
+ Acc.
+
+
+-spec encode_codepoint_binary(integer(), fun()) -> binary().
+encode_codepoint_binary(C, Fun) ->
+ case Fun(C) of
+ false -> percent_encode_binary(C);
+ true -> <<C>>
+ end.
+
+
+-spec percent_encode_binary(integer()) -> binary().
+percent_encode_binary(Code) ->
+ percent_encode_binary(<<Code/utf8>>, <<>>).
+
+
+percent_encode_binary(<<A:4,B:4,Rest/binary>>, Acc) ->
+ percent_encode_binary(Rest, <<Acc/binary,$%,(?DEC2HEX(A)),(?DEC2HEX(B))>>);
+percent_encode_binary(<<>>, Acc) ->
+ Acc.
+
+
+%%-------------------------------------------------------------------------
+%%-------------------------------------------------------------------------
+validate_scheme([]) -> true;
+validate_scheme([H|T]) ->
+ case is_scheme(H) of
+ true -> validate_scheme(T);
+ false -> false
+ end;
+validate_scheme(<<>>) -> true;
+validate_scheme(<<H, Rest/binary>>) ->
+ case is_scheme(H) of
+ true -> validate_scheme(Rest);
+ false -> false
+ end.
+
+
+%%-------------------------------------------------------------------------
+%% Classifies hostname into the following categories:
+%% regname, ipv4 - address does not contain reserved characters to be
+%% percent-encoded
+%% ipv6 - address does not contain reserved characters but it shall be
+%% encolsed in brackets
+%% other - address shall be percent-encoded
+%%-------------------------------------------------------------------------
+classify_host([]) -> other;
+classify_host(Addr) when is_binary(Addr) ->
+ A = unicode:characters_to_list(Addr),
+ classify_host_ipv6(A);
+classify_host(Addr) ->
+ classify_host_ipv6(Addr).
+
+classify_host_ipv6(Addr) ->
+ case is_ipv6_address(Addr) of
+ true -> ipv6;
+ false -> classify_host_ipv4(Addr)
+ end.
+
+classify_host_ipv4(Addr) ->
+ case is_ipv4_address(Addr) of
+ true -> ipv4;
+ false -> classify_host_regname(Addr)
+ end.
+
+classify_host_regname([]) -> regname;
+classify_host_regname([H|T]) ->
+ case is_reg_name(H) of
+ true -> classify_host_regname(T);
+ false -> other
+ end.
+
+is_ipv4_address(Addr) ->
+ case inet:parse_ipv4strict_address(Addr) of
+ {ok, _} -> true;
+ {error, _} -> false
+ end.
+
+is_ipv6_address(Addr) ->
+ case inet:parse_ipv6strict_address(Addr) of
+ {ok, _} -> true;
+ {error, _} -> false
+ end.
+
+bracket_ipv6(Addr) when is_binary(Addr) ->
+ concat(<<$[,Addr/binary>>,<<$]>>);
+bracket_ipv6(Addr) when is_list(Addr) ->
+ [$[|Addr] ++ "]".
+
+
+%%-------------------------------------------------------------------------
+%% Helper funtions for recompose
+%%-------------------------------------------------------------------------
+
+%%-------------------------------------------------------------------------
+%% Checks if input Map has valid combination of fields that can be
+%% recomposed into a URI.
+%%
+%% The implementation is based on a decision tree that fulfills the
+%% following rules:
+%% - 'path' shall always be present in the input map
+%% URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
+%% hier-part = "//" authority path-abempty
+%% / path-absolute
+%% / path-rootless
+%% / path-empty
+%% - 'host' shall be present in the input map when 'path' starts with
+%% two slashes ("//")
+%% path = path-abempty ; begins with "/" or is empty
+%% / path-absolute ; begins with "/" but not "//"
+%% / path-noscheme ; begins with a non-colon segment
+%% / path-rootless ; begins with a segment
+%% / path-empty ; zero characters
+%% path-abempty = *( "/" segment )
+%% segment = *pchar
+%% - 'host' shall be present if userinfo or port is present in input map
+%% authority = [ userinfo "@" ] host [ ":" port ]
+%% - All fields shall be valid (scheme, userinfo, host, port, path, query
+%% or fragment).
+%%-------------------------------------------------------------------------
+is_valid_map(#{path := Path} = Map) ->
+ ((starts_with_two_slash(Path) andalso is_valid_map_host(Map))
+ orelse
+ (maps:is_key(userinfo, Map) andalso is_valid_map_host(Map))
+ orelse
+ (maps:is_key(port, Map) andalso is_valid_map_host(Map))
+ orelse
+ all_fields_valid(Map));
+is_valid_map(#{}) ->
+ false.
+
+
+is_valid_map_host(Map) ->
+ maps:is_key(host, Map) andalso all_fields_valid(Map).
+
+
+all_fields_valid(Map) ->
+ Fun = fun(scheme, _, Acc) -> Acc;
+ (userinfo, _, Acc) -> Acc;
+ (host, _, Acc) -> Acc;
+ (port, _, Acc) -> Acc;
+ (path, _, Acc) -> Acc;
+ (query, _, Acc) -> Acc;
+ (fragment, _, Acc) -> Acc;
+ (_, _, _) -> false
+ end,
+ maps:fold(Fun, true, Map).
+
+
+starts_with_two_slash([$/,$/|_]) ->
+ true;
+starts_with_two_slash(?STRING_REST("//", _)) ->
+ true;
+starts_with_two_slash(_) -> false.
+
+
+update_scheme(#{scheme := Scheme}, _) ->
+ add_colon_postfix(encode_scheme(Scheme));
+update_scheme(#{}, _) ->
+ empty.
+
+
+update_userinfo(#{userinfo := Userinfo}, empty) ->
+ add_auth_prefix(encode_userinfo(Userinfo));
+update_userinfo(#{userinfo := Userinfo}, URI) ->
+ concat(URI,add_auth_prefix(encode_userinfo(Userinfo)));
+update_userinfo(#{}, empty) ->
+ empty;
+update_userinfo(#{}, URI) ->
+ URI.
+
+
+update_host(#{host := Host}, empty) ->
+ add_auth_prefix(encode_host(Host));
+update_host(#{host := Host} = Map, URI) ->
+ concat(URI,add_host_prefix(Map, encode_host(Host)));
+update_host(#{}, empty) ->
+ empty;
+update_host(#{}, URI) ->
+ URI.
+
+
+%% URI cannot be empty for ports. E.g. ":8080" is not a valid URI
+update_port(#{port := undefined}, URI) ->
+ concat(URI, <<":">>);
+update_port(#{port := Port}, URI) ->
+ concat(URI,add_colon(encode_port(Port)));
+update_port(#{}, URI) ->
+ URI.
+
+
+update_path(#{path := Path}, empty) ->
+ encode_path(Path);
+update_path(#{path := Path}, URI) ->
+ concat(URI,encode_path(Path));
+update_path(#{}, empty) ->
+ empty;
+update_path(#{}, URI) ->
+ URI.
+
+
+update_query(#{query := Query}, empty) ->
+ encode_query(Query);
+update_query(#{query := Query}, URI) ->
+ concat(URI,add_question_mark(encode_query(Query)));
+update_query(#{}, empty) ->
+ empty;
+update_query(#{}, URI) ->
+ URI.
+
+
+update_fragment(#{fragment := Fragment}, empty) ->
+ add_hashmark(encode_fragment(Fragment));
+update_fragment(#{fragment := Fragment}, URI) ->
+ concat(URI,add_hashmark(encode_fragment(Fragment)));
+update_fragment(#{}, empty) ->
+ "";
+update_fragment(#{}, URI) ->
+ URI.
+
+%%-------------------------------------------------------------------------
+%% Concatenates its arguments that can be lists and binaries.
+%% The result is a list if at least one of its argument is a list and
+%% binary otherwise.
+%%-------------------------------------------------------------------------
+concat(A, B) when is_binary(A), is_binary(B) ->
+ <<A/binary, B/binary>>;
+concat(A, B) when is_binary(A), is_list(B) ->
+ unicode:characters_to_list(A) ++ B;
+concat(A, B) when is_list(A) ->
+ A ++ maybe_to_list(B).
+
+add_hashmark(Comp) when is_binary(Comp) ->
+ <<$#, Comp/binary>>;
+add_hashmark(Comp) when is_list(Comp) ->
+ [$#|Comp].
+
+add_question_mark(Comp) when is_binary(Comp) ->
+ <<$?, Comp/binary>>;
+add_question_mark(Comp) when is_list(Comp) ->
+ [$?|Comp].
+
+add_colon(Comp) when is_binary(Comp) ->
+ <<$:, Comp/binary>>.
+
+add_colon_postfix(Comp) when is_binary(Comp) ->
+ <<Comp/binary,$:>>;
+add_colon_postfix(Comp) when is_list(Comp) ->
+ Comp ++ ":".
+
+add_auth_prefix(Comp) when is_binary(Comp) ->
+ <<"//", Comp/binary>>;
+add_auth_prefix(Comp) when is_list(Comp) ->
+ [$/,$/|Comp].
+
+add_host_prefix(#{userinfo := _}, Host) when is_binary(Host) ->
+ <<$@,Host/binary>>;
+add_host_prefix(#{}, Host) when is_binary(Host) ->
+ <<"//",Host/binary>>;
+add_host_prefix(#{userinfo := _}, Host) when is_list(Host) ->
+ [$@|Host];
+add_host_prefix(#{}, Host) when is_list(Host) ->
+ [$/,$/|Host].
+
+maybe_to_list(Comp) when is_binary(Comp) -> unicode:characters_to_list(Comp);
+maybe_to_list(Comp) -> Comp.
+
+encode_port(Port) ->
+ integer_to_binary(Port).
+
+%%-------------------------------------------------------------------------
+%% Helper functions for transcode
+%%-------------------------------------------------------------------------
+
+%%-------------------------------------------------------------------------
+%% uri_string:transcode(<<"x%00%00%00%F6"/utf32>>).
+%% 1. Convert (transcode/2) input to list form (list of unicode codepoints)
+%% "x%00%00%00%F6"
+%% 2. Accumulate characters until percent-encoded segment (transcode/4).
+%% Acc = "x"
+%% 3. Convert percent-encoded triplets to binary form (transcode_pct/4)
+%% <<0,0,0,246>>
+%% 4. Transcode in-encoded binary to out-encoding (utf32 -> utf8):
+%% <<195,182>>
+%% 5. Percent-encode out-encoded binary:
+%% <<"%C3%B6"/utf8>> = <<37,67,51,37,66,54>>
+%% 6. Convert binary to list form, reverse it and append the accumulator
+%% "6B%3C%" + "x"
+%% 7. Reverse Acc and return it
+%%-------------------------------------------------------------------------
+transcode([$%,_C0,_C1|_Rest] = L, Acc, InEnc, OutEnc) ->
+ transcode_pct(L, Acc, <<>>, InEnc, OutEnc);
+transcode([_C|_Rest] = L, Acc, InEnc, OutEnc) ->
+ transcode(L, Acc, [], InEnc, OutEnc).
+%%
+transcode([$%,_C0,_C1|_Rest] = L, Acc, List, InEncoding, OutEncoding) ->
+ transcode_pct(L, List ++ Acc, <<>>, InEncoding, OutEncoding);
+transcode([C|Rest], Acc, List, InEncoding, OutEncoding) ->
+ transcode(Rest, Acc, [C|List], InEncoding, OutEncoding);
+transcode([], Acc, List, _InEncoding, _OutEncoding) ->
+ lists:reverse(List ++ Acc).
+
+
+%% Transcode percent-encoded segment
+transcode_pct([$%,C0,C1|Rest] = L, Acc, B, InEncoding, OutEncoding) ->
+ case is_hex_digit(C0) andalso is_hex_digit(C1) of
+ true ->
+ Int = ?HEX2DEC(C0)*16+?HEX2DEC(C1),
+ transcode_pct(Rest, Acc, <<B/binary, Int>>, InEncoding, OutEncoding);
+ false -> throw({error, invalid_percent_encoding,L})
+ end;
+transcode_pct([_C|_Rest] = L, Acc, B, InEncoding, OutEncoding) ->
+ OutBinary = convert_to_binary(B, InEncoding, OutEncoding),
+ PctEncUtf8 = percent_encode_segment(OutBinary),
+ Out = lists:reverse(convert_to_list(PctEncUtf8, utf8)),
+ transcode(L, Out ++ Acc, [], InEncoding, OutEncoding);
+transcode_pct([], Acc, B, InEncoding, OutEncoding) ->
+ OutBinary = convert_to_binary(B, InEncoding, OutEncoding),
+ PctEncUtf8 = percent_encode_segment(OutBinary),
+ Out = convert_to_list(PctEncUtf8, utf8),
+ lists:reverse(Acc) ++ Out.
+
+
+%% Convert to binary
+convert_to_binary(Binary, InEncoding, OutEncoding) ->
+ case unicode:characters_to_binary(Binary, InEncoding, OutEncoding) of
+ {error, _List, RestData} ->
+ throw({error, invalid_input, RestData});
+ {incomplete, _List, RestData} ->
+ throw({error, invalid_input, RestData});
+ Result ->
+ Result
+ end.
+
+
+%% Convert to list
+convert_to_list(Binary, InEncoding) ->
+ case unicode:characters_to_list(Binary, InEncoding) of
+ {error, _List, RestData} ->
+ throw({error, invalid_input, RestData});
+ {incomplete, _List, RestData} ->
+ throw({error, invalid_input, RestData});
+ Result ->
+ Result
+ end.
+
+
+%% Flatten input list
+flatten_list([], _) ->
+ [];
+flatten_list(L, InEnc) ->
+ flatten_list(L, InEnc, []).
+%%
+flatten_list([H|T], InEnc, Acc) when is_binary(H) ->
+ L = convert_to_list(H, InEnc),
+ flatten_list(T, InEnc, lists:reverse(L) ++ Acc);
+flatten_list([H|T], InEnc, Acc) when is_list(H) ->
+ flatten_list(H ++ T, InEnc, Acc);
+flatten_list([H|T], InEnc, Acc) ->
+ flatten_list(T, InEnc, [H|Acc]);
+flatten_list([], _InEnc, Acc) ->
+ lists:reverse(Acc);
+flatten_list(Arg, _, _) ->
+ throw({error, invalid_input, Arg}).
+
+
+percent_encode_segment(Segment) ->
+ percent_encode_binary(Segment, <<>>).
+
+
+%%-------------------------------------------------------------------------
+%% Helper functions for normalize
+%%-------------------------------------------------------------------------
+
+%% 6.2.2.1. Case Normalization
+normalize_case(#{scheme := Scheme, host := Host} = Map) ->
+ Map#{scheme => to_lower(Scheme),
+ host => to_lower(Host)};
+normalize_case(#{host := Host} = Map) ->
+ Map#{host => to_lower(Host)};
+normalize_case(#{scheme := Scheme} = Map) ->
+ Map#{scheme => to_lower(Scheme)};
+normalize_case(#{} = Map) ->
+ Map.
+
+
+to_lower(Cs) when is_list(Cs) ->
+ B = convert_to_binary(Cs, utf8, utf8),
+ convert_to_list(to_lower(B), utf8);
+to_lower(Cs) when is_binary(Cs) ->
+ to_lower(Cs, <<>>).
+%%
+to_lower(<<C,Cs/binary>>, Acc) when $A =< C, C =< $Z ->
+ to_lower(Cs, <<Acc/binary,(C + 32)>>);
+to_lower(<<C,Cs/binary>>, Acc) ->
+ to_lower(Cs, <<Acc/binary,C>>);
+to_lower(<<>>, Acc) ->
+ Acc.
+
+
+%% 6.2.2.3. Path Segment Normalization
+%% 5.2.4. Remove Dot Segments
+normalize_path_segment(Map) ->
+ Path = maps:get(path, Map, undefined),
+ Map#{path => remove_dot_segments(Path)}.
+
+
+remove_dot_segments(Path) when is_binary(Path) ->
+ remove_dot_segments(Path, <<>>);
+remove_dot_segments(Path) when is_list(Path) ->
+ B = convert_to_binary(Path, utf8, utf8),
+ B1 = remove_dot_segments(B, <<>>),
+ convert_to_list(B1, utf8).
+%%
+remove_dot_segments(<<>>, Output) ->
+ Output;
+remove_dot_segments(<<"../",T/binary>>, Output) ->
+ remove_dot_segments(T, Output);
+remove_dot_segments(<<"./",T/binary>>, Output) ->
+ remove_dot_segments(T, Output);
+remove_dot_segments(<<"/./",T/binary>>, Output) ->
+ remove_dot_segments(<<$/,T/binary>>, Output);
+remove_dot_segments(<<"/.">>, Output) ->
+ remove_dot_segments(<<$/>>, Output);
+remove_dot_segments(<<"/../",T/binary>>, Output) ->
+ Out1 = remove_last_segment(Output),
+ remove_dot_segments(<<$/,T/binary>>, Out1);
+remove_dot_segments(<<"/..">>, Output) ->
+ Out1 = remove_last_segment(Output),
+ remove_dot_segments(<<$/>>, Out1);
+remove_dot_segments(<<$.>>, Output) ->
+ remove_dot_segments(<<>>, Output);
+remove_dot_segments(<<"..">>, Output) ->
+ remove_dot_segments(<<>>, Output);
+remove_dot_segments(Input, Output) ->
+ {First, Rest} = first_path_segment(Input),
+ remove_dot_segments(Rest, <<Output/binary,First/binary>>).
+
+
+first_path_segment(Input) ->
+ F = first_path_segment(Input, <<>>),
+ split_binary(Input, byte_size(F)).
+%%
+first_path_segment(<<$/,T/binary>>, Acc) ->
+ first_path_segment_end(<<T/binary>>, <<Acc/binary,$/>>);
+first_path_segment(<<C,T/binary>>, Acc) ->
+ first_path_segment_end(<<T/binary>>, <<Acc/binary,C>>).
+
+
+first_path_segment_end(<<>>, Acc) ->
+ Acc;
+first_path_segment_end(<<$/,_/binary>>, Acc) ->
+ Acc;
+first_path_segment_end(<<C,T/binary>>, Acc) ->
+ first_path_segment_end(<<T/binary>>, <<Acc/binary,C>>).
+
+
+remove_last_segment(<<>>) ->
+ <<>>;
+remove_last_segment(B) ->
+ {Init, Last} = split_binary(B, byte_size(B) - 1),
+ case Last of
+ <<$/>> ->
+ Init;
+ _Char ->
+ remove_last_segment(Init)
+ end.
+
+
+%% RFC 3986, 6.2.3. Scheme-Based Normalization
+normalize_scheme_based(Map) ->
+ Scheme = maps:get(scheme, Map, undefined),
+ Port = maps:get(port, Map, undefined),
+ Path= maps:get(path, Map, undefined),
+ normalize_scheme_based(Map, Scheme, Port, Path).
+%%
+normalize_scheme_based(Map, Scheme, Port, Path)
+ when Scheme =:= "http"; Scheme =:= <<"http">> ->
+ normalize_http(Map, Port, Path);
+normalize_scheme_based(Map, Scheme, Port, Path)
+ when Scheme =:= "https"; Scheme =:= <<"https">> ->
+ normalize_https(Map, Port, Path);
+normalize_scheme_based(Map, Scheme, Port, _Path)
+ when Scheme =:= "ftp"; Scheme =:= <<"ftp">> ->
+ normalize_ftp(Map, Port);
+normalize_scheme_based(Map, Scheme, Port, _Path)
+ when Scheme =:= "ssh"; Scheme =:= <<"ssh">> ->
+ normalize_ssh_sftp(Map, Port);
+normalize_scheme_based(Map, Scheme, Port, _Path)
+ when Scheme =:= "sftp"; Scheme =:= <<"sftp">> ->
+ normalize_ssh_sftp(Map, Port);
+normalize_scheme_based(Map, Scheme, Port, _Path)
+ when Scheme =:= "tftp"; Scheme =:= <<"tftp">> ->
+ normalize_tftp(Map, Port);
+normalize_scheme_based(Map, _, _, _) ->
+ Map.
+
+
+normalize_http(Map, Port, Path) ->
+ M1 = normalize_port(Map, Port, 80),
+ normalize_http_path(M1, Path).
+
+
+normalize_https(Map, Port, Path) ->
+ M1 = normalize_port(Map, Port, 443),
+ normalize_http_path(M1, Path).
+
+
+normalize_ftp(Map, Port) ->
+ normalize_port(Map, Port, 21).
+
+
+normalize_ssh_sftp(Map, Port) ->
+ normalize_port(Map, Port, 22).
+
+
+normalize_tftp(Map, Port) ->
+ normalize_port(Map, Port, 69).
+
+
+normalize_port(Map, Port, Default) ->
+ case Port of
+ Default ->
+ maps:remove(port, Map);
+ _Else ->
+ Map
+ end.
+
+
+normalize_http_path(Map, Path) ->
+ case Path of
+ "" ->
+ Map#{path => "/"};
+ <<>> ->
+ Map#{path => <<"/">>};
+ _Else ->
+ Map
+ end.
diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile
index 7b79dcf04d..8490770f3d 100644
--- a/lib/stdlib/test/Makefile
+++ b/lib/stdlib/test/Makefile
@@ -69,6 +69,7 @@ MODULES= \
sets_test_lib \
sofs_SUITE \
stdlib_SUITE \
+ stdlib_bench_SUITE \
string_SUITE \
supervisor_1 \
supervisor_2 \
@@ -86,6 +87,7 @@ MODULES= \
timer_simple_SUITE \
unicode_SUITE \
unicode_util_SUITE \
+ uri_string_SUITE \
win32reg_SUITE \
y2k_SUITE \
select_SUITE \
@@ -146,7 +148,7 @@ release_spec: opt
release_tests_spec: make_emakefile
$(INSTALL_DIR) "$(RELSYSDIR)"
- $(INSTALL_DATA) stdlib.spec $(EMAKEFILE) \
+ $(INSTALL_DATA) stdlib.spec stdlib_bench.spec $(EMAKEFILE) \
$(ERL_FILES) $(COVERFILE) "$(RELSYSDIR)"
chmod -R u+w "$(RELSYSDIR)"
@tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
diff --git a/lib/stdlib/test/erl_internal_SUITE.erl b/lib/stdlib/test/erl_internal_SUITE.erl
index 789a9d4363..7d9df1f989 100644
--- a/lib/stdlib/test/erl_internal_SUITE.erl
+++ b/lib/stdlib/test/erl_internal_SUITE.erl
@@ -80,7 +80,7 @@ callbacks(application) ->
callbacks(gen_server) ->
[{init,1}, {handle_call,3}, {handle_cast,2},
{handle_info,2}, {terminate,2}, {code_change,3},
- {format_status,2}];
+ {format_status,2}, {handle_continue, 2}];
callbacks(gen_fsm) ->
[{init,1}, {handle_event,3}, {handle_sync_event,4},
{handle_info,3}, {terminate,3}, {code_change,4},
@@ -101,7 +101,7 @@ callbacks(supervisor) ->
optional_callbacks(application) ->
[];
optional_callbacks(gen_server) ->
- [{handle_info, 2}, {terminate, 2}, {code_change, 3}, {format_status, 2}];
+ [{handle_info, 2}, {handle_continue, 2}, {terminate, 2}, {code_change, 3}, {format_status, 2}];
optional_callbacks(gen_fsm) ->
[{handle_info, 3}, {terminate, 3}, {code_change, 4}, {format_status, 2}];
optional_callbacks(gen_event) ->
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 05451a83fb..f329a07b7a 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -1682,7 +1682,7 @@ do_random_test() ->
ets:delete(Set),
verify_etsmem(EtsMem).
-%% Ttest various variants of update_element.
+%% Test various variants of update_element.
update_element(Config) when is_list(Config) ->
EtsMem = etsmem(),
repeat_for_opts(fun update_element_opts/1),
@@ -2283,13 +2283,8 @@ write_concurrency(Config) when is_list(Config) ->
NoHashMem = ets:info(No7,memory),
NoHashMem = ets:info(No8,memory),
- case erlang:system_info(smp_support) of
- true ->
- true = YesMem > NoHashMem,
- true = YesMem > NoTreeMem;
- false ->
- true = YesMem =:= NoHashMem
- end,
+ true = YesMem > NoHashMem,
+ true = YesMem > NoTreeMem,
{'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency,foo}])),
{'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency}])),
@@ -5912,16 +5907,11 @@ add_lists([E1|T1], [E2|T2], Acc) ->
run_smp_workers(InitF,ExecF,FiniF,Laps) ->
run_smp_workers(InitF,ExecF,FiniF,Laps, 0).
run_smp_workers(InitF,ExecF,FiniF,Laps, Exclude) ->
- case erlang:system_info(smp_support) of
- true ->
- case erlang:system_info(schedulers_online) of
- N when N > Exclude ->
- run_workers_do(InitF,ExecF,FiniF,Laps, N - Exclude);
- _ ->
- {skipped, "Too few schedulers online"}
- end;
- false ->
- {skipped,"No smp support"}
+ case erlang:system_info(schedulers_online) of
+ N when N > Exclude ->
+ run_workers_do(InitF,ExecF,FiniF,Laps, N - Exclude);
+ _ ->
+ {skipped, "Too few schedulers online"}
end.
run_sched_workers(InitF,ExecF,FiniF,Laps) ->
@@ -6231,11 +6221,9 @@ spawn_monitor_with_pid(Pid, Fun, N) ->
only_if_smp(Func) ->
only_if_smp(2, Func).
only_if_smp(Schedulers, Func) ->
- case {erlang:system_info(smp_support),
- erlang:system_info(schedulers_online)} of
- {false,_} -> {skip,"No smp support"};
- {true,N} when N < Schedulers -> {skip,"Too few schedulers online"};
- {true,_} -> Func()
+ case erlang:system_info(schedulers_online) of
+ N when N < Schedulers -> {skip,"Too few schedulers online"};
+ _ -> Func()
end.
%% Copy-paste from emulator/test/binary_SUITE.erl
diff --git a/lib/stdlib/test/filelib_SUITE.erl b/lib/stdlib/test/filelib_SUITE.erl
index c94821bc75..1236fe45f4 100644
--- a/lib/stdlib/test/filelib_SUITE.erl
+++ b/lib/stdlib/test/filelib_SUITE.erl
@@ -120,7 +120,7 @@ wcc(Wc, Error) ->
do_wildcard_1(Dir, Wcf0) ->
do_wildcard_2(Dir, Wcf0),
Wcf = fun(Wc0) ->
- Wc = filename:join(Dir, Wc0),
+ Wc = Dir ++ "/" ++ Wc0,
L = Wcf0(Wc),
[subtract_dir(N, Dir) || N <- L]
end,
@@ -268,8 +268,37 @@ do_wildcard_9(Dir, Wcf) ->
%% Cleanup.
del(Files),
[ok = file:del_dir(D) || D <- lists:reverse(Dirs)],
- ok.
+ do_wildcard_10(Dir, Wcf).
+
+%% ERL-451/OTP-14577: Escape characters using \\.
+do_wildcard_10(Dir, Wcf) ->
+ All0 = ["{abc}","abc","def","---","z--","@a,b","@c"],
+ All = case os:type() of
+ {unix,_} ->
+ %% '?' is allowed in file names on Unix, but
+ %% not on Windows.
+ ["?q"|All0];
+ _ ->
+ All0
+ end,
+ Files = mkfiles(lists:reverse(All), Dir),
+
+ ["{abc}"] = Wcf("\\{a*"),
+ ["{abc}"] = Wcf("\\{abc}"),
+ ["abc","def","z--"] = Wcf("[a-z]*"),
+ ["---","abc","z--"] = Wcf("[a\\-z]*"),
+ ["@a,b","@c"] = Wcf("@{a\\,b,c}"),
+ ["@c"] = Wcf("@{a,b,c}"),
+
+ case os:type() of
+ {unix,_} ->
+ ["?q"] = Wcf("\\?q");
+ _ ->
+ [] = Wcf("\\?q")
+ end,
+ del(Files),
+ ok.
fold_files(Config) when is_list(Config) ->
Dir = filename:join(proplists:get_value(priv_dir, Config), "fold_files"),
diff --git a/lib/stdlib/test/gen_server_SUITE.erl b/lib/stdlib/test/gen_server_SUITE.erl
index 2e9dc4d4fb..2bc220fef2 100644
--- a/lib/stdlib/test/gen_server_SUITE.erl
+++ b/lib/stdlib/test/gen_server_SUITE.erl
@@ -27,7 +27,7 @@
-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
init_per_group/2,end_per_group/2]).
-export([start/1, crash/1, call/1, cast/1, cast_fast/1,
- info/1, abcast/1, multicall/1, multicall_down/1,
+ continue/1, info/1, abcast/1, multicall/1, multicall_down/1,
call_remote1/1, call_remote2/1, call_remote3/1,
call_remote_n1/1, call_remote_n2/1, call_remote_n3/1, spec_init/1,
spec_init_local_registered_parent/1,
@@ -37,7 +37,8 @@
get_state/1, replace_state/1, call_with_huge_message_queue/1,
undef_handle_call/1, undef_handle_cast/1, undef_handle_info/1,
undef_init/1, undef_code_change/1, undef_terminate1/1,
- undef_terminate2/1, undef_in_terminate/1, undef_in_handle_info/1
+ undef_terminate2/1, undef_in_terminate/1, undef_in_handle_info/1,
+ undef_handle_continue/1
]).
-export([stop1/1, stop2/1, stop3/1, stop4/1, stop5/1, stop6/1, stop7/1,
@@ -52,7 +53,7 @@
%% The gen_server behaviour
--export([init/1, handle_call/3, handle_cast/2,
+-export([init/1, handle_call/3, handle_cast/2, handle_continue/2,
handle_info/2, code_change/3, terminate/2, format_status/2]).
suite() ->
@@ -61,7 +62,7 @@ suite() ->
all() ->
[start, {group,stop}, crash, call, cast, cast_fast, info, abcast,
- multicall, multicall_down, call_remote1, call_remote2,
+ continue, multicall, multicall_down, call_remote1, call_remote2,
call_remote3, call_remote_n1, call_remote_n2,
call_remote_n3, spec_init,
spec_init_local_registered_parent,
@@ -76,7 +77,7 @@ groups() ->
[{stop, [],
[stop1, stop2, stop3, stop4, stop5, stop6, stop7, stop8, stop9, stop10]},
{undef_callbacks, [],
- [undef_handle_call, undef_handle_cast, undef_handle_info,
+ [undef_handle_call, undef_handle_cast, undef_handle_info, undef_handle_continue,
undef_init, undef_code_change, undef_terminate1, undef_terminate2]}].
@@ -458,6 +459,47 @@ call(Config) when is_list(Config) ->
ok.
%% --------------------------------------
+%% Test handle_continue.
+%% --------------------------------------
+
+continue(Config) when is_list(Config) ->
+ {ok, Pid} = gen_server:start_link(gen_server_SUITE, {continue, self()}, []),
+ [{Pid, continue}, {Pid, after_continue}] = read_replies(Pid),
+
+ gen_server:call(Pid, {continue_reply, self()}),
+ [{Pid, continue}, {Pid, after_continue}] = read_replies(Pid),
+
+ gen_server:call(Pid, {continue_noreply, self()}),
+ [{Pid, continue}, {Pid, after_continue}] = read_replies(Pid),
+
+ gen_server:cast(Pid, {continue_noreply, self()}),
+ [{Pid, continue}, {Pid, after_continue}] = read_replies(Pid),
+
+ Pid ! {continue_noreply, self()},
+ [{Pid, continue}, {Pid, after_continue}] = read_replies(Pid),
+
+ Pid ! {continue_continue, self()},
+ [{Pid, before_continue}, {Pid, continue}, {Pid, after_continue}] = read_replies(Pid),
+
+ Ref = monitor(process, Pid),
+ Pid ! continue_stop,
+ verify_down_reason(Ref, Pid, normal).
+
+read_replies(Pid) ->
+ receive
+ {Pid, ack} -> read_replies()
+ after
+ 1000 -> ct:fail({continue, ack})
+ end.
+
+read_replies() ->
+ receive
+ Msg -> [Msg | read_replies()]
+ after
+ 0 -> []
+ end.
+
+%% --------------------------------------
%% Test call to nonexisting processes on remote nodes
%% --------------------------------------
@@ -1346,7 +1388,7 @@ echo_loop() ->
%% Test the default implementation of terminate if the callback module
%% does not export it
undef_terminate1(Config) when is_list(Config) ->
- {ok, Server} = gen_server:start(oc_server, [], []),
+ {ok, Server} = oc_server:start(),
MRef = monitor(process, Server),
ok = gen_server:stop(Server),
ok = verify_down_reason(MRef, Server, normal).
@@ -1354,7 +1396,7 @@ undef_terminate1(Config) when is_list(Config) ->
%% Test the default implementation of terminate if the callback module
%% does not export it
undef_terminate2(Config) when is_list(Config) ->
- {ok, Server} = gen_server:start(oc_server, [], []),
+ {ok, Server} = oc_server:start(),
MRef = monitor(process, Server),
ok = gen_server:stop(Server, {error, test}, infinity),
ok = verify_down_reason(MRef, Server, {error, test}).
@@ -1377,7 +1419,7 @@ undef_init(_Config) ->
%% The upgrade should fail if code_change is expected in the callback module
%% but not exported, but the server should continue with the old code
undef_code_change(Config) when is_list(Config) ->
- {ok, Server} = gen_server:start(oc_server, [], []),
+ {ok, Server} = oc_server:start(),
{error, {'EXIT', {undef, [{oc_server, code_change, [_, _, _], _}|_]}}}
= fake_upgrade(Server, ?MODULE),
true = is_process_alive(Server).
@@ -1385,7 +1427,7 @@ undef_code_change(Config) when is_list(Config) ->
%% The server should crash if the handle_call callback is
%% not exported in the callback module
undef_handle_call(_Config) ->
- {ok, Server} = gen_server:start(oc_server, [], []),
+ {ok, Server} = oc_server:start(),
try
gen_server:call(Server, call_msg),
ct:fail(should_crash)
@@ -1397,17 +1439,25 @@ undef_handle_call(_Config) ->
%% The server should crash if the handle_cast callback is
%% not exported in the callback module
undef_handle_cast(_Config) ->
- {ok, Server} = gen_server:start(oc_server, [], []),
+ {ok, Server} = oc_server:start(),
MRef = monitor(process, Server),
gen_server:cast(Server, cast_msg),
verify_undef_down(MRef, Server, oc_server, handle_cast),
ok.
+%% The server should crash if the handle_continue callback is
+%% not exported in the callback module
+undef_handle_continue(_Config) ->
+ {ok, Server} = oc_server:start(continue),
+ MRef = monitor(process, Server),
+ verify_undef_down(MRef, Server, oc_server, handle_continue),
+ ok.
+
%% The server should log but not crash if the handle_info callback is
%% calling an undefined function
undef_handle_info(Config) when is_list(Config) ->
error_logger_forwarder:register(),
- {ok, Server} = gen_server:start(oc_server, [], []),
+ {ok, Server} = oc_server:start(),
Server ! hej,
wait_until_processed(Server, hej, 10),
true = is_process_alive(Server),
@@ -1570,8 +1620,11 @@ init(hibernate) ->
init(sleep) ->
ct:sleep(1000),
{ok, []};
+init({continue, Pid}) ->
+ self() ! {after_continue, Pid},
+ {ok, [], {continue, {message, Pid}}};
init({state,State}) ->
- {ok, State}.
+ {ok,State}.
handle_call(started_p, _From, State) ->
io:format("FROZ"),
@@ -1604,6 +1657,12 @@ handle_call(shutdown_reason, _From, _State) ->
handle_call({call_undef_fun, Mod, Fun}, _From, State) ->
Mod:Fun(),
{reply, ok, State};
+handle_call({continue_reply, Pid}, _From, State) ->
+ self() ! {after_continue, Pid},
+ {reply, ok, State, {continue, {message, Pid}}};
+handle_call({continue_noreply, Pid}, From, State) ->
+ self() ! {after_continue, Pid},
+ {noreply, State, {continue, {message, Pid, From}}};
handle_call(stop_shutdown_reason, _From, State) ->
{stop,{shutdown,stop_reason},State}.
@@ -1620,6 +1679,9 @@ handle_cast(hibernate_later, _State) ->
handle_cast({call_undef_fun, Mod, Fun}, State) ->
Mod:Fun(),
{noreply, State};
+handle_cast({continue_noreply, Pid}, State) ->
+ self() ! {after_continue, Pid},
+ {noreply, State, {continue, {message, Pid}}};
handle_cast({From, stop}, State) ->
io:format("BAZ"),
{stop, {From,stopped}, State}.
@@ -1657,9 +1719,34 @@ handle_info(continue, From) ->
{noreply, []};
handle_info({From, stop}, State) ->
{stop, {From,stopped_info}, State};
+handle_info({after_continue, Pid}, State) ->
+ Pid ! {self(), after_continue},
+ Pid ! {self(), ack},
+ {noreply, State};
+handle_info({continue_noreply, Pid}, State) ->
+ self() ! {after_continue, Pid},
+ {noreply, State, {continue, {message, Pid}}};
+handle_info({continue_continue, Pid}, State) ->
+ {noreply, State, {continue, {continue, Pid}}};
+handle_info(continue_stop, State) ->
+ {noreply, State, {continue, stop}};
handle_info(_Info, State) ->
{noreply, State}.
+handle_continue({continue, Pid}, State) ->
+ Pid ! {self(), before_continue},
+ self() ! {after_continue, Pid},
+ {noreply, State, {continue, {message, Pid}}};
+handle_continue(stop, State) ->
+ {stop, normal, State};
+handle_continue({message, Pid}, State) ->
+ Pid ! {self(), continue},
+ {noreply, State};
+handle_continue({message, Pid, From}, State) ->
+ Pid ! {self(), continue},
+ gen_server:reply(From, ok),
+ {noreply, State}.
+
code_change(_OldVsn,
{new, {undef_in_code_change, {Mod, Fun}}} = State,
_Extra) ->
diff --git a/lib/stdlib/test/gen_server_SUITE_data/oc_server.erl b/lib/stdlib/test/gen_server_SUITE_data/oc_server.erl
index 4ba37987f3..7b92a49bf6 100644
--- a/lib/stdlib/test/gen_server_SUITE_data/oc_server.erl
+++ b/lib/stdlib/test/gen_server_SUITE_data/oc_server.erl
@@ -22,7 +22,7 @@
-behaviour(gen_server).
%% API
--export([start/0]).
+-export([start/0, start/1]).
%% gen_server callbacks
-export([init/1]).
@@ -30,8 +30,12 @@
-record(state, {}).
start() ->
- gen_server:start({local, ?MODULE}, ?MODULE, [], []).
+ gen_server:start(?MODULE, ok, []).
-init([]) ->
- {ok, #state{}}.
+start(continue) ->
+ gen_server:start(?MODULE, continue, []).
+init(ok) ->
+ {ok, #state{}};
+init(continue) ->
+ {ok, #state{}, {continue, continue}}.
diff --git a/lib/stdlib/test/property_test/README b/lib/stdlib/test/property_test/README
new file mode 100644
index 0000000000..57602bf719
--- /dev/null
+++ b/lib/stdlib/test/property_test/README
@@ -0,0 +1,12 @@
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% %%%
+%%% WARNING %%%
+%%% %%%
+%%% This is experimental code which may be changed or removed %%%
+%%% anytime without any warning. %%%
+%%% %%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+The test in this directory are written assuming that the user has a QuickCheck license. They are to be run manually. Some may be possible to be run with other tools, e.g. PropEr.
+
diff --git a/lib/stdlib/test/property_test/uri_string_recompose.erl b/lib/stdlib/test/property_test/uri_string_recompose.erl
new file mode 100644
index 0000000000..e51a671172
--- /dev/null
+++ b/lib/stdlib/test/property_test/uri_string_recompose.erl
@@ -0,0 +1,361 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(uri_string_recompose).
+
+-compile(export_all).
+
+-proptest(eqc).
+-proptest([triq,proper]).
+
+-ifndef(EQC).
+-ifndef(PROPER).
+-ifndef(TRIQ).
+-define(EQC,true).
+-endif.
+-endif.
+-endif.
+
+-ifdef(EQC).
+-include_lib("eqc/include/eqc.hrl").
+-define(MOD_eqc,eqc).
+
+-else.
+-ifdef(PROPER).
+-include_lib("proper/include/proper.hrl").
+-define(MOD_eqc,proper).
+
+-else.
+-ifdef(TRIQ).
+-define(MOD_eqc,triq).
+-include_lib("triq/include/triq.hrl").
+
+-endif.
+-endif.
+-endif.
+
+
+-define(STRING_REST(MatchStr, Rest), <<MatchStr/utf8, Rest/binary>>).
+
+-define(SCHEME, {scheme, scheme()}).
+-define(USER, {userinfo, unicode()}).
+-define(HOST, {host, host_map()}).
+-define(PORT, {port, port()}).
+-define(PATH_ABE, {path, path_abempty_map()}).
+-define(PATH_ABS, {path, path_absolute_map()}).
+-define(PATH_NOS, {path, path_noscheme_map()}).
+-define(PATH_ROO, {path, path_rootless_map()}).
+-define(PATH_EMP, {path, path_empty_map()}).
+-define(QUERY, {query, query_map()}).
+-define(FRAGMENT, {fragment, fragment_map()}).
+
+
+%%%========================================================================
+%%% Properties
+%%%========================================================================
+
+prop_recompose() ->
+ ?FORALL(Map, map(),
+ Map =:= uri_string:parse(uri_string:recompose(Map))
+ ).
+
+%% Stats
+prop_map_key_length_collect() ->
+ ?FORALL(List, map(),
+ collect(length(maps:keys(List)), true)).
+
+prop_map_collect() ->
+ ?FORALL(List, map(),
+ collect(lists:sort(maps:keys(List)), true)).
+
+prop_scheme_collect() ->
+ ?FORALL(List, scheme(),
+ collect(length(List), true)).
+
+
+%%%========================================================================
+%%% Generators
+%%%========================================================================
+
+map() ->
+ ?LET(Gen, comp_proplist(), proplist_to_map(Gen)).
+
+comp_proplist() ->
+ frequency([
+ {2, [?SCHEME,?PATH_ABS]},
+ {2, [?SCHEME,?PATH_ROO]},
+ {2, [?SCHEME,?PATH_EMP]},
+ {2, [?SCHEME,?HOST,?PATH_ABE]},
+ {2, [?SCHEME,?USER,?HOST,?PATH_ABE]},
+ {2, [?SCHEME,?HOST,?PORT,?PATH_ABE]},
+ {2, [?SCHEME,?USER,?HOST,?PORT,?PATH_ABE]},
+
+ {2, [?PATH_ABS]},
+ {2, [?PATH_NOS]},
+ {2, [?PATH_EMP]},
+ {2, [?HOST,?PATH_ABE]},
+ {2, [?USER,?HOST,?PATH_ABE]},
+ {2, [?HOST,?PORT,?PATH_ABE]},
+ {2, [?USER,?HOST,?PORT,?PATH_ABE]},
+
+
+ {2, [?SCHEME,?PATH_ABS,?QUERY]},
+ {2, [?SCHEME,?PATH_ROO,?QUERY]},
+ {2, [?SCHEME,?PATH_EMP,?QUERY]},
+ {2, [?SCHEME,?HOST,?PATH_ABE,?QUERY]},
+ {2, [?SCHEME,?USER,?HOST,?PATH_ABE,?QUERY]},
+ {2, [?SCHEME,?HOST,?PORT,?PATH_ABE,?QUERY]},
+ {2, [?SCHEME,?USER,?HOST,?PORT,?PATH_ABE,?QUERY]},
+
+ {2, [?PATH_ABS,?QUERY]},
+ {2, [?PATH_NOS,?QUERY]},
+ {2, [?PATH_EMP,?QUERY]},
+ {2, [?HOST,?PATH_ABE,?QUERY]},
+ {2, [?USER,?HOST,?PATH_ABE,?QUERY]},
+ {2, [?HOST,?PORT,?PATH_ABE,?QUERY]},
+ {2, [?USER,?HOST,?PORT,?PATH_ABE,?QUERY]},
+
+
+ {2, [?SCHEME,?PATH_ABS,?FRAGMENT]},
+ {2, [?SCHEME,?PATH_ROO,?FRAGMENT]},
+ {2, [?SCHEME,?PATH_EMP,?FRAGMENT]},
+ {2, [?SCHEME,?HOST,?PATH_ABE,?FRAGMENT]},
+ {2, [?SCHEME,?USER,?HOST,?PATH_ABE,?FRAGMENT]},
+ {2, [?SCHEME,?HOST,?PORT,?PATH_ABE,?FRAGMENT]},
+ {2, [?SCHEME,?USER,?HOST,?PORT,?PATH_ABE,?FRAGMENT]},
+
+ {2, [?PATH_ABS,?FRAGMENT]},
+ {2, [?PATH_NOS,?FRAGMENT]},
+ {2, [?PATH_EMP,?FRAGMENT]},
+ {2, [?HOST,?PATH_ABE,?FRAGMENT]},
+ {2, [?USER,?HOST,?PATH_ABE,?FRAGMENT]},
+ {2, [?HOST,?PORT,?PATH_ABE,?FRAGMENT]},
+ {2, [?USER,?HOST,?PORT,?PATH_ABE,?FRAGMENT]},
+
+
+ {2, [?SCHEME,?PATH_ABS,?QUERY,?FRAGMENT]},
+ {2, [?SCHEME,?PATH_ROO,?QUERY,?FRAGMENT]},
+ {2, [?SCHEME,?PATH_EMP,?QUERY,?FRAGMENT]},
+ {2, [?SCHEME,?HOST,?PATH_ABE,?QUERY,?FRAGMENT]},
+ {2, [?SCHEME,?USER,?HOST,?PATH_ABE,?QUERY,?FRAGMENT]},
+ {2, [?SCHEME,?HOST,?PORT,?PATH_ABE,?QUERY,?FRAGMENT]},
+ {2, [?SCHEME,?USER,?HOST,?PORT,?PATH_ABE,?QUERY,?FRAGMENT]},
+
+ {2, [?PATH_ABS,?QUERY,?FRAGMENT]},
+ {2, [?PATH_NOS,?QUERY,?FRAGMENT]},
+ {2, [?PATH_EMP,?QUERY,?FRAGMENT]},
+ {2, [?HOST,?PATH_ABE,?QUERY,?FRAGMENT]},
+ {2, [?USER,?HOST,?PATH_ABE,?QUERY,?FRAGMENT]},
+ {2, [?HOST,?PORT,?PATH_ABE,?QUERY,?FRAGMENT]},
+ {2, [?USER,?HOST,?PORT,?PATH_ABE,?QUERY,?FRAGMENT]}
+ ]).
+
+
+%%-------------------------------------------------------------------------
+%% Path
+%%-------------------------------------------------------------------------
+path_abempty_map() ->
+ frequency([{90, path_abe_map()},
+ {10, path_empty_map()}]).
+
+path_abe_map() ->
+ ?SIZED(Length, path_abe_map(Length, [])).
+%%
+path_abe_map(0, Segments) ->
+ ?LET(Gen, Segments, lists:append(Gen));
+path_abe_map(N, Segments) ->
+ path_abe_map(N-1, [slash(),segment()|Segments]).
+
+
+path_absolute_map() ->
+ ?SIZED(Length, path_absolute_map(Length, [])).
+%%
+path_absolute_map(0, Segments) ->
+ ?LET(Gen, [slash(),segment_nz()|Segments], lists:append(Gen));
+path_absolute_map(N, Segments) ->
+ path_absolute_map(N-1, [slash(),segment()|Segments]).
+
+
+path_noscheme_map() ->
+ ?SIZED(Length, path_noscheme_map(Length, [])).
+%%
+path_noscheme_map(0, Segments) ->
+ ?LET(Gen, [segment_nz_nc()|Segments], lists:append(Gen));
+path_noscheme_map(N, Segments) ->
+ path_noscheme_map(N-1, [slash(),segment()|Segments]).
+
+path_rootless_map() ->
+ ?SIZED(Length, path_rootless_map(Length, [])).
+%%
+path_rootless_map(0, Segments) ->
+ ?LET(Gen, [segment_nz()|Segments], lists:append(Gen));
+path_rootless_map(N, Segments) ->
+ path_rootless_map(N-1, [slash(),segment()|Segments]).
+
+
+segment_nz() ->
+ non_empty(segment()).
+
+segment_nz_nc() ->
+ non_empty(list(frequency([{30, unreserved()},
+ {10, sub_delims()},
+ {10, unicode_char()},
+ {5, oneof([$@])}
+ ]))).
+
+
+segment() ->
+ list(frequency([{30, unreserved()},
+ {10, sub_delims()},
+ {10, unicode_char()},
+ {5, oneof([$:, $@])}
+ ])).
+
+slash() ->
+ "/".
+
+path_empty_map() ->
+ "".
+
+
+%%-------------------------------------------------------------------------
+%% Path
+%%-------------------------------------------------------------------------
+host_map() ->
+ frequency([{30, reg_name()},
+ {30, ip_address()}
+ ]).
+
+
+reg_name() ->
+ list(frequency([{30, alpha()},
+ {10, sub_delims()},
+ {10, unicode_char()}
+ ])).
+
+ip_address() ->
+ oneof(["127.0.0.1", "::127.0.0.1",
+ "2001:0db8:0000:0000:0000:0000:1428:07ab",
+ "2001:0db8:0000:0000:0000::1428:07ab",
+ "2001:0db8:0:0:0:0:1428:07ab",
+ "2001:0db8:0::0:1428:07ab"]).
+
+%% Generating only reg-names
+host_uri() ->
+ non_empty(list(frequency([{30, unreserved()},
+ {10, sub_delims()},
+ {10, pct_encoded()}
+ ]))).
+
+%%-------------------------------------------------------------------------
+%% Port, Query, Fragment
+%%-------------------------------------------------------------------------
+port() ->
+ frequency([{10, undefined},
+ {10, range(1,65535)}
+ ]).
+
+query_map() ->
+ unicode().
+
+
+query_uri() ->
+ [$?| non_empty(list(frequency([{20, pchar()},
+ {5, oneof([$/, $?])} % punctuation
+ ])))].
+
+fragment_map() ->
+ unicode().
+
+fragment_uri() ->
+ [$?| non_empty(list(frequency([{20, pchar()},
+ {5, oneof([$/, $?])} % punctuation
+ ])))].
+
+
+%%-------------------------------------------------------------------------
+%% Scheme
+%%-------------------------------------------------------------------------
+scheme() ->
+ ?SIZED(Length, scheme_start(Length, [])).
+%%
+scheme_start(0, L) ->
+ ?LET(Gen, L, lists:reverse(Gen));
+scheme_start(N, L) ->
+ scheme(N-1,[alpha()|L]).
+
+scheme(0, L) ->
+ ?LET(Gen, L, lists:reverse(Gen));
+scheme(N, L) ->
+ scheme(N-1, [scheme_char()|L]).
+
+
+%%-------------------------------------------------------------------------
+%% Misc
+%%-------------------------------------------------------------------------
+unicode() ->
+ list(frequency([{20, alpha()}, % alpha
+ {10, digit()}, % digit
+ {10, unicode_char()} % unicode
+ ])).
+
+scheme_char() ->
+ frequency([{20, alpha()}, % alpha
+ {20, digit()}, % digit
+ {5, oneof([$+, $-, $.])} % punctuation
+ ]).
+
+sub_delims() ->
+ oneof([$!, $$, $&, $', $(, $),
+ $*, $+, $,,$;, $=]).
+
+pchar() ->
+ frequency([{20, unreserved()},
+ {5, pct_encoded()},
+ {5, sub_delims()},
+ {1, oneof([$:, $@])} % punctuation
+ ]).
+
+unreserved() ->
+ frequency([{20, alpha()},
+ {5, digit()},
+ {1, oneof([$-, $., $_, $~])} % punctuation
+ ]).
+
+unicode_char() ->
+ range(913, 1023).
+
+alpha() ->
+ frequency([{20, range($a, $z)}, % letters
+ {20, range($A, $Z)}]). % letters
+
+digit() ->
+ range($0, $9). % numbers
+
+pct_encoded() ->
+ oneof(["%C3%A4", "%C3%A5", "%C3%B6"]).
+
+
+%%%========================================================================
+%%% Helpers
+%%%========================================================================
+proplist_to_map(L) ->
+ lists:foldl(fun({K,V},M) -> M#{K => V};
+ (_,M) -> M
+ end, #{}, L).
diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl
index 432293b656..ef4f9faad9 100644
--- a/lib/stdlib/test/rand_SUITE.erl
+++ b/lib/stdlib/test/rand_SUITE.erl
@@ -29,11 +29,14 @@
basic_stats_uniform_1/1, basic_stats_uniform_2/1,
basic_stats_standard_normal/1,
basic_stats_normal/1,
+ uniform_real_conv/1,
plugin/1, measure/1,
reference_jump_state/1, reference_jump_procdict/1]).
-export([test/0, gen/1]).
+-export([uniform_real_gen/1, uniform_gen/2]).
+
-include_lib("common_test/include/ct.hrl").
-define(LOOP, 1000000).
@@ -46,7 +49,7 @@ all() ->
[seed, interval_int, interval_float,
api_eq,
reference,
- {group, basic_stats},
+ {group, basic_stats}, uniform_real_conv,
plugin, measure,
{group, reference_jump}
].
@@ -80,7 +83,7 @@ test() ->
end, Tests).
algs() ->
- [exs64, exsplus, exsp, exrop, exs1024, exs1024s].
+ [exrop, exsp, exs1024s, exs64, exsplus, exs1024].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -101,7 +104,7 @@ seed_1(Alg) ->
_ = rand:uniform(),
S00 = get(rand_seed),
erase(),
- _ = rand:uniform(),
+ _ = rand:uniform_real(),
false = S00 =:= get(rand_seed), %% hopefully
%% Choosing algo and seed
@@ -228,11 +231,13 @@ interval_float(Config) when is_list(Config) ->
interval_float_1(0) -> ok;
interval_float_1(N) ->
X = rand:uniform(),
+ Y = rand:uniform_real(),
if
- 0.0 =< X, X < 1.0 ->
+ 0.0 =< X, X < 1.0, 0.0 < Y, Y < 1.0 ->
ok;
true ->
- io:format("X=~p 0=<~p<1.0~n", [X,X]),
+ io:format("X=~p 0.0=<~p<1.0~n", [X,X]),
+ io:format("Y=~p 0.0<~p<1.0~n", [Y,Y]),
exit({X, rand:export_seed()})
end,
interval_float_1(N-1).
@@ -334,7 +339,13 @@ basic_stats_normal(Config) when is_list(Config) ->
IntendedMeanVariancePairs).
basic_uniform_1(N, S0, Sum, A0) when N > 0 ->
- {X,S} = rand:uniform_s(S0),
+ {X,S} =
+ case N band 1 of
+ 0 ->
+ rand:uniform_s(S0);
+ 1 ->
+ rand:uniform_real_s(S0)
+ end,
I = trunc(X*100),
A = array:set(I, 1+array:get(I,A0), A0),
basic_uniform_1(N-1, S, Sum+X, A);
@@ -400,6 +411,137 @@ normal_s(Mean, Variance, State0) ->
rand:normal_s(Mean, Variance, State0).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% White box test of the conversion to float
+
+uniform_real_conv(Config) when is_list(Config) ->
+ [begin
+%% ct:pal("~13.16.0bx~3.16.0b: ~p~n", [M,E,Gen]),
+ uniform_real_conv_check(M, E, Gen)
+ end || {M, E, Gen} <- uniform_real_conv_data()],
+ uniform_real_scan(0),
+ uniform_real_scan(3).
+
+uniform_real_conv_data() ->
+ [{16#fffffffffffff, -1, [16#3ffffffffffffff]},
+ {16#fffffffffffff, -1, [16#3ffffffffffffe0]},
+ {16#ffffffffffffe, -1, [16#3ffffffffffffdf]},
+ %%
+ {16#0000000000000, -1, [16#200000000000000]},
+ {16#fffffffffffff, -2, [16#1ffffffffffffff]},
+ {16#fffffffffffff, -2, [16#1fffffffffffff0]},
+ {16#ffffffffffffe, -2, [16#1ffffffffffffef]},
+ %%
+ {16#0000000000000, -2, [16#100000000000000]},
+ {16#fffffffffffff, -3, [16#0ffffffffffffff]},
+ {16#fffffffffffff, -3, [16#0fffffffffffff8]},
+ {16#ffffffffffffe, -3, [16#0fffffffffffff7]},
+ %%
+ {16#0000000000000, -3, [16#080000000000000]},
+ {16#fffffffffffff, -4, [16#07fffffffffffff]},
+ {16#fffffffffffff, -4, [16#07ffffffffffffc]},
+ {16#ffffffffffffe, -4, [16#07ffffffffffffb]},
+ %%
+ {16#0000000000000, -4, [16#040000000000000]},
+ {16#fffffffffffff, -5, [16#03fffffffffffff,16#3ffffffffffffff]},
+ {16#fffffffffffff, -5, [16#03ffffffffffffe,16#200000000000000]},
+ {16#ffffffffffffe, -5, [16#03fffffffffffff,16#1ffffffffffffff]},
+ {16#ffffffffffffe, -5, [16#03fffffffffffff,16#100000000000000]},
+ %%
+ {16#0000000000001, -56, [16#000000000000007,16#00000000000007f]},
+ {16#0000000000001, -56, [16#000000000000004,16#000000000000040]},
+ {16#0000000000000, -57, [16#000000000000003,16#20000000000001f]},
+ {16#0000000000000, -57, [16#000000000000000,16#200000000000000]},
+ {16#fffffffffffff, -58, [16#000000000000003,16#1ffffffffffffff]},
+ {16#fffffffffffff, -58, [16#000000000000000,16#1fffffffffffff0]},
+ {16#ffffffffffffe, -58, [16#000000000000000,16#1ffffffffffffef]},
+ {16#ffffffffffffe, -58, [16#000000000000000,16#1ffffffffffffe0]},
+ %%
+ {16#0000000000000, -58, [16#000000000000000,16#10000000000000f]},
+ {16#0000000000000, -58, [16#000000000000000,16#100000000000000]},
+ {2#11001100000000000000000000000000000000000011000000011, % 53 bits
+ -1022,
+ [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, % 18 zeros
+ 2#1100110000000000000000000000000000000000001 bsl 2, % 43 bits
+ 2#1000000011 bsl (56-10+2)]}, % 10 bits
+ {0, -1, % 0.5 after retry
+ [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, % 18 zeros
+ 2#111111111111111111111111111111111111111111 bsl 2, % 42 bits - retry
+ 16#200000000000003]}]. % 0.5
+
+-define(UNIFORM_REAL_SCAN_PATTERN, (16#19000000000009)). % 53 bits
+-define(UNIFORM_REAL_SCAN_NUMBER, (1021)).
+
+uniform_real_scan_template(K) ->
+ <<0:?UNIFORM_REAL_SCAN_NUMBER,
+ ?UNIFORM_REAL_SCAN_PATTERN:53,K:2,0:1>>.
+
+uniform_real_scan(K) ->
+ Templ = uniform_real_scan_template(K),
+ N = ?UNIFORM_REAL_SCAN_NUMBER,
+ uniform_real_scan(Templ, N, K).
+
+uniform_real_scan(Templ, N, K) when 0 =< N ->
+ <<_:N/bits,T/bits>> = Templ,
+ Data = uniform_real_scan_data(T, K),
+ uniform_real_conv_check(
+ ?UNIFORM_REAL_SCAN_PATTERN, N - 1 - ?UNIFORM_REAL_SCAN_NUMBER, Data),
+ uniform_real_scan(Templ, N - 1, K);
+uniform_real_scan(_, _, _) ->
+ ok.
+
+uniform_real_scan_data(Templ, K) ->
+ case Templ of
+ <<X:56, T/bits>> ->
+ B = rand:bc64(X),
+ [(X bsl 2) bor K |
+ if
+ 53 =< B ->
+ [];
+ true ->
+ uniform_real_scan_data(T, K)
+ end];
+ _ ->
+ <<X:56, _/bits>> = <<Templ/bits, 0:56>>,
+ [(X bsl 2) bor K]
+ end.
+
+uniform_real_conv_check(M, E, Gen) ->
+ <<F/float>> = <<0:1, (E + 16#3ff):11, M:52>>,
+ try uniform_real_gen(Gen) of
+ F -> F;
+ FF ->
+ ct:pal(
+ "~s =/= ~s: ~s~n",
+ [rand:float2str(FF), rand:float2str(F),
+ [["16#",integer_to_list(G,16),$\s]||G<-Gen]]),
+ ct:fail({neq, FF, F})
+ catch
+ Error:Reason ->
+ ct:pal(
+ "~w:~p ~s: ~s~n",
+ [Error, Reason, rand:float2str(F),
+ [["16#",integer_to_list(G,16),$\s]||G<-Gen]]),
+ ct:fail({Error, Reason, F, erlang:get_stacktrace()})
+ end.
+
+
+uniform_real_gen(Gen) ->
+ State = rand_state(Gen),
+ {F, {#{type := rand_SUITE_list},[]}} = rand:uniform_real_s(State),
+ F.
+
+uniform_gen(Range, Gen) ->
+ State = rand_state(Gen),
+ {N, {#{type := rand_SUITE_list},[]}} = rand:uniform_s(Range, State),
+ N.
+
+%% Loaded dice for white box tests
+rand_state(Gen) ->
+ {#{type => rand_SUITE_list, bits => 58, weak_low_bits => 1,
+ next => fun ([H|T]) -> {H, T} end},
+ Gen}.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Test that the user can write algorithms.
plugin(Config) when is_list(Config) ->
@@ -459,213 +601,289 @@ measure(Config) ->
{skip,{will_not_run_in_scaled_time,Scale}}
end.
+-define(CHECK_UNIFORM_RANGE(Gen, Range, X, St),
+ case (Gen) of
+ {(X), (St)} when is_integer(X), 1 =< (X), (X) =< (Range) ->
+ St
+ end).
+-define(CHECK_UNIFORM(Gen, X, St),
+ case (Gen) of
+ {(X), (St)} when is_float(X), 0.0 =< (X), (X) < 1.0 ->
+ St
+ end).
+-define(CHECK_UNIFORM_NZ(Gen, X, St),
+ case (Gen) of
+ {(X), (St)} when is_float(X), 0.0 < (X), (X) =< 1.0 ->
+ St
+ end).
+-define(CHECK_NORMAL(Gen, X, St),
+ case (Gen) of
+ {(X), (St)} when is_float(X) ->
+ St
+ end).
+
do_measure(_Config) ->
- Algos =
+ Algs =
+ algs() ++
try crypto:strong_rand_bytes(1) of
- <<_>> -> [crypto64, crypto]
+ <<_>> -> [crypto64, crypto_cache, crypto]
catch
error:low_entropy -> [];
error:undef -> []
- end ++ algs(),
+ end,
%%
- ct:pal("RNG uniform integer performance~n",[]),
- TMark1 =
+ ct:pal("~nRNG uniform integer range 10000 performance~n",[]),
+ _ =
measure_1(
- random,
fun (_) -> 10000 end,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
+ %%
+ ct:pal("~nRNG uniform integer 32 bit performance~n",[]),
_ =
- [measure_1(
- Algo,
- fun (_) -> 10000 end,
- TMark1,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (_) -> 1 bsl 32 end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG uniform integer half range performance~n",[]),
- HalfRangeFun = fun (State) -> half_range(State) end,
- TMark2 =
- measure_1(
- random,
- HalfRangeFun,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
_ =
- [measure_1(
- Algo,
- HalfRangeFun,
- TMark2,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
- %%
- ct:pal("~nRNG uniform integer half range + 1 performance~n",[]),
- HalfRangePlus1Fun = fun (State) -> half_range(State) + 1 end,
- TMark3 =
measure_1(
- random,
- HalfRangePlus1Fun,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
+ fun (State) -> half_range(State) end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
+ %%
+ ct:pal("~nRNG uniform integer half range + 1 performance~n",[]),
_ =
- [measure_1(
- Algo,
- HalfRangePlus1Fun,
- TMark3,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (State) -> half_range(State) + 1 end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG uniform integer full range - 1 performance~n",[]),
- FullRangeMinus1Fun = fun (State) -> (half_range(State) bsl 1) - 1 end,
- TMark4 =
- measure_1(
- random,
- FullRangeMinus1Fun,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
_ =
- [measure_1(
- Algo,
- FullRangeMinus1Fun,
- TMark4,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (State) -> (half_range(State) bsl 1) - 1 end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG uniform integer full range performance~n",[]),
- FullRangeFun = fun (State) -> half_range(State) bsl 1 end,
- TMark5 =
- measure_1(
- random,
- FullRangeFun,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
_ =
- [measure_1(
- Algo,
- FullRangeFun,
- TMark5,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (State) -> half_range(State) bsl 1 end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG uniform integer full range + 1 performance~n",[]),
- FullRangePlus1Fun = fun (State) -> (half_range(State) bsl 1) + 1 end,
- TMark6 =
- measure_1(
- random,
- FullRangePlus1Fun,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
_ =
- [measure_1(
- Algo,
- FullRangePlus1Fun,
- TMark6,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (State) -> (half_range(State) bsl 1) + 1 end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG uniform integer double range performance~n",[]),
- DoubleRangeFun = fun (State) -> half_range(State) bsl 2 end,
- TMark7 =
- measure_1(
- random,
- DoubleRangeFun,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
_ =
- [measure_1(
- Algo,
- DoubleRangeFun,
- TMark7,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (State) ->
+ half_range(State) bsl 2
+ end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG uniform integer double range + 1 performance~n",[]),
- DoubleRangePlus1Fun = fun (State) -> (half_range(State) bsl 2) + 1 end,
- TMark8 =
+ _ =
measure_1(
- random,
- DoubleRangePlus1Fun,
- undefined,
- fun (Range, State) ->
- {int, random:uniform_s(Range, State)}
- end),
+ fun (State) ->
+ (half_range(State) bsl 2) + 1
+ end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
+ %%
+ ct:pal("~nRNG uniform integer 64 bit performance~n",[]),
_ =
- [measure_1(
- Algo,
- DoubleRangePlus1Fun,
- TMark8,
- fun (Range, State) ->
- {int, rand:uniform_s(Range, State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (_) -> 1 bsl 64 end,
+ fun (State, Range, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM_RANGE(
+ Mod:uniform_s(Range, St0), Range,
+ X, St1)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG uniform float performance~n",[]),
- TMark9 =
+ _ =
measure_1(
- random,
fun (_) -> 0 end,
- undefined,
- fun (_, State) ->
- {uniform, random:uniform_s(State)}
- end),
+ fun (State, _, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM(Mod:uniform_s(St0), X, St)
+ end,
+ State)
+ end,
+ Algs),
+ %%
+ ct:pal("~nRNG uniform_real float performance~n",[]),
_ =
- [measure_1(
- Algo,
- fun (_) -> 0 end,
- TMark9,
- fun (_, State) ->
- {uniform, rand:uniform_s(State)}
- end) || Algo <- Algos],
+ measure_1(
+ fun (_) -> 0 end,
+ fun (State, _, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_UNIFORM(Mod:uniform_real_s(St0), X, St)
+ end,
+ State)
+ end,
+ Algs),
%%
ct:pal("~nRNG normal float performance~n",[]),
- io:format("~.12w: not implemented (too few bits)~n", [random]),
- _ = [measure_1(
- Algo,
- fun (_) -> 0 end,
- TMark9,
- fun (_, State) ->
- {normal, rand:normal_s(State)}
- end) || Algo <- Algos],
+ [TMarkNormalFloat|_] =
+ measure_1(
+ fun (_) -> 0 end,
+ fun (State, _, Mod) ->
+ measure_loop(
+ fun (St0) ->
+ ?CHECK_NORMAL(Mod:normal_s(St0), X, St1)
+ end,
+ State)
+ end,
+ Algs),
+ %% Just for fun try an implementation of the Box-Muller
+ %% transformation for creating normal distribution floats
+ %% to compare with our Ziggurat implementation.
+ %% Generates two numbers per call that we add so they
+ %% will not be optimized away. Hence the benchmark time
+ %% is twice what it should be.
+ TwoPi = 2 * math:pi(),
+ _ =
+ measure_1(
+ fun (_) -> 0 end,
+ fun (State, _, Mod) ->
+ measure_loop(
+ fun (State0) ->
+ {U1, State1} = Mod:uniform_real_s(State0),
+ {U2, State2} = Mod:uniform_s(State1),
+ R = math:sqrt(-2.0 * math:log(U1)),
+ T = TwoPi * U2,
+ Z0 = R * math:cos(T),
+ Z1 = R * math:sin(T),
+ ?CHECK_NORMAL({Z0 + Z1, State2}, X, State3)
+ end,
+ State)
+ end,
+ exrop, TMarkNormalFloat),
+ ok.
+
+-define(LOOP_MEASURE, (?LOOP div 5)).
+
+measure_loop(Fun, State) ->
+ measure_loop(Fun, State, ?LOOP_MEASURE).
+%%
+measure_loop(Fun, State, N) when 0 < N ->
+ measure_loop(Fun, Fun(State), N-1);
+measure_loop(_, _, _) ->
ok.
-measure_1(Algo, RangeFun, TMark, Gen) ->
+measure_1(RangeFun, Fun, Algs) ->
+ TMark = measure_1(RangeFun, Fun, hd(Algs), undefined),
+ [TMark] ++
+ [measure_1(RangeFun, Fun, Alg, TMark) || Alg <- tl(Algs)].
+
+measure_1(RangeFun, Fun, Alg, TMark) ->
Parent = self(),
- Seed =
- case Algo of
+ {Mod, State} =
+ case Alg of
crypto64 ->
- crypto64_seed();
+ {rand, crypto64_seed()};
+ crypto_cache ->
+ {rand, crypto:rand_seed_alg(crypto_cache)};
crypto ->
- crypto:rand_seed_s();
+ {rand, crypto:rand_seed_s()};
random ->
- random:seed(os:timestamp()), get(random_seed);
+ {random, random:seed(os:timestamp()), get(random_seed)};
_ ->
- rand:seed_s(Algo)
+ {rand, rand:seed_s(Alg)}
end,
- Range = RangeFun(Seed),
+ Range = RangeFun(State),
Pid = spawn_link(
fun() ->
- Fun = fun() -> measure_2(?LOOP, Range, Seed, Gen) end,
- {Time, ok} = timer:tc(Fun),
+ {Time, ok} = timer:tc(fun () -> Fun(State, Range, Mod) end),
Percent =
case TMark of
undefined -> 100;
@@ -673,7 +891,8 @@ measure_1(Algo, RangeFun, TMark, Gen) ->
end,
io:format(
"~.12w: ~p ns ~p% [16#~.16b]~n",
- [Algo, (Time * 1000 + 500) div ?LOOP, Percent, Range]),
+ [Alg, (Time * 1000 + 500) div ?LOOP_MEASURE,
+ Percent, Range]),
Parent ! {self(), Time},
normal
end),
@@ -681,21 +900,6 @@ measure_1(Algo, RangeFun, TMark, Gen) ->
{Pid, Msg} -> Msg
end.
-measure_2(N, Range, State0, Fun) when N > 0 ->
- case Fun(Range, State0) of
- {int, {Random, State}}
- when is_integer(Random), Random >= 1, Random =< Range ->
- measure_2(N-1, Range, State, Fun);
- {uniform, {Random, State}}
- when is_float(Random), 0.0 =< Random, Random < 1.0 ->
- measure_2(N-1, Range, State, Fun);
- {normal, {Random, State}} when is_float(Random) ->
- measure_2(N-1, Range, State, Fun);
- Res ->
- exit({error, Res, State0})
- end;
-measure_2(0, _, _, _) -> ok.
-
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% The jump sequence tests has two parts
%% for those with the functional API (jump/1)
diff --git a/lib/stdlib/test/stdlib.spec b/lib/stdlib/test/stdlib.spec
index 3768e494b2..91712b8963 100644
--- a/lib/stdlib/test/stdlib.spec
+++ b/lib/stdlib/test/stdlib.spec
@@ -1 +1,2 @@
{suites,"../stdlib_test",all}.
+{skip_suites,"../stdlib_test",stdlib_bench_SUITE, "bench only"}.
diff --git a/lib/stdlib/test/stdlib_bench.spec b/lib/stdlib/test/stdlib_bench.spec
new file mode 100644
index 0000000000..a5d1e1db80
--- /dev/null
+++ b/lib/stdlib/test/stdlib_bench.spec
@@ -0,0 +1,7 @@
+%% Needed to compile ,unicode_util_SUITE and string_SUITE...
+{cases,"../stdlib_test",unicode_util_SUITE, []}.
+{cases,"../stdlib_test",string_SUITE, []}.
+{skip_suites,"../stdlib_test",unicode_util_SUITE, "bench only"}.
+{skip_suites,"../stdlib_test",string_SUITE, "bench only"}.
+
+{suites,"../stdlib_test",[stdlib_bench_SUITE]}.
diff --git a/lib/stdlib/test/stdlib_bench_SUITE.erl b/lib/stdlib/test/stdlib_bench_SUITE.erl
new file mode 100644
index 0000000000..8670e7029c
--- /dev/null
+++ b/lib/stdlib/test/stdlib_bench_SUITE.erl
@@ -0,0 +1,107 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+-module(stdlib_bench_SUITE).
+-compile([export_all, nowarn_export_all]).
+-include_lib("common_test/include/ct_event.hrl").
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}].
+
+
+all() ->
+ [{group,unicode}].
+
+groups() ->
+ [{unicode,[{repeat,5}],
+ [norm_nfc_list, norm_nfc_deep_l, norm_nfc_binary,
+ string_lexemes_list, string_lexemes_binary
+ ]}].
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_testcase(_Func, Conf) ->
+ Conf.
+
+end_per_testcase(_Func, _Conf) ->
+ ok.
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+-define(REPEAT_NORM, 5).
+
+norm_nfc_list(Config) ->
+ Bin = norm_data(Config),
+ {_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, list, Bin, ?REPEAT_NORM),
+ report(1000.0*Res / Mean).
+
+norm_nfc_deep_l(Config) ->
+ Bin = norm_data(Config),
+ {_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, deep_l, Bin, ?REPEAT_NORM),
+ report(1000.0*Res / Mean).
+
+norm_nfc_binary(Config) ->
+ Bin = norm_data(Config),
+ {_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, binary, Bin, ?REPEAT_NORM),
+ report(1000.0*Res / Mean).
+
+
+string_lexemes_list(Config) ->
+ %% Use nth_lexeme instead of lexemes to avoid building a result of
+ %% large lists which causes large differences between test runs, gc?
+ Bin = norm_data(Config),
+ Fun = fun(Str) -> string:nth_lexeme(Str, 200000, [$;,$\n,$\r]), 200000 end,
+ {_N, Mean, _Stddev, Res} = string_SUITE:time_func(Fun, list, Bin, 15),
+ report(1000.0*Res / Mean).
+
+string_lexemes_binary(Config) ->
+ %% Use nth_lexeme instead of lexemes to avoid building a result of
+ %% large lists which causes large differences between test runs, gc?
+ Bin = norm_data(Config),
+ Fun = fun(Str) -> string:nth_lexeme(Str, 200000, [$;,$\n,$\r]), 200000 end,
+ {_N, Mean, _Stddev, Res} = string_SUITE:time_func(Fun, binary, Bin, ?REPEAT_NORM),
+ report(1000.0*Res / Mean).
+
+%%%
+report(Tps) ->
+ ct_event:notify(#event{name = benchmark_data,
+ data = [{suite,"stdlib_unicode"},{value,round(Tps)}]}),
+ Tps.
+
+norm_data(Config) ->
+ DataDir0 = proplists:get_value(data_dir, Config),
+ DataDir = filename:join(lists:droplast(filename:split(DataDir0))),
+ File = filename:join([DataDir,"unicode_util_SUITE_data","NormalizationTest.txt"]),
+ {ok, Bin} = file:read_file(File),
+ Bin.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
diff --git a/lib/stdlib/test/string_SUITE.erl b/lib/stdlib/test/string_SUITE.erl
index 90f980c0e5..05f18ef238 100644
--- a/lib/stdlib/test/string_SUITE.erl
+++ b/lib/stdlib/test/string_SUITE.erl
@@ -47,7 +47,7 @@
-export([to_upper_to_lower/1]).
%% Run tests when debugging them
--export([debug/0]).
+-export([debug/0, time_func/4]).
suite() ->
[{ct_hooks,[ts_install_cth]},
@@ -728,7 +728,7 @@ do_measure(TestDir) ->
{ok, Bin} = file:read_file(File),
io:format("~p~n",[byte_size(Bin)]),
Do = fun(Name, Func, Mode) ->
- {N, Mean, Stddev, _} = time_func(Func, Mode, Bin),
+ {N, Mean, Stddev, _} = time_func(Func, Mode, Bin, 50),
io:format("~10w ~6w ~6.2fms ±~4.2fms #~.2w gc included~n",
[Name, Mode, Mean/1000, Stddev/1000, N])
end,
@@ -938,19 +938,19 @@ needs_check(_) -> true.
%%%% Timer stuff
-time_func(Fun, Mode, Bin) ->
+time_func(Fun, Mode, Bin, Repeat) ->
timer:sleep(100), %% Let emulator catch up and clean things before test runs
Self = self(),
Pid = spawn_link(fun() ->
Str = mode(Mode, Bin),
- Self ! {self(),time_func(0,0,0, Fun, Str, undefined)}
+ Self ! {self(),time_func(0,0,0, Fun, Str, undefined, Repeat)}
end),
receive {Pid,Msg} -> Msg end.
-time_func(N,Sum,SumSq, Fun, Str, _) when N < 50 ->
+time_func(N,Sum,SumSq, Fun, Str, _, Repeat) when N < Repeat ->
{Time, Res} = timer:tc(fun() -> Fun(Str) end),
- time_func(N+1,Sum+Time,SumSq+Time*Time, Fun, Str, Res);
-time_func(N,Sum,SumSq, _, _, Res) ->
+ time_func(N+1,Sum+Time,SumSq+Time*Time, Fun, Str, Res, Repeat);
+time_func(N,Sum,SumSq, _, _, Res, _) ->
Mean = round(Sum / N),
Stdev = round(math:sqrt((SumSq - (Sum*Sum/N))/(N - 1))),
{N, Mean, Stdev, Res}.
diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl
index 03c24c7027..7dba0a2fd0 100644
--- a/lib/stdlib/test/unicode_util_SUITE.erl
+++ b/lib/stdlib/test/unicode_util_SUITE.erl
@@ -29,7 +29,9 @@
get/1,
count/1]).
--export([debug/0, id/1, bin_split/1, uc_loaded_size/0]).
+-export([debug/0, id/1, bin_split/1, uc_loaded_size/0,
+ time_count/4 %% Used by stdlib_bench_SUITE
+ ]).
suite() ->
[{ct_hooks,[ts_install_cth]},
@@ -323,7 +325,7 @@ do_measure(Config) ->
File = DataDir ++ "/NormalizationTest.txt",
{ok, Bin} = file:read_file(File),
Do = fun(Func, Mode) ->
- {N, Mean, Stddev, Res} = time_count(Func, Mode, Bin),
+ {N, Mean, Stddev, Res} = time_count(Func, Mode, Bin, 10),
io:format("~4w ~6w ~.10w ~.6wms ±~.2wms #~.2w~n",
[Func, Mode, Res, Mean div 1000, Stddev div 1000, N])
end,
@@ -345,19 +347,19 @@ uc_loaded_size([_|Rest]) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-time_count(Fun, Mode, Bin) ->
+time_count(Fun, Mode, Bin, Repeat) ->
timer:sleep(100), %% Let emulator catch up and clean things before test runs
Self = self(),
Pid = spawn_link(fun() ->
Str = mode(Mode, Bin),
- Self ! {self(),do_count(0,0,0, Fun, Str, undefined)}
+ Self ! {self(),do_count(0,0,0, Fun, Str, undefined, Repeat)}
end),
receive {Pid,Msg} -> Msg end.
-do_count(N,Sum,SumSq, Fun, Str, _) when N < 10 ->
+do_count(N,Sum,SumSq, Fun, Str, _, Repeat) when N < Repeat ->
{Time, Res} = do_count(Fun, Str),
- do_count(N+1,Sum+Time,SumSq+Time*Time, Fun, Str, Res);
-do_count(N,Sum,SumSq, _, _, Res) ->
+ do_count(N+1,Sum+Time,SumSq+Time*Time, Fun, Str, Res, Repeat);
+do_count(N,Sum,SumSq, _, _, Res, _) ->
Mean = round(Sum / N),
Stdev = round(math:sqrt((SumSq - (Sum*Sum/N))/(N - 1))),
{N, Mean, Stdev, Res}.
diff --git a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt
index 4bb4b1369b..d7d8f90de0 100644
--- a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt
+++ b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt
@@ -1,23 +1,24 @@
-# GraphemeBreakTest-9.0.0.txt
-# Date: 2016-06-02, 18:28:17 GMT
-# © 2016 Unicode®, Inc.
+# GraphemeBreakTest-10.0.0.txt
+# Date: 2017-04-14, 05:40:29 GMT
+# © 2017 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# Unicode Character Database
# For documentation, see http://www.unicode.org/reports/tr44/
#
-# Default Grapheme Break Test
+# Default Grapheme_Cluster_Break Test
#
# Format:
-# <string> (# <comment>)?
-# <string> contains hex Unicode code points, with
-# ÷ wherever there is a break opportunity, and
+# <string> (# <comment>)?
+# <string> contains hex Unicode code points, with
+# ÷ wherever there is a break opportunity, and
# × wherever there is not.
# <comment> the format can change, but currently it shows:
# - the sample character name
# - (x) the Grapheme_Cluster_Break property value for the sample character
-# - [x] the rule that determines whether there is a break or not
+# - [x] the rule that determines whether there is a break or not,
+# as listed in the Rules section of GraphemeBreakTest.html
#
# These samples may be extended or changed in the future.
#
@@ -53,8 +54,8 @@
÷ 0020 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 0020 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 0020 × 0308 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 0020 ÷ 2764 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 0020 × 0308 ÷ 2764 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 0020 ÷ 2640 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 0020 × 0308 ÷ 2640 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 0020 ÷ 1F466 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0020 × 0308 ÷ 1F466 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0020 ÷ 0378 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -93,8 +94,8 @@
÷ 000D ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 000D ÷ 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 000D ÷ 0308 × 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 000D ÷ 2764 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 000D ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 000D ÷ 2640 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 000D ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 000D ÷ 1F466 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] BOY (EBG) ÷ [0.3]
÷ 000D ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 000D ÷ 0378 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3]
@@ -133,8 +134,8 @@
÷ 000A ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 000A ÷ 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 000A ÷ 0308 × 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 000A ÷ 2764 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 000A ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 000A ÷ 2640 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 000A ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 000A ÷ 1F466 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] BOY (EBG) ÷ [0.3]
÷ 000A ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 000A ÷ 0378 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3]
@@ -173,8 +174,8 @@
÷ 0001 ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 0001 ÷ 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 0001 ÷ 0308 × 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 0001 ÷ 2764 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 0001 ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 0001 ÷ 2640 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 0001 ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 0001 ÷ 1F466 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] BOY (EBG) ÷ [0.3]
÷ 0001 ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0001 ÷ 0378 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3]
@@ -213,8 +214,8 @@
÷ 0300 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 0300 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 0300 × 0308 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 0300 ÷ 2764 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 0300 × 0308 ÷ 2764 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 0300 ÷ 2640 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 0300 × 0308 ÷ 2640 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 0300 ÷ 1F466 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0300 × 0308 ÷ 1F466 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0300 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -253,8 +254,8 @@
÷ 0600 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 0600 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 0600 × 0308 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 0600 × 2764 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 0600 × 0308 ÷ 2764 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 0600 × 2640 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 0600 × 0308 ÷ 2640 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 0600 × 1F466 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] BOY (EBG) ÷ [0.3]
÷ 0600 × 0308 ÷ 1F466 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0600 × 0378 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] <reserved-0378> (Other) ÷ [0.3]
@@ -293,8 +294,8 @@
÷ 0903 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 0903 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 0903 × 0308 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 0903 ÷ 2764 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 0903 × 0308 ÷ 2764 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 0903 ÷ 2640 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 0903 × 0308 ÷ 2640 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 0903 ÷ 1F466 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0903 × 0308 ÷ 1F466 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0903 ÷ 0378 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -333,8 +334,8 @@
÷ 1100 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 1100 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 1100 × 0308 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 1100 ÷ 2764 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 1100 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 1100 ÷ 2640 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 1100 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 1100 ÷ 1F466 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1100 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1100 ÷ 0378 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -373,8 +374,8 @@
÷ 1160 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 1160 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 1160 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 1160 ÷ 2764 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 1160 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 1160 ÷ 2640 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 1160 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 1160 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1160 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1160 ÷ 0378 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -413,8 +414,8 @@
÷ 11A8 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 11A8 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 11A8 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 11A8 ÷ 2764 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 11A8 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 11A8 ÷ 2640 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 11A8 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 11A8 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 11A8 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 11A8 ÷ 0378 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -453,8 +454,8 @@
÷ AC00 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ AC00 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ AC00 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ AC00 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ AC00 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ AC00 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ AC00 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ AC00 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ AC00 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ AC00 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -493,8 +494,8 @@
÷ AC01 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ AC01 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ AC01 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ AC01 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ AC01 × 0308 ÷ 2764 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ AC01 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ AC01 × 0308 ÷ 2640 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ AC01 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ AC01 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ AC01 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -533,8 +534,8 @@
÷ 1F1E6 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 1F1E6 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 1F1E6 × 0308 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 1F1E6 ÷ 2764 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 1F1E6 × 0308 ÷ 2764 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 1F1E6 ÷ 2640 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 1F1E6 × 0308 ÷ 2640 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 1F1E6 ÷ 1F466 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1F1E6 × 0308 ÷ 1F466 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1F1E6 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -573,8 +574,8 @@
÷ 261D × 0308 × 1F3FB ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 261D × 200D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 261D × 0308 × 200D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 261D ÷ 2764 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 261D × 0308 ÷ 2764 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 261D ÷ 2640 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 261D × 0308 ÷ 2640 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 261D ÷ 1F466 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 261D × 0308 ÷ 1F466 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 261D ÷ 0378 ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -613,8 +614,8 @@
÷ 1F3FB × 0308 ÷ 1F3FB ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 1F3FB × 200D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 1F3FB × 0308 × 200D ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 1F3FB ÷ 2764 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 1F3FB × 0308 ÷ 2764 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 1F3FB ÷ 2640 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 1F3FB × 0308 ÷ 2640 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 1F3FB ÷ 1F466 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1F3FB × 0308 ÷ 1F466 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1F3FB ÷ 0378 ÷ # ÷ [0.2] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -653,54 +654,54 @@
÷ 200D × 0308 ÷ 1F3FB ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 200D × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 200D × 0308 × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 200D × 2764 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 200D × 0308 ÷ 2764 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 200D × 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 200D × 0308 ÷ 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 200D × 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) ÷ [0.3]
÷ 200D × 0308 ÷ 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 200D ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
÷ 200D × 0308 ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
÷ 200D ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3]
÷ 200D × 0308 ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3]
-÷ 2764 ÷ 0020 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 2764 × 0308 ÷ 0020 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]
-÷ 2764 ÷ 000D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 2764 × 0308 ÷ 000D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
-÷ 2764 ÷ 000A ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 2764 × 0308 ÷ 000A ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
-÷ 2764 ÷ 0001 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 2764 × 0308 ÷ 0001 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
-÷ 2764 × 0300 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]
-÷ 2764 × 0308 × 0300 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]
-÷ 2764 ÷ 0600 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 2764 × 0308 ÷ 0600 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
-÷ 2764 × 0903 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 2764 × 0308 × 0903 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]
-÷ 2764 ÷ 1100 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 2764 × 0308 ÷ 1100 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
-÷ 2764 ÷ 1160 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 2764 × 0308 ÷ 1160 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
-÷ 2764 ÷ 11A8 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 2764 × 0308 ÷ 11A8 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
-÷ 2764 ÷ AC00 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 2764 × 0308 ÷ AC00 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
-÷ 2764 ÷ AC01 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 2764 × 0308 ÷ AC01 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
-÷ 2764 ÷ 1F1E6 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 2764 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
-÷ 2764 ÷ 261D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3]
-÷ 2764 × 0308 ÷ 261D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3]
-÷ 2764 ÷ 1F3FB ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
-÷ 2764 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
-÷ 2764 × 200D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 2764 × 0308 × 200D ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 2764 ÷ 2764 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 2764 × 0308 ÷ 2764 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 2764 ÷ 1F466 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] BOY (EBG) ÷ [0.3]
-÷ 2764 × 0308 ÷ 1F466 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
-÷ 2764 ÷ 0378 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 2764 × 0308 ÷ 0378 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
-÷ 2764 ÷ D800 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3]
-÷ 2764 × 0308 ÷ D800 ÷ # ÷ [0.2] HEAVY BLACK HEART (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3]
+÷ 2640 ÷ 0020 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] SPACE (Other) ÷ [0.3]
+÷ 2640 × 0308 ÷ 0020 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]
+÷ 2640 ÷ 000D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
+÷ 2640 × 0308 ÷ 000D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
+÷ 2640 ÷ 000A ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
+÷ 2640 × 0308 ÷ 000A ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3]
+÷ 2640 ÷ 0001 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
+÷ 2640 × 0308 ÷ 0001 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3]
+÷ 2640 × 0300 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]
+÷ 2640 × 0308 × 0300 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]
+÷ 2640 ÷ 0600 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
+÷ 2640 × 0308 ÷ 0600 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3]
+÷ 2640 × 0903 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]
+÷ 2640 × 0308 × 0903 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]
+÷ 2640 ÷ 1100 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
+÷ 2640 × 0308 ÷ 1100 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]
+÷ 2640 ÷ 1160 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
+÷ 2640 × 0308 ÷ 1160 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]
+÷ 2640 ÷ 11A8 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
+÷ 2640 × 0308 ÷ 11A8 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]
+÷ 2640 ÷ AC00 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
+÷ 2640 × 0308 ÷ AC00 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]
+÷ 2640 ÷ AC01 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
+÷ 2640 × 0308 ÷ AC01 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]
+÷ 2640 ÷ 1F1E6 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
+÷ 2640 × 0308 ÷ 1F1E6 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3]
+÷ 2640 ÷ 261D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3]
+÷ 2640 × 0308 ÷ 261D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3]
+÷ 2640 ÷ 1F3FB ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
+÷ 2640 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
+÷ 2640 × 200D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
+÷ 2640 × 0308 × 200D ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
+÷ 2640 ÷ 2640 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 2640 × 0308 ÷ 2640 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 2640 ÷ 1F466 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] BOY (EBG) ÷ [0.3]
+÷ 2640 × 0308 ÷ 1F466 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
+÷ 2640 ÷ 0378 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
+÷ 2640 × 0308 ÷ 0378 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
+÷ 2640 ÷ D800 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3]
+÷ 2640 × 0308 ÷ D800 ÷ # ÷ [0.2] FEMALE SIGN (Glue_After_Zwj) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3]
÷ 1F466 ÷ 0020 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] SPACE (Other) ÷ [0.3]
÷ 1F466 × 0308 ÷ 0020 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]
÷ 1F466 ÷ 000D ÷ # ÷ [0.2] BOY (EBG) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3]
@@ -733,8 +734,8 @@
÷ 1F466 × 0308 × 1F3FB ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 1F466 × 200D ÷ # ÷ [0.2] BOY (EBG) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 1F466 × 0308 × 200D ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 1F466 ÷ 2764 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 1F466 × 0308 ÷ 2764 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 1F466 ÷ 2640 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 1F466 × 0308 ÷ 2640 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 1F466 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1F466 × 0308 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 1F466 ÷ 0378 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -773,8 +774,8 @@
÷ 0378 × 0308 ÷ 1F3FB ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 0378 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ 0378 × 0308 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ 0378 ÷ 2764 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ 0378 × 0308 ÷ 2764 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 0378 ÷ 2640 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ 0378 × 0308 ÷ 2640 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 0378 ÷ 1F466 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0378 × 0308 ÷ 1F466 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ 0378 ÷ 0378 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3]
@@ -813,8 +814,8 @@
÷ D800 ÷ 0308 ÷ 1F3FB ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ D800 ÷ 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
÷ D800 ÷ 0308 × 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] ZERO WIDTH JOINER (ZWJ) ÷ [0.3]
-÷ D800 ÷ 2764 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
-÷ D800 ÷ 0308 ÷ 2764 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ D800 ÷ 2640 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
+÷ D800 ÷ 0308 ÷ 2640 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ D800 ÷ 1F466 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] BOY (EBG) ÷ [0.3]
÷ D800 ÷ 0308 ÷ 1F466 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] BOY (EBG) ÷ [0.3]
÷ D800 ÷ 0378 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3]
@@ -840,7 +841,7 @@
÷ 261D × 1F3FB ÷ 261D ÷ # ÷ [0.2] WHITE UP POINTING INDEX (E_Base) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [999.0] WHITE UP POINTING INDEX (E_Base) ÷ [0.3]
÷ 1F466 × 1F3FB ÷ # ÷ [0.2] BOY (EBG) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
÷ 200D × 1F466 × 1F3FB ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) × [10.0] EMOJI MODIFIER FITZPATRICK TYPE-1-2 (E_Modifier) ÷ [0.3]
-÷ 200D × 2764 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] HEAVY BLACK HEART (Glue_After_Zwj) ÷ [0.3]
+÷ 200D × 2640 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] FEMALE SIGN (Glue_After_Zwj) ÷ [0.3]
÷ 200D × 1F466 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ) × [11.0] BOY (EBG) ÷ [0.3]
÷ 1F466 ÷ 1F466 ÷ # ÷ [0.2] BOY (EBG) ÷ [999.0] BOY (EBG) ÷ [0.3]
#
diff --git a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt
index 05efcf5a44..6715446aba 100644
--- a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt
+++ b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt
@@ -1,25 +1,28 @@
-# LineBreakTest-9.0.0.txt
-# Date: 2016-06-18, 00:42:06 GMT
-# © 2016 Unicode®, Inc.
+# LineBreakTest-10.0.0.txt
+# Date: 2017-04-14, 05:40:30 GMT
+# © 2017 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# Unicode Character Database
# For documentation, see http://www.unicode.org/reports/tr44/
#
-# Default Line Break Test
+# Default Line_Break Test
#
# Format:
-# <string> (# <comment>)?
-# <string> contains hex Unicode code points, with
-# ÷ wherever there is a break opportunity, and
+# <string> (# <comment>)?
+# <string> contains hex Unicode code points, with
+# ÷ wherever there is a break opportunity, and
# × wherever there is not.
# <comment> the format can change, but currently it shows:
# - the sample character name
# - (x) the Line_Break property value for the sample character
-# - [x] the rule that determines whether there is a break or not
-# Note: The Line Break tests use tailoring of numbers described in Example 7 of Section 8.2 Examples of Customization.
-# They also differ from the results produced by a pair table implementation in sequences like: ZW SP CL.
+# - [x] the rule that determines whether there is a break or not,
+# as listed in the Rules section of LineBreakTest.html
+#
+# Note:
+# The Line_Break tests use tailoring of numbers described in
+# Example 7 of Section 8.2, "Examples of Customization" of UAX #14.
#
# These samples may be extended or changed in the future.
#
diff --git a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt
index e133fa8a78..71f2371c5e 100644
--- a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt
+++ b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt
@@ -1,6 +1,6 @@
-# NormalizationTest-9.0.0.txt
-# Date: 2016-04-04, 11:41:55 GMT
-# © 2016 Unicode®, Inc.
+# NormalizationTest-10.0.0.txt
+# Date: 2017-03-08, 08:41:55 GMT
+# © 2017 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
@@ -17653,6 +17653,10 @@ FFEE;FFEE;FFEE;25CB;25CB; # (○; ○; ○; ○; ○; ) HALFWIDTH WHITE CIRCLE
0061 0CBC 3099 093C 0334 0062;0061 0334 0CBC 093C 3099 0062;0061 0334 0CBC 093C 3099 0062;0061 0334 0CBC 093C 3099 0062;0061 0334 0CBC 093C 3099 0062; # (a◌಼◌゙◌़◌̴b; a◌̴◌಼◌़◌゙b; a◌̴◌಼◌़◌゙b; a◌̴◌಼◌़◌゙b; a◌̴◌಼◌़◌゙b; ) LATIN SMALL LETTER A, KANNADA SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B
0061 05B0 094D 3099 0CCD 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062; # (a◌ְ◌्◌゙◌್b; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, KANNADA SIGN VIRAMA, LATIN SMALL LETTER B
0061 0CCD 05B0 094D 3099 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062; # (a◌್◌ְ◌्◌゙b; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; ) LATIN SMALL LETTER A, KANNADA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
+0061 05B0 094D 3099 0D3B 0062;0061 3099 094D 0D3B 05B0 0062;0061 3099 094D 0D3B 05B0 0062;0061 3099 094D 0D3B 05B0 0062;0061 3099 094D 0D3B 05B0 0062; # (a◌ְ◌्◌゙◌഻b; a◌゙◌्◌഻◌ְb; a◌゙◌्◌഻◌ְb; a◌゙◌्◌഻◌ְb; a◌゙◌्◌഻◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MALAYALAM SIGN VERTICAL BAR VIRAMA, LATIN SMALL LETTER B
+0061 0D3B 05B0 094D 3099 0062;0061 3099 0D3B 094D 05B0 0062;0061 3099 0D3B 094D 05B0 0062;0061 3099 0D3B 094D 05B0 0062;0061 3099 0D3B 094D 05B0 0062; # (a◌഻◌ְ◌्◌゙b; a◌゙◌഻◌्◌ְb; a◌゙◌഻◌्◌ְb; a◌゙◌഻◌्◌ְb; a◌゙◌഻◌्◌ְb; ) LATIN SMALL LETTER A, MALAYALAM SIGN VERTICAL BAR VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
+0061 05B0 094D 3099 0D3C 0062;0061 3099 094D 0D3C 05B0 0062;0061 3099 094D 0D3C 05B0 0062;0061 3099 094D 0D3C 05B0 0062;0061 3099 094D 0D3C 05B0 0062; # (a◌ְ◌्◌゙◌഼b; a◌゙◌्◌഼◌ְb; a◌゙◌्◌഼◌ְb; a◌゙◌्◌഼◌ְb; a◌゙◌्◌഼◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MALAYALAM SIGN CIRCULAR VIRAMA, LATIN SMALL LETTER B
+0061 0D3C 05B0 094D 3099 0062;0061 3099 0D3C 094D 05B0 0062;0061 3099 0D3C 094D 05B0 0062;0061 3099 0D3C 094D 05B0 0062;0061 3099 0D3C 094D 05B0 0062; # (a◌഼◌ְ◌्◌゙b; a◌゙◌഼◌्◌ְb; a◌゙◌഼◌्◌ְb; a◌゙◌഼◌्◌ְb; a◌゙◌഼◌्◌ְb; ) LATIN SMALL LETTER A, MALAYALAM SIGN CIRCULAR VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
0061 05B0 094D 3099 0D4D 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062; # (a◌ְ◌्◌゙◌്b; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MALAYALAM SIGN VIRAMA, LATIN SMALL LETTER B
0061 0D4D 05B0 094D 3099 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062; # (a◌്◌ְ◌्◌゙b; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; ) LATIN SMALL LETTER A, MALAYALAM SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
0061 05B0 094D 3099 0DCA 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062; # (a◌ְ◌्◌゙◌්b; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, SINHALA SIGN AL-LAKUNA, LATIN SMALL LETTER B
@@ -17999,6 +18003,14 @@ FFEE;FFEE;FFEE;25CB;25CB; # (○; ○; ○; ○; ○; ) HALFWIDTH WHITE CIRCLE
0061 1DF4 0315 0300 05AE 0062;0061 05AE 1DF4 0300 0315 0062;0061 05AE 1DF4 0300 0315 0062;0061 05AE 1DF4 0300 0315 0062;0061 05AE 1DF4 0300 0315 0062; # (a◌ᷴ◌̕◌̀◌֮b; a◌֮◌ᷴ◌̀◌̕b; a◌֮◌ᷴ◌̀◌̕b; a◌֮◌ᷴ◌̀◌̕b; a◌֮◌ᷴ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER U WITH DIAERESIS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B
0061 0315 0300 05AE 1DF5 0062;00E0 05AE 1DF5 0315 0062;0061 05AE 0300 1DF5 0315 0062;00E0 05AE 1DF5 0315 0062;0061 05AE 0300 1DF5 0315 0062; # (a◌̕◌̀◌֮◌᷵b; à◌֮◌᷵◌̕b; a◌֮◌̀◌᷵◌̕b; à◌֮◌᷵◌̕b; a◌֮◌̀◌᷵◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING UP TACK ABOVE, LATIN SMALL LETTER B
0061 1DF5 0315 0300 05AE 0062;0061 05AE 1DF5 0300 0315 0062;0061 05AE 1DF5 0300 0315 0062;0061 05AE 1DF5 0300 0315 0062;0061 05AE 1DF5 0300 0315 0062; # (a◌᷵◌̕◌̀◌֮b; a◌֮◌᷵◌̀◌̕b; a◌֮◌᷵◌̀◌̕b; a◌֮◌᷵◌̀◌̕b; a◌֮◌᷵◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING UP TACK ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B
+0061 035C 0315 0300 1DF6 0062;00E0 0315 1DF6 035C 0062;0061 0300 0315 1DF6 035C 0062;00E0 0315 1DF6 035C 0062;0061 0300 0315 1DF6 035C 0062; # (a◌͜◌̕◌̀◌᷶b; à◌̕◌᷶◌͜b; a◌̀◌̕◌᷶◌͜b; à◌̕◌᷶◌͜b; a◌̀◌̕◌᷶◌͜b; ) LATIN SMALL LETTER A, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, COMBINING KAVYKA ABOVE RIGHT, LATIN SMALL LETTER B
+0061 1DF6 035C 0315 0300 0062;00E0 1DF6 0315 035C 0062;0061 0300 1DF6 0315 035C 0062;00E0 1DF6 0315 035C 0062;0061 0300 1DF6 0315 035C 0062; # (a◌᷶◌͜◌̕◌̀b; à◌᷶◌̕◌͜b; a◌̀◌᷶◌̕◌͜b; à◌᷶◌̕◌͜b; a◌̀◌᷶◌̕◌͜b; ) LATIN SMALL LETTER A, COMBINING KAVYKA ABOVE RIGHT, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B
+0061 0300 05AE 1D16D 1DF7 0062;00E0 1D16D 05AE 1DF7 0062;0061 1D16D 05AE 1DF7 0300 0062;00E0 1D16D 05AE 1DF7 0062;0061 1D16D 05AE 1DF7 0300 0062; # (a◌̀◌𝅭֮◌᷷b; à𝅭◌֮◌᷷b; a𝅭◌֮◌᷷◌̀b; à𝅭◌֮◌᷷b; a𝅭◌֮◌᷷◌̀b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, COMBINING KAVYKA ABOVE LEFT, LATIN SMALL LETTER B
+0061 1DF7 0300 05AE 1D16D 0062;00E0 1D16D 1DF7 05AE 0062;0061 1D16D 1DF7 05AE 0300 0062;00E0 1D16D 1DF7 05AE 0062;0061 1D16D 1DF7 05AE 0300 0062; # (a◌᷷◌̀◌𝅭֮b; à𝅭◌᷷◌֮b; a𝅭◌᷷◌֮◌̀b; à𝅭◌᷷◌֮b; a𝅭◌᷷◌֮◌̀b; ) LATIN SMALL LETTER A, COMBINING KAVYKA ABOVE LEFT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B
+0061 0300 05AE 1D16D 1DF8 0062;00E0 1D16D 05AE 1DF8 0062;0061 1D16D 05AE 1DF8 0300 0062;00E0 1D16D 05AE 1DF8 0062;0061 1D16D 05AE 1DF8 0300 0062; # (a◌̀◌𝅭֮◌᷸b; à𝅭◌֮◌᷸b; a𝅭◌֮◌᷸◌̀b; à𝅭◌֮◌᷸b; a𝅭◌֮◌᷸◌̀b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, COMBINING DOT ABOVE LEFT, LATIN SMALL LETTER B
+0061 1DF8 0300 05AE 1D16D 0062;00E0 1D16D 1DF8 05AE 0062;0061 1D16D 1DF8 05AE 0300 0062;00E0 1D16D 1DF8 05AE 0062;0061 1D16D 1DF8 05AE 0300 0062; # (a◌᷸◌̀◌𝅭֮b; à𝅭◌᷸◌֮b; a𝅭◌᷸◌֮◌̀b; à𝅭◌᷸◌֮b; a𝅭◌᷸◌֮◌̀b; ) LATIN SMALL LETTER A, COMBINING DOT ABOVE LEFT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B
+0061 059A 0316 302A 1DF9 0062;0061 302A 0316 1DF9 059A 0062;0061 302A 0316 1DF9 059A 0062;0061 302A 0316 1DF9 059A 0062;0061 302A 0316 1DF9 059A 0062; # (a◌֚◌̖◌〪◌᷹b; a◌〪◌̖◌᷹◌֚b; a◌〪◌̖◌᷹◌֚b; a◌〪◌̖◌᷹◌֚b; a◌〪◌̖◌᷹◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING WIDE INVERTED BRIDGE BELOW, LATIN SMALL LETTER B
+0061 1DF9 059A 0316 302A 0062;0061 302A 1DF9 0316 059A 0062;0061 302A 1DF9 0316 059A 0062;0061 302A 1DF9 0316 059A 0062;0061 302A 1DF9 0316 059A 0062; # (a◌᷹◌֚◌̖◌〪b; a◌〪◌᷹◌̖◌֚b; a◌〪◌᷹◌̖◌֚b; a◌〪◌᷹◌̖◌֚b; a◌〪◌᷹◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING WIDE INVERTED BRIDGE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B
0061 0315 0300 05AE 1DFB 0062;00E0 05AE 1DFB 0315 0062;0061 05AE 0300 1DFB 0315 0062;00E0 05AE 1DFB 0315 0062;0061 05AE 0300 1DFB 0315 0062; # (a◌̕◌̀◌֮◌᷻b; à◌֮◌᷻◌̕b; a◌֮◌̀◌᷻◌̕b; à◌֮◌᷻◌̕b; a◌֮◌̀◌᷻◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DELETION MARK, LATIN SMALL LETTER B
0061 1DFB 0315 0300 05AE 0062;0061 05AE 1DFB 0300 0315 0062;0061 05AE 1DFB 0300 0315 0062;0061 05AE 1DFB 0300 0315 0062;0061 05AE 1DFB 0300 0315 0062; # (a◌᷻◌̕◌̀◌֮b; a◌֮◌᷻◌̀◌̕b; a◌֮◌᷻◌̀◌̕b; a◌֮◌᷻◌̀◌̕b; a◌֮◌᷻◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DELETION MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B
0061 035D 035C 0315 1DFC 0062;0061 0315 035C 1DFC 035D 0062;0061 0315 035C 1DFC 035D 0062;0061 0315 035C 1DFC 035D 0062;0061 0315 035C 1DFC 035D 0062; # (a◌͝◌͜◌̕◌᷼b; a◌̕◌͜◌᷼◌͝b; a◌̕◌͜◌᷼◌͝b; a◌̕◌͜◌᷼◌͝b; a◌̕◌͜◌᷼◌͝b; ) LATIN SMALL LETTER A, COMBINING DOUBLE BREVE, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING DOUBLE INVERTED BREVE BELOW, LATIN SMALL LETTER B
@@ -18397,8 +18409,20 @@ FFEE;FFEE;FFEE;25CB;25CB; # (○; ○; ○; ○; ○; ) HALFWIDTH WHITE CIRCLE
0061 116B7 3099 093C 0334 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062;0061 0334 116B7 093C 3099 0062; # (a◌𑚷◌゙◌़◌̴b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; a◌̴◌𑚷◌़◌゙b; ) LATIN SMALL LETTER A, TAKRI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B
0061 05B0 094D 3099 1172B 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062;0061 3099 094D 1172B 05B0 0062; # (a◌ְ◌्◌゙◌𑜫b; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; a◌゙◌्◌𑜫◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, AHOM SIGN KILLER, LATIN SMALL LETTER B
0061 1172B 05B0 094D 3099 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062;0061 3099 1172B 094D 05B0 0062; # (a◌𑜫◌ְ◌्◌゙b; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; a◌゙◌𑜫◌्◌ְb; ) LATIN SMALL LETTER A, AHOM SIGN KILLER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
+0061 05B0 094D 3099 11A34 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062; # (a◌ְ◌्◌゙◌𑨴b; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SIGN VIRAMA, LATIN SMALL LETTER B
+0061 11A34 05B0 094D 3099 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062; # (a◌𑨴◌ְ◌्◌゙b; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; ) LATIN SMALL LETTER A, ZANABAZAR SQUARE SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
+0061 05B0 094D 3099 11A47 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062; # (a◌ְ◌्◌゙◌𑩇b; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SUBJOINER, LATIN SMALL LETTER B
+0061 11A47 05B0 094D 3099 0062;0061 3099 11A47 094D 05B0 0062;0061 3099 11A47 094D 05B0 0062;0061 3099 11A47 094D 05B0 0062;0061 3099 11A47 094D 05B0 0062; # (a◌𑩇◌ְ◌्◌゙b; a◌゙◌𑩇◌्◌ְb; a◌゙◌𑩇◌्◌ְb; a◌゙◌𑩇◌्◌ְb; a◌゙◌𑩇◌्◌ְb; ) LATIN SMALL LETTER A, ZANABAZAR SQUARE SUBJOINER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
+0061 05B0 094D 3099 11A99 0062;0061 3099 094D 11A99 05B0 0062;0061 3099 094D 11A99 05B0 0062;0061 3099 094D 11A99 05B0 0062;0061 3099 094D 11A99 05B0 0062; # (a◌ְ◌्◌゙◌𑪙b; a◌゙◌्◌𑪙◌ְb; a◌゙◌्◌𑪙◌ְb; a◌゙◌्◌𑪙◌ְb; a◌゙◌्◌𑪙◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, SOYOMBO SUBJOINER, LATIN SMALL LETTER B
+0061 11A99 05B0 094D 3099 0062;0061 3099 11A99 094D 05B0 0062;0061 3099 11A99 094D 05B0 0062;0061 3099 11A99 094D 05B0 0062;0061 3099 11A99 094D 05B0 0062; # (a◌𑪙◌ְ◌्◌゙b; a◌゙◌𑪙◌्◌ְb; a◌゙◌𑪙◌्◌ְb; a◌゙◌𑪙◌्◌ְb; a◌゙◌𑪙◌्◌ְb; ) LATIN SMALL LETTER A, SOYOMBO SUBJOINER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
0061 05B0 094D 3099 11C3F 0062;0061 3099 094D 11C3F 05B0 0062;0061 3099 094D 11C3F 05B0 0062;0061 3099 094D 11C3F 05B0 0062;0061 3099 094D 11C3F 05B0 0062; # (a◌ְ◌्◌゙◌𑰿b; a◌゙◌्◌𑰿◌ְb; a◌゙◌्◌𑰿◌ְb; a◌゙◌्◌𑰿◌ְb; a◌゙◌्◌𑰿◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, BHAIKSUKI SIGN VIRAMA, LATIN SMALL LETTER B
0061 11C3F 05B0 094D 3099 0062;0061 3099 11C3F 094D 05B0 0062;0061 3099 11C3F 094D 05B0 0062;0061 3099 11C3F 094D 05B0 0062;0061 3099 11C3F 094D 05B0 0062; # (a◌𑰿◌ְ◌्◌゙b; a◌゙◌𑰿◌्◌ְb; a◌゙◌𑰿◌्◌ְb; a◌゙◌𑰿◌्◌ְb; a◌゙◌𑰿◌्◌ְb; ) LATIN SMALL LETTER A, BHAIKSUKI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
+0061 3099 093C 0334 11D42 0062;0061 0334 093C 11D42 3099 0062;0061 0334 093C 11D42 3099 0062;0061 0334 093C 11D42 3099 0062;0061 0334 093C 11D42 3099 0062; # (a◌゙◌़◌̴◌𑵂b; a◌̴◌़◌𑵂◌゙b; a◌̴◌़◌𑵂◌゙b; a◌̴◌़◌𑵂◌゙b; a◌̴◌़◌𑵂◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, MASARAM GONDI SIGN NUKTA, LATIN SMALL LETTER B
+0061 11D42 3099 093C 0334 0062;0061 0334 11D42 093C 3099 0062;0061 0334 11D42 093C 3099 0062;0061 0334 11D42 093C 3099 0062;0061 0334 11D42 093C 3099 0062; # (a◌𑵂◌゙◌़◌̴b; a◌̴◌𑵂◌़◌゙b; a◌̴◌𑵂◌़◌゙b; a◌̴◌𑵂◌़◌゙b; a◌̴◌𑵂◌़◌゙b; ) LATIN SMALL LETTER A, MASARAM GONDI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B
+0061 05B0 094D 3099 11D44 0062;0061 3099 094D 11D44 05B0 0062;0061 3099 094D 11D44 05B0 0062;0061 3099 094D 11D44 05B0 0062;0061 3099 094D 11D44 05B0 0062; # (a◌ְ◌्◌゙◌𑵄b; a◌゙◌्◌𑵄◌ְb; a◌゙◌्◌𑵄◌ְb; a◌゙◌्◌𑵄◌ְb; a◌゙◌्◌𑵄◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MASARAM GONDI SIGN HALANTA, LATIN SMALL LETTER B
+0061 11D44 05B0 094D 3099 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062;0061 3099 11D44 094D 05B0 0062; # (a◌𑵄◌ְ◌्◌゙b; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; a◌゙◌𑵄◌्◌ְb; ) LATIN SMALL LETTER A, MASARAM GONDI SIGN HALANTA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
+0061 05B0 094D 3099 11D45 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062;0061 3099 094D 11D45 05B0 0062; # (a◌ְ◌्◌゙◌𑵅b; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; a◌゙◌्◌𑵅◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MASARAM GONDI VIRAMA, LATIN SMALL LETTER B
+0061 11D45 05B0 094D 3099 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062; # (a◌𑵅◌ְ◌्◌゙b; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; a◌゙◌𑵅◌्◌ְb; ) LATIN SMALL LETTER A, MASARAM GONDI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B
0061 093C 0334 16AF0 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062;0061 0334 16AF0 093C 0062; # (a◌़◌̴◌𖫰b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; a◌̴◌𖫰◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING HIGH TONE, LATIN SMALL LETTER B
0061 16AF0 093C 0334 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062;0061 16AF0 0334 093C 0062; # (a◌𖫰◌़◌̴b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; a◌𖫰◌̴◌़b; ) LATIN SMALL LETTER A, BASSA VAH COMBINING HIGH TONE, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B
0061 093C 0334 16AF1 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062;0061 0334 16AF1 093C 0062; # (a◌़◌̴◌𖫱b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; a◌̴◌𖫱◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING LOW TONE, LATIN SMALL LETTER B
diff --git a/lib/stdlib/test/uri_string_SUITE.erl b/lib/stdlib/test/uri_string_SUITE.erl
new file mode 100644
index 0000000000..c625da56c6
--- /dev/null
+++ b/lib/stdlib/test/uri_string_SUITE.erl
@@ -0,0 +1,844 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(uri_string_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-export([all/0, suite/0,groups/0,
+ normalize/1,
+ parse_binary_fragment/1, parse_binary_host/1, parse_binary_host_ipv4/1,
+ parse_binary_host_ipv6/1,
+ parse_binary_path/1, parse_binary_pct_encoded_fragment/1, parse_binary_pct_encoded_query/1,
+ parse_binary_pct_encoded_userinfo/1, parse_binary_port/1,
+ parse_binary_query/1, parse_binary_scheme/1, parse_binary_userinfo/1,
+ parse_fragment/1, parse_host/1, parse_host_ipv4/1, parse_host_ipv6/1,
+ parse_path/1, parse_pct_encoded_fragment/1, parse_pct_encoded_query/1,
+ parse_pct_encoded_userinfo/1, parse_port/1,
+ parse_query/1, parse_scheme/1, parse_userinfo/1,
+ parse_list/1, parse_binary/1, parse_mixed/1, parse_relative/1,
+ parse_special/1, parse_special2/1, parse_negative/1,
+ recompose_fragment/1, recompose_parse_fragment/1,
+ recompose_query/1, recompose_parse_query/1,
+ recompose_path/1, recompose_parse_path/1,
+ recompose_autogen/1, parse_recompose_autogen/1,
+ transcode_basic/1, transcode_options/1, transcode_mixed/1, transcode_negative/1
+ ]).
+
+
+-define(SCHEME, "foo").
+-define(USERINFO, "åsa").
+-define(USERINFO_ENC, "%C3%A5sa").
+-define(HOST, "älvsjö").
+-define(HOST_ENC, "%C3%A4lvsj%C3%B6").
+-define(IPV6, "::127.0.0.1").
+-define(IPV6_ENC, "[::127.0.0.1]").
+-define(PORT, 8042).
+-define(PORT_ENC, ":8042").
+-define(PATH, "/där").
+-define(PATH_ENC, "/d%C3%A4r").
+-define(QUERY, "name=örn").
+-define(QUERY_ENC, "?name=%C3%B6rn").
+-define(FRAGMENT, "näsa").
+-define(FRAGMENT_ENC, "#n%C3%A4sa").
+
+
+suite() ->
+ [{timetrap,{minutes,1}}].
+
+all() ->
+ [
+ normalize,
+ parse_binary_scheme,
+ parse_binary_userinfo,
+ parse_binary_pct_encoded_userinfo,
+ parse_binary_host,
+ parse_binary_host_ipv4,
+ parse_binary_host_ipv6,
+ parse_binary_port,
+ parse_binary_path,
+ parse_binary_query,
+ parse_binary_pct_encoded_query,
+ parse_binary_fragment,
+ parse_binary_pct_encoded_fragment,
+ parse_scheme,
+ parse_userinfo,
+ parse_pct_encoded_userinfo,
+ parse_host,
+ parse_host_ipv4,
+ parse_host_ipv6,
+ parse_port,
+ parse_path,
+ parse_query,
+ parse_pct_encoded_query,
+ parse_fragment,
+ parse_pct_encoded_fragment,
+ parse_list,
+ parse_binary,
+ parse_mixed,
+ parse_relative,
+ parse_special,
+ parse_special2,
+ parse_negative,
+ recompose_fragment,
+ recompose_parse_fragment,
+ recompose_query,
+ recompose_parse_query,
+ recompose_path,
+ recompose_parse_path,
+ recompose_autogen,
+ parse_recompose_autogen,
+ transcode_basic,
+ transcode_options,
+ transcode_mixed,
+ transcode_negative
+ ].
+
+groups() ->
+ [].
+
+
+%%-------------------------------------------------------------------------
+%% Helper functions
+%%-------------------------------------------------------------------------
+uri_combinations() ->
+ [[Sch,Usr,Hst,Prt,Pat,Qry,Frg] ||
+ Sch <- [fun update_scheme/1, fun update_scheme_binary/1, none],
+ Usr <- [fun update_userinfo/1, fun update_userinfo_binary/1, none],
+ Hst <- [fun update_host/1, fun update_host_binary/1,
+ fun update_ipv6/1, fun update_ipv6_binary/1, none],
+ Prt <- [fun update_port/1, none],
+ Pat <- [fun update_path/1, fun update_path_binary/1],
+ Qry <- [fun update_query/1,fun update_query_binary/1, none],
+ Frg <- [fun update_fragment/1, fun update_fragment_binary/1, none],
+ not (Usr =:= none andalso Hst =:= none andalso Prt =/= none),
+ not (Usr =/= none andalso Hst =:= none andalso Prt =:= none),
+ not (Usr =/= none andalso Hst =:= none andalso Prt =/= none)].
+
+
+generate_test_vector(Comb) ->
+ Fun = fun (F, {Map, URI}) when is_function(F) -> F({Map, URI});
+ (_, Map) -> Map
+ end,
+ lists:foldl(Fun, {#{}, empty}, Comb).
+
+generate_test_vectors(L) ->
+ lists:map(fun generate_test_vector/1, L).
+
+update_fragment({In, empty}) ->
+ {In#{fragment => ?FRAGMENT}, ?FRAGMENT_ENC};
+update_fragment({In, Out}) when is_list(Out) ->
+ {In#{fragment => ?FRAGMENT}, Out ++ ?FRAGMENT_ENC};
+update_fragment({In, Out}) when is_binary(Out) ->
+ {In#{fragment => ?FRAGMENT}, binary_to_list(Out) ++ ?FRAGMENT_ENC}.
+
+update_fragment_binary({In, empty}) ->
+ {In#{fragment => <<?FRAGMENT/utf8>>}, <<?FRAGMENT_ENC>>};
+update_fragment_binary({In, Out}) when is_list(Out) ->
+ {In#{fragment => <<?FRAGMENT/utf8>>}, Out ++ ?FRAGMENT_ENC};
+update_fragment_binary({In, Out}) when is_binary(Out) ->
+ {In#{fragment => <<?FRAGMENT/utf8>>}, <<Out/binary,?FRAGMENT_ENC>>}.
+
+
+update_query({In, empty}) ->
+ {In#{query => ?QUERY}, ?QUERY_ENC};
+update_query({In, Out}) when is_list(Out) ->
+ {In#{query => ?QUERY}, Out ++ ?QUERY_ENC};
+update_query({In, Out}) when is_binary(Out) ->
+ {In#{query => ?QUERY}, binary_to_list(Out) ++ ?QUERY_ENC}.
+
+update_query_binary({In, empty}) ->
+ {In#{query => <<?QUERY/utf8>>}, <<?QUERY_ENC>>};
+update_query_binary({In, Out}) when is_list(Out) ->
+ {In#{query => <<?QUERY/utf8>>}, Out ++ ?QUERY_ENC};
+update_query_binary({In, Out}) when is_binary(Out) ->
+ {In#{query => <<?QUERY/utf8>>}, <<Out/binary,?QUERY_ENC>>}.
+
+update_path({In, empty}) ->
+ {In#{path => ?PATH}, ?PATH_ENC};
+update_path({In, Out}) when is_list(Out) ->
+ {In#{path => ?PATH}, Out ++ ?PATH_ENC};
+update_path({In, Out}) when is_binary(Out) ->
+ {In#{path => ?PATH}, binary_to_list(Out) ++ ?PATH_ENC}.
+
+update_path_binary({In, empty}) ->
+ {In#{path => <<?PATH/utf8>>}, <<?PATH_ENC>>};
+update_path_binary({In, Out}) when is_list(Out) ->
+ {In#{path => <<?PATH/utf8>>}, Out ++ ?PATH_ENC};
+update_path_binary({In, Out}) when is_binary(Out) ->
+ {In#{path => <<?PATH/utf8>>}, <<Out/binary,?PATH_ENC>>}.
+
+update_port({In, Out}) when is_list(Out) ->
+ {In#{port => ?PORT}, Out ++ ?PORT_ENC};
+update_port({In, Out}) when is_binary(Out) ->
+ {In#{port => ?PORT}, <<Out/binary,?PORT_ENC>>}.
+
+update_host({In, empty}) ->
+ {In#{host => ?HOST}, "//" ++ ?HOST_ENC};
+update_host({In, Out}) when is_list(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => ?HOST}, Out ++ [$@|?HOST_ENC]};
+ false -> {In#{host => ?HOST}, Out ++ [$/,$/|?HOST_ENC]}
+ end;
+update_host({In, Out}) when is_binary(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => ?HOST}, binary_to_list(Out) ++ [$@|?HOST_ENC]};
+ false -> {In#{host => ?HOST}, binary_to_list(Out) ++ [$/,$/|?HOST_ENC]}
+ end.
+
+update_host_binary({In, empty}) ->
+ {In#{host => <<?HOST/utf8>>}, <<"//",?HOST_ENC>>};
+update_host_binary({In, Out}) when is_list(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => <<?HOST/utf8>>}, Out ++ [$@|?HOST_ENC]};
+ false -> {In#{host => <<?HOST/utf8>>}, Out ++ [$/,$/|?HOST_ENC]}
+ end;
+update_host_binary({In, Out}) when is_binary(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => <<?HOST/utf8>>}, <<Out/binary,$@,?HOST_ENC>>};
+ false-> {In#{host => <<?HOST/utf8>>}, <<Out/binary,"//",?HOST_ENC>>}
+ end.
+
+update_ipv6({In, empty}) ->
+ {In#{host => ?IPV6}, "//" ++ ?IPV6_ENC};
+update_ipv6({In, Out}) when is_list(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => ?IPV6}, Out ++ [$@|?IPV6_ENC]};
+ false -> {In#{host => ?IPV6}, Out ++ [$/,$/|?IPV6_ENC]}
+ end;
+update_ipv6({In, Out}) when is_binary(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => ?IPV6}, binary_to_list(Out) ++ [$@|?IPV6_ENC]};
+ false -> {In#{host => ?IPV6}, binary_to_list(Out) ++ [$/,$/|?IPV6_ENC]}
+ end.
+
+update_ipv6_binary({In, empty}) ->
+ {In#{host => <<?IPV6/utf8>>}, <<"//",?IPV6_ENC>>};
+update_ipv6_binary({In, Out}) when is_list(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => <<?IPV6/utf8>>}, Out ++ [$@|?IPV6_ENC]};
+ false -> {In#{host => <<?IPV6/utf8>>}, Out ++ [$/,$/|?IPV6_ENC]}
+ end;
+update_ipv6_binary({In, Out}) when is_binary(Out) ->
+ case maps:is_key(userinfo, In) of
+ true -> {In#{host => <<?IPV6/utf8>>}, <<Out/binary,$@,?IPV6_ENC>>};
+ false-> {In#{host => <<?IPV6/utf8>>}, <<Out/binary,"//",?IPV6_ENC>>}
+ end.
+
+update_userinfo({In, empty}) ->
+ {In#{userinfo => ?USERINFO}, "//" ++ ?USERINFO_ENC};
+update_userinfo({In, Out}) when is_list(Out) ->
+ {In#{userinfo => ?USERINFO}, Out ++ "//" ++ ?USERINFO_ENC};
+update_userinfo({In, Out}) when is_binary(Out) ->
+ {In#{userinfo => ?USERINFO}, binary_to_list(Out) ++ "//" ++ ?USERINFO_ENC}.
+
+update_userinfo_binary({In, empty}) ->
+ {In#{userinfo => <<?USERINFO/utf8>>}, <<"//",?USERINFO_ENC>>};
+update_userinfo_binary({In, Out}) when is_list(Out) ->
+ {In#{userinfo => <<?USERINFO/utf8>>}, Out ++ "//" ++ ?USERINFO_ENC};
+update_userinfo_binary({In, Out}) when is_binary(Out) ->
+ {In#{userinfo => <<?USERINFO/utf8>>}, <<Out/binary,"//",?USERINFO_ENC>>}.
+
+update_scheme({In, empty}) ->
+ {In#{scheme => ?SCHEME}, ?SCHEME ++ ":"}.
+
+update_scheme_binary({In, empty}) ->
+ {In#{scheme => <<?SCHEME/utf8>>}, <<?SCHEME,$:>>}.
+
+
+%% Test recompose on a generated test vector
+run_test_recompose({#{}, empty}) ->
+ try "" = uri_string:recompose(#{}) of
+ _ -> ok
+ catch
+ _:_ -> error({test_failed, #{}, ""})
+ end;
+run_test_recompose({Map, URI}) ->
+ try URI = uri_string:recompose(Map) of
+ URI -> ok
+ catch
+ _:_ -> error({test_failed, Map, URI})
+ end.
+
+%% Test parse - recompose on a generated test vector
+run_test_parse_recompose({#{}, empty}) ->
+ try "" = uri_string:recompose(uri_string:parse("")) of
+ _ -> ok
+ catch
+ _:_ -> error({test_failed, #{}, ""})
+ end;
+run_test_parse_recompose({Map, URI}) ->
+ try URI = uri_string:recompose(uri_string:parse(URI)) of
+ URI -> ok
+ catch
+ _:_ -> error({test_failed, Map, URI})
+ end.
+
+
+%%-------------------------------------------------------------------------
+%% Parse tests
+%%-------------------------------------------------------------------------
+
+parse_binary_scheme(_Config) ->
+ #{} = uri_string:parse(<<>>),
+ #{path := <<"foo">>} = uri_string:parse(<<"foo">>),
+ #{scheme := <<"foo">>} = uri_string:parse(<<"foo:">>),
+ #{scheme := <<"foo">>, path := <<"bar:nisse">>} = uri_string:parse(<<"foo:bar:nisse">>),
+ #{scheme := <<"foo">>, host := <<"">>} = uri_string:parse(<<"foo://">>),
+ #{scheme := <<"foo">>, host := <<"">>, path := <<"/">>} = uri_string:parse(<<"foo:///">>),
+ #{scheme := <<"foo">>, host := <<"">>, path := <<"//">>} = uri_string:parse(<<"foo:////">>),
+
+ #{path := <<"/">>} = uri_string:parse(<<"/">>),
+ #{host := <<>>} = uri_string:parse(<<"//">>),
+ #{host := <<>>, path := <<"/">>} = uri_string:parse(<<"///">>).
+
+parse_binary_userinfo(_Config) ->
+ #{scheme := <<"user">>, path := <<"password@localhost">>} =
+ uri_string:parse(<<"user:password@localhost">>),
+ #{path := <<"user@">>} = uri_string:parse(<<"user@">>),
+ #{path := <<"/user@">>} = uri_string:parse(<<"/user@">>),
+ #{path := <<"user@localhost">>} = uri_string:parse(<<"user@localhost">>),
+ #{userinfo := <<"user">>, host := <<"localhost">>} = uri_string:parse(<<"//user@localhost">>),
+ #{userinfo := <<"user:password">>, host := <<"localhost">>} =
+ uri_string:parse(<<"//user:password@localhost">>),
+ #{scheme := <<"foo">>, path := <<"/user@">>} =
+ uri_string:parse(<<"foo:/user@">>),
+ #{scheme := <<"foo">>, userinfo := <<"user">>, host := <<"localhost">>} =
+ uri_string:parse(<<"foo://user@localhost">>),
+ #{scheme := <<"foo">>, userinfo := <<"user:password">>, host := <<"localhost">>} =
+ uri_string:parse(<<"foo://user:password@localhost">>).
+
+parse_binary_pct_encoded_userinfo(_Config) ->
+ #{scheme := <<"user">>, path := <<"合@気道"/utf8>>} =
+ uri_string:parse(<<"user:%E5%90%88@%E6%B0%97%E9%81%93">>),
+ #{path := <<"合気道@"/utf8>>} = uri_string:parse(<<"%E5%90%88%E6%B0%97%E9%81%93@">>),
+ #{path := <<"/合気道@"/utf8>>} = uri_string:parse(<<"/%E5%90%88%E6%B0%97%E9%81%93@">>),
+ #{path := <<"合@気道"/utf8>>} = uri_string:parse(<<"%E5%90%88@%E6%B0%97%E9%81%93">>),
+ #{userinfo := <<"合"/utf8>>, host := <<"気道"/utf8>>} =
+ uri_string:parse(<<"//%E5%90%88@%E6%B0%97%E9%81%93">>),
+ #{userinfo := <<"合:気"/utf8>>, host := <<"道"/utf8>>} =
+ uri_string:parse(<<"//%E5%90%88:%E6%B0%97@%E9%81%93">>),
+ #{scheme := <<"foo">>, path := <<"/合気道@"/utf8>>} =
+ uri_string:parse(<<"foo:/%E5%90%88%E6%B0%97%E9%81%93@">>),
+ #{scheme := <<"foo">>, userinfo := <<"合"/utf8>>, host := <<"気道"/utf8>>} =
+ uri_string:parse(<<"foo://%E5%90%88@%E6%B0%97%E9%81%93">>),
+ #{scheme := <<"foo">>, userinfo := <<"合:気"/utf8>>, host := <<"道"/utf8>>} =
+ uri_string:parse(<<"foo://%E5%90%88:%E6%B0%97@%E9%81%93">>),
+ {error,invalid_uri,"@"} = uri_string:parse(<<"//%E5%90%88@%E6%B0%97%E9%81%93@">>),
+ {error,invalid_uri,":"} = uri_string:parse(<<"foo://%E5%90%88@%E6%B0%97%E9%81%93@">>).
+
+parse_binary_host(_Config) ->
+ #{host := <<"hostname">>} = uri_string:parse(<<"//hostname">>),
+ #{host := <<"hostname">>,scheme := <<"foo">>} = uri_string:parse(<<"foo://hostname">>),
+ #{host := <<"hostname">>,scheme := <<"foo">>, userinfo := <<"user">>} =
+ uri_string:parse(<<"foo://user@hostname">>).
+
+parse_binary_host_ipv4(_Config) ->
+ #{host := <<"127.0.0.1">>} = uri_string:parse(<<"//127.0.0.1">>),
+ #{host := <<"127.0.0.1">>, path := <<"/over/there">>} =
+ uri_string:parse(<<"//127.0.0.1/over/there">>),
+ #{host := <<"127.0.0.1">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"//127.0.0.1?name=ferret">>),
+ #{host := <<"127.0.0.1">>, fragment := <<"nose">>} = uri_string:parse(<<"//127.0.0.1#nose">>),
+ {error,invalid_uri,"x"} = uri_string:parse(<<"//127.0.0.x">>),
+ {error,invalid_uri,"1227.0.0.1"} = uri_string:parse(<<"//1227.0.0.1">>).
+
+parse_binary_host_ipv6(_Config) ->
+ #{host := <<"::127.0.0.1">>} = uri_string:parse(<<"//[::127.0.0.1]">>),
+ #{host := <<"2001:0db8:0000:0000:0000:0000:1428:07ab">>} =
+ uri_string:parse(<<"//[2001:0db8:0000:0000:0000:0000:1428:07ab]">>),
+ #{host := <<"::127.0.0.1">>, path := <<"/over/there">>} =
+ uri_string:parse(<<"//[::127.0.0.1]/over/there">>),
+ #{host := <<"::127.0.0.1">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"//[::127.0.0.1]?name=ferret">>),
+ #{host := <<"::127.0.0.1">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"//[::127.0.0.1]#nose">>),
+ {error,invalid_uri,"x"} = uri_string:parse(<<"//[::127.0.0.x]">>),
+ {error,invalid_uri,"::1227.0.0.1"} = uri_string:parse(<<"//[::1227.0.0.1]">>),
+ {error,invalid_uri,"G"} = uri_string:parse(<<"//[2001:0db8:0000:0000:0000:0000:1428:G7ab]">>).
+
+parse_binary_port(_Config) ->
+ #{path:= <<"/:8042">>} =
+ uri_string:parse(<<"/:8042">>),
+ #{host:= <<>>, port := 8042} =
+ uri_string:parse(<<"//:8042">>),
+ #{host := <<"example.com">>, port:= 8042} =
+ uri_string:parse(<<"//example.com:8042">>),
+ #{scheme := <<"foo">>, path := <<"/:8042">>} =
+ uri_string:parse(<<"foo:/:8042">>),
+ #{scheme := <<"foo">>, host := <<>>, port := 8042} =
+ uri_string:parse(<<"foo://:8042">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, port := 8042} =
+ uri_string:parse(<<"foo://example.com:8042">>),
+ {error,invalid_uri,":"} = uri_string:parse(":600"),
+ {error,invalid_uri,"x"} = uri_string:parse("//:8042x").
+
+parse_binary_path(_Config) ->
+ #{path := <<"over/there">>} = uri_string:parse(<<"over/there">>),
+ #{path := <<"/over/there">>} = uri_string:parse(<<"/over/there">>),
+ #{scheme := <<"foo">>, path := <<"/over/there">>} =
+ uri_string:parse(<<"foo:/over/there">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, path := <<"/over/there">>} =
+ uri_string:parse(<<"foo://example.com/over/there">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, path := <<"/over/there">>, port := 8042} =
+ uri_string:parse(<<"foo://example.com:8042/over/there">>).
+
+parse_binary_query(_Config) ->
+ #{scheme := <<"foo">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"foo:?name=ferret">>),
+ #{scheme := <<"foo">>, path:= <<"over/there">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"foo:over/there?name=ferret">>),
+ #{scheme := <<"foo">>, path:= <<"/over/there">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"foo:/over/there?name=ferret">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"foo://example.com?name=ferret">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, path := <<"/">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"foo://example.com/?name=ferret">>),
+
+ #{path := <<>>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"?name=ferret">>),
+ #{path := <<"over/there">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"over/there?name=ferret">>),
+ #{path := <<"/">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"/?name=ferret">>),
+ #{path := <<"/over/there">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"/over/there?name=ferret">>),
+ #{host := <<"example.com">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"//example.com?name=ferret">>),
+ #{host := <<"example.com">>, path := <<"/">>, query := <<"name=ferret">>} =
+ uri_string:parse(<<"//example.com/?name=ferret">>).
+
+parse_binary_pct_encoded_query(_Config) ->
+ #{scheme := <<"foo">>, host := <<"example.com">>, path := <<"/">>,
+ query := <<"name=合気道"/utf8>>} =
+ uri_string:parse(<<"foo://example.com/?name=%E5%90%88%E6%B0%97%E9%81%93">>),
+ #{host := <<"example.com">>, path := <<"/">>, query := <<"name=合気道"/utf8>>} =
+ uri_string:parse(<<"//example.com/?name=%E5%90%88%E6%B0%97%E9%81%93">>).
+
+parse_binary_fragment(_Config) ->
+ #{scheme := <<"foo">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo:#nose">>),
+ #{scheme := <<"foo">>, path:= <<"over/there">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo:over/there#nose">>),
+ #{scheme := <<"foo">>, path:= <<"/over/there">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo:/over/there#nose">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo://example.com#nose">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, path := <<"/">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo://example.com/#nose">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo://example.com#nose">>),
+
+ #{fragment := <<"nose">>} =
+ uri_string:parse(<<"#nose">>),
+ #{path := <<"over/there">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"over/there#nose">>),
+ #{path := <<"/">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"/#nose">>),
+ #{path := <<"/over/there">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"/over/there#nose">>),
+ #{host := <<"example.com">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"//example.com#nose">>),
+ #{host := <<"example.com">>, path := <<"/">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"//example.com/#nose">>).
+
+parse_binary_pct_encoded_fragment(_Config) ->
+ #{scheme := <<"foo">>, host := <<"example.com">>, fragment := <<"合気道"/utf8>>} =
+ uri_string:parse(<<"foo://example.com#%E5%90%88%E6%B0%97%E9%81%93">>),
+ #{host := <<"example.com">>, path := <<"/">>, fragment := <<"合気道"/utf8>>} =
+ uri_string:parse(<<"//example.com/#%E5%90%88%E6%B0%97%E9%81%93">>).
+
+parse_scheme(_Config) ->
+ #{} = uri_string:parse(""),
+ #{path := "foo"} = uri_string:parse("foo"),
+ #{scheme := "foo"} = uri_string:parse("foo:"),
+ #{scheme := "foo", path := "bar:nisse"} = uri_string:parse("foo:bar:nisse"),
+ #{scheme := "foo", host := ""} = uri_string:parse("foo://"),
+ #{scheme := "foo", host := "", path := "/"} = uri_string:parse("foo:///"),
+ #{scheme := "foo", host := "", path := "//"} = uri_string:parse("foo:////"),
+
+ #{path := "/"} = uri_string:parse("/"),
+ #{host := ""} = uri_string:parse("//"),
+ #{host := "", path := "/"} = uri_string:parse("///").
+
+parse_userinfo(_Config) ->
+ #{scheme := "user", path := "password@localhost"} = uri_string:parse("user:password@localhost"),
+ #{path := "user@"} = uri_string:parse("user@"),
+ #{path := "/user@"} = uri_string:parse("/user@"),
+ #{path := "user@localhost"} = uri_string:parse("user@localhost"),
+ #{userinfo := "user", host := "localhost"} = uri_string:parse("//user@localhost"),
+ #{userinfo := "user:password", host := "localhost"} =
+ uri_string:parse("//user:password@localhost"),
+ #{scheme := "foo", path := "/user@"} =
+ uri_string:parse("foo:/user@"),
+ #{scheme := "foo", userinfo := "user", host := "localhost"} =
+ uri_string:parse("foo://user@localhost"),
+ #{scheme := "foo", userinfo := "user:password", host := "localhost"} =
+ uri_string:parse("foo://user:password@localhost").
+
+parse_pct_encoded_userinfo(_Config) ->
+ #{scheme := "user", path := "合@気道"} =
+ uri_string:parse("user:%E5%90%88@%E6%B0%97%E9%81%93"),
+ #{path := "合気道@"} = uri_string:parse("%E5%90%88%E6%B0%97%E9%81%93@"),
+ #{path := "/合気道@"} = uri_string:parse("/%E5%90%88%E6%B0%97%E9%81%93@"),
+ #{path := "合@気道"} = uri_string:parse("%E5%90%88@%E6%B0%97%E9%81%93"),
+ #{userinfo := "合", host := "気道"} =
+ uri_string:parse("//%E5%90%88@%E6%B0%97%E9%81%93"),
+ #{userinfo := "合:気", host := "道"} =
+ uri_string:parse("//%E5%90%88:%E6%B0%97@%E9%81%93"),
+ #{scheme := "foo", path := "/合気道@"} =
+ uri_string:parse("foo:/%E5%90%88%E6%B0%97%E9%81%93@"),
+ #{scheme := "foo", userinfo := "合", host := "気道"} =
+ uri_string:parse("foo://%E5%90%88@%E6%B0%97%E9%81%93"),
+ #{scheme := "foo", userinfo := "合:気", host := "道"} =
+ uri_string:parse("foo://%E5%90%88:%E6%B0%97@%E9%81%93"),
+ {error,invalid_uri,"@"} = uri_string:parse("//%E5%90%88@%E6%B0%97%E9%81%93@"),
+ {error,invalid_uri,":"} = uri_string:parse("foo://%E5%90%88@%E6%B0%97%E9%81%93@").
+
+
+parse_host(_Config) ->
+ #{host := "hostname"} = uri_string:parse("//hostname"),
+ #{host := "hostname",scheme := "foo"} = uri_string:parse("foo://hostname"),
+ #{host := "hostname",scheme := "foo", userinfo := "user"} =
+ uri_string:parse("foo://user@hostname").
+
+parse_host_ipv4(_Config) ->
+ #{host := "127.0.0.1"} = uri_string:parse("//127.0.0.1"),
+ #{host := "2001:0db8:0000:0000:0000:0000:1428:07ab"} =
+ uri_string:parse("//[2001:0db8:0000:0000:0000:0000:1428:07ab]"),
+ #{host := "127.0.0.1", path := "/over/there"} = uri_string:parse("//127.0.0.1/over/there"),
+ #{host := "127.0.0.1", query := "name=ferret"} = uri_string:parse("//127.0.0.1?name=ferret"),
+ #{host := "127.0.0.1", fragment := "nose"} = uri_string:parse("//127.0.0.1#nose"),
+ {error,invalid_uri,"x"} = uri_string:parse("//127.0.0.x"),
+ {error,invalid_uri,"1227.0.0.1"} = uri_string:parse("//1227.0.0.1").
+
+parse_host_ipv6(_Config) ->
+ #{host := "::127.0.0.1"} = uri_string:parse("//[::127.0.0.1]"),
+ #{host := "::127.0.0.1", path := "/over/there"} = uri_string:parse("//[::127.0.0.1]/over/there"),
+ #{host := "::127.0.0.1", query := "name=ferret"} =
+ uri_string:parse("//[::127.0.0.1]?name=ferret"),
+ #{host := "::127.0.0.1", fragment := "nose"} = uri_string:parse("//[::127.0.0.1]#nose"),
+ {error,invalid_uri,"x"} = uri_string:parse("//[::127.0.0.x]"),
+ {error,invalid_uri,"::1227.0.0.1"} = uri_string:parse("//[::1227.0.0.1]"),
+ {error,invalid_uri,"G"} = uri_string:parse("//[2001:0db8:0000:0000:0000:0000:1428:G7ab]").
+
+parse_port(_Config) ->
+ #{path:= "/:8042"} =
+ uri_string:parse("/:8042"),
+ #{host:= "", port := 8042} =
+ uri_string:parse("//:8042"),
+ #{host := "example.com", port:= 8042} =
+ uri_string:parse("//example.com:8042"),
+ #{scheme := "foo", path := "/:8042"} =
+ uri_string:parse("foo:/:8042"),
+ #{scheme := "foo", host := "", port := 8042} =
+ uri_string:parse("foo://:8042"),
+ #{scheme := "foo", host := "example.com", port := 8042} =
+ uri_string:parse("foo://example.com:8042").
+
+parse_path(_Config) ->
+ #{path := "over/there"} = uri_string:parse("over/there"),
+ #{path := "/over/there"} = uri_string:parse("/over/there"),
+ #{scheme := "foo", path := "/over/there"} =
+ uri_string:parse("foo:/over/there"),
+ #{scheme := "foo", host := "example.com", path := "/over/there"} =
+ uri_string:parse("foo://example.com/over/there"),
+ #{scheme := "foo", host := "example.com", path := "/over/there", port := 8042} =
+ uri_string:parse("foo://example.com:8042/over/there").
+
+parse_query(_Config) ->
+ #{scheme := "foo", query := "name=ferret"} =
+ uri_string:parse("foo:?name=ferret"),
+ #{scheme := "foo", path:= "over/there", query := "name=ferret"} =
+ uri_string:parse("foo:over/there?name=ferret"),
+ #{scheme := "foo", path:= "/over/there", query := "name=ferret"} =
+ uri_string:parse("foo:/over/there?name=ferret"),
+ #{scheme := "foo", host := "example.com", query := "name=ferret"} =
+ uri_string:parse("foo://example.com?name=ferret"),
+ #{scheme := "foo", host := "example.com", path := "/", query := "name=ferret"} =
+ uri_string:parse("foo://example.com/?name=ferret"),
+
+ #{path := "", query := "name=ferret"} =
+ uri_string:parse("?name=ferret"),
+ #{path := "over/there", query := "name=ferret"} =
+ uri_string:parse("over/there?name=ferret"),
+ #{path := "/", query := "name=ferret"} =
+ uri_string:parse("/?name=ferret"),
+ #{path := "/over/there", query := "name=ferret"} =
+ uri_string:parse("/over/there?name=ferret"),
+ #{host := "example.com", query := "name=ferret"} =
+ uri_string:parse("//example.com?name=ferret"),
+ #{host := "example.com", path := "/", query := "name=ferret"} =
+ uri_string:parse("//example.com/?name=ferret").
+
+parse_pct_encoded_query(_Config) ->
+ #{scheme := "foo", host := "example.com", path := "/",
+ query := "name=合気道"} =
+ uri_string:parse("foo://example.com/?name=%E5%90%88%E6%B0%97%E9%81%93"),
+ #{host := "example.com", path := "/", query := "name=合気道"} =
+ uri_string:parse("//example.com/?name=%E5%90%88%E6%B0%97%E9%81%93").
+
+parse_fragment(_Config) ->
+ #{scheme := "foo", fragment := "nose"} =
+ uri_string:parse("foo:#nose"),
+ #{scheme := "foo", path:= "over/there", fragment := "nose"} =
+ uri_string:parse("foo:over/there#nose"),
+ #{scheme := "foo", path:= "/over/there", fragment := "nose"} =
+ uri_string:parse("foo:/over/there#nose"),
+ #{scheme := "foo", host := "example.com", fragment := "nose"} =
+ uri_string:parse("foo://example.com#nose"),
+ #{scheme := "foo", host := "example.com", path := "/", fragment := "nose"} =
+ uri_string:parse("foo://example.com/#nose"),
+ #{scheme := "foo", host := "example.com", fragment := "nose"} =
+ uri_string:parse("foo://example.com#nose"),
+
+ #{fragment := "nose"} =
+ uri_string:parse("#nose"),
+ #{path := "over/there", fragment := "nose"} =
+ uri_string:parse("over/there#nose"),
+ #{path := "/", fragment := "nose"} =
+ uri_string:parse("/#nose"),
+ #{path := "/over/there", fragment := "nose"} =
+ uri_string:parse("/over/there#nose"),
+ #{host := "example.com", fragment := "nose"} =
+ uri_string:parse("//example.com#nose"),
+ #{host := "example.com", path := "/", fragment := "nose"} =
+ uri_string:parse("//example.com/#nose").
+
+parse_pct_encoded_fragment(_Config) ->
+ #{scheme := "foo", host := "example.com", fragment := "合気道"} =
+ uri_string:parse("foo://example.com#%E5%90%88%E6%B0%97%E9%81%93"),
+ #{host := "example.com", path := "/", fragment := "合気道"} =
+ uri_string:parse("//example.com/#%E5%90%88%E6%B0%97%E9%81%93").
+
+parse_list(_Config) ->
+ #{scheme := "foo", path := "bar:nisse"} = uri_string:parse("foo:bar:nisse"),
+ #{scheme := "foo", host := "example.com", port := 8042,
+ path := "/over/there", query := "name=ferret", fragment := "nose"} =
+ uri_string:parse("foo://example.com:8042/over/there?name=ferret#nose"),
+ #{scheme := "foo", userinfo := "admin:admin", host := "example.com", port := 8042,
+ path := "/over/there", query := "name=ferret", fragment := "nose"} =
+ uri_string:parse("foo://admin:[email protected]:8042/over/there?name=ferret#nose").
+
+parse_binary(_Config) ->
+ #{scheme := <<"foo">>, path := <<"bar:nisse">>} = uri_string:parse(<<"foo:bar:nisse">>),
+ #{scheme := <<"foo">>, host := <<"example.com">>, port := 8042,
+ path := <<"/over/there">>, query := <<"name=ferret">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo://example.com:8042/over/there?name=ferret#nose">>),
+ #{scheme := <<"foo">>, userinfo := <<"admin:admin">>, host := <<"example.com">>, port := 8042,
+ path := <<"/over/there">>, query := <<"name=ferret">>, fragment := <<"nose">>} =
+ uri_string:parse(<<"foo://admin:[email protected]:8042/over/there?name=ferret#nose">>).
+
+
+parse_mixed(_Config) ->
+ #{scheme := "foo", path := "bar"} =
+ uri_string:parse(lists:append("fo",<<"o:bar">>)),
+ #{scheme := "foo", path := "bar"} =
+ uri_string:parse(lists:append("foo:b",<<"ar">>)),
+ #{scheme := "foo", path := "bar:bar"} =
+ uri_string:parse([[102],[111,111],<<":bar">>,58,98,97,114]).
+
+parse_relative(_Config) ->
+ #{path := "/path"} =
+ uri_string:parse(lists:append("/pa",<<"th">>)),
+ #{path := "foo"} =
+ uri_string:parse(lists:append("fo",<<"o">>)).
+
+parse_special(_Config) ->
+ #{host := [],query := []} = uri_string:parse("//?"),
+ #{fragment := [],host := []} = uri_string:parse("//#"),
+ #{host := [],query := [],scheme := "foo"} = uri_string:parse("foo://?"),
+ #{fragment := [],host := [],scheme := "foo"} = uri_string:parse("foo://#"),
+ #{host := <<>>, path := <<"/">>} = uri_string:parse(<<"///">>),
+ #{host := <<"hostname">>} = uri_string:parse(<<"//hostname">>),
+ #{host := <<>>, path := <<"/hostname">>} = uri_string:parse(<<"///hostname">>),
+ #{host := [],path := "/",query := []} = uri_string:parse("///?"),
+ #{fragment := [],host := [],path := "/"} = uri_string:parse("///#"),
+ #{host := "foo",query := []} = uri_string:parse("//foo?"),
+ #{fragment := [],host := "foo"} = uri_string:parse("//foo#"),
+ #{host := "foo",path := "/"} = uri_string:parse("//foo/"),
+ #{host := "foo",query := [],scheme := "http"} = uri_string:parse("http://foo?"),
+ #{fragment := [],host := "foo",scheme := "http"} = uri_string:parse("http://foo#"),
+ #{host := "foo",path := "/",scheme := "http"} = uri_string:parse("http://foo/"),
+ #{fragment := [],host := "host",port := 80,scheme := "http"} = uri_string:parse("http://host:80#"),
+ #{host := "host",port := 80,query := [],scheme := "http"} = uri_string:parse("http://host:80?"),
+ #{path := [],query := []} = uri_string:parse("?"),
+ #{path := [],query := "?"} = uri_string:parse("??"),
+ #{path := [],query := "??"} = uri_string:parse("???").
+
+parse_special2(_Config) ->
+ #{host := [],path := "/",port := 1,scheme := "a"} = uri_string:parse("a://:1/"),
+ #{path := "/a/",scheme := "a"} = uri_string:parse("a:/a/"),
+ #{host := [],path := [],userinfo := []} = uri_string:parse("//@"),
+ #{host := [],path := [],scheme := "foo",userinfo := []} = uri_string:parse("foo://@"),
+ #{host := [],path := "/",userinfo := []} = uri_string:parse("//@/"),
+ #{host := [],path := "/",scheme := "foo",userinfo := []} = uri_string:parse("foo://@/"),
+ #{host := "localhost",path := "/",port := undefined} = uri_string:parse("//localhost:/"),
+ #{host := [],path := [],port := undefined} = uri_string:parse("//:").
+
+parse_negative(_Config) ->
+ {error,invalid_uri,"å"} = uri_string:parse("å"),
+ {error,invalid_uri,"å"} = uri_string:parse("aå:/foo"),
+ {error,invalid_uri,":"} = uri_string:parse("foo://usär@host"),
+ {error,invalid_uri,"ö"} = uri_string:parse("//host/path?foö=bar"),
+ {error,invalid_uri,"ö"} = uri_string:parse("//host/path#foö"),
+ {error,invalid_uri,"127.256.0.1"} = uri_string:parse("//127.256.0.1"),
+ {error,invalid_uri,":::127.0.0.1"} = uri_string:parse("//[:::127.0.0.1]"),
+ {error,invalid_utf8,<<0,0,0,246>>} = uri_string:parse("//%00%00%00%F6"),
+ {error,invalid_uri,"A"} = uri_string:parse("//localhost:A8").
+
+
+%%-------------------------------------------------------------------------
+%% Recompose tests
+%%-------------------------------------------------------------------------
+recompose_fragment(_Config) ->
+ <<?FRAGMENT_ENC>> = uri_string:recompose(#{fragment => <<?FRAGMENT/utf8>>, path => <<>>}),
+ ?FRAGMENT_ENC = uri_string:recompose(#{fragment => ?FRAGMENT, path => ""}).
+
+recompose_parse_fragment(_Config) ->
+ <<?FRAGMENT_ENC>> = uri_string:recompose(uri_string:parse(<<?FRAGMENT_ENC>>)),
+ ?FRAGMENT_ENC = uri_string:recompose(uri_string:parse(?FRAGMENT_ENC)).
+
+recompose_query(_Config) ->
+ <<?QUERY_ENC>> =
+ uri_string:recompose(#{query => <<?QUERY/utf8>>, path => <<>>}),
+ <<?QUERY_ENC?FRAGMENT_ENC>> =
+ uri_string:recompose(#{query => <<?QUERY/utf8>>,
+ fragment => <<?FRAGMENT/utf8>>,
+ path => <<>>}),
+ "?name=%C3%B6rn" =
+ uri_string:recompose(#{query => "name=örn", path => ""}),
+ "?name=%C3%B6rn#n%C3%A4sa" =
+ uri_string:recompose(#{query => "name=örn",
+ fragment => "näsa",
+ path => ""}).
+
+recompose_parse_query(_Config) ->
+ <<"?name=%C3%B6rn">> = uri_string:recompose(uri_string:parse(<<"?name=%C3%B6rn">>)),
+ <<"?name=%C3%B6rn#n%C3%A4sa">> =
+ uri_string:recompose(uri_string:parse(<<"?name=%C3%B6rn#n%C3%A4sa">>)),
+ "?name=%C3%B6rn" = uri_string:recompose(uri_string:parse("?name=%C3%B6rn")),
+ "?name=%C3%B6rn#n%C3%A4sa" = uri_string:recompose(uri_string:parse("?name=%C3%B6rn#n%C3%A4sa")).
+
+recompose_path(_Config) ->
+ <<"/d%C3%A4r">> =
+ uri_string:recompose(#{path => <<"/där"/utf8>>}),
+ <<"/d%C3%A4r#n%C3%A4sa">> =
+ uri_string:recompose(#{path => <<"/där"/utf8>>,
+ fragment => <<"näsa"/utf8>>}),
+ <<"/d%C3%A4r?name=%C3%B6rn">> =
+ uri_string:recompose(#{path => <<"/där"/utf8>>,
+ query => <<"name=örn"/utf8>>}),
+ <<"/d%C3%A4r?name=%C3%B6rn#n%C3%A4sa">> =
+ uri_string:recompose(#{path => <<"/där"/utf8>>,
+ query => <<"name=örn"/utf8>>,
+ fragment => <<"näsa"/utf8>>}),
+
+
+ "/d%C3%A4r" =
+ uri_string:recompose(#{path => "/där"}),
+ "/d%C3%A4r#n%C3%A4sa" =
+ uri_string:recompose(#{path => "/där",
+ fragment => "näsa"}),
+ "/d%C3%A4r?name=%C3%B6rn" =
+ uri_string:recompose(#{path => "/där",
+ query => "name=örn"}),
+ "/d%C3%A4r?name=%C3%B6rn#n%C3%A4sa" =
+ uri_string:recompose(#{path => "/där",
+ query => "name=örn",
+ fragment => "näsa"}).
+
+
+recompose_parse_path(_Config) ->
+ <<"/d%C3%A4r">> =
+ uri_string:recompose(uri_string:parse(<<"/d%C3%A4r">>)),
+ <<"/d%C3%A4r#n%C3%A4sa">> =
+ uri_string:recompose(uri_string:parse(<<"/d%C3%A4r#n%C3%A4sa">>)),
+ <<"/d%C3%A4r?name=%C3%B6rn">> =
+ uri_string:recompose(uri_string:parse(<<"/d%C3%A4r?name=%C3%B6rn">>)),
+
+ "/d%C3%A4r" =
+ uri_string:recompose(uri_string:parse("/d%C3%A4r")),
+ "/d%C3%A4r#n%C3%A4sa" =
+ uri_string:recompose(uri_string:parse("/d%C3%A4r#n%C3%A4sa")),
+ "/d%C3%A4r?name=%C3%B6rn" =
+ uri_string:recompose(uri_string:parse("/d%C3%A4r?name=%C3%B6rn")).
+
+
+recompose_autogen(_Config) ->
+ Tests = generate_test_vectors(uri_combinations()),
+ lists:map(fun run_test_recompose/1, Tests).
+
+parse_recompose_autogen(_Config) ->
+ Tests = generate_test_vectors(uri_combinations()),
+ lists:map(fun run_test_parse_recompose/1, Tests).
+
+transcode_basic(_Config) ->
+ <<"foo%C3%B6bar"/utf8>> =
+ uri_string:transcode(<<"foo%00%00%00%F6bar"/utf32>>, [{in_encoding, utf32},{out_encoding, utf8}]),
+ "foo%C3%B6bar" =
+ uri_string:transcode("foo%00%00%00%F6bar", [{in_encoding, utf32},{out_encoding, utf8}]),
+ <<"foo%00%00%00%F6bar"/utf32>> =
+ uri_string:transcode(<<"foo%C3%B6bar"/utf8>>, [{in_encoding, utf8},{out_encoding, utf32}]),
+ "foo%00%00%00%F6bar" =
+ uri_string:transcode("foo%C3%B6bar", [{in_encoding, utf8},{out_encoding, utf32}]),
+ "foo%C3%B6bar" =
+ uri_string:transcode("foo%F6bar", [{in_encoding, latin1},{out_encoding, utf8}]).
+
+transcode_options(_Config) ->
+ <<"foo%C3%B6bar"/utf8>> =
+ uri_string:transcode(<<"foo%C3%B6bar"/utf8>>, []),
+ <<"foo%C3%B6bar"/utf8>> =
+ uri_string:transcode(<<"foo%00%00%00%F6bar"/utf32>>, [{in_encoding, utf32}]),
+ <<"foo%00%00%00%F6bar"/utf32>> =
+ uri_string:transcode(<<"foo%C3%B6bar"/utf8>>, [{out_encoding, utf32}]).
+
+transcode_mixed(_Config) ->
+ "foo%00%00%00%F6bar" =
+ uri_string:transcode(["foo",<<"%C3%B6"/utf8>>,<<"ba"/utf8>>,"r"], [{out_encoding, utf32}]),
+ "foo%00%00%00%F6bar" =
+ uri_string:transcode(["foo",<<"%C3%"/utf8>>,<<"B6ba"/utf8>>,"r"], [{out_encoding, utf32}]),
+ "foo%C3%B6bar" =
+ uri_string:transcode(["foo%00", <<"%00%0"/utf32>>,<<"0%F"/utf32>>,"6bar"], [{in_encoding, utf32},{out_encoding, utf8}]).
+
+transcode_negative(_Config) ->
+ {error,invalid_percent_encoding,"%BXbar"} =
+ uri_string:transcode(<<"foo%C3%BXbar"/utf8>>, [{in_encoding, utf8},{out_encoding, utf32}]),
+ {error,invalid_input,<<"ö">>} =
+ uri_string:transcode("foo%F6bar", [{in_encoding, utf8},{out_encoding, utf8}]).
+
+normalize(_Config) ->
+ "/a/g" = uri_string:normalize("/a/b/c/./../../g"),
+ <<"mid/6">> = uri_string:normalize(<<"mid/content=5/../6">>),
+ "http://localhost-%C3%B6rebro/a/g" =
+ uri_string:normalize("http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g"),
+ <<"http://localhost-%C3%B6rebro/a/g">> =
+ uri_string:normalize(<<"http://localhos%74-%c3%b6rebro:80/a/b/c/./../../g">>),
+ <<"https://localhost/">> =
+ uri_string:normalize(<<"https://localhost:443">>),
+ <<"https://localhost:445/">> =
+ uri_string:normalize(<<"https://localhost:445">>),
+ <<"ftp://localhost">> =
+ uri_string:normalize(<<"ftp://localhost:21">>),
+ <<"ssh://localhost">> =
+ uri_string:normalize(<<"ssh://localhost:22">>),
+ <<"sftp://localhost">> =
+ uri_string:normalize(<<"sftp://localhost:22">>),
+ <<"tftp://localhost">> =
+ uri_string:normalize(<<"tftp://localhost:69">>).
diff --git a/lib/compiler/src/v3_life.hrl b/lib/stdlib/test/uri_string_property_test_SUITE.erl
index 5c76312067..ae2c61c7aa 100644
--- a/lib/compiler/src/v3_life.hrl
+++ b/lib/stdlib/test/uri_string_property_test_SUITE.erl
@@ -1,8 +1,8 @@
%%
%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 1999-2016. All Rights Reserved.
-%%
+%%
+%% Copyright Ericsson AB 2008-2017. All Rights Reserved.
+%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
@@ -14,16 +14,26 @@
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
-%%
+%%
%% %CopyrightEnd%
%%
-%% This record contains variable life-time annotation for a
-%% kernel expression. Added by v3_life, used by v3_codegen.
+-module(uri_string_property_test_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+-compile(export_all).
+
+all() -> [recompose].
--type vdb_entry() :: {atom(),non_neg_integer(),non_neg_integer()}.
+init_per_suite(Config) ->
+ ct_property_test:init_per_suite(Config).
--record(l, {ke, %Kernel expression
- i=0 :: non_neg_integer(), %Op number
- vdb=[] :: [vdb_entry()], %Variable database
- a=[] :: [term()]}). %Core annotation
+end_per_suite(Config) ->
+ Config.
+%%%========================================================================
+%%% Test suites
+%%%========================================================================
+recompose(Config) ->
+ ct_property_test:quickcheck(
+ uri_string_recompose:prop_recompose(),
+ Config).
diff --git a/lib/stdlib/uc_spec/CaseFolding.txt b/lib/stdlib/uc_spec/CaseFolding.txt
index 372ee68bd8..efdf18e441 100644
--- a/lib/stdlib/uc_spec/CaseFolding.txt
+++ b/lib/stdlib/uc_spec/CaseFolding.txt
@@ -1,6 +1,6 @@
-# CaseFolding-9.0.0.txt
-# Date: 2016-03-02, 18:54:54 GMT
-# © 2016 Unicode®, Inc.
+# CaseFolding-10.0.0.txt
+# Date: 2017-04-14, 05:40:18 GMT
+# © 2017 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
@@ -24,7 +24,7 @@
#
# NOTE: case folding does not preserve normalization formats!
#
-# For information on case folding, including how to have case folding
+# For information on case folding, including how to have case folding
# preserve normalization formats, see Section 3.13 Default Case Algorithms in
# The Unicode Standard.
#
diff --git a/lib/stdlib/uc_spec/CompositionExclusions.txt b/lib/stdlib/uc_spec/CompositionExclusions.txt
index 1999ed1328..ff42508686 100644
--- a/lib/stdlib/uc_spec/CompositionExclusions.txt
+++ b/lib/stdlib/uc_spec/CompositionExclusions.txt
@@ -1,6 +1,6 @@
-# CompositionExclusions-9.0.0.txt
-# Date: 2016-01-21, 22:00:00 GMT [KW, LI]
-# © 2016 Unicode®, Inc.
+# CompositionExclusions-10.0.0.txt
+# Date: 2017-02-15, 00:00:00 GMT [KW, LI]
+# © 2017 Unicode®, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# Unicode Character Database
diff --git a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt
index c5e94a3762..32bb12e47e 100644
--- a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt
+++ b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt
@@ -1,6 +1,6 @@
-# GraphemeBreakProperty-9.0.0.txt
-# Date: 2016-06-03, 22:23:55 GMT
-# © 2016 Unicode®, Inc.
+# GraphemeBreakProperty-10.0.0.txt
+# Date: 2017-03-12, 07:03:41 GMT
+# © 2017 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
@@ -25,8 +25,11 @@
0D4E ; Prepend # Lo MALAYALAM LETTER DOT REPH
110BD ; Prepend # Cf KAITHI NUMBER SIGN
111C2..111C3 ; Prepend # Lo [2] SHARADA SIGN JIHVAMULIYA..SHARADA SIGN UPADHMANIYA
+11A3A ; Prepend # Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA
+11A86..11A89 ; Prepend # Lo [4] SOYOMBO CLUSTER-INITIAL LETTER RA..SOYOMBO CLUSTER-INITIAL LETTER SA
+11D46 ; Prepend # Lo MASARAM GONDI REPHA
-# Total code points: 13
+# Total code points: 19
# ================================================
@@ -126,6 +129,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
0AC7..0AC8 ; Extend # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI
0ACD ; Extend # Mn GUJARATI SIGN VIRAMA
0AE2..0AE3 ; Extend # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL
+0AFA..0AFF ; Extend # Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE
0B01 ; Extend # Mn ORIYA SIGN CANDRABINDU
0B3C ; Extend # Mn ORIYA SIGN NUKTA
0B3E ; Extend # Mc ORIYA VOWEL SIGN AA
@@ -154,7 +158,8 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
0CCC..0CCD ; Extend # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA
0CD5..0CD6 ; Extend # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK
0CE2..0CE3 ; Extend # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL
-0D01 ; Extend # Mn MALAYALAM SIGN CANDRABINDU
+0D00..0D01 ; Extend # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU
+0D3B..0D3C ; Extend # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA
0D3E ; Extend # Mc MALAYALAM VOWEL SIGN AA
0D41..0D44 ; Extend # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR
0D4D ; Extend # Mn MALAYALAM SIGN VIRAMA
@@ -243,7 +248,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF>
1CED ; Extend # Mn VEDIC SIGN TIRYAK
1CF4 ; Extend # Mn VEDIC TONE CANDRA ABOVE
1CF8..1CF9 ; Extend # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE
-1DC0..1DF5 ; Extend # Mn [54] COMBINING DOTTED GRAVE ACCENT..COMBINING UP TACK ABOVE
+1DC0..1DF9 ; Extend # Mn [58] COMBINING DOTTED GRAVE ACCENT..COMBINING WIDE INVERTED BRIDGE BELOW
1DFB..1DFF ; Extend # Mn [5] COMBINING DELETION MARK..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW
200C ; Extend # Cf ZERO WIDTH NON-JOINER
20D0..20DC ; Extend # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE
@@ -353,6 +358,15 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
1171D..1171F ; Extend # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA
11722..11725 ; Extend # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
11727..1172B ; Extend # Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER
+11A01..11A06 ; Extend # Mn [6] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL SIGN O
+11A09..11A0A ; Extend # Mn [2] ZANABAZAR SQUARE VOWEL SIGN REVERSED I..ZANABAZAR SQUARE VOWEL LENGTH MARK
+11A33..11A38 ; Extend # Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA
+11A3B..11A3E ; Extend # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA
+11A47 ; Extend # Mn ZANABAZAR SQUARE SUBJOINER
+11A51..11A56 ; Extend # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE
+11A59..11A5B ; Extend # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK
+11A8A..11A96 ; Extend # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA
+11A98..11A99 ; Extend # Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER
11C30..11C36 ; Extend # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L
11C38..11C3D ; Extend # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA
11C3F ; Extend # Mn BHAIKSUKI SIGN VIRAMA
@@ -360,6 +374,11 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
11CAA..11CB0 ; Extend # Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA
11CB2..11CB3 ; Extend # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E
11CB5..11CB6 ; Extend # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU
+11D31..11D36 ; Extend # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R
+11D3A ; Extend # Mn MASARAM GONDI VOWEL SIGN E
+11D3C..11D3D ; Extend # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O
+11D3F..11D45 ; Extend # Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA
+11D47 ; Extend # Mn MASARAM GONDI RA-KARA
16AF0..16AF4 ; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
16B30..16B36 ; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
16F8F..16F92 ; Extend # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW
@@ -387,7 +406,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT
E0020..E007F ; Extend # Cf [96] TAG SPACE..CANCEL TAG
E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
-# Total code points: 1828
+# Total code points: 1901
# ================================================
@@ -472,6 +491,7 @@ E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
1C34..1C35 ; SpacingMark # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG
1CE1 ; SpacingMark # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA
1CF2..1CF3 ; SpacingMark # Mc [2] VEDIC SIGN ARDHAVISARGA..VEDIC SIGN ROTATED ARDHAVISARGA
+1CF7 ; SpacingMark # Mc VEDIC SIGN ATIKRAMA
A823..A824 ; SpacingMark # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I
A827 ; SpacingMark # Mc SYLOTI NAGRI VOWEL SIGN OO
A880..A881 ; SpacingMark # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA
@@ -529,6 +549,10 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK
116B6 ; SpacingMark # Mc TAKRI SIGN VIRAMA
11720..11721 ; SpacingMark # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA
11726 ; SpacingMark # Mc AHOM VOWEL SIGN E
+11A07..11A08 ; SpacingMark # Mc [2] ZANABAZAR SQUARE VOWEL SIGN AI..ZANABAZAR SQUARE VOWEL SIGN AU
+11A39 ; SpacingMark # Mc ZANABAZAR SQUARE SIGN VISARGA
+11A57..11A58 ; SpacingMark # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU
+11A97 ; SpacingMark # Mc SOYOMBO SIGN VISARGA
11C2F ; SpacingMark # Mc BHAIKSUKI VOWEL SIGN AA
11C3E ; SpacingMark # Mc BHAIKSUKI SIGN VISARGA
11CA9 ; SpacingMark # Mc MARCHEN SUBJOINED LETTER YA
@@ -538,7 +562,7 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK
1D166 ; SpacingMark # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM
1D16D ; SpacingMark # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT
-# Total code points: 341
+# Total code points: 348
# ================================================
@@ -1375,8 +1399,9 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH
26F9 ; E_Base # So PERSON WITH BALL
270A..270D ; E_Base # So [4] RAISED FIST..WRITING HAND
1F385 ; E_Base # So FATHER CHRISTMAS
-1F3C3..1F3C4 ; E_Base # So [2] RUNNER..SURFER
-1F3CA..1F3CB ; E_Base # So [2] SWIMMER..WEIGHT LIFTER
+1F3C2..1F3C4 ; E_Base # So [3] SNOWBOARDER..SURFER
+1F3C7 ; E_Base # So HORSE RACING
+1F3CA..1F3CC ; E_Base # So [3] SWIMMER..GOLFER
1F442..1F443 ; E_Base # So [2] EAR..NOSE
1F446..1F450 ; E_Base # So [11] WHITE UP POINTING BACKHAND INDEX..OPEN HANDS SIGN
1F46E ; E_Base # So POLICE OFFICER
@@ -1385,7 +1410,7 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH
1F481..1F483 ; E_Base # So [3] INFORMATION DESK PERSON..DANCER
1F485..1F487 ; E_Base # So [3] NAIL POLISH..HAIRCUT
1F4AA ; E_Base # So FLEXED BICEPS
-1F575 ; E_Base # So SLEUTH OR SPY
+1F574..1F575 ; E_Base # So [2] MAN IN BUSINESS SUIT LEVITATING..SLEUTH OR SPY
1F57A ; E_Base # So MAN DANCING
1F590 ; E_Base # So RAISED HAND WITH FINGERS SPLAYED
1F595..1F596 ; E_Base # So [2] REVERSED HAND WITH MIDDLE FINGER EXTENDED..RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS
@@ -1394,13 +1419,15 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH
1F6A3 ; E_Base # So ROWBOAT
1F6B4..1F6B6 ; E_Base # So [3] BICYCLIST..PEDESTRIAN
1F6C0 ; E_Base # So BATH
-1F918..1F91E ; E_Base # So [7] SIGN OF THE HORNS..HAND WITH INDEX AND MIDDLE FINGERS CROSSED
+1F6CC ; E_Base # So SLEEPING ACCOMMODATION
+1F918..1F91C ; E_Base # So [5] SIGN OF THE HORNS..RIGHT-FACING FIST
+1F91E..1F91F ; E_Base # So [2] HAND WITH INDEX AND MIDDLE FINGERS CROSSED..I LOVE YOU HAND SIGN
1F926 ; E_Base # So FACE PALM
-1F930 ; E_Base # So PREGNANT WOMAN
-1F933..1F939 ; E_Base # So [7] SELFIE..JUGGLING
-1F93C..1F93E ; E_Base # So [3] WRESTLERS..HANDBALL
+1F930..1F939 ; E_Base # So [10] PREGNANT WOMAN..JUGGLING
+1F93D..1F93E ; E_Base # So [2] WATER POLO..HANDBALL
+1F9D1..1F9DD ; E_Base # So [13] ADULT..ELF
-# Total code points: 79
+# Total code points: 98
# ================================================
@@ -1416,11 +1443,28 @@ D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH
# ================================================
+2640 ; Glue_After_Zwj # So FEMALE SIGN
+2642 ; Glue_After_Zwj # So MALE SIGN
+2695..2696 ; Glue_After_Zwj # So [2] STAFF OF AESCULAPIUS..SCALES
+2708 ; Glue_After_Zwj # So AIRPLANE
2764 ; Glue_After_Zwj # So HEAVY BLACK HEART
+1F308 ; Glue_After_Zwj # So RAINBOW
+1F33E ; Glue_After_Zwj # So EAR OF RICE
+1F373 ; Glue_After_Zwj # So COOKING
+1F393 ; Glue_After_Zwj # So GRADUATION CAP
+1F3A4 ; Glue_After_Zwj # So MICROPHONE
+1F3A8 ; Glue_After_Zwj # So ARTIST PALETTE
+1F3EB ; Glue_After_Zwj # So SCHOOL
+1F3ED ; Glue_After_Zwj # So FACTORY
1F48B ; Glue_After_Zwj # So KISS MARK
+1F4BB..1F4BC ; Glue_After_Zwj # So [2] PERSONAL COMPUTER..BRIEFCASE
+1F527 ; Glue_After_Zwj # So WRENCH
+1F52C ; Glue_After_Zwj # So MICROSCOPE
1F5E8 ; Glue_After_Zwj # So LEFT SPEECH BUBBLE
+1F680 ; Glue_After_Zwj # So ROCKET
+1F692 ; Glue_After_Zwj # So FIRE ENGINE
-# Total code points: 3
+# Total code points: 22
# ================================================
diff --git a/lib/stdlib/uc_spec/PropList.txt b/lib/stdlib/uc_spec/PropList.txt
index a8c0da7135..9a2d0e4b1c 100644
--- a/lib/stdlib/uc_spec/PropList.txt
+++ b/lib/stdlib/uc_spec/PropList.txt
@@ -1,6 +1,6 @@
-# PropList-9.0.0.txt
-# Date: 2016-06-01, 10:34:30 GMT
-# © 2016 Unicode®, Inc.
+# PropList-10.0.0.txt
+# Date: 2017-03-10, 08:25:30 GMT
+# © 2017 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
@@ -199,6 +199,9 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA
115C9..115D7 ; Terminal_Punctuation # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES
11641..11642 ; Terminal_Punctuation # Po [2] MODI DANDA..MODI DOUBLE DANDA
1173C..1173E ; Terminal_Punctuation # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI
+11A42..11A43 ; Terminal_Punctuation # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD
+11A9B..11A9C ; Terminal_Punctuation # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD
+11AA1..11AA2 ; Terminal_Punctuation # Po [2] SOYOMBO TERMINAL MARK-1..SOYOMBO TERMINAL MARK-2
11C41..11C43 ; Terminal_Punctuation # Po [3] BHAIKSUKI DANDA..BHAIKSUKI WORD SEPARATOR
11C71 ; Terminal_Punctuation # Po MARCHEN MARK SHAD
12470..12474 ; Terminal_Punctuation # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON
@@ -209,7 +212,7 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA
1BC9F ; Terminal_Punctuation # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP
1DA87..1DA8A ; Terminal_Punctuation # Po [4] SIGNWRITING COMMA..SIGNWRITING COLON
-# Total code points: 246
+# Total code points: 252
# ================================================
@@ -471,6 +474,7 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L
0AC9 ; Other_Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O
0ACB..0ACC ; Other_Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU
0AE2..0AE3 ; Other_Alphabetic # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL
+0AFA..0AFC ; Other_Alphabetic # Mn [3] GUJARATI SIGN SUKUN..GUJARATI SIGN MADDAH
0B01 ; Other_Alphabetic # Mn ORIYA SIGN CANDRABINDU
0B02..0B03 ; Other_Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA
0B3E ; Other_Alphabetic # Mc ORIYA VOWEL SIGN AA
@@ -508,7 +512,7 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L
0CCC ; Other_Alphabetic # Mn KANNADA VOWEL SIGN AU
0CD5..0CD6 ; Other_Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK
0CE2..0CE3 ; Other_Alphabetic # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL
-0D01 ; Other_Alphabetic # Mn MALAYALAM SIGN CANDRABINDU
+0D00..0D01 ; Other_Alphabetic # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU
0D02..0D03 ; Other_Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA
0D3E..0D40 ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II
0D41..0D44 ; Other_Alphabetic # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR
@@ -726,6 +730,17 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA
11722..11725 ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU
11726 ; Other_Alphabetic # Mc AHOM VOWEL SIGN E
11727..1172A ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN AW..AHOM VOWEL SIGN AM
+11A01..11A06 ; Other_Alphabetic # Mn [6] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL SIGN O
+11A07..11A08 ; Other_Alphabetic # Mc [2] ZANABAZAR SQUARE VOWEL SIGN AI..ZANABAZAR SQUARE VOWEL SIGN AU
+11A09..11A0A ; Other_Alphabetic # Mn [2] ZANABAZAR SQUARE VOWEL SIGN REVERSED I..ZANABAZAR SQUARE VOWEL LENGTH MARK
+11A35..11A38 ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE SIGN CANDRABINDU..ZANABAZAR SQUARE SIGN ANUSVARA
+11A39 ; Other_Alphabetic # Mc ZANABAZAR SQUARE SIGN VISARGA
+11A3B..11A3E ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA
+11A51..11A56 ; Other_Alphabetic # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE
+11A57..11A58 ; Other_Alphabetic # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU
+11A59..11A5B ; Other_Alphabetic # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK
+11A8A..11A96 ; Other_Alphabetic # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA
+11A97 ; Other_Alphabetic # Mc SOYOMBO SIGN VISARGA
11C2F ; Other_Alphabetic # Mc BHAIKSUKI VOWEL SIGN AA
11C30..11C36 ; Other_Alphabetic # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L
11C38..11C3D ; Other_Alphabetic # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA
@@ -737,6 +752,12 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA
11CB2..11CB3 ; Other_Alphabetic # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E
11CB4 ; Other_Alphabetic # Mc MARCHEN VOWEL SIGN O
11CB5..11CB6 ; Other_Alphabetic # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU
+11D31..11D36 ; Other_Alphabetic # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R
+11D3A ; Other_Alphabetic # Mn MASARAM GONDI VOWEL SIGN E
+11D3C..11D3D ; Other_Alphabetic # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O
+11D3F..11D41 ; Other_Alphabetic # Mn [3] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI SIGN VISARGA
+11D43 ; Other_Alphabetic # Mn MASARAM GONDI SIGN CANDRA
+11D47 ; Other_Alphabetic # Mn MASARAM GONDI RA-KARA
16B30..16B36 ; Other_Alphabetic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
16F51..16F7E ; Other_Alphabetic # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG
1BC9E ; Other_Alphabetic # Mn DUPLOYAN DOUBLE MARK
@@ -750,7 +771,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA
1F150..1F169 ; Other_Alphabetic # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z
1F170..1F189 ; Other_Alphabetic # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z
-# Total code points: 1238
+# Total code points: 1300
# ================================================
@@ -759,18 +780,20 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA
3021..3029 ; Ideographic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE
3038..303A ; Ideographic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY
3400..4DB5 ; Ideographic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5
-4E00..9FD5 ; Ideographic # Lo [20950] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FD5
+4E00..9FEA ; Ideographic # Lo [20971] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEA
F900..FA6D ; Ideographic # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D
FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9
17000..187EC ; Ideographic # Lo [6125] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187EC
18800..18AF2 ; Ideographic # Lo [755] TANGUT COMPONENT-001..TANGUT COMPONENT-755
+1B170..1B2FB ; Ideographic # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB
20000..2A6D6 ; Ideographic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6
2A700..2B734 ; Ideographic # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734
2B740..2B81D ; Ideographic # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; Ideographic # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
+2CEB0..2EBE0 ; Ideographic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2F800..2FA1D ; Ideographic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
-# Total code points: 88284
+# Total code points: 96174
# ================================================
@@ -826,12 +849,14 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM
0A4D ; Diacritic # Mn GURMUKHI SIGN VIRAMA
0ABC ; Diacritic # Mn GUJARATI SIGN NUKTA
0ACD ; Diacritic # Mn GUJARATI SIGN VIRAMA
+0AFD..0AFF ; Diacritic # Mn [3] GUJARATI SIGN THREE-DOT NUKTA ABOVE..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE
0B3C ; Diacritic # Mn ORIYA SIGN NUKTA
0B4D ; Diacritic # Mn ORIYA SIGN VIRAMA
0BCD ; Diacritic # Mn TAMIL SIGN VIRAMA
0C4D ; Diacritic # Mn TELUGU SIGN VIRAMA
0CBC ; Diacritic # Mn KANNADA SIGN NUKTA
0CCD ; Diacritic # Mn KANNADA SIGN VIRAMA
+0D3B..0D3C ; Diacritic # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA
0D4D ; Diacritic # Mn MALAYALAM SIGN VIRAMA
0DCA ; Diacritic # Mn SINHALA SIGN AL-LAKUNA
0E47..0E4C ; Diacritic # Mn [6] THAI CHARACTER MAITAIKHU..THAI CHARACTER THANTHAKHAT
@@ -871,10 +896,11 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM
1CE2..1CE8 ; Diacritic # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL
1CED ; Diacritic # Mn VEDIC SIGN TIRYAK
1CF4 ; Diacritic # Mn VEDIC TONE CANDRA ABOVE
+1CF7 ; Diacritic # Mc VEDIC SIGN ATIKRAMA
1CF8..1CF9 ; Diacritic # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE
1D2C..1D6A ; Diacritic # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI
1DC4..1DCF ; Diacritic # Mn [12] COMBINING MACRON-ACUTE..COMBINING ZIGZAG BELOW
-1DF5 ; Diacritic # Mn COMBINING UP TACK ABOVE
+1DF5..1DF9 ; Diacritic # Mn [5] COMBINING UP TACK ABOVE..COMBINING WIDE INVERTED BRIDGE BELOW
1DFD..1DFF ; Diacritic # Mn [3] COMBINING ALMOST EQUAL TO BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW
1FBD ; Diacritic # Sk GREEK KORONIS
1FBF..1FC1 ; Diacritic # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI
@@ -947,7 +973,12 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON
116B6 ; Diacritic # Mc TAKRI SIGN VIRAMA
116B7 ; Diacritic # Mn TAKRI SIGN NUKTA
1172B ; Diacritic # Mn AHOM SIGN KILLER
+11A34 ; Diacritic # Mn ZANABAZAR SQUARE SIGN VIRAMA
+11A47 ; Diacritic # Mn ZANABAZAR SQUARE SUBJOINER
+11A99 ; Diacritic # Mn SOYOMBO SUBJOINER
11C3F ; Diacritic # Mn BHAIKSUKI SIGN VIRAMA
+11D42 ; Diacritic # Mn MASARAM GONDI SIGN NUKTA
+11D44..11D45 ; Diacritic # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA
16AF0..16AF4 ; Diacritic # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
16F8F..16F92 ; Diacritic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW
16F93..16F9F ; Diacritic # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8
@@ -960,7 +991,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON
1E944..1E946 ; Diacritic # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK
1E948..1E94A ; Diacritic # Mn [3] ADLAM CONSONANT MODIFIER..ADLAM NUKTA
-# Total code points: 782
+# Total code points: 798
# ================================================
@@ -989,11 +1020,12 @@ AAF3..AAF4 ; Extender # Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETE
FF70 ; Extender # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK
1135D ; Extender # Lo GRANTHA SIGN PLUTA
115C6..115C8 ; Extender # Po [3] SIDDHAM REPETITION MARK-1..SIDDHAM REPETITION MARK-3
+11A98 ; Extender # Mn SOYOMBO GEMINATION MARK
16B42..16B43 ; Extender # Lm [2] PAHAWH HMONG SIGN VOS NRUA..PAHAWH HMONG SIGN IB YAM
-16FE0 ; Extender # Lm TANGUT ITERATION MARK
+16FE0..16FE1 ; Extender # Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK
1E944..1E946 ; Extender # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK
-# Total code points: 42
+# Total code points: 44
# ================================================
@@ -1105,7 +1137,7 @@ E0020..E007F ; Other_Grapheme_Extend # Cf [96] TAG SPACE..CANCEL TAG
# ================================================
3400..4DB5 ; Unified_Ideograph # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5
-4E00..9FD5 ; Unified_Ideograph # Lo [20950] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FD5
+4E00..9FEA ; Unified_Ideograph # Lo [20971] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEA
FA0E..FA0F ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA0E..CJK COMPATIBILITY IDEOGRAPH-FA0F
FA11 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA11
FA13..FA14 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA13..CJK COMPATIBILITY IDEOGRAPH-FA14
@@ -1117,8 +1149,9 @@ FA27..FA29 ; Unified_Ideograph # Lo [3] CJK COMPATIBILITY IDEOGRAPH-FA27..C
2A700..2B734 ; Unified_Ideograph # Lo [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734
2B740..2B81D ; Unified_Ideograph # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; Unified_Ideograph # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
+2CEB0..2EBE0 ; Unified_Ideograph # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
-# Total code points: 80388
+# Total code points: 87882
# ================================================
@@ -1277,6 +1310,8 @@ FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP
115C9..115D7 ; Sentence_Terminal # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES
11641..11642 ; Sentence_Terminal # Po [2] MODI DANDA..MODI DOUBLE DANDA
1173C..1173E ; Sentence_Terminal # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI
+11A42..11A43 ; Sentence_Terminal # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD
+11A9B..11A9C ; Sentence_Terminal # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD
11C41..11C42 ; Sentence_Terminal # Po [2] BHAIKSUKI DANDA..BHAIKSUKI DOUBLE DANDA
16A6E..16A6F ; Sentence_Terminal # Po [2] MRO DANDA..MRO DOUBLE DANDA
16AF5 ; Sentence_Terminal # Po BASSA VAH FULL STOP
@@ -1285,7 +1320,7 @@ FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP
1BC9F ; Sentence_Terminal # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP
1DA88 ; Sentence_Terminal # Po SIGNWRITING FULL STOP
-# Total code points: 124
+# Total code points: 128
# ================================================
@@ -1402,9 +1437,7 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S
239B..23B3 ; Pattern_Syntax # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM
23B4..23DB ; Pattern_Syntax # So [40] TOP SQUARE BRACKET..FUSE
23DC..23E1 ; Pattern_Syntax # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET
-23E2..23FE ; Pattern_Syntax # So [29] WHITE TRAPEZIUM..POWER SLEEP SYMBOL
-23FF ; Pattern_Syntax # Cn <reserved-23FF>
-2400..2426 ; Pattern_Syntax # So [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO
+23E2..2426 ; Pattern_Syntax # So [69] WHITE TRAPEZIUM..SYMBOL FOR SUBSTITUTE FORM TWO
2427..243F ; Pattern_Syntax # Cn [25] <reserved-2427>..<reserved-243F>
2440..244A ; Pattern_Syntax # So [11] OCR HOOK..OCR DOUBLE BACKSLASH
244B..245F ; Pattern_Syntax # Cn [21] <reserved-244B>..<reserved-245F>
@@ -1492,8 +1525,8 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S
2BBA..2BBC ; Pattern_Syntax # Cn [3] <reserved-2BBA>..<reserved-2BBC>
2BBD..2BC8 ; Pattern_Syntax # So [12] BALLOT BOX WITH LIGHT X..BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED
2BC9 ; Pattern_Syntax # Cn <reserved-2BC9>
-2BCA..2BD1 ; Pattern_Syntax # So [8] TOP HALF BLACK CIRCLE..UNCERTAINTY SIGN
-2BD2..2BEB ; Pattern_Syntax # Cn [26] <reserved-2BD2>..<reserved-2BEB>
+2BCA..2BD2 ; Pattern_Syntax # So [9] TOP HALF BLACK CIRCLE..GROUP MARK
+2BD3..2BEB ; Pattern_Syntax # Cn [25] <reserved-2BD3>..<reserved-2BEB>
2BEC..2BEF ; Pattern_Syntax # So [4] LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS..DOWNWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS
2BF0..2BFF ; Pattern_Syntax # Cn [16] <reserved-2BF0>..<reserved-2BFF>
2E00..2E01 ; Pattern_Syntax # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER
@@ -1533,8 +1566,8 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S
2E40 ; Pattern_Syntax # Pd DOUBLE HYPHEN
2E41 ; Pattern_Syntax # Po REVERSED COMMA
2E42 ; Pattern_Syntax # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK
-2E43..2E44 ; Pattern_Syntax # Po [2] DASH WITH LEFT UPTURN..DOUBLE SUSPENSION MARK
-2E45..2E7F ; Pattern_Syntax # Cn [59] <reserved-2E45>..<reserved-2E7F>
+2E43..2E49 ; Pattern_Syntax # Po [7] DASH WITH LEFT UPTURN..DOUBLE STACKED COMMA
+2E4A..2E7F ; Pattern_Syntax # Cn [54] <reserved-2E4A>..<reserved-2E7F>
3001..3003 ; Pattern_Syntax # Po [3] IDEOGRAPHIC COMMA..DITTO MARK
3008 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET
3009 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET
@@ -1576,4 +1609,10 @@ FE45..FE46 ; Pattern_Syntax # Po [2] SESAME DOT..WHITE SESAME DOT
# Total code points: 10
+# ================================================
+
+1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z
+
+# Total code points: 26
+
# EOF
diff --git a/lib/stdlib/uc_spec/SpecialCasing.txt b/lib/stdlib/uc_spec/SpecialCasing.txt
index b23fa7f768..b9ba0d81c1 100644
--- a/lib/stdlib/uc_spec/SpecialCasing.txt
+++ b/lib/stdlib/uc_spec/SpecialCasing.txt
@@ -1,6 +1,6 @@
-# SpecialCasing-9.0.0.txt
-# Date: 2016-03-02, 18:55:13 GMT
-# © 2016 Unicode®, Inc.
+# SpecialCasing-10.0.0.txt
+# Date: 2017-04-14, 05:40:43 GMT
+# © 2017 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
@@ -197,7 +197,7 @@ FB17; FB17; 0544 056D; 0544 053D; # ARMENIAN SMALL LIGATURE MEN XEH
# ================================================================================
# Conditional Mappings
-# The remainder of this file provides conditional casing data used to produce
+# The remainder of this file provides conditional casing data used to produce
# full case mappings.
# ================================================================================
# Language-Insensitive Mappings
diff --git a/lib/stdlib/uc_spec/UnicodeData.txt b/lib/stdlib/uc_spec/UnicodeData.txt
index a756976461..d89c64f526 100644
--- a/lib/stdlib/uc_spec/UnicodeData.txt
+++ b/lib/stdlib/uc_spec/UnicodeData.txt
@@ -2072,6 +2072,17 @@
085A;MANDAIC VOCALIZATION MARK;Mn;220;NSM;;;;;N;;;;;
085B;MANDAIC GEMINATION MARK;Mn;220;NSM;;;;;N;;;;;
085E;MANDAIC PUNCTUATION;Po;0;R;;;;;N;;;;;
+0860;SYRIAC LETTER MALAYALAM NGA;Lo;0;AL;;;;;N;;;;;
+0861;SYRIAC LETTER MALAYALAM JA;Lo;0;AL;;;;;N;;;;;
+0862;SYRIAC LETTER MALAYALAM NYA;Lo;0;AL;;;;;N;;;;;
+0863;SYRIAC LETTER MALAYALAM TTA;Lo;0;AL;;;;;N;;;;;
+0864;SYRIAC LETTER MALAYALAM NNA;Lo;0;AL;;;;;N;;;;;
+0865;SYRIAC LETTER MALAYALAM NNNA;Lo;0;AL;;;;;N;;;;;
+0866;SYRIAC LETTER MALAYALAM BHA;Lo;0;AL;;;;;N;;;;;
+0867;SYRIAC LETTER MALAYALAM RA;Lo;0;AL;;;;;N;;;;;
+0868;SYRIAC LETTER MALAYALAM LLA;Lo;0;AL;;;;;N;;;;;
+0869;SYRIAC LETTER MALAYALAM LLLA;Lo;0;AL;;;;;N;;;;;
+086A;SYRIAC LETTER MALAYALAM SSA;Lo;0;AL;;;;;N;;;;;
08A0;ARABIC LETTER BEH WITH SMALL V BELOW;Lo;0;AL;;;;;N;;;;;
08A1;ARABIC LETTER BEH WITH HAMZA ABOVE;Lo;0;AL;;;;;N;;;;;
08A2;ARABIC LETTER JEEM WITH TWO DOTS ABOVE;Lo;0;AL;;;;;N;;;;;
@@ -2366,6 +2377,8 @@
09F9;BENGALI CURRENCY DENOMINATOR SIXTEEN;No;0;L;;;;16;N;;;;;
09FA;BENGALI ISSHAR;So;0;L;;;;;N;;;;;
09FB;BENGALI GANDA MARK;Sc;0;ET;;;;;N;;;;;
+09FC;BENGALI LETTER VEDIC ANUSVARA;Lo;0;L;;;;;N;;;;;
+09FD;BENGALI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;;
0A01;GURMUKHI SIGN ADAK BINDI;Mn;0;NSM;;;;;N;;;;;
0A02;GURMUKHI SIGN BINDI;Mn;0;NSM;;;;;N;;;;;
0A03;GURMUKHI SIGN VISARGA;Mc;0;L;;;;;N;;;;;
@@ -2530,6 +2543,12 @@
0AF0;GUJARATI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;;
0AF1;GUJARATI RUPEE SIGN;Sc;0;ET;;;;;N;;;;;
0AF9;GUJARATI LETTER ZHA;Lo;0;L;;;;;N;;;;;
+0AFA;GUJARATI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;;
+0AFB;GUJARATI SIGN SHADDA;Mn;0;NSM;;;;;N;;;;;
+0AFC;GUJARATI SIGN MADDAH;Mn;0;NSM;;;;;N;;;;;
+0AFD;GUJARATI SIGN THREE-DOT NUKTA ABOVE;Mn;0;NSM;;;;;N;;;;;
+0AFE;GUJARATI SIGN CIRCLE NUKTA ABOVE;Mn;0;NSM;;;;;N;;;;;
+0AFF;GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE;Mn;0;NSM;;;;;N;;;;;
0B01;ORIYA SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;
0B02;ORIYA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;;
0B03;ORIYA SIGN VISARGA;Mc;0;L;;;;;N;;;;;
@@ -2876,6 +2895,7 @@
0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;;
0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;;
0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;;
+0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;;
0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;
0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;;
0D03;MALAYALAM SIGN VISARGA;Mc;0;L;;;;;N;;;;;
@@ -2931,6 +2951,8 @@
0D38;MALAYALAM LETTER SA;Lo;0;L;;;;;N;;;;;
0D39;MALAYALAM LETTER HA;Lo;0;L;;;;;N;;;;;
0D3A;MALAYALAM LETTER TTTA;Lo;0;L;;;;;N;;;;;
+0D3B;MALAYALAM SIGN VERTICAL BAR VIRAMA;Mn;9;NSM;;;;;N;;;;;
+0D3C;MALAYALAM SIGN CIRCULAR VIRAMA;Mn;9;NSM;;;;;N;;;;;
0D3D;MALAYALAM SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;;
0D3E;MALAYALAM VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;
0D3F;MALAYALAM VOWEL SIGN I;Mc;0;L;;;;;N;;;;;
@@ -6413,6 +6435,7 @@
1CF4;VEDIC TONE CANDRA ABOVE;Mn;230;NSM;;;;;N;;;;;
1CF5;VEDIC SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;;
1CF6;VEDIC SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;;
+1CF7;VEDIC SIGN ATIKRAMA;Mc;0;L;;;;;N;;;;;
1CF8;VEDIC TONE RING ABOVE;Mn;230;NSM;;;;;N;;;;;
1CF9;VEDIC TONE DOUBLE RING ABOVE;Mn;230;NSM;;;;;N;;;;;
1D00;LATIN LETTER SMALL CAPITAL A;Ll;0;L;;;;;N;;;;;
@@ -6661,6 +6684,10 @@
1DF3;COMBINING LATIN SMALL LETTER O WITH DIAERESIS;Mn;230;NSM;;;;;N;;;;;
1DF4;COMBINING LATIN SMALL LETTER U WITH DIAERESIS;Mn;230;NSM;;;;;N;;;;;
1DF5;COMBINING UP TACK ABOVE;Mn;230;NSM;;;;;N;;;;;
+1DF6;COMBINING KAVYKA ABOVE RIGHT;Mn;232;NSM;;;;;N;;;;;
+1DF7;COMBINING KAVYKA ABOVE LEFT;Mn;228;NSM;;;;;N;;;;;
+1DF8;COMBINING DOT ABOVE LEFT;Mn;228;NSM;;;;;N;;;;;
+1DF9;COMBINING WIDE INVERTED BRIDGE BELOW;Mn;220;NSM;;;;;N;;;;;
1DFB;COMBINING DELETION MARK;Mn;230;NSM;;;;;N;;;;;
1DFC;COMBINING DOUBLE INVERTED BREVE BELOW;Mn;233;NSM;;;;;N;;;;;
1DFD;COMBINING ALMOST EQUAL TO BELOW;Mn;220;NSM;;;;;N;;;;;
@@ -7339,6 +7366,7 @@
20BC;MANAT SIGN;Sc;0;ET;;;;;N;;;;;
20BD;RUBLE SIGN;Sc;0;ET;;;;;N;;;;;
20BE;LARI SIGN;Sc;0;ET;;;;;N;;;;;
+20BF;BITCOIN SIGN;Sc;0;ET;;;;;N;;;;;
20D0;COMBINING LEFT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING LEFT HARPOON ABOVE;;;;
20D1;COMBINING RIGHT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING RIGHT HARPOON ABOVE;;;;
20D2;COMBINING LONG VERTICAL LINE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING LONG VERTICAL BAR OVERLAY;;;;
@@ -8135,6 +8163,7 @@
23FC;POWER ON-OFF SYMBOL;So;0;ON;;;;;N;;;;;
23FD;POWER ON SYMBOL;So;0;ON;;;;;N;;;;;
23FE;POWER SLEEP SYMBOL;So;0;ON;;;;;N;;;;;
+23FF;OBSERVER EYE SYMBOL;So;0;ON;;;;;N;;;;;
2400;SYMBOL FOR NULL;So;0;ON;;;;;N;GRAPHIC FOR NULL;;;;
2401;SYMBOL FOR START OF HEADING;So;0;ON;;;;;N;GRAPHIC FOR START OF HEADING;;;;
2402;SYMBOL FOR START OF TEXT;So;0;ON;;;;;N;GRAPHIC FOR START OF TEXT;;;;
@@ -10083,6 +10112,7 @@
2BCF;ROTATED WHITE FOUR POINTED CUSP;So;0;ON;;;;;N;;;;;
2BD0;SQUARE POSITION INDICATOR;So;0;ON;;;;;N;;;;;
2BD1;UNCERTAINTY SIGN;So;0;ON;;;;;N;;;;;
+2BD2;GROUP MARK;So;0;ON;;;;;N;;;;;
2BEC;LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;;
2BED;UPWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;;
2BEE;RIGHTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS;So;0;ON;;;;;N;;;;;
@@ -10615,6 +10645,11 @@
2E42;DOUBLE LOW-REVERSED-9 QUOTATION MARK;Ps;0;ON;;;;;N;;;;;
2E43;DASH WITH LEFT UPTURN;Po;0;ON;;;;;N;;;;;
2E44;DOUBLE SUSPENSION MARK;Po;0;ON;;;;;N;;;;;
+2E45;INVERTED LOW KAVYKA;Po;0;ON;;;;;N;;;;;
+2E46;INVERTED LOW KAVYKA WITH KAVYKA ABOVE;Po;0;ON;;;;;N;;;;;
+2E47;LOW KAVYKA;Po;0;ON;;;;;N;;;;;
+2E48;LOW KAVYKA WITH DOT;Po;0;ON;;;;;N;;;;;
+2E49;DOUBLE STACKED COMMA;Po;0;ON;;;;;N;;;;;
2E80;CJK RADICAL REPEAT;So;0;ON;;;;;N;;;;;
2E81;CJK RADICAL CLIFF;So;0;ON;;;;;N;;;;;
2E82;CJK RADICAL SECOND ONE;So;0;ON;;;;;N;;;;;
@@ -11250,6 +11285,7 @@
312B;BOPOMOFO LETTER NG;Lo;0;L;;;;;N;;;;;
312C;BOPOMOFO LETTER GN;Lo;0;L;;;;;N;;;;;
312D;BOPOMOFO LETTER IH;Lo;0;L;;;;;N;;;;;
+312E;BOPOMOFO LETTER O WITH DOT ABOVE;Lo;0;L;;;;;N;;;;;
3131;HANGUL LETTER KIYEOK;Lo;0;L;<compat> 1100;;;;N;HANGUL LETTER GIYEOG;;;;
3132;HANGUL LETTER SSANGKIYEOK;Lo;0;L;<compat> 1101;;;;N;HANGUL LETTER SSANG GIYEOG;;;;
3133;HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
@@ -12016,7 +12052,7 @@
4DFE;HEXAGRAM FOR AFTER COMPLETION;So;0;ON;;;;;N;;;;;
4DFF;HEXAGRAM FOR BEFORE COMPLETION;So;0;ON;;;;;N;;;;;
4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
-9FD5;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
+9FEA;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
A000;YI SYLLABLE IT;Lo;0;L;;;;;N;;;;;
A001;YI SYLLABLE IX;Lo;0;L;;;;;N;;;;;
A002;YI SYLLABLE I;Lo;0;L;;;;;N;;;;;
@@ -17093,6 +17129,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
10321;OLD ITALIC NUMERAL FIVE;No;0;L;;;;5;N;;;;;
10322;OLD ITALIC NUMERAL TEN;No;0;L;;;;10;N;;;;;
10323;OLD ITALIC NUMERAL FIFTY;No;0;L;;;;50;N;;;;;
+1032D;OLD ITALIC LETTER YE;Lo;0;L;;;;;N;;;;;
+1032E;OLD ITALIC LETTER NORTHERN TSE;Lo;0;L;;;;;N;;;;;
+1032F;OLD ITALIC LETTER SOUTHERN TSE;Lo;0;L;;;;;N;;;;;
10330;GOTHIC LETTER AHSA;Lo;0;L;;;;;N;;;;;
10331;GOTHIC LETTER BAIRKAN;Lo;0;L;;;;;N;;;;;
10332;GOTHIC LETTER GIBA;Lo;0;L;;;;;N;;;;;
@@ -20068,6 +20107,158 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
118F1;WARANG CITI NUMBER EIGHTY;No;0;L;;;;80;N;;;;;
118F2;WARANG CITI NUMBER NINETY;No;0;L;;;;90;N;;;;;
118FF;WARANG CITI OM;Lo;0;L;;;;;N;;;;;
+11A00;ZANABAZAR SQUARE LETTER A;Lo;0;L;;;;;N;;;;;
+11A01;ZANABAZAR SQUARE VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;;
+11A02;ZANABAZAR SQUARE VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;;
+11A03;ZANABAZAR SQUARE VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;
+11A04;ZANABAZAR SQUARE VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;;
+11A05;ZANABAZAR SQUARE VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;;
+11A06;ZANABAZAR SQUARE VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;;
+11A07;ZANABAZAR SQUARE VOWEL SIGN AI;Mc;0;L;;;;;N;;;;;
+11A08;ZANABAZAR SQUARE VOWEL SIGN AU;Mc;0;L;;;;;N;;;;;
+11A09;ZANABAZAR SQUARE VOWEL SIGN REVERSED I;Mn;0;NSM;;;;;N;;;;;
+11A0A;ZANABAZAR SQUARE VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;;
+11A0B;ZANABAZAR SQUARE LETTER KA;Lo;0;L;;;;;N;;;;;
+11A0C;ZANABAZAR SQUARE LETTER KHA;Lo;0;L;;;;;N;;;;;
+11A0D;ZANABAZAR SQUARE LETTER GA;Lo;0;L;;;;;N;;;;;
+11A0E;ZANABAZAR SQUARE LETTER GHA;Lo;0;L;;;;;N;;;;;
+11A0F;ZANABAZAR SQUARE LETTER NGA;Lo;0;L;;;;;N;;;;;
+11A10;ZANABAZAR SQUARE LETTER CA;Lo;0;L;;;;;N;;;;;
+11A11;ZANABAZAR SQUARE LETTER CHA;Lo;0;L;;;;;N;;;;;
+11A12;ZANABAZAR SQUARE LETTER JA;Lo;0;L;;;;;N;;;;;
+11A13;ZANABAZAR SQUARE LETTER NYA;Lo;0;L;;;;;N;;;;;
+11A14;ZANABAZAR SQUARE LETTER TTA;Lo;0;L;;;;;N;;;;;
+11A15;ZANABAZAR SQUARE LETTER TTHA;Lo;0;L;;;;;N;;;;;
+11A16;ZANABAZAR SQUARE LETTER DDA;Lo;0;L;;;;;N;;;;;
+11A17;ZANABAZAR SQUARE LETTER DDHA;Lo;0;L;;;;;N;;;;;
+11A18;ZANABAZAR SQUARE LETTER NNA;Lo;0;L;;;;;N;;;;;
+11A19;ZANABAZAR SQUARE LETTER TA;Lo;0;L;;;;;N;;;;;
+11A1A;ZANABAZAR SQUARE LETTER THA;Lo;0;L;;;;;N;;;;;
+11A1B;ZANABAZAR SQUARE LETTER DA;Lo;0;L;;;;;N;;;;;
+11A1C;ZANABAZAR SQUARE LETTER DHA;Lo;0;L;;;;;N;;;;;
+11A1D;ZANABAZAR SQUARE LETTER NA;Lo;0;L;;;;;N;;;;;
+11A1E;ZANABAZAR SQUARE LETTER PA;Lo;0;L;;;;;N;;;;;
+11A1F;ZANABAZAR SQUARE LETTER PHA;Lo;0;L;;;;;N;;;;;
+11A20;ZANABAZAR SQUARE LETTER BA;Lo;0;L;;;;;N;;;;;
+11A21;ZANABAZAR SQUARE LETTER BHA;Lo;0;L;;;;;N;;;;;
+11A22;ZANABAZAR SQUARE LETTER MA;Lo;0;L;;;;;N;;;;;
+11A23;ZANABAZAR SQUARE LETTER TSA;Lo;0;L;;;;;N;;;;;
+11A24;ZANABAZAR SQUARE LETTER TSHA;Lo;0;L;;;;;N;;;;;
+11A25;ZANABAZAR SQUARE LETTER DZA;Lo;0;L;;;;;N;;;;;
+11A26;ZANABAZAR SQUARE LETTER DZHA;Lo;0;L;;;;;N;;;;;
+11A27;ZANABAZAR SQUARE LETTER ZHA;Lo;0;L;;;;;N;;;;;
+11A28;ZANABAZAR SQUARE LETTER ZA;Lo;0;L;;;;;N;;;;;
+11A29;ZANABAZAR SQUARE LETTER -A;Lo;0;L;;;;;N;;;;;
+11A2A;ZANABAZAR SQUARE LETTER YA;Lo;0;L;;;;;N;;;;;
+11A2B;ZANABAZAR SQUARE LETTER RA;Lo;0;L;;;;;N;;;;;
+11A2C;ZANABAZAR SQUARE LETTER LA;Lo;0;L;;;;;N;;;;;
+11A2D;ZANABAZAR SQUARE LETTER VA;Lo;0;L;;;;;N;;;;;
+11A2E;ZANABAZAR SQUARE LETTER SHA;Lo;0;L;;;;;N;;;;;
+11A2F;ZANABAZAR SQUARE LETTER SSA;Lo;0;L;;;;;N;;;;;
+11A30;ZANABAZAR SQUARE LETTER SA;Lo;0;L;;;;;N;;;;;
+11A31;ZANABAZAR SQUARE LETTER HA;Lo;0;L;;;;;N;;;;;
+11A32;ZANABAZAR SQUARE LETTER KSSA;Lo;0;L;;;;;N;;;;;
+11A33;ZANABAZAR SQUARE FINAL CONSONANT MARK;Mn;0;NSM;;;;;N;;;;;
+11A34;ZANABAZAR SQUARE SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;
+11A35;ZANABAZAR SQUARE SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;
+11A36;ZANABAZAR SQUARE SIGN CANDRABINDU WITH ORNAMENT;Mn;0;NSM;;;;;N;;;;;
+11A37;ZANABAZAR SQUARE SIGN CANDRA WITH ORNAMENT;Mn;0;NSM;;;;;N;;;;;
+11A38;ZANABAZAR SQUARE SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;
+11A39;ZANABAZAR SQUARE SIGN VISARGA;Mc;0;L;;;;;N;;;;;
+11A3A;ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA;Lo;0;L;;;;;N;;;;;
+11A3B;ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA;Mn;0;NSM;;;;;N;;;;;
+11A3C;ZANABAZAR SQUARE CLUSTER-FINAL LETTER RA;Mn;0;NSM;;;;;N;;;;;
+11A3D;ZANABAZAR SQUARE CLUSTER-FINAL LETTER LA;Mn;0;NSM;;;;;N;;;;;
+11A3E;ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA;Mn;0;NSM;;;;;N;;;;;
+11A3F;ZANABAZAR SQUARE INITIAL HEAD MARK;Po;0;L;;;;;N;;;;;
+11A40;ZANABAZAR SQUARE CLOSING HEAD MARK;Po;0;L;;;;;N;;;;;
+11A41;ZANABAZAR SQUARE MARK TSHEG;Po;0;L;;;;;N;;;;;
+11A42;ZANABAZAR SQUARE MARK SHAD;Po;0;L;;;;;N;;;;;
+11A43;ZANABAZAR SQUARE MARK DOUBLE SHAD;Po;0;L;;;;;N;;;;;
+11A44;ZANABAZAR SQUARE MARK LONG TSHEG;Po;0;L;;;;;N;;;;;
+11A45;ZANABAZAR SQUARE INITIAL DOUBLE-LINED HEAD MARK;Po;0;L;;;;;N;;;;;
+11A46;ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK;Po;0;L;;;;;N;;;;;
+11A47;ZANABAZAR SQUARE SUBJOINER;Mn;9;NSM;;;;;N;;;;;
+11A50;SOYOMBO LETTER A;Lo;0;L;;;;;N;;;;;
+11A51;SOYOMBO VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;;
+11A52;SOYOMBO VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;;
+11A53;SOYOMBO VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;
+11A54;SOYOMBO VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;;
+11A55;SOYOMBO VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;;
+11A56;SOYOMBO VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;;
+11A57;SOYOMBO VOWEL SIGN AI;Mc;0;L;;;;;N;;;;;
+11A58;SOYOMBO VOWEL SIGN AU;Mc;0;L;;;;;N;;;;;
+11A59;SOYOMBO VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;
+11A5A;SOYOMBO VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;;
+11A5B;SOYOMBO VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;;
+11A5C;SOYOMBO LETTER KA;Lo;0;L;;;;;N;;;;;
+11A5D;SOYOMBO LETTER KHA;Lo;0;L;;;;;N;;;;;
+11A5E;SOYOMBO LETTER GA;Lo;0;L;;;;;N;;;;;
+11A5F;SOYOMBO LETTER GHA;Lo;0;L;;;;;N;;;;;
+11A60;SOYOMBO LETTER NGA;Lo;0;L;;;;;N;;;;;
+11A61;SOYOMBO LETTER CA;Lo;0;L;;;;;N;;;;;
+11A62;SOYOMBO LETTER CHA;Lo;0;L;;;;;N;;;;;
+11A63;SOYOMBO LETTER JA;Lo;0;L;;;;;N;;;;;
+11A64;SOYOMBO LETTER JHA;Lo;0;L;;;;;N;;;;;
+11A65;SOYOMBO LETTER NYA;Lo;0;L;;;;;N;;;;;
+11A66;SOYOMBO LETTER TTA;Lo;0;L;;;;;N;;;;;
+11A67;SOYOMBO LETTER TTHA;Lo;0;L;;;;;N;;;;;
+11A68;SOYOMBO LETTER DDA;Lo;0;L;;;;;N;;;;;
+11A69;SOYOMBO LETTER DDHA;Lo;0;L;;;;;N;;;;;
+11A6A;SOYOMBO LETTER NNA;Lo;0;L;;;;;N;;;;;
+11A6B;SOYOMBO LETTER TA;Lo;0;L;;;;;N;;;;;
+11A6C;SOYOMBO LETTER THA;Lo;0;L;;;;;N;;;;;
+11A6D;SOYOMBO LETTER DA;Lo;0;L;;;;;N;;;;;
+11A6E;SOYOMBO LETTER DHA;Lo;0;L;;;;;N;;;;;
+11A6F;SOYOMBO LETTER NA;Lo;0;L;;;;;N;;;;;
+11A70;SOYOMBO LETTER PA;Lo;0;L;;;;;N;;;;;
+11A71;SOYOMBO LETTER PHA;Lo;0;L;;;;;N;;;;;
+11A72;SOYOMBO LETTER BA;Lo;0;L;;;;;N;;;;;
+11A73;SOYOMBO LETTER BHA;Lo;0;L;;;;;N;;;;;
+11A74;SOYOMBO LETTER MA;Lo;0;L;;;;;N;;;;;
+11A75;SOYOMBO LETTER TSA;Lo;0;L;;;;;N;;;;;
+11A76;SOYOMBO LETTER TSHA;Lo;0;L;;;;;N;;;;;
+11A77;SOYOMBO LETTER DZA;Lo;0;L;;;;;N;;;;;
+11A78;SOYOMBO LETTER ZHA;Lo;0;L;;;;;N;;;;;
+11A79;SOYOMBO LETTER ZA;Lo;0;L;;;;;N;;;;;
+11A7A;SOYOMBO LETTER -A;Lo;0;L;;;;;N;;;;;
+11A7B;SOYOMBO LETTER YA;Lo;0;L;;;;;N;;;;;
+11A7C;SOYOMBO LETTER RA;Lo;0;L;;;;;N;;;;;
+11A7D;SOYOMBO LETTER LA;Lo;0;L;;;;;N;;;;;
+11A7E;SOYOMBO LETTER VA;Lo;0;L;;;;;N;;;;;
+11A7F;SOYOMBO LETTER SHA;Lo;0;L;;;;;N;;;;;
+11A80;SOYOMBO LETTER SSA;Lo;0;L;;;;;N;;;;;
+11A81;SOYOMBO LETTER SA;Lo;0;L;;;;;N;;;;;
+11A82;SOYOMBO LETTER HA;Lo;0;L;;;;;N;;;;;
+11A83;SOYOMBO LETTER KSSA;Lo;0;L;;;;;N;;;;;
+11A86;SOYOMBO CLUSTER-INITIAL LETTER RA;Lo;0;L;;;;;N;;;;;
+11A87;SOYOMBO CLUSTER-INITIAL LETTER LA;Lo;0;L;;;;;N;;;;;
+11A88;SOYOMBO CLUSTER-INITIAL LETTER SHA;Lo;0;L;;;;;N;;;;;
+11A89;SOYOMBO CLUSTER-INITIAL LETTER SA;Lo;0;L;;;;;N;;;;;
+11A8A;SOYOMBO FINAL CONSONANT SIGN G;Mn;0;NSM;;;;;N;;;;;
+11A8B;SOYOMBO FINAL CONSONANT SIGN K;Mn;0;NSM;;;;;N;;;;;
+11A8C;SOYOMBO FINAL CONSONANT SIGN NG;Mn;0;NSM;;;;;N;;;;;
+11A8D;SOYOMBO FINAL CONSONANT SIGN D;Mn;0;NSM;;;;;N;;;;;
+11A8E;SOYOMBO FINAL CONSONANT SIGN N;Mn;0;NSM;;;;;N;;;;;
+11A8F;SOYOMBO FINAL CONSONANT SIGN B;Mn;0;NSM;;;;;N;;;;;
+11A90;SOYOMBO FINAL CONSONANT SIGN M;Mn;0;NSM;;;;;N;;;;;
+11A91;SOYOMBO FINAL CONSONANT SIGN R;Mn;0;NSM;;;;;N;;;;;
+11A92;SOYOMBO FINAL CONSONANT SIGN L;Mn;0;NSM;;;;;N;;;;;
+11A93;SOYOMBO FINAL CONSONANT SIGN SH;Mn;0;NSM;;;;;N;;;;;
+11A94;SOYOMBO FINAL CONSONANT SIGN S;Mn;0;NSM;;;;;N;;;;;
+11A95;SOYOMBO FINAL CONSONANT SIGN -A;Mn;0;NSM;;;;;N;;;;;
+11A96;SOYOMBO SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;
+11A97;SOYOMBO SIGN VISARGA;Mc;0;L;;;;;N;;;;;
+11A98;SOYOMBO GEMINATION MARK;Mn;0;NSM;;;;;N;;;;;
+11A99;SOYOMBO SUBJOINER;Mn;9;NSM;;;;;N;;;;;
+11A9A;SOYOMBO MARK TSHEG;Po;0;L;;;;;N;;;;;
+11A9B;SOYOMBO MARK SHAD;Po;0;L;;;;;N;;;;;
+11A9C;SOYOMBO MARK DOUBLE SHAD;Po;0;L;;;;;N;;;;;
+11A9E;SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME;Po;0;L;;;;;N;;;;;
+11A9F;SOYOMBO HEAD MARK WITH MOON AND SUN AND FLAME;Po;0;L;;;;;N;;;;;
+11AA0;SOYOMBO HEAD MARK WITH MOON AND SUN;Po;0;L;;;;;N;;;;;
+11AA1;SOYOMBO TERMINAL MARK-1;Po;0;L;;;;;N;;;;;
+11AA2;SOYOMBO TERMINAL MARK-2;Po;0;L;;;;;N;;;;;
11AC0;PAU CIN HAU LETTER PA;Lo;0;L;;;;;N;;;;;
11AC1;PAU CIN HAU LETTER KA;Lo;0;L;;;;;N;;;;;
11AC2;PAU CIN HAU LETTER LA;Lo;0;L;;;;;N;;;;;
@@ -20290,6 +20481,81 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
11CB4;MARCHEN VOWEL SIGN O;Mc;0;L;;;;;N;;;;;
11CB5;MARCHEN SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;
11CB6;MARCHEN SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;
+11D00;MASARAM GONDI LETTER A;Lo;0;L;;;;;N;;;;;
+11D01;MASARAM GONDI LETTER AA;Lo;0;L;;;;;N;;;;;
+11D02;MASARAM GONDI LETTER I;Lo;0;L;;;;;N;;;;;
+11D03;MASARAM GONDI LETTER II;Lo;0;L;;;;;N;;;;;
+11D04;MASARAM GONDI LETTER U;Lo;0;L;;;;;N;;;;;
+11D05;MASARAM GONDI LETTER UU;Lo;0;L;;;;;N;;;;;
+11D06;MASARAM GONDI LETTER E;Lo;0;L;;;;;N;;;;;
+11D08;MASARAM GONDI LETTER AI;Lo;0;L;;;;;N;;;;;
+11D09;MASARAM GONDI LETTER O;Lo;0;L;;;;;N;;;;;
+11D0B;MASARAM GONDI LETTER AU;Lo;0;L;;;;;N;;;;;
+11D0C;MASARAM GONDI LETTER KA;Lo;0;L;;;;;N;;;;;
+11D0D;MASARAM GONDI LETTER KHA;Lo;0;L;;;;;N;;;;;
+11D0E;MASARAM GONDI LETTER GA;Lo;0;L;;;;;N;;;;;
+11D0F;MASARAM GONDI LETTER GHA;Lo;0;L;;;;;N;;;;;
+11D10;MASARAM GONDI LETTER NGA;Lo;0;L;;;;;N;;;;;
+11D11;MASARAM GONDI LETTER CA;Lo;0;L;;;;;N;;;;;
+11D12;MASARAM GONDI LETTER CHA;Lo;0;L;;;;;N;;;;;
+11D13;MASARAM GONDI LETTER JA;Lo;0;L;;;;;N;;;;;
+11D14;MASARAM GONDI LETTER JHA;Lo;0;L;;;;;N;;;;;
+11D15;MASARAM GONDI LETTER NYA;Lo;0;L;;;;;N;;;;;
+11D16;MASARAM GONDI LETTER TTA;Lo;0;L;;;;;N;;;;;
+11D17;MASARAM GONDI LETTER TTHA;Lo;0;L;;;;;N;;;;;
+11D18;MASARAM GONDI LETTER DDA;Lo;0;L;;;;;N;;;;;
+11D19;MASARAM GONDI LETTER DDHA;Lo;0;L;;;;;N;;;;;
+11D1A;MASARAM GONDI LETTER NNA;Lo;0;L;;;;;N;;;;;
+11D1B;MASARAM GONDI LETTER TA;Lo;0;L;;;;;N;;;;;
+11D1C;MASARAM GONDI LETTER THA;Lo;0;L;;;;;N;;;;;
+11D1D;MASARAM GONDI LETTER DA;Lo;0;L;;;;;N;;;;;
+11D1E;MASARAM GONDI LETTER DHA;Lo;0;L;;;;;N;;;;;
+11D1F;MASARAM GONDI LETTER NA;Lo;0;L;;;;;N;;;;;
+11D20;MASARAM GONDI LETTER PA;Lo;0;L;;;;;N;;;;;
+11D21;MASARAM GONDI LETTER PHA;Lo;0;L;;;;;N;;;;;
+11D22;MASARAM GONDI LETTER BA;Lo;0;L;;;;;N;;;;;
+11D23;MASARAM GONDI LETTER BHA;Lo;0;L;;;;;N;;;;;
+11D24;MASARAM GONDI LETTER MA;Lo;0;L;;;;;N;;;;;
+11D25;MASARAM GONDI LETTER YA;Lo;0;L;;;;;N;;;;;
+11D26;MASARAM GONDI LETTER RA;Lo;0;L;;;;;N;;;;;
+11D27;MASARAM GONDI LETTER LA;Lo;0;L;;;;;N;;;;;
+11D28;MASARAM GONDI LETTER VA;Lo;0;L;;;;;N;;;;;
+11D29;MASARAM GONDI LETTER SHA;Lo;0;L;;;;;N;;;;;
+11D2A;MASARAM GONDI LETTER SSA;Lo;0;L;;;;;N;;;;;
+11D2B;MASARAM GONDI LETTER SA;Lo;0;L;;;;;N;;;;;
+11D2C;MASARAM GONDI LETTER HA;Lo;0;L;;;;;N;;;;;
+11D2D;MASARAM GONDI LETTER LLA;Lo;0;L;;;;;N;;;;;
+11D2E;MASARAM GONDI LETTER KSSA;Lo;0;L;;;;;N;;;;;
+11D2F;MASARAM GONDI LETTER JNYA;Lo;0;L;;;;;N;;;;;
+11D30;MASARAM GONDI LETTER TRA;Lo;0;L;;;;;N;;;;;
+11D31;MASARAM GONDI VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;;
+11D32;MASARAM GONDI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;;
+11D33;MASARAM GONDI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;;
+11D34;MASARAM GONDI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;
+11D35;MASARAM GONDI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;;
+11D36;MASARAM GONDI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;
+11D3A;MASARAM GONDI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;;
+11D3C;MASARAM GONDI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;;
+11D3D;MASARAM GONDI VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;;
+11D3F;MASARAM GONDI VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;;
+11D40;MASARAM GONDI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;
+11D41;MASARAM GONDI SIGN VISARGA;Mn;0;NSM;;;;;N;;;;;
+11D42;MASARAM GONDI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;;
+11D43;MASARAM GONDI SIGN CANDRA;Mn;0;NSM;;;;;N;;;;;
+11D44;MASARAM GONDI SIGN HALANTA;Mn;9;NSM;;;;;N;;;;;
+11D45;MASARAM GONDI VIRAMA;Mn;9;NSM;;;;;N;;;;;
+11D46;MASARAM GONDI REPHA;Lo;0;L;;;;;N;;;;;
+11D47;MASARAM GONDI RA-KARA;Mn;0;NSM;;;;;N;;;;;
+11D50;MASARAM GONDI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;;
+11D51;MASARAM GONDI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;;
+11D52;MASARAM GONDI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;;
+11D53;MASARAM GONDI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;;
+11D54;MASARAM GONDI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;;
+11D55;MASARAM GONDI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;;
+11D56;MASARAM GONDI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;;
+11D57;MASARAM GONDI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;;
+11D58;MASARAM GONDI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;;
+11D59;MASARAM GONDI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;;
12000;CUNEIFORM SIGN A;Lo;0;L;;;;;N;;;;;
12001;CUNEIFORM SIGN A TIMES A;Lo;0;L;;;;;N;;;;;
12002;CUNEIFORM SIGN A TIMES BAD;Lo;0;L;;;;;N;;;;;
@@ -24087,6 +24353,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
16F9E;MIAO LETTER REFORMED TONE-6;Lm;0;L;;;;;N;;;;;
16F9F;MIAO LETTER REFORMED TONE-8;Lm;0;L;;;;;N;;;;;
16FE0;TANGUT ITERATION MARK;Lm;0;L;;;;;N;;;;;
+16FE1;NUSHU ITERATION MARK;Lm;0;L;;;;;N;;;;;
17000;<Tangut Ideograph, First>;Lo;0;L;;;;;N;;;;;
187EC;<Tangut Ideograph, Last>;Lo;0;L;;;;;N;;;;;
18800;TANGUT COMPONENT-001;Lo;0;L;;;;;N;;;;;
@@ -24846,6 +25113,687 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
18AF2;TANGUT COMPONENT-755;Lo;0;L;;;;;N;;;;;
1B000;KATAKANA LETTER ARCHAIC E;Lo;0;L;;;;;N;;;;;
1B001;HIRAGANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;;
+1B002;HENTAIGANA LETTER A-1;Lo;0;L;;;;;N;;;;;
+1B003;HENTAIGANA LETTER A-2;Lo;0;L;;;;;N;;;;;
+1B004;HENTAIGANA LETTER A-3;Lo;0;L;;;;;N;;;;;
+1B005;HENTAIGANA LETTER A-WO;Lo;0;L;;;;;N;;;;;
+1B006;HENTAIGANA LETTER I-1;Lo;0;L;;;;;N;;;;;
+1B007;HENTAIGANA LETTER I-2;Lo;0;L;;;;;N;;;;;
+1B008;HENTAIGANA LETTER I-3;Lo;0;L;;;;;N;;;;;
+1B009;HENTAIGANA LETTER I-4;Lo;0;L;;;;;N;;;;;
+1B00A;HENTAIGANA LETTER U-1;Lo;0;L;;;;;N;;;;;
+1B00B;HENTAIGANA LETTER U-2;Lo;0;L;;;;;N;;;;;
+1B00C;HENTAIGANA LETTER U-3;Lo;0;L;;;;;N;;;;;
+1B00D;HENTAIGANA LETTER U-4;Lo;0;L;;;;;N;;;;;
+1B00E;HENTAIGANA LETTER U-5;Lo;0;L;;;;;N;;;;;
+1B00F;HENTAIGANA LETTER E-2;Lo;0;L;;;;;N;;;;;
+1B010;HENTAIGANA LETTER E-3;Lo;0;L;;;;;N;;;;;
+1B011;HENTAIGANA LETTER E-4;Lo;0;L;;;;;N;;;;;
+1B012;HENTAIGANA LETTER E-5;Lo;0;L;;;;;N;;;;;
+1B013;HENTAIGANA LETTER E-6;Lo;0;L;;;;;N;;;;;
+1B014;HENTAIGANA LETTER O-1;Lo;0;L;;;;;N;;;;;
+1B015;HENTAIGANA LETTER O-2;Lo;0;L;;;;;N;;;;;
+1B016;HENTAIGANA LETTER O-3;Lo;0;L;;;;;N;;;;;
+1B017;HENTAIGANA LETTER KA-1;Lo;0;L;;;;;N;;;;;
+1B018;HENTAIGANA LETTER KA-2;Lo;0;L;;;;;N;;;;;
+1B019;HENTAIGANA LETTER KA-3;Lo;0;L;;;;;N;;;;;
+1B01A;HENTAIGANA LETTER KA-4;Lo;0;L;;;;;N;;;;;
+1B01B;HENTAIGANA LETTER KA-5;Lo;0;L;;;;;N;;;;;
+1B01C;HENTAIGANA LETTER KA-6;Lo;0;L;;;;;N;;;;;
+1B01D;HENTAIGANA LETTER KA-7;Lo;0;L;;;;;N;;;;;
+1B01E;HENTAIGANA LETTER KA-8;Lo;0;L;;;;;N;;;;;
+1B01F;HENTAIGANA LETTER KA-9;Lo;0;L;;;;;N;;;;;
+1B020;HENTAIGANA LETTER KA-10;Lo;0;L;;;;;N;;;;;
+1B021;HENTAIGANA LETTER KA-11;Lo;0;L;;;;;N;;;;;
+1B022;HENTAIGANA LETTER KA-KE;Lo;0;L;;;;;N;;;;;
+1B023;HENTAIGANA LETTER KI-1;Lo;0;L;;;;;N;;;;;
+1B024;HENTAIGANA LETTER KI-2;Lo;0;L;;;;;N;;;;;
+1B025;HENTAIGANA LETTER KI-3;Lo;0;L;;;;;N;;;;;
+1B026;HENTAIGANA LETTER KI-4;Lo;0;L;;;;;N;;;;;
+1B027;HENTAIGANA LETTER KI-5;Lo;0;L;;;;;N;;;;;
+1B028;HENTAIGANA LETTER KI-6;Lo;0;L;;;;;N;;;;;
+1B029;HENTAIGANA LETTER KI-7;Lo;0;L;;;;;N;;;;;
+1B02A;HENTAIGANA LETTER KI-8;Lo;0;L;;;;;N;;;;;
+1B02B;HENTAIGANA LETTER KU-1;Lo;0;L;;;;;N;;;;;
+1B02C;HENTAIGANA LETTER KU-2;Lo;0;L;;;;;N;;;;;
+1B02D;HENTAIGANA LETTER KU-3;Lo;0;L;;;;;N;;;;;
+1B02E;HENTAIGANA LETTER KU-4;Lo;0;L;;;;;N;;;;;
+1B02F;HENTAIGANA LETTER KU-5;Lo;0;L;;;;;N;;;;;
+1B030;HENTAIGANA LETTER KU-6;Lo;0;L;;;;;N;;;;;
+1B031;HENTAIGANA LETTER KU-7;Lo;0;L;;;;;N;;;;;
+1B032;HENTAIGANA LETTER KE-1;Lo;0;L;;;;;N;;;;;
+1B033;HENTAIGANA LETTER KE-2;Lo;0;L;;;;;N;;;;;
+1B034;HENTAIGANA LETTER KE-3;Lo;0;L;;;;;N;;;;;
+1B035;HENTAIGANA LETTER KE-4;Lo;0;L;;;;;N;;;;;
+1B036;HENTAIGANA LETTER KE-5;Lo;0;L;;;;;N;;;;;
+1B037;HENTAIGANA LETTER KE-6;Lo;0;L;;;;;N;;;;;
+1B038;HENTAIGANA LETTER KO-1;Lo;0;L;;;;;N;;;;;
+1B039;HENTAIGANA LETTER KO-2;Lo;0;L;;;;;N;;;;;
+1B03A;HENTAIGANA LETTER KO-3;Lo;0;L;;;;;N;;;;;
+1B03B;HENTAIGANA LETTER KO-KI;Lo;0;L;;;;;N;;;;;
+1B03C;HENTAIGANA LETTER SA-1;Lo;0;L;;;;;N;;;;;
+1B03D;HENTAIGANA LETTER SA-2;Lo;0;L;;;;;N;;;;;
+1B03E;HENTAIGANA LETTER SA-3;Lo;0;L;;;;;N;;;;;
+1B03F;HENTAIGANA LETTER SA-4;Lo;0;L;;;;;N;;;;;
+1B040;HENTAIGANA LETTER SA-5;Lo;0;L;;;;;N;;;;;
+1B041;HENTAIGANA LETTER SA-6;Lo;0;L;;;;;N;;;;;
+1B042;HENTAIGANA LETTER SA-7;Lo;0;L;;;;;N;;;;;
+1B043;HENTAIGANA LETTER SA-8;Lo;0;L;;;;;N;;;;;
+1B044;HENTAIGANA LETTER SI-1;Lo;0;L;;;;;N;;;;;
+1B045;HENTAIGANA LETTER SI-2;Lo;0;L;;;;;N;;;;;
+1B046;HENTAIGANA LETTER SI-3;Lo;0;L;;;;;N;;;;;
+1B047;HENTAIGANA LETTER SI-4;Lo;0;L;;;;;N;;;;;
+1B048;HENTAIGANA LETTER SI-5;Lo;0;L;;;;;N;;;;;
+1B049;HENTAIGANA LETTER SI-6;Lo;0;L;;;;;N;;;;;
+1B04A;HENTAIGANA LETTER SU-1;Lo;0;L;;;;;N;;;;;
+1B04B;HENTAIGANA LETTER SU-2;Lo;0;L;;;;;N;;;;;
+1B04C;HENTAIGANA LETTER SU-3;Lo;0;L;;;;;N;;;;;
+1B04D;HENTAIGANA LETTER SU-4;Lo;0;L;;;;;N;;;;;
+1B04E;HENTAIGANA LETTER SU-5;Lo;0;L;;;;;N;;;;;
+1B04F;HENTAIGANA LETTER SU-6;Lo;0;L;;;;;N;;;;;
+1B050;HENTAIGANA LETTER SU-7;Lo;0;L;;;;;N;;;;;
+1B051;HENTAIGANA LETTER SU-8;Lo;0;L;;;;;N;;;;;
+1B052;HENTAIGANA LETTER SE-1;Lo;0;L;;;;;N;;;;;
+1B053;HENTAIGANA LETTER SE-2;Lo;0;L;;;;;N;;;;;
+1B054;HENTAIGANA LETTER SE-3;Lo;0;L;;;;;N;;;;;
+1B055;HENTAIGANA LETTER SE-4;Lo;0;L;;;;;N;;;;;
+1B056;HENTAIGANA LETTER SE-5;Lo;0;L;;;;;N;;;;;
+1B057;HENTAIGANA LETTER SO-1;Lo;0;L;;;;;N;;;;;
+1B058;HENTAIGANA LETTER SO-2;Lo;0;L;;;;;N;;;;;
+1B059;HENTAIGANA LETTER SO-3;Lo;0;L;;;;;N;;;;;
+1B05A;HENTAIGANA LETTER SO-4;Lo;0;L;;;;;N;;;;;
+1B05B;HENTAIGANA LETTER SO-5;Lo;0;L;;;;;N;;;;;
+1B05C;HENTAIGANA LETTER SO-6;Lo;0;L;;;;;N;;;;;
+1B05D;HENTAIGANA LETTER SO-7;Lo;0;L;;;;;N;;;;;
+1B05E;HENTAIGANA LETTER TA-1;Lo;0;L;;;;;N;;;;;
+1B05F;HENTAIGANA LETTER TA-2;Lo;0;L;;;;;N;;;;;
+1B060;HENTAIGANA LETTER TA-3;Lo;0;L;;;;;N;;;;;
+1B061;HENTAIGANA LETTER TA-4;Lo;0;L;;;;;N;;;;;
+1B062;HENTAIGANA LETTER TI-1;Lo;0;L;;;;;N;;;;;
+1B063;HENTAIGANA LETTER TI-2;Lo;0;L;;;;;N;;;;;
+1B064;HENTAIGANA LETTER TI-3;Lo;0;L;;;;;N;;;;;
+1B065;HENTAIGANA LETTER TI-4;Lo;0;L;;;;;N;;;;;
+1B066;HENTAIGANA LETTER TI-5;Lo;0;L;;;;;N;;;;;
+1B067;HENTAIGANA LETTER TI-6;Lo;0;L;;;;;N;;;;;
+1B068;HENTAIGANA LETTER TI-7;Lo;0;L;;;;;N;;;;;
+1B069;HENTAIGANA LETTER TU-1;Lo;0;L;;;;;N;;;;;
+1B06A;HENTAIGANA LETTER TU-2;Lo;0;L;;;;;N;;;;;
+1B06B;HENTAIGANA LETTER TU-3;Lo;0;L;;;;;N;;;;;
+1B06C;HENTAIGANA LETTER TU-4;Lo;0;L;;;;;N;;;;;
+1B06D;HENTAIGANA LETTER TU-TO;Lo;0;L;;;;;N;;;;;
+1B06E;HENTAIGANA LETTER TE-1;Lo;0;L;;;;;N;;;;;
+1B06F;HENTAIGANA LETTER TE-2;Lo;0;L;;;;;N;;;;;
+1B070;HENTAIGANA LETTER TE-3;Lo;0;L;;;;;N;;;;;
+1B071;HENTAIGANA LETTER TE-4;Lo;0;L;;;;;N;;;;;
+1B072;HENTAIGANA LETTER TE-5;Lo;0;L;;;;;N;;;;;
+1B073;HENTAIGANA LETTER TE-6;Lo;0;L;;;;;N;;;;;
+1B074;HENTAIGANA LETTER TE-7;Lo;0;L;;;;;N;;;;;
+1B075;HENTAIGANA LETTER TE-8;Lo;0;L;;;;;N;;;;;
+1B076;HENTAIGANA LETTER TE-9;Lo;0;L;;;;;N;;;;;
+1B077;HENTAIGANA LETTER TO-1;Lo;0;L;;;;;N;;;;;
+1B078;HENTAIGANA LETTER TO-2;Lo;0;L;;;;;N;;;;;
+1B079;HENTAIGANA LETTER TO-3;Lo;0;L;;;;;N;;;;;
+1B07A;HENTAIGANA LETTER TO-4;Lo;0;L;;;;;N;;;;;
+1B07B;HENTAIGANA LETTER TO-5;Lo;0;L;;;;;N;;;;;
+1B07C;HENTAIGANA LETTER TO-6;Lo;0;L;;;;;N;;;;;
+1B07D;HENTAIGANA LETTER TO-RA;Lo;0;L;;;;;N;;;;;
+1B07E;HENTAIGANA LETTER NA-1;Lo;0;L;;;;;N;;;;;
+1B07F;HENTAIGANA LETTER NA-2;Lo;0;L;;;;;N;;;;;
+1B080;HENTAIGANA LETTER NA-3;Lo;0;L;;;;;N;;;;;
+1B081;HENTAIGANA LETTER NA-4;Lo;0;L;;;;;N;;;;;
+1B082;HENTAIGANA LETTER NA-5;Lo;0;L;;;;;N;;;;;
+1B083;HENTAIGANA LETTER NA-6;Lo;0;L;;;;;N;;;;;
+1B084;HENTAIGANA LETTER NA-7;Lo;0;L;;;;;N;;;;;
+1B085;HENTAIGANA LETTER NA-8;Lo;0;L;;;;;N;;;;;
+1B086;HENTAIGANA LETTER NA-9;Lo;0;L;;;;;N;;;;;
+1B087;HENTAIGANA LETTER NI-1;Lo;0;L;;;;;N;;;;;
+1B088;HENTAIGANA LETTER NI-2;Lo;0;L;;;;;N;;;;;
+1B089;HENTAIGANA LETTER NI-3;Lo;0;L;;;;;N;;;;;
+1B08A;HENTAIGANA LETTER NI-4;Lo;0;L;;;;;N;;;;;
+1B08B;HENTAIGANA LETTER NI-5;Lo;0;L;;;;;N;;;;;
+1B08C;HENTAIGANA LETTER NI-6;Lo;0;L;;;;;N;;;;;
+1B08D;HENTAIGANA LETTER NI-7;Lo;0;L;;;;;N;;;;;
+1B08E;HENTAIGANA LETTER NI-TE;Lo;0;L;;;;;N;;;;;
+1B08F;HENTAIGANA LETTER NU-1;Lo;0;L;;;;;N;;;;;
+1B090;HENTAIGANA LETTER NU-2;Lo;0;L;;;;;N;;;;;
+1B091;HENTAIGANA LETTER NU-3;Lo;0;L;;;;;N;;;;;
+1B092;HENTAIGANA LETTER NE-1;Lo;0;L;;;;;N;;;;;
+1B093;HENTAIGANA LETTER NE-2;Lo;0;L;;;;;N;;;;;
+1B094;HENTAIGANA LETTER NE-3;Lo;0;L;;;;;N;;;;;
+1B095;HENTAIGANA LETTER NE-4;Lo;0;L;;;;;N;;;;;
+1B096;HENTAIGANA LETTER NE-5;Lo;0;L;;;;;N;;;;;
+1B097;HENTAIGANA LETTER NE-6;Lo;0;L;;;;;N;;;;;
+1B098;HENTAIGANA LETTER NE-KO;Lo;0;L;;;;;N;;;;;
+1B099;HENTAIGANA LETTER NO-1;Lo;0;L;;;;;N;;;;;
+1B09A;HENTAIGANA LETTER NO-2;Lo;0;L;;;;;N;;;;;
+1B09B;HENTAIGANA LETTER NO-3;Lo;0;L;;;;;N;;;;;
+1B09C;HENTAIGANA LETTER NO-4;Lo;0;L;;;;;N;;;;;
+1B09D;HENTAIGANA LETTER NO-5;Lo;0;L;;;;;N;;;;;
+1B09E;HENTAIGANA LETTER HA-1;Lo;0;L;;;;;N;;;;;
+1B09F;HENTAIGANA LETTER HA-2;Lo;0;L;;;;;N;;;;;
+1B0A0;HENTAIGANA LETTER HA-3;Lo;0;L;;;;;N;;;;;
+1B0A1;HENTAIGANA LETTER HA-4;Lo;0;L;;;;;N;;;;;
+1B0A2;HENTAIGANA LETTER HA-5;Lo;0;L;;;;;N;;;;;
+1B0A3;HENTAIGANA LETTER HA-6;Lo;0;L;;;;;N;;;;;
+1B0A4;HENTAIGANA LETTER HA-7;Lo;0;L;;;;;N;;;;;
+1B0A5;HENTAIGANA LETTER HA-8;Lo;0;L;;;;;N;;;;;
+1B0A6;HENTAIGANA LETTER HA-9;Lo;0;L;;;;;N;;;;;
+1B0A7;HENTAIGANA LETTER HA-10;Lo;0;L;;;;;N;;;;;
+1B0A8;HENTAIGANA LETTER HA-11;Lo;0;L;;;;;N;;;;;
+1B0A9;HENTAIGANA LETTER HI-1;Lo;0;L;;;;;N;;;;;
+1B0AA;HENTAIGANA LETTER HI-2;Lo;0;L;;;;;N;;;;;
+1B0AB;HENTAIGANA LETTER HI-3;Lo;0;L;;;;;N;;;;;
+1B0AC;HENTAIGANA LETTER HI-4;Lo;0;L;;;;;N;;;;;
+1B0AD;HENTAIGANA LETTER HI-5;Lo;0;L;;;;;N;;;;;
+1B0AE;HENTAIGANA LETTER HI-6;Lo;0;L;;;;;N;;;;;
+1B0AF;HENTAIGANA LETTER HI-7;Lo;0;L;;;;;N;;;;;
+1B0B0;HENTAIGANA LETTER HU-1;Lo;0;L;;;;;N;;;;;
+1B0B1;HENTAIGANA LETTER HU-2;Lo;0;L;;;;;N;;;;;
+1B0B2;HENTAIGANA LETTER HU-3;Lo;0;L;;;;;N;;;;;
+1B0B3;HENTAIGANA LETTER HE-1;Lo;0;L;;;;;N;;;;;
+1B0B4;HENTAIGANA LETTER HE-2;Lo;0;L;;;;;N;;;;;
+1B0B5;HENTAIGANA LETTER HE-3;Lo;0;L;;;;;N;;;;;
+1B0B6;HENTAIGANA LETTER HE-4;Lo;0;L;;;;;N;;;;;
+1B0B7;HENTAIGANA LETTER HE-5;Lo;0;L;;;;;N;;;;;
+1B0B8;HENTAIGANA LETTER HE-6;Lo;0;L;;;;;N;;;;;
+1B0B9;HENTAIGANA LETTER HE-7;Lo;0;L;;;;;N;;;;;
+1B0BA;HENTAIGANA LETTER HO-1;Lo;0;L;;;;;N;;;;;
+1B0BB;HENTAIGANA LETTER HO-2;Lo;0;L;;;;;N;;;;;
+1B0BC;HENTAIGANA LETTER HO-3;Lo;0;L;;;;;N;;;;;
+1B0BD;HENTAIGANA LETTER HO-4;Lo;0;L;;;;;N;;;;;
+1B0BE;HENTAIGANA LETTER HO-5;Lo;0;L;;;;;N;;;;;
+1B0BF;HENTAIGANA LETTER HO-6;Lo;0;L;;;;;N;;;;;
+1B0C0;HENTAIGANA LETTER HO-7;Lo;0;L;;;;;N;;;;;
+1B0C1;HENTAIGANA LETTER HO-8;Lo;0;L;;;;;N;;;;;
+1B0C2;HENTAIGANA LETTER MA-1;Lo;0;L;;;;;N;;;;;
+1B0C3;HENTAIGANA LETTER MA-2;Lo;0;L;;;;;N;;;;;
+1B0C4;HENTAIGANA LETTER MA-3;Lo;0;L;;;;;N;;;;;
+1B0C5;HENTAIGANA LETTER MA-4;Lo;0;L;;;;;N;;;;;
+1B0C6;HENTAIGANA LETTER MA-5;Lo;0;L;;;;;N;;;;;
+1B0C7;HENTAIGANA LETTER MA-6;Lo;0;L;;;;;N;;;;;
+1B0C8;HENTAIGANA LETTER MA-7;Lo;0;L;;;;;N;;;;;
+1B0C9;HENTAIGANA LETTER MI-1;Lo;0;L;;;;;N;;;;;
+1B0CA;HENTAIGANA LETTER MI-2;Lo;0;L;;;;;N;;;;;
+1B0CB;HENTAIGANA LETTER MI-3;Lo;0;L;;;;;N;;;;;
+1B0CC;HENTAIGANA LETTER MI-4;Lo;0;L;;;;;N;;;;;
+1B0CD;HENTAIGANA LETTER MI-5;Lo;0;L;;;;;N;;;;;
+1B0CE;HENTAIGANA LETTER MI-6;Lo;0;L;;;;;N;;;;;
+1B0CF;HENTAIGANA LETTER MI-7;Lo;0;L;;;;;N;;;;;
+1B0D0;HENTAIGANA LETTER MU-1;Lo;0;L;;;;;N;;;;;
+1B0D1;HENTAIGANA LETTER MU-2;Lo;0;L;;;;;N;;;;;
+1B0D2;HENTAIGANA LETTER MU-3;Lo;0;L;;;;;N;;;;;
+1B0D3;HENTAIGANA LETTER MU-4;Lo;0;L;;;;;N;;;;;
+1B0D4;HENTAIGANA LETTER ME-1;Lo;0;L;;;;;N;;;;;
+1B0D5;HENTAIGANA LETTER ME-2;Lo;0;L;;;;;N;;;;;
+1B0D6;HENTAIGANA LETTER ME-MA;Lo;0;L;;;;;N;;;;;
+1B0D7;HENTAIGANA LETTER MO-1;Lo;0;L;;;;;N;;;;;
+1B0D8;HENTAIGANA LETTER MO-2;Lo;0;L;;;;;N;;;;;
+1B0D9;HENTAIGANA LETTER MO-3;Lo;0;L;;;;;N;;;;;
+1B0DA;HENTAIGANA LETTER MO-4;Lo;0;L;;;;;N;;;;;
+1B0DB;HENTAIGANA LETTER MO-5;Lo;0;L;;;;;N;;;;;
+1B0DC;HENTAIGANA LETTER MO-6;Lo;0;L;;;;;N;;;;;
+1B0DD;HENTAIGANA LETTER YA-1;Lo;0;L;;;;;N;;;;;
+1B0DE;HENTAIGANA LETTER YA-2;Lo;0;L;;;;;N;;;;;
+1B0DF;HENTAIGANA LETTER YA-3;Lo;0;L;;;;;N;;;;;
+1B0E0;HENTAIGANA LETTER YA-4;Lo;0;L;;;;;N;;;;;
+1B0E1;HENTAIGANA LETTER YA-5;Lo;0;L;;;;;N;;;;;
+1B0E2;HENTAIGANA LETTER YA-YO;Lo;0;L;;;;;N;;;;;
+1B0E3;HENTAIGANA LETTER YU-1;Lo;0;L;;;;;N;;;;;
+1B0E4;HENTAIGANA LETTER YU-2;Lo;0;L;;;;;N;;;;;
+1B0E5;HENTAIGANA LETTER YU-3;Lo;0;L;;;;;N;;;;;
+1B0E6;HENTAIGANA LETTER YU-4;Lo;0;L;;;;;N;;;;;
+1B0E7;HENTAIGANA LETTER YO-1;Lo;0;L;;;;;N;;;;;
+1B0E8;HENTAIGANA LETTER YO-2;Lo;0;L;;;;;N;;;;;
+1B0E9;HENTAIGANA LETTER YO-3;Lo;0;L;;;;;N;;;;;
+1B0EA;HENTAIGANA LETTER YO-4;Lo;0;L;;;;;N;;;;;
+1B0EB;HENTAIGANA LETTER YO-5;Lo;0;L;;;;;N;;;;;
+1B0EC;HENTAIGANA LETTER YO-6;Lo;0;L;;;;;N;;;;;
+1B0ED;HENTAIGANA LETTER RA-1;Lo;0;L;;;;;N;;;;;
+1B0EE;HENTAIGANA LETTER RA-2;Lo;0;L;;;;;N;;;;;
+1B0EF;HENTAIGANA LETTER RA-3;Lo;0;L;;;;;N;;;;;
+1B0F0;HENTAIGANA LETTER RA-4;Lo;0;L;;;;;N;;;;;
+1B0F1;HENTAIGANA LETTER RI-1;Lo;0;L;;;;;N;;;;;
+1B0F2;HENTAIGANA LETTER RI-2;Lo;0;L;;;;;N;;;;;
+1B0F3;HENTAIGANA LETTER RI-3;Lo;0;L;;;;;N;;;;;
+1B0F4;HENTAIGANA LETTER RI-4;Lo;0;L;;;;;N;;;;;
+1B0F5;HENTAIGANA LETTER RI-5;Lo;0;L;;;;;N;;;;;
+1B0F6;HENTAIGANA LETTER RI-6;Lo;0;L;;;;;N;;;;;
+1B0F7;HENTAIGANA LETTER RI-7;Lo;0;L;;;;;N;;;;;
+1B0F8;HENTAIGANA LETTER RU-1;Lo;0;L;;;;;N;;;;;
+1B0F9;HENTAIGANA LETTER RU-2;Lo;0;L;;;;;N;;;;;
+1B0FA;HENTAIGANA LETTER RU-3;Lo;0;L;;;;;N;;;;;
+1B0FB;HENTAIGANA LETTER RU-4;Lo;0;L;;;;;N;;;;;
+1B0FC;HENTAIGANA LETTER RU-5;Lo;0;L;;;;;N;;;;;
+1B0FD;HENTAIGANA LETTER RU-6;Lo;0;L;;;;;N;;;;;
+1B0FE;HENTAIGANA LETTER RE-1;Lo;0;L;;;;;N;;;;;
+1B0FF;HENTAIGANA LETTER RE-2;Lo;0;L;;;;;N;;;;;
+1B100;HENTAIGANA LETTER RE-3;Lo;0;L;;;;;N;;;;;
+1B101;HENTAIGANA LETTER RE-4;Lo;0;L;;;;;N;;;;;
+1B102;HENTAIGANA LETTER RO-1;Lo;0;L;;;;;N;;;;;
+1B103;HENTAIGANA LETTER RO-2;Lo;0;L;;;;;N;;;;;
+1B104;HENTAIGANA LETTER RO-3;Lo;0;L;;;;;N;;;;;
+1B105;HENTAIGANA LETTER RO-4;Lo;0;L;;;;;N;;;;;
+1B106;HENTAIGANA LETTER RO-5;Lo;0;L;;;;;N;;;;;
+1B107;HENTAIGANA LETTER RO-6;Lo;0;L;;;;;N;;;;;
+1B108;HENTAIGANA LETTER WA-1;Lo;0;L;;;;;N;;;;;
+1B109;HENTAIGANA LETTER WA-2;Lo;0;L;;;;;N;;;;;
+1B10A;HENTAIGANA LETTER WA-3;Lo;0;L;;;;;N;;;;;
+1B10B;HENTAIGANA LETTER WA-4;Lo;0;L;;;;;N;;;;;
+1B10C;HENTAIGANA LETTER WA-5;Lo;0;L;;;;;N;;;;;
+1B10D;HENTAIGANA LETTER WI-1;Lo;0;L;;;;;N;;;;;
+1B10E;HENTAIGANA LETTER WI-2;Lo;0;L;;;;;N;;;;;
+1B10F;HENTAIGANA LETTER WI-3;Lo;0;L;;;;;N;;;;;
+1B110;HENTAIGANA LETTER WI-4;Lo;0;L;;;;;N;;;;;
+1B111;HENTAIGANA LETTER WI-5;Lo;0;L;;;;;N;;;;;
+1B112;HENTAIGANA LETTER WE-1;Lo;0;L;;;;;N;;;;;
+1B113;HENTAIGANA LETTER WE-2;Lo;0;L;;;;;N;;;;;
+1B114;HENTAIGANA LETTER WE-3;Lo;0;L;;;;;N;;;;;
+1B115;HENTAIGANA LETTER WE-4;Lo;0;L;;;;;N;;;;;
+1B116;HENTAIGANA LETTER WO-1;Lo;0;L;;;;;N;;;;;
+1B117;HENTAIGANA LETTER WO-2;Lo;0;L;;;;;N;;;;;
+1B118;HENTAIGANA LETTER WO-3;Lo;0;L;;;;;N;;;;;
+1B119;HENTAIGANA LETTER WO-4;Lo;0;L;;;;;N;;;;;
+1B11A;HENTAIGANA LETTER WO-5;Lo;0;L;;;;;N;;;;;
+1B11B;HENTAIGANA LETTER WO-6;Lo;0;L;;;;;N;;;;;
+1B11C;HENTAIGANA LETTER WO-7;Lo;0;L;;;;;N;;;;;
+1B11D;HENTAIGANA LETTER N-MU-MO-1;Lo;0;L;;;;;N;;;;;
+1B11E;HENTAIGANA LETTER N-MU-MO-2;Lo;0;L;;;;;N;;;;;
+1B170;NUSHU CHARACTER-1B170;Lo;0;L;;;;;N;;;;;
+1B171;NUSHU CHARACTER-1B171;Lo;0;L;;;;;N;;;;;
+1B172;NUSHU CHARACTER-1B172;Lo;0;L;;;;;N;;;;;
+1B173;NUSHU CHARACTER-1B173;Lo;0;L;;;;;N;;;;;
+1B174;NUSHU CHARACTER-1B174;Lo;0;L;;;;;N;;;;;
+1B175;NUSHU CHARACTER-1B175;Lo;0;L;;;;;N;;;;;
+1B176;NUSHU CHARACTER-1B176;Lo;0;L;;;;;N;;;;;
+1B177;NUSHU CHARACTER-1B177;Lo;0;L;;;;;N;;;;;
+1B178;NUSHU CHARACTER-1B178;Lo;0;L;;;;;N;;;;;
+1B179;NUSHU CHARACTER-1B179;Lo;0;L;;;;;N;;;;;
+1B17A;NUSHU CHARACTER-1B17A;Lo;0;L;;;;;N;;;;;
+1B17B;NUSHU CHARACTER-1B17B;Lo;0;L;;;;;N;;;;;
+1B17C;NUSHU CHARACTER-1B17C;Lo;0;L;;;;;N;;;;;
+1B17D;NUSHU CHARACTER-1B17D;Lo;0;L;;;;;N;;;;;
+1B17E;NUSHU CHARACTER-1B17E;Lo;0;L;;;;;N;;;;;
+1B17F;NUSHU CHARACTER-1B17F;Lo;0;L;;;;;N;;;;;
+1B180;NUSHU CHARACTER-1B180;Lo;0;L;;;;;N;;;;;
+1B181;NUSHU CHARACTER-1B181;Lo;0;L;;;;;N;;;;;
+1B182;NUSHU CHARACTER-1B182;Lo;0;L;;;;;N;;;;;
+1B183;NUSHU CHARACTER-1B183;Lo;0;L;;;;;N;;;;;
+1B184;NUSHU CHARACTER-1B184;Lo;0;L;;;;;N;;;;;
+1B185;NUSHU CHARACTER-1B185;Lo;0;L;;;;;N;;;;;
+1B186;NUSHU CHARACTER-1B186;Lo;0;L;;;;;N;;;;;
+1B187;NUSHU CHARACTER-1B187;Lo;0;L;;;;;N;;;;;
+1B188;NUSHU CHARACTER-1B188;Lo;0;L;;;;;N;;;;;
+1B189;NUSHU CHARACTER-1B189;Lo;0;L;;;;;N;;;;;
+1B18A;NUSHU CHARACTER-1B18A;Lo;0;L;;;;;N;;;;;
+1B18B;NUSHU CHARACTER-1B18B;Lo;0;L;;;;;N;;;;;
+1B18C;NUSHU CHARACTER-1B18C;Lo;0;L;;;;;N;;;;;
+1B18D;NUSHU CHARACTER-1B18D;Lo;0;L;;;;;N;;;;;
+1B18E;NUSHU CHARACTER-1B18E;Lo;0;L;;;;;N;;;;;
+1B18F;NUSHU CHARACTER-1B18F;Lo;0;L;;;;;N;;;;;
+1B190;NUSHU CHARACTER-1B190;Lo;0;L;;;;;N;;;;;
+1B191;NUSHU CHARACTER-1B191;Lo;0;L;;;;;N;;;;;
+1B192;NUSHU CHARACTER-1B192;Lo;0;L;;;;;N;;;;;
+1B193;NUSHU CHARACTER-1B193;Lo;0;L;;;;;N;;;;;
+1B194;NUSHU CHARACTER-1B194;Lo;0;L;;;;;N;;;;;
+1B195;NUSHU CHARACTER-1B195;Lo;0;L;;;;;N;;;;;
+1B196;NUSHU CHARACTER-1B196;Lo;0;L;;;;;N;;;;;
+1B197;NUSHU CHARACTER-1B197;Lo;0;L;;;;;N;;;;;
+1B198;NUSHU CHARACTER-1B198;Lo;0;L;;;;;N;;;;;
+1B199;NUSHU CHARACTER-1B199;Lo;0;L;;;;;N;;;;;
+1B19A;NUSHU CHARACTER-1B19A;Lo;0;L;;;;;N;;;;;
+1B19B;NUSHU CHARACTER-1B19B;Lo;0;L;;;;;N;;;;;
+1B19C;NUSHU CHARACTER-1B19C;Lo;0;L;;;;;N;;;;;
+1B19D;NUSHU CHARACTER-1B19D;Lo;0;L;;;;;N;;;;;
+1B19E;NUSHU CHARACTER-1B19E;Lo;0;L;;;;;N;;;;;
+1B19F;NUSHU CHARACTER-1B19F;Lo;0;L;;;;;N;;;;;
+1B1A0;NUSHU CHARACTER-1B1A0;Lo;0;L;;;;;N;;;;;
+1B1A1;NUSHU CHARACTER-1B1A1;Lo;0;L;;;;;N;;;;;
+1B1A2;NUSHU CHARACTER-1B1A2;Lo;0;L;;;;;N;;;;;
+1B1A3;NUSHU CHARACTER-1B1A3;Lo;0;L;;;;;N;;;;;
+1B1A4;NUSHU CHARACTER-1B1A4;Lo;0;L;;;;;N;;;;;
+1B1A5;NUSHU CHARACTER-1B1A5;Lo;0;L;;;;;N;;;;;
+1B1A6;NUSHU CHARACTER-1B1A6;Lo;0;L;;;;;N;;;;;
+1B1A7;NUSHU CHARACTER-1B1A7;Lo;0;L;;;;;N;;;;;
+1B1A8;NUSHU CHARACTER-1B1A8;Lo;0;L;;;;;N;;;;;
+1B1A9;NUSHU CHARACTER-1B1A9;Lo;0;L;;;;;N;;;;;
+1B1AA;NUSHU CHARACTER-1B1AA;Lo;0;L;;;;;N;;;;;
+1B1AB;NUSHU CHARACTER-1B1AB;Lo;0;L;;;;;N;;;;;
+1B1AC;NUSHU CHARACTER-1B1AC;Lo;0;L;;;;;N;;;;;
+1B1AD;NUSHU CHARACTER-1B1AD;Lo;0;L;;;;;N;;;;;
+1B1AE;NUSHU CHARACTER-1B1AE;Lo;0;L;;;;;N;;;;;
+1B1AF;NUSHU CHARACTER-1B1AF;Lo;0;L;;;;;N;;;;;
+1B1B0;NUSHU CHARACTER-1B1B0;Lo;0;L;;;;;N;;;;;
+1B1B1;NUSHU CHARACTER-1B1B1;Lo;0;L;;;;;N;;;;;
+1B1B2;NUSHU CHARACTER-1B1B2;Lo;0;L;;;;;N;;;;;
+1B1B3;NUSHU CHARACTER-1B1B3;Lo;0;L;;;;;N;;;;;
+1B1B4;NUSHU CHARACTER-1B1B4;Lo;0;L;;;;;N;;;;;
+1B1B5;NUSHU CHARACTER-1B1B5;Lo;0;L;;;;;N;;;;;
+1B1B6;NUSHU CHARACTER-1B1B6;Lo;0;L;;;;;N;;;;;
+1B1B7;NUSHU CHARACTER-1B1B7;Lo;0;L;;;;;N;;;;;
+1B1B8;NUSHU CHARACTER-1B1B8;Lo;0;L;;;;;N;;;;;
+1B1B9;NUSHU CHARACTER-1B1B9;Lo;0;L;;;;;N;;;;;
+1B1BA;NUSHU CHARACTER-1B1BA;Lo;0;L;;;;;N;;;;;
+1B1BB;NUSHU CHARACTER-1B1BB;Lo;0;L;;;;;N;;;;;
+1B1BC;NUSHU CHARACTER-1B1BC;Lo;0;L;;;;;N;;;;;
+1B1BD;NUSHU CHARACTER-1B1BD;Lo;0;L;;;;;N;;;;;
+1B1BE;NUSHU CHARACTER-1B1BE;Lo;0;L;;;;;N;;;;;
+1B1BF;NUSHU CHARACTER-1B1BF;Lo;0;L;;;;;N;;;;;
+1B1C0;NUSHU CHARACTER-1B1C0;Lo;0;L;;;;;N;;;;;
+1B1C1;NUSHU CHARACTER-1B1C1;Lo;0;L;;;;;N;;;;;
+1B1C2;NUSHU CHARACTER-1B1C2;Lo;0;L;;;;;N;;;;;
+1B1C3;NUSHU CHARACTER-1B1C3;Lo;0;L;;;;;N;;;;;
+1B1C4;NUSHU CHARACTER-1B1C4;Lo;0;L;;;;;N;;;;;
+1B1C5;NUSHU CHARACTER-1B1C5;Lo;0;L;;;;;N;;;;;
+1B1C6;NUSHU CHARACTER-1B1C6;Lo;0;L;;;;;N;;;;;
+1B1C7;NUSHU CHARACTER-1B1C7;Lo;0;L;;;;;N;;;;;
+1B1C8;NUSHU CHARACTER-1B1C8;Lo;0;L;;;;;N;;;;;
+1B1C9;NUSHU CHARACTER-1B1C9;Lo;0;L;;;;;N;;;;;
+1B1CA;NUSHU CHARACTER-1B1CA;Lo;0;L;;;;;N;;;;;
+1B1CB;NUSHU CHARACTER-1B1CB;Lo;0;L;;;;;N;;;;;
+1B1CC;NUSHU CHARACTER-1B1CC;Lo;0;L;;;;;N;;;;;
+1B1CD;NUSHU CHARACTER-1B1CD;Lo;0;L;;;;;N;;;;;
+1B1CE;NUSHU CHARACTER-1B1CE;Lo;0;L;;;;;N;;;;;
+1B1CF;NUSHU CHARACTER-1B1CF;Lo;0;L;;;;;N;;;;;
+1B1D0;NUSHU CHARACTER-1B1D0;Lo;0;L;;;;;N;;;;;
+1B1D1;NUSHU CHARACTER-1B1D1;Lo;0;L;;;;;N;;;;;
+1B1D2;NUSHU CHARACTER-1B1D2;Lo;0;L;;;;;N;;;;;
+1B1D3;NUSHU CHARACTER-1B1D3;Lo;0;L;;;;;N;;;;;
+1B1D4;NUSHU CHARACTER-1B1D4;Lo;0;L;;;;;N;;;;;
+1B1D5;NUSHU CHARACTER-1B1D5;Lo;0;L;;;;;N;;;;;
+1B1D6;NUSHU CHARACTER-1B1D6;Lo;0;L;;;;;N;;;;;
+1B1D7;NUSHU CHARACTER-1B1D7;Lo;0;L;;;;;N;;;;;
+1B1D8;NUSHU CHARACTER-1B1D8;Lo;0;L;;;;;N;;;;;
+1B1D9;NUSHU CHARACTER-1B1D9;Lo;0;L;;;;;N;;;;;
+1B1DA;NUSHU CHARACTER-1B1DA;Lo;0;L;;;;;N;;;;;
+1B1DB;NUSHU CHARACTER-1B1DB;Lo;0;L;;;;;N;;;;;
+1B1DC;NUSHU CHARACTER-1B1DC;Lo;0;L;;;;;N;;;;;
+1B1DD;NUSHU CHARACTER-1B1DD;Lo;0;L;;;;;N;;;;;
+1B1DE;NUSHU CHARACTER-1B1DE;Lo;0;L;;;;;N;;;;;
+1B1DF;NUSHU CHARACTER-1B1DF;Lo;0;L;;;;;N;;;;;
+1B1E0;NUSHU CHARACTER-1B1E0;Lo;0;L;;;;;N;;;;;
+1B1E1;NUSHU CHARACTER-1B1E1;Lo;0;L;;;;;N;;;;;
+1B1E2;NUSHU CHARACTER-1B1E2;Lo;0;L;;;;;N;;;;;
+1B1E3;NUSHU CHARACTER-1B1E3;Lo;0;L;;;;;N;;;;;
+1B1E4;NUSHU CHARACTER-1B1E4;Lo;0;L;;;;;N;;;;;
+1B1E5;NUSHU CHARACTER-1B1E5;Lo;0;L;;;;;N;;;;;
+1B1E6;NUSHU CHARACTER-1B1E6;Lo;0;L;;;;;N;;;;;
+1B1E7;NUSHU CHARACTER-1B1E7;Lo;0;L;;;;;N;;;;;
+1B1E8;NUSHU CHARACTER-1B1E8;Lo;0;L;;;;;N;;;;;
+1B1E9;NUSHU CHARACTER-1B1E9;Lo;0;L;;;;;N;;;;;
+1B1EA;NUSHU CHARACTER-1B1EA;Lo;0;L;;;;;N;;;;;
+1B1EB;NUSHU CHARACTER-1B1EB;Lo;0;L;;;;;N;;;;;
+1B1EC;NUSHU CHARACTER-1B1EC;Lo;0;L;;;;;N;;;;;
+1B1ED;NUSHU CHARACTER-1B1ED;Lo;0;L;;;;;N;;;;;
+1B1EE;NUSHU CHARACTER-1B1EE;Lo;0;L;;;;;N;;;;;
+1B1EF;NUSHU CHARACTER-1B1EF;Lo;0;L;;;;;N;;;;;
+1B1F0;NUSHU CHARACTER-1B1F0;Lo;0;L;;;;;N;;;;;
+1B1F1;NUSHU CHARACTER-1B1F1;Lo;0;L;;;;;N;;;;;
+1B1F2;NUSHU CHARACTER-1B1F2;Lo;0;L;;;;;N;;;;;
+1B1F3;NUSHU CHARACTER-1B1F3;Lo;0;L;;;;;N;;;;;
+1B1F4;NUSHU CHARACTER-1B1F4;Lo;0;L;;;;;N;;;;;
+1B1F5;NUSHU CHARACTER-1B1F5;Lo;0;L;;;;;N;;;;;
+1B1F6;NUSHU CHARACTER-1B1F6;Lo;0;L;;;;;N;;;;;
+1B1F7;NUSHU CHARACTER-1B1F7;Lo;0;L;;;;;N;;;;;
+1B1F8;NUSHU CHARACTER-1B1F8;Lo;0;L;;;;;N;;;;;
+1B1F9;NUSHU CHARACTER-1B1F9;Lo;0;L;;;;;N;;;;;
+1B1FA;NUSHU CHARACTER-1B1FA;Lo;0;L;;;;;N;;;;;
+1B1FB;NUSHU CHARACTER-1B1FB;Lo;0;L;;;;;N;;;;;
+1B1FC;NUSHU CHARACTER-1B1FC;Lo;0;L;;;;;N;;;;;
+1B1FD;NUSHU CHARACTER-1B1FD;Lo;0;L;;;;;N;;;;;
+1B1FE;NUSHU CHARACTER-1B1FE;Lo;0;L;;;;;N;;;;;
+1B1FF;NUSHU CHARACTER-1B1FF;Lo;0;L;;;;;N;;;;;
+1B200;NUSHU CHARACTER-1B200;Lo;0;L;;;;;N;;;;;
+1B201;NUSHU CHARACTER-1B201;Lo;0;L;;;;;N;;;;;
+1B202;NUSHU CHARACTER-1B202;Lo;0;L;;;;;N;;;;;
+1B203;NUSHU CHARACTER-1B203;Lo;0;L;;;;;N;;;;;
+1B204;NUSHU CHARACTER-1B204;Lo;0;L;;;;;N;;;;;
+1B205;NUSHU CHARACTER-1B205;Lo;0;L;;;;;N;;;;;
+1B206;NUSHU CHARACTER-1B206;Lo;0;L;;;;;N;;;;;
+1B207;NUSHU CHARACTER-1B207;Lo;0;L;;;;;N;;;;;
+1B208;NUSHU CHARACTER-1B208;Lo;0;L;;;;;N;;;;;
+1B209;NUSHU CHARACTER-1B209;Lo;0;L;;;;;N;;;;;
+1B20A;NUSHU CHARACTER-1B20A;Lo;0;L;;;;;N;;;;;
+1B20B;NUSHU CHARACTER-1B20B;Lo;0;L;;;;;N;;;;;
+1B20C;NUSHU CHARACTER-1B20C;Lo;0;L;;;;;N;;;;;
+1B20D;NUSHU CHARACTER-1B20D;Lo;0;L;;;;;N;;;;;
+1B20E;NUSHU CHARACTER-1B20E;Lo;0;L;;;;;N;;;;;
+1B20F;NUSHU CHARACTER-1B20F;Lo;0;L;;;;;N;;;;;
+1B210;NUSHU CHARACTER-1B210;Lo;0;L;;;;;N;;;;;
+1B211;NUSHU CHARACTER-1B211;Lo;0;L;;;;;N;;;;;
+1B212;NUSHU CHARACTER-1B212;Lo;0;L;;;;;N;;;;;
+1B213;NUSHU CHARACTER-1B213;Lo;0;L;;;;;N;;;;;
+1B214;NUSHU CHARACTER-1B214;Lo;0;L;;;;;N;;;;;
+1B215;NUSHU CHARACTER-1B215;Lo;0;L;;;;;N;;;;;
+1B216;NUSHU CHARACTER-1B216;Lo;0;L;;;;;N;;;;;
+1B217;NUSHU CHARACTER-1B217;Lo;0;L;;;;;N;;;;;
+1B218;NUSHU CHARACTER-1B218;Lo;0;L;;;;;N;;;;;
+1B219;NUSHU CHARACTER-1B219;Lo;0;L;;;;;N;;;;;
+1B21A;NUSHU CHARACTER-1B21A;Lo;0;L;;;;;N;;;;;
+1B21B;NUSHU CHARACTER-1B21B;Lo;0;L;;;;;N;;;;;
+1B21C;NUSHU CHARACTER-1B21C;Lo;0;L;;;;;N;;;;;
+1B21D;NUSHU CHARACTER-1B21D;Lo;0;L;;;;;N;;;;;
+1B21E;NUSHU CHARACTER-1B21E;Lo;0;L;;;;;N;;;;;
+1B21F;NUSHU CHARACTER-1B21F;Lo;0;L;;;;;N;;;;;
+1B220;NUSHU CHARACTER-1B220;Lo;0;L;;;;;N;;;;;
+1B221;NUSHU CHARACTER-1B221;Lo;0;L;;;;;N;;;;;
+1B222;NUSHU CHARACTER-1B222;Lo;0;L;;;;;N;;;;;
+1B223;NUSHU CHARACTER-1B223;Lo;0;L;;;;;N;;;;;
+1B224;NUSHU CHARACTER-1B224;Lo;0;L;;;;;N;;;;;
+1B225;NUSHU CHARACTER-1B225;Lo;0;L;;;;;N;;;;;
+1B226;NUSHU CHARACTER-1B226;Lo;0;L;;;;;N;;;;;
+1B227;NUSHU CHARACTER-1B227;Lo;0;L;;;;;N;;;;;
+1B228;NUSHU CHARACTER-1B228;Lo;0;L;;;;;N;;;;;
+1B229;NUSHU CHARACTER-1B229;Lo;0;L;;;;;N;;;;;
+1B22A;NUSHU CHARACTER-1B22A;Lo;0;L;;;;;N;;;;;
+1B22B;NUSHU CHARACTER-1B22B;Lo;0;L;;;;;N;;;;;
+1B22C;NUSHU CHARACTER-1B22C;Lo;0;L;;;;;N;;;;;
+1B22D;NUSHU CHARACTER-1B22D;Lo;0;L;;;;;N;;;;;
+1B22E;NUSHU CHARACTER-1B22E;Lo;0;L;;;;;N;;;;;
+1B22F;NUSHU CHARACTER-1B22F;Lo;0;L;;;;;N;;;;;
+1B230;NUSHU CHARACTER-1B230;Lo;0;L;;;;;N;;;;;
+1B231;NUSHU CHARACTER-1B231;Lo;0;L;;;;;N;;;;;
+1B232;NUSHU CHARACTER-1B232;Lo;0;L;;;;;N;;;;;
+1B233;NUSHU CHARACTER-1B233;Lo;0;L;;;;;N;;;;;
+1B234;NUSHU CHARACTER-1B234;Lo;0;L;;;;;N;;;;;
+1B235;NUSHU CHARACTER-1B235;Lo;0;L;;;;;N;;;;;
+1B236;NUSHU CHARACTER-1B236;Lo;0;L;;;;;N;;;;;
+1B237;NUSHU CHARACTER-1B237;Lo;0;L;;;;;N;;;;;
+1B238;NUSHU CHARACTER-1B238;Lo;0;L;;;;;N;;;;;
+1B239;NUSHU CHARACTER-1B239;Lo;0;L;;;;;N;;;;;
+1B23A;NUSHU CHARACTER-1B23A;Lo;0;L;;;;;N;;;;;
+1B23B;NUSHU CHARACTER-1B23B;Lo;0;L;;;;;N;;;;;
+1B23C;NUSHU CHARACTER-1B23C;Lo;0;L;;;;;N;;;;;
+1B23D;NUSHU CHARACTER-1B23D;Lo;0;L;;;;;N;;;;;
+1B23E;NUSHU CHARACTER-1B23E;Lo;0;L;;;;;N;;;;;
+1B23F;NUSHU CHARACTER-1B23F;Lo;0;L;;;;;N;;;;;
+1B240;NUSHU CHARACTER-1B240;Lo;0;L;;;;;N;;;;;
+1B241;NUSHU CHARACTER-1B241;Lo;0;L;;;;;N;;;;;
+1B242;NUSHU CHARACTER-1B242;Lo;0;L;;;;;N;;;;;
+1B243;NUSHU CHARACTER-1B243;Lo;0;L;;;;;N;;;;;
+1B244;NUSHU CHARACTER-1B244;Lo;0;L;;;;;N;;;;;
+1B245;NUSHU CHARACTER-1B245;Lo;0;L;;;;;N;;;;;
+1B246;NUSHU CHARACTER-1B246;Lo;0;L;;;;;N;;;;;
+1B247;NUSHU CHARACTER-1B247;Lo;0;L;;;;;N;;;;;
+1B248;NUSHU CHARACTER-1B248;Lo;0;L;;;;;N;;;;;
+1B249;NUSHU CHARACTER-1B249;Lo;0;L;;;;;N;;;;;
+1B24A;NUSHU CHARACTER-1B24A;Lo;0;L;;;;;N;;;;;
+1B24B;NUSHU CHARACTER-1B24B;Lo;0;L;;;;;N;;;;;
+1B24C;NUSHU CHARACTER-1B24C;Lo;0;L;;;;;N;;;;;
+1B24D;NUSHU CHARACTER-1B24D;Lo;0;L;;;;;N;;;;;
+1B24E;NUSHU CHARACTER-1B24E;Lo;0;L;;;;;N;;;;;
+1B24F;NUSHU CHARACTER-1B24F;Lo;0;L;;;;;N;;;;;
+1B250;NUSHU CHARACTER-1B250;Lo;0;L;;;;;N;;;;;
+1B251;NUSHU CHARACTER-1B251;Lo;0;L;;;;;N;;;;;
+1B252;NUSHU CHARACTER-1B252;Lo;0;L;;;;;N;;;;;
+1B253;NUSHU CHARACTER-1B253;Lo;0;L;;;;;N;;;;;
+1B254;NUSHU CHARACTER-1B254;Lo;0;L;;;;;N;;;;;
+1B255;NUSHU CHARACTER-1B255;Lo;0;L;;;;;N;;;;;
+1B256;NUSHU CHARACTER-1B256;Lo;0;L;;;;;N;;;;;
+1B257;NUSHU CHARACTER-1B257;Lo;0;L;;;;;N;;;;;
+1B258;NUSHU CHARACTER-1B258;Lo;0;L;;;;;N;;;;;
+1B259;NUSHU CHARACTER-1B259;Lo;0;L;;;;;N;;;;;
+1B25A;NUSHU CHARACTER-1B25A;Lo;0;L;;;;;N;;;;;
+1B25B;NUSHU CHARACTER-1B25B;Lo;0;L;;;;;N;;;;;
+1B25C;NUSHU CHARACTER-1B25C;Lo;0;L;;;;;N;;;;;
+1B25D;NUSHU CHARACTER-1B25D;Lo;0;L;;;;;N;;;;;
+1B25E;NUSHU CHARACTER-1B25E;Lo;0;L;;;;;N;;;;;
+1B25F;NUSHU CHARACTER-1B25F;Lo;0;L;;;;;N;;;;;
+1B260;NUSHU CHARACTER-1B260;Lo;0;L;;;;;N;;;;;
+1B261;NUSHU CHARACTER-1B261;Lo;0;L;;;;;N;;;;;
+1B262;NUSHU CHARACTER-1B262;Lo;0;L;;;;;N;;;;;
+1B263;NUSHU CHARACTER-1B263;Lo;0;L;;;;;N;;;;;
+1B264;NUSHU CHARACTER-1B264;Lo;0;L;;;;;N;;;;;
+1B265;NUSHU CHARACTER-1B265;Lo;0;L;;;;;N;;;;;
+1B266;NUSHU CHARACTER-1B266;Lo;0;L;;;;;N;;;;;
+1B267;NUSHU CHARACTER-1B267;Lo;0;L;;;;;N;;;;;
+1B268;NUSHU CHARACTER-1B268;Lo;0;L;;;;;N;;;;;
+1B269;NUSHU CHARACTER-1B269;Lo;0;L;;;;;N;;;;;
+1B26A;NUSHU CHARACTER-1B26A;Lo;0;L;;;;;N;;;;;
+1B26B;NUSHU CHARACTER-1B26B;Lo;0;L;;;;;N;;;;;
+1B26C;NUSHU CHARACTER-1B26C;Lo;0;L;;;;;N;;;;;
+1B26D;NUSHU CHARACTER-1B26D;Lo;0;L;;;;;N;;;;;
+1B26E;NUSHU CHARACTER-1B26E;Lo;0;L;;;;;N;;;;;
+1B26F;NUSHU CHARACTER-1B26F;Lo;0;L;;;;;N;;;;;
+1B270;NUSHU CHARACTER-1B270;Lo;0;L;;;;;N;;;;;
+1B271;NUSHU CHARACTER-1B271;Lo;0;L;;;;;N;;;;;
+1B272;NUSHU CHARACTER-1B272;Lo;0;L;;;;;N;;;;;
+1B273;NUSHU CHARACTER-1B273;Lo;0;L;;;;;N;;;;;
+1B274;NUSHU CHARACTER-1B274;Lo;0;L;;;;;N;;;;;
+1B275;NUSHU CHARACTER-1B275;Lo;0;L;;;;;N;;;;;
+1B276;NUSHU CHARACTER-1B276;Lo;0;L;;;;;N;;;;;
+1B277;NUSHU CHARACTER-1B277;Lo;0;L;;;;;N;;;;;
+1B278;NUSHU CHARACTER-1B278;Lo;0;L;;;;;N;;;;;
+1B279;NUSHU CHARACTER-1B279;Lo;0;L;;;;;N;;;;;
+1B27A;NUSHU CHARACTER-1B27A;Lo;0;L;;;;;N;;;;;
+1B27B;NUSHU CHARACTER-1B27B;Lo;0;L;;;;;N;;;;;
+1B27C;NUSHU CHARACTER-1B27C;Lo;0;L;;;;;N;;;;;
+1B27D;NUSHU CHARACTER-1B27D;Lo;0;L;;;;;N;;;;;
+1B27E;NUSHU CHARACTER-1B27E;Lo;0;L;;;;;N;;;;;
+1B27F;NUSHU CHARACTER-1B27F;Lo;0;L;;;;;N;;;;;
+1B280;NUSHU CHARACTER-1B280;Lo;0;L;;;;;N;;;;;
+1B281;NUSHU CHARACTER-1B281;Lo;0;L;;;;;N;;;;;
+1B282;NUSHU CHARACTER-1B282;Lo;0;L;;;;;N;;;;;
+1B283;NUSHU CHARACTER-1B283;Lo;0;L;;;;;N;;;;;
+1B284;NUSHU CHARACTER-1B284;Lo;0;L;;;;;N;;;;;
+1B285;NUSHU CHARACTER-1B285;Lo;0;L;;;;;N;;;;;
+1B286;NUSHU CHARACTER-1B286;Lo;0;L;;;;;N;;;;;
+1B287;NUSHU CHARACTER-1B287;Lo;0;L;;;;;N;;;;;
+1B288;NUSHU CHARACTER-1B288;Lo;0;L;;;;;N;;;;;
+1B289;NUSHU CHARACTER-1B289;Lo;0;L;;;;;N;;;;;
+1B28A;NUSHU CHARACTER-1B28A;Lo;0;L;;;;;N;;;;;
+1B28B;NUSHU CHARACTER-1B28B;Lo;0;L;;;;;N;;;;;
+1B28C;NUSHU CHARACTER-1B28C;Lo;0;L;;;;;N;;;;;
+1B28D;NUSHU CHARACTER-1B28D;Lo;0;L;;;;;N;;;;;
+1B28E;NUSHU CHARACTER-1B28E;Lo;0;L;;;;;N;;;;;
+1B28F;NUSHU CHARACTER-1B28F;Lo;0;L;;;;;N;;;;;
+1B290;NUSHU CHARACTER-1B290;Lo;0;L;;;;;N;;;;;
+1B291;NUSHU CHARACTER-1B291;Lo;0;L;;;;;N;;;;;
+1B292;NUSHU CHARACTER-1B292;Lo;0;L;;;;;N;;;;;
+1B293;NUSHU CHARACTER-1B293;Lo;0;L;;;;;N;;;;;
+1B294;NUSHU CHARACTER-1B294;Lo;0;L;;;;;N;;;;;
+1B295;NUSHU CHARACTER-1B295;Lo;0;L;;;;;N;;;;;
+1B296;NUSHU CHARACTER-1B296;Lo;0;L;;;;;N;;;;;
+1B297;NUSHU CHARACTER-1B297;Lo;0;L;;;;;N;;;;;
+1B298;NUSHU CHARACTER-1B298;Lo;0;L;;;;;N;;;;;
+1B299;NUSHU CHARACTER-1B299;Lo;0;L;;;;;N;;;;;
+1B29A;NUSHU CHARACTER-1B29A;Lo;0;L;;;;;N;;;;;
+1B29B;NUSHU CHARACTER-1B29B;Lo;0;L;;;;;N;;;;;
+1B29C;NUSHU CHARACTER-1B29C;Lo;0;L;;;;;N;;;;;
+1B29D;NUSHU CHARACTER-1B29D;Lo;0;L;;;;;N;;;;;
+1B29E;NUSHU CHARACTER-1B29E;Lo;0;L;;;;;N;;;;;
+1B29F;NUSHU CHARACTER-1B29F;Lo;0;L;;;;;N;;;;;
+1B2A0;NUSHU CHARACTER-1B2A0;Lo;0;L;;;;;N;;;;;
+1B2A1;NUSHU CHARACTER-1B2A1;Lo;0;L;;;;;N;;;;;
+1B2A2;NUSHU CHARACTER-1B2A2;Lo;0;L;;;;;N;;;;;
+1B2A3;NUSHU CHARACTER-1B2A3;Lo;0;L;;;;;N;;;;;
+1B2A4;NUSHU CHARACTER-1B2A4;Lo;0;L;;;;;N;;;;;
+1B2A5;NUSHU CHARACTER-1B2A5;Lo;0;L;;;;;N;;;;;
+1B2A6;NUSHU CHARACTER-1B2A6;Lo;0;L;;;;;N;;;;;
+1B2A7;NUSHU CHARACTER-1B2A7;Lo;0;L;;;;;N;;;;;
+1B2A8;NUSHU CHARACTER-1B2A8;Lo;0;L;;;;;N;;;;;
+1B2A9;NUSHU CHARACTER-1B2A9;Lo;0;L;;;;;N;;;;;
+1B2AA;NUSHU CHARACTER-1B2AA;Lo;0;L;;;;;N;;;;;
+1B2AB;NUSHU CHARACTER-1B2AB;Lo;0;L;;;;;N;;;;;
+1B2AC;NUSHU CHARACTER-1B2AC;Lo;0;L;;;;;N;;;;;
+1B2AD;NUSHU CHARACTER-1B2AD;Lo;0;L;;;;;N;;;;;
+1B2AE;NUSHU CHARACTER-1B2AE;Lo;0;L;;;;;N;;;;;
+1B2AF;NUSHU CHARACTER-1B2AF;Lo;0;L;;;;;N;;;;;
+1B2B0;NUSHU CHARACTER-1B2B0;Lo;0;L;;;;;N;;;;;
+1B2B1;NUSHU CHARACTER-1B2B1;Lo;0;L;;;;;N;;;;;
+1B2B2;NUSHU CHARACTER-1B2B2;Lo;0;L;;;;;N;;;;;
+1B2B3;NUSHU CHARACTER-1B2B3;Lo;0;L;;;;;N;;;;;
+1B2B4;NUSHU CHARACTER-1B2B4;Lo;0;L;;;;;N;;;;;
+1B2B5;NUSHU CHARACTER-1B2B5;Lo;0;L;;;;;N;;;;;
+1B2B6;NUSHU CHARACTER-1B2B6;Lo;0;L;;;;;N;;;;;
+1B2B7;NUSHU CHARACTER-1B2B7;Lo;0;L;;;;;N;;;;;
+1B2B8;NUSHU CHARACTER-1B2B8;Lo;0;L;;;;;N;;;;;
+1B2B9;NUSHU CHARACTER-1B2B9;Lo;0;L;;;;;N;;;;;
+1B2BA;NUSHU CHARACTER-1B2BA;Lo;0;L;;;;;N;;;;;
+1B2BB;NUSHU CHARACTER-1B2BB;Lo;0;L;;;;;N;;;;;
+1B2BC;NUSHU CHARACTER-1B2BC;Lo;0;L;;;;;N;;;;;
+1B2BD;NUSHU CHARACTER-1B2BD;Lo;0;L;;;;;N;;;;;
+1B2BE;NUSHU CHARACTER-1B2BE;Lo;0;L;;;;;N;;;;;
+1B2BF;NUSHU CHARACTER-1B2BF;Lo;0;L;;;;;N;;;;;
+1B2C0;NUSHU CHARACTER-1B2C0;Lo;0;L;;;;;N;;;;;
+1B2C1;NUSHU CHARACTER-1B2C1;Lo;0;L;;;;;N;;;;;
+1B2C2;NUSHU CHARACTER-1B2C2;Lo;0;L;;;;;N;;;;;
+1B2C3;NUSHU CHARACTER-1B2C3;Lo;0;L;;;;;N;;;;;
+1B2C4;NUSHU CHARACTER-1B2C4;Lo;0;L;;;;;N;;;;;
+1B2C5;NUSHU CHARACTER-1B2C5;Lo;0;L;;;;;N;;;;;
+1B2C6;NUSHU CHARACTER-1B2C6;Lo;0;L;;;;;N;;;;;
+1B2C7;NUSHU CHARACTER-1B2C7;Lo;0;L;;;;;N;;;;;
+1B2C8;NUSHU CHARACTER-1B2C8;Lo;0;L;;;;;N;;;;;
+1B2C9;NUSHU CHARACTER-1B2C9;Lo;0;L;;;;;N;;;;;
+1B2CA;NUSHU CHARACTER-1B2CA;Lo;0;L;;;;;N;;;;;
+1B2CB;NUSHU CHARACTER-1B2CB;Lo;0;L;;;;;N;;;;;
+1B2CC;NUSHU CHARACTER-1B2CC;Lo;0;L;;;;;N;;;;;
+1B2CD;NUSHU CHARACTER-1B2CD;Lo;0;L;;;;;N;;;;;
+1B2CE;NUSHU CHARACTER-1B2CE;Lo;0;L;;;;;N;;;;;
+1B2CF;NUSHU CHARACTER-1B2CF;Lo;0;L;;;;;N;;;;;
+1B2D0;NUSHU CHARACTER-1B2D0;Lo;0;L;;;;;N;;;;;
+1B2D1;NUSHU CHARACTER-1B2D1;Lo;0;L;;;;;N;;;;;
+1B2D2;NUSHU CHARACTER-1B2D2;Lo;0;L;;;;;N;;;;;
+1B2D3;NUSHU CHARACTER-1B2D3;Lo;0;L;;;;;N;;;;;
+1B2D4;NUSHU CHARACTER-1B2D4;Lo;0;L;;;;;N;;;;;
+1B2D5;NUSHU CHARACTER-1B2D5;Lo;0;L;;;;;N;;;;;
+1B2D6;NUSHU CHARACTER-1B2D6;Lo;0;L;;;;;N;;;;;
+1B2D7;NUSHU CHARACTER-1B2D7;Lo;0;L;;;;;N;;;;;
+1B2D8;NUSHU CHARACTER-1B2D8;Lo;0;L;;;;;N;;;;;
+1B2D9;NUSHU CHARACTER-1B2D9;Lo;0;L;;;;;N;;;;;
+1B2DA;NUSHU CHARACTER-1B2DA;Lo;0;L;;;;;N;;;;;
+1B2DB;NUSHU CHARACTER-1B2DB;Lo;0;L;;;;;N;;;;;
+1B2DC;NUSHU CHARACTER-1B2DC;Lo;0;L;;;;;N;;;;;
+1B2DD;NUSHU CHARACTER-1B2DD;Lo;0;L;;;;;N;;;;;
+1B2DE;NUSHU CHARACTER-1B2DE;Lo;0;L;;;;;N;;;;;
+1B2DF;NUSHU CHARACTER-1B2DF;Lo;0;L;;;;;N;;;;;
+1B2E0;NUSHU CHARACTER-1B2E0;Lo;0;L;;;;;N;;;;;
+1B2E1;NUSHU CHARACTER-1B2E1;Lo;0;L;;;;;N;;;;;
+1B2E2;NUSHU CHARACTER-1B2E2;Lo;0;L;;;;;N;;;;;
+1B2E3;NUSHU CHARACTER-1B2E3;Lo;0;L;;;;;N;;;;;
+1B2E4;NUSHU CHARACTER-1B2E4;Lo;0;L;;;;;N;;;;;
+1B2E5;NUSHU CHARACTER-1B2E5;Lo;0;L;;;;;N;;;;;
+1B2E6;NUSHU CHARACTER-1B2E6;Lo;0;L;;;;;N;;;;;
+1B2E7;NUSHU CHARACTER-1B2E7;Lo;0;L;;;;;N;;;;;
+1B2E8;NUSHU CHARACTER-1B2E8;Lo;0;L;;;;;N;;;;;
+1B2E9;NUSHU CHARACTER-1B2E9;Lo;0;L;;;;;N;;;;;
+1B2EA;NUSHU CHARACTER-1B2EA;Lo;0;L;;;;;N;;;;;
+1B2EB;NUSHU CHARACTER-1B2EB;Lo;0;L;;;;;N;;;;;
+1B2EC;NUSHU CHARACTER-1B2EC;Lo;0;L;;;;;N;;;;;
+1B2ED;NUSHU CHARACTER-1B2ED;Lo;0;L;;;;;N;;;;;
+1B2EE;NUSHU CHARACTER-1B2EE;Lo;0;L;;;;;N;;;;;
+1B2EF;NUSHU CHARACTER-1B2EF;Lo;0;L;;;;;N;;;;;
+1B2F0;NUSHU CHARACTER-1B2F0;Lo;0;L;;;;;N;;;;;
+1B2F1;NUSHU CHARACTER-1B2F1;Lo;0;L;;;;;N;;;;;
+1B2F2;NUSHU CHARACTER-1B2F2;Lo;0;L;;;;;N;;;;;
+1B2F3;NUSHU CHARACTER-1B2F3;Lo;0;L;;;;;N;;;;;
+1B2F4;NUSHU CHARACTER-1B2F4;Lo;0;L;;;;;N;;;;;
+1B2F5;NUSHU CHARACTER-1B2F5;Lo;0;L;;;;;N;;;;;
+1B2F6;NUSHU CHARACTER-1B2F6;Lo;0;L;;;;;N;;;;;
+1B2F7;NUSHU CHARACTER-1B2F7;Lo;0;L;;;;;N;;;;;
+1B2F8;NUSHU CHARACTER-1B2F8;Lo;0;L;;;;;N;;;;;
+1B2F9;NUSHU CHARACTER-1B2F9;Lo;0;L;;;;;N;;;;;
+1B2FA;NUSHU CHARACTER-1B2FA;Lo;0;L;;;;;N;;;;;
+1B2FB;NUSHU CHARACTER-1B2FB;Lo;0;L;;;;;N;;;;;
1BC00;DUPLOYAN LETTER H;Lo;0;L;;;;;N;;;;;
1BC01;DUPLOYAN LETTER X;Lo;0;L;;;;;N;;;;;
1BC02;DUPLOYAN LETTER P;Lo;0;L;;;;;N;;;;;
@@ -28269,6 +29217,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F248;TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557;So;0;L;<compat> 3014 6557 3015;;;;N;;;;;
1F250;CIRCLED IDEOGRAPH ADVANTAGE;So;0;L;<circle> 5F97;;;;N;;;;;
1F251;CIRCLED IDEOGRAPH ACCEPT;So;0;L;<circle> 53EF;;;;N;;;;;
+1F260;ROUNDED SYMBOL FOR FU;So;0;ON;;;;;N;;;;;
+1F261;ROUNDED SYMBOL FOR LU;So;0;ON;;;;;N;;;;;
+1F262;ROUNDED SYMBOL FOR SHOU;So;0;ON;;;;;N;;;;;
+1F263;ROUNDED SYMBOL FOR XI;So;0;ON;;;;;N;;;;;
+1F264;ROUNDED SYMBOL FOR SHUANGXI;So;0;ON;;;;;N;;;;;
+1F265;ROUNDED SYMBOL FOR CAI;So;0;ON;;;;;N;;;;;
1F300;CYCLONE;So;0;ON;;;;;N;;;;;
1F301;FOGGY;So;0;ON;;;;;N;;;;;
1F302;CLOSED UMBRELLA;So;0;ON;;;;;N;;;;;
@@ -29248,6 +30202,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F6D0;PLACE OF WORSHIP;So;0;ON;;;;;N;;;;;
1F6D1;OCTAGONAL SIGN;So;0;ON;;;;;N;;;;;
1F6D2;SHOPPING TROLLEY;So;0;ON;;;;;N;;;;;
+1F6D3;STUPA;So;0;ON;;;;;N;;;;;
+1F6D4;PAGODA;So;0;ON;;;;;N;;;;;
1F6E0;HAMMER AND WRENCH;So;0;ON;;;;;N;;;;;
1F6E1;SHIELD;So;0;ON;;;;;N;;;;;
1F6E2;OIL DRUM;So;0;ON;;;;;N;;;;;
@@ -29268,6 +30224,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F6F4;SCOOTER;So;0;ON;;;;;N;;;;;
1F6F5;MOTOR SCOOTER;So;0;ON;;;;;N;;;;;
1F6F6;CANOE;So;0;ON;;;;;N;;;;;
+1F6F7;SLED;So;0;ON;;;;;N;;;;;
+1F6F8;FLYING SAUCER;So;0;ON;;;;;N;;;;;
1F700;ALCHEMICAL SYMBOL FOR QUINTESSENCE;So;0;ON;;;;;N;;;;;
1F701;ALCHEMICAL SYMBOL FOR AIR;So;0;ON;;;;;N;;;;;
1F702;ALCHEMICAL SYMBOL FOR FIRE;So;0;ON;;;;;N;;;;;
@@ -29617,6 +30575,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F8AB;RIGHTWARDS FRONT-TILTED SHADOWED WHITE ARROW;So;0;ON;;;;;N;;;;;
1F8AC;WHITE ARROW SHAFT WIDTH ONE;So;0;ON;;;;;N;;;;;
1F8AD;WHITE ARROW SHAFT WIDTH TWO THIRDS;So;0;ON;;;;;N;;;;;
+1F900;CIRCLED CROSS FORMEE WITH FOUR DOTS;So;0;ON;;;;;N;;;;;
+1F901;CIRCLED CROSS FORMEE WITH TWO DOTS;So;0;ON;;;;;N;;;;;
+1F902;CIRCLED CROSS FORMEE;So;0;ON;;;;;N;;;;;
+1F903;LEFT HALF CIRCLE WITH FOUR DOTS;So;0;ON;;;;;N;;;;;
+1F904;LEFT HALF CIRCLE WITH THREE DOTS;So;0;ON;;;;;N;;;;;
+1F905;LEFT HALF CIRCLE WITH TWO DOTS;So;0;ON;;;;;N;;;;;
+1F906;LEFT HALF CIRCLE WITH DOT;So;0;ON;;;;;N;;;;;
+1F907;LEFT HALF CIRCLE;So;0;ON;;;;;N;;;;;
+1F908;DOWNWARD FACING HOOK;So;0;ON;;;;;N;;;;;
+1F909;DOWNWARD FACING NOTCHED HOOK;So;0;ON;;;;;N;;;;;
+1F90A;DOWNWARD FACING HOOK WITH DOT;So;0;ON;;;;;N;;;;;
+1F90B;DOWNWARD FACING NOTCHED HOOK WITH DOT;So;0;ON;;;;;N;;;;;
1F910;ZIPPER-MOUTH FACE;So;0;ON;;;;;N;;;;;
1F911;MONEY-MOUTH FACE;So;0;ON;;;;;N;;;;;
1F912;FACE WITH THERMOMETER;So;0;ON;;;;;N;;;;;
@@ -29632,6 +30602,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F91C;RIGHT-FACING FIST;So;0;ON;;;;;N;;;;;
1F91D;HANDSHAKE;So;0;ON;;;;;N;;;;;
1F91E;HAND WITH INDEX AND MIDDLE FINGERS CROSSED;So;0;ON;;;;;N;;;;;
+1F91F;I LOVE YOU HAND SIGN;So;0;ON;;;;;N;;;;;
1F920;FACE WITH COWBOY HAT;So;0;ON;;;;;N;;;;;
1F921;CLOWN FACE;So;0;ON;;;;;N;;;;;
1F922;NAUSEATED FACE;So;0;ON;;;;;N;;;;;
@@ -29640,7 +30611,17 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F925;LYING FACE;So;0;ON;;;;;N;;;;;
1F926;FACE PALM;So;0;ON;;;;;N;;;;;
1F927;SNEEZING FACE;So;0;ON;;;;;N;;;;;
+1F928;FACE WITH ONE EYEBROW RAISED;So;0;ON;;;;;N;;;;;
+1F929;GRINNING FACE WITH STAR EYES;So;0;ON;;;;;N;;;;;
+1F92A;GRINNING FACE WITH ONE LARGE AND ONE SMALL EYE;So;0;ON;;;;;N;;;;;
+1F92B;FACE WITH FINGER COVERING CLOSED LIPS;So;0;ON;;;;;N;;;;;
+1F92C;SERIOUS FACE WITH SYMBOLS COVERING MOUTH;So;0;ON;;;;;N;;;;;
+1F92D;SMILING FACE WITH SMILING EYES AND HAND COVERING MOUTH;So;0;ON;;;;;N;;;;;
+1F92E;FACE WITH OPEN MOUTH VOMITING;So;0;ON;;;;;N;;;;;
+1F92F;SHOCKED FACE WITH EXPLODING HEAD;So;0;ON;;;;;N;;;;;
1F930;PREGNANT WOMAN;So;0;ON;;;;;N;;;;;
+1F931;BREAST-FEEDING;So;0;ON;;;;;N;;;;;
+1F932;PALMS UP TOGETHER;So;0;ON;;;;;N;;;;;
1F933;SELFIE;So;0;ON;;;;;N;;;;;
1F934;PRINCE;So;0;ON;;;;;N;;;;;
1F935;MAN IN TUXEDO;So;0;ON;;;;;N;;;;;
@@ -29665,6 +30646,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F949;THIRD PLACE MEDAL;So;0;ON;;;;;N;;;;;
1F94A;BOXING GLOVE;So;0;ON;;;;;N;;;;;
1F94B;MARTIAL ARTS UNIFORM;So;0;ON;;;;;N;;;;;
+1F94C;CURLING STONE;So;0;ON;;;;;N;;;;;
1F950;CROISSANT;So;0;ON;;;;;N;;;;;
1F951;AVOCADO;So;0;ON;;;;;N;;;;;
1F952;CUCUMBER;So;0;ON;;;;;N;;;;;
@@ -29680,6 +30662,19 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F95C;PEANUTS;So;0;ON;;;;;N;;;;;
1F95D;KIWIFRUIT;So;0;ON;;;;;N;;;;;
1F95E;PANCAKES;So;0;ON;;;;;N;;;;;
+1F95F;DUMPLING;So;0;ON;;;;;N;;;;;
+1F960;FORTUNE COOKIE;So;0;ON;;;;;N;;;;;
+1F961;TAKEOUT BOX;So;0;ON;;;;;N;;;;;
+1F962;CHOPSTICKS;So;0;ON;;;;;N;;;;;
+1F963;BOWL WITH SPOON;So;0;ON;;;;;N;;;;;
+1F964;CUP WITH STRAW;So;0;ON;;;;;N;;;;;
+1F965;COCONUT;So;0;ON;;;;;N;;;;;
+1F966;BROCCOLI;So;0;ON;;;;;N;;;;;
+1F967;PIE;So;0;ON;;;;;N;;;;;
+1F968;PRETZEL;So;0;ON;;;;;N;;;;;
+1F969;CUT OF MEAT;So;0;ON;;;;;N;;;;;
+1F96A;SANDWICH;So;0;ON;;;;;N;;;;;
+1F96B;CANNED FOOD;So;0;ON;;;;;N;;;;;
1F980;CRAB;So;0;ON;;;;;N;;;;;
1F981;LION FACE;So;0;ON;;;;;N;;;;;
1F982;SCORPION;So;0;ON;;;;;N;;;;;
@@ -29698,7 +30693,36 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F98F;RHINOCEROS;So;0;ON;;;;;N;;;;;
1F990;SHRIMP;So;0;ON;;;;;N;;;;;
1F991;SQUID;So;0;ON;;;;;N;;;;;
+1F992;GIRAFFE FACE;So;0;ON;;;;;N;;;;;
+1F993;ZEBRA FACE;So;0;ON;;;;;N;;;;;
+1F994;HEDGEHOG;So;0;ON;;;;;N;;;;;
+1F995;SAUROPOD;So;0;ON;;;;;N;;;;;
+1F996;T-REX;So;0;ON;;;;;N;;;;;
+1F997;CRICKET;So;0;ON;;;;;N;;;;;
1F9C0;CHEESE WEDGE;So;0;ON;;;;;N;;;;;
+1F9D0;FACE WITH MONOCLE;So;0;ON;;;;;N;;;;;
+1F9D1;ADULT;So;0;ON;;;;;N;;;;;
+1F9D2;CHILD;So;0;ON;;;;;N;;;;;
+1F9D3;OLDER ADULT;So;0;ON;;;;;N;;;;;
+1F9D4;BEARDED PERSON;So;0;ON;;;;;N;;;;;
+1F9D5;PERSON WITH HEADSCARF;So;0;ON;;;;;N;;;;;
+1F9D6;PERSON IN STEAMY ROOM;So;0;ON;;;;;N;;;;;
+1F9D7;PERSON CLIMBING;So;0;ON;;;;;N;;;;;
+1F9D8;PERSON IN LOTUS POSITION;So;0;ON;;;;;N;;;;;
+1F9D9;MAGE;So;0;ON;;;;;N;;;;;
+1F9DA;FAIRY;So;0;ON;;;;;N;;;;;
+1F9DB;VAMPIRE;So;0;ON;;;;;N;;;;;
+1F9DC;MERPERSON;So;0;ON;;;;;N;;;;;
+1F9DD;ELF;So;0;ON;;;;;N;;;;;
+1F9DE;GENIE;So;0;ON;;;;;N;;;;;
+1F9DF;ZOMBIE;So;0;ON;;;;;N;;;;;
+1F9E0;BRAIN;So;0;ON;;;;;N;;;;;
+1F9E1;ORANGE HEART;So;0;ON;;;;;N;;;;;
+1F9E2;BILLED CAP;So;0;ON;;;;;N;;;;;
+1F9E3;SCARF;So;0;ON;;;;;N;;;;;
+1F9E4;GLOVES;So;0;ON;;;;;N;;;;;
+1F9E5;COAT;So;0;ON;;;;;N;;;;;
+1F9E6;SOCKS;So;0;ON;;;;;N;;;;;
20000;<CJK Ideograph Extension B, First>;Lo;0;L;;;;;N;;;;;
2A6D6;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;;
2A700;<CJK Ideograph Extension C, First>;Lo;0;L;;;;;N;;;;;
@@ -29707,6 +30731,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
2B81D;<CJK Ideograph Extension D, Last>;Lo;0;L;;;;;N;;;;;
2B820;<CJK Ideograph Extension E, First>;Lo;0;L;;;;;N;;;;;
2CEA1;<CJK Ideograph Extension E, Last>;Lo;0;L;;;;;N;;;;;
+2CEB0;<CJK Ideograph Extension F, First>;Lo;0;L;;;;;N;;;;;
+2EBE0;<CJK Ideograph Extension F, Last>;Lo;0;L;;;;;N;;;;;
2F800;CJK COMPATIBILITY IDEOGRAPH-2F800;Lo;0;L;4E3D;;;;N;;;;;
2F801;CJK COMPATIBILITY IDEOGRAPH-2F801;Lo;0;L;4E38;;;;N;;;;;
2F802;CJK COMPATIBILITY IDEOGRAPH-2F802;Lo;0;L;4E41;;;;N;;;;;
diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript
index fefd7d3b70..674e5a0628 100755
--- a/lib/stdlib/uc_spec/gen_unicode_mod.escript
+++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript
@@ -65,7 +65,7 @@ main(_) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
parse_unicode_data(Line0, Acc) ->
- Line = string:strip(Line0, right, $\n),
+ Line = string:chomp(Line0),
[CodePoint,Name,_Cat,Class,_BiDi,Decomp,
_N1,_N2,_N3,_BDMirror,_Uni1,_Iso|Case] = tokens(Line, ";"),
{Dec,Comp} = case to_decomp(Decomp) of
@@ -78,14 +78,14 @@ parse_unicode_data(Line0, Acc) ->
|Acc].
to_class(String) ->
- list_to_integer(string:strip(String, both)).
+ list_to_integer(string:trim(String, both)).
to_decomp("") -> [];
to_decomp("<" ++ Str) ->
- [Tag,Rest] = string:tokens(Str, ">"),
+ [Tag,Rest] = string:lexemes(Str, ">"),
{list_to_atom(Tag), to_decomp(Rest)};
to_decomp(CodePoints) ->
- CPL = string:tokens(CodePoints, " "),
+ CPL = string:lexemes(CodePoints, " "),
[hex_to_int(CP) || CP <- CPL].
to_case(["","",""]) -> [];
@@ -105,20 +105,20 @@ parse_special_casing(Line, Table) ->
array:set(CP, Entry#cp{cs=Case}, Table).
to_scase([Lower,Title,Upper|_]) ->
- {unlist([hex_to_int(CP) || CP <- string:strip(string:tokens(Upper, " "), both)]),
- unlist([hex_to_int(CP) || CP <- string:strip(string:tokens(Lower, " "), both)]),
- unlist([hex_to_int(CP) || CP <- string:strip(string:tokens(Title, " "), both)]),
+ {unlist([hex_to_int(CP) || CP <- string:lexemes(Upper, " ")]),
+ unlist([hex_to_int(CP) || CP <- string:lexemes(Lower, " ")]),
+ unlist([hex_to_int(CP) || CP <- string:lexemes(Title, " ")]),
[]}.
parse_case_folding(Line, Table) ->
[CodePoint, Class0, CaseStr |_Comments] = tokens(Line, ";"),
- Class = string:strip(Class0, both),
+ Class = string:trim(Class0, both),
if Class =:= "T" -> Table; %% Do not support localization yet
Class =:= "S" -> Table; %% Ignore simple
true ->
CP = hex_to_int(CodePoint),
Case = unlist([hex_to_int(CPC) ||
- CPC <- string:strip(string:tokens(CaseStr, " "), both)]),
+ CPC <- string:lexemes(CaseStr, " ")]),
#cp{cs={U,L,T,_}} = Entry = array:get(CP, Table),
array:set(CP, Entry#cp{cs={U,L,T,Case}}, Table)
end.
@@ -186,7 +186,7 @@ gen_static(Fd) ->
" {U,L} -> #{upper=>U,lower=>L,title=>U,fold=>L};\n"
" {U,L,T,F} -> #{upper=>U,lower=>L,title=>T,fold=>F}\n"
" end.\n\n"),
- io:put_chars(Fd, "spec_version() -> {9,0}.\n\n\n"),
+ io:put_chars(Fd, "spec_version() -> {10,0}.\n\n\n"),
io:put_chars(Fd, "class(Codepoint) -> {CCC,_,_} = unicode_table(Codepoint),\n CCC.\n\n"),
io:put_chars(Fd, "-spec uppercase(unicode:chardata()) -> "
"maybe_improper_list(gc(),unicode:chardata()).\n"),
@@ -869,10 +869,10 @@ optimize_ranges_1(Rs) ->
hex_to_int([]) -> [];
hex_to_int(HexStr) ->
- list_to_integer(string:strip(HexStr, both), 16).
+ list_to_integer(string:trim(HexStr, both), 16).
to_atom(Str) ->
- list_to_atom(string:to_lower(string:strip(Str, both))).
+ list_to_atom(string:lowercase(string:trim(Str, both))).
foldl(Fun, Acc, Fd) ->
Get = fun() -> file:read_line(Fd) end,
@@ -892,7 +892,7 @@ foldl_1(Fun, Acc, Get) ->
-%% Differs from string:tokens, it returns empty string as token between two delimiters
+%% Differs from string:lexemes, it returns empty string as token between two delimiters
tokens(S, [C]) ->
tokens(lists:reverse(S), C, []).
diff --git a/lib/syntax_tools/src/Makefile b/lib/syntax_tools/src/Makefile
index 8325db45a8..c21d2f49c8 100644
--- a/lib/syntax_tools/src/Makefile
+++ b/lib/syntax_tools/src/Makefile
@@ -75,7 +75,7 @@ $(EBIN)/%.$(EMULATOR):%.erl
# special rules and dependencies to apply the transform to itself
$(EBIN)/merl_transform.beam: $(EBIN)/merl.beam ./merl_transform.beam \
- ../include/merl.hrl \
+ ../include/merl.hrl $(EBIN)/erl_comment_scan.beam \
$(EBIN)/erl_syntax.beam $(EBIN)/erl_syntax_lib.beam
./merl_transform.beam: ./merl_transform.erl $(EBIN)/merl.beam \
../include/merl.hrl
diff --git a/lib/syntax_tools/src/erl_comment_scan.erl b/lib/syntax_tools/src/erl_comment_scan.erl
index 07e501e553..e3eb95b819 100644
--- a/lib/syntax_tools/src/erl_comment_scan.erl
+++ b/lib/syntax_tools/src/erl_comment_scan.erl
@@ -208,7 +208,7 @@ scan_comment([], Cs1, L, Col, M, Ack) ->
seen_comment(Cs, Cs1, L, Col, M, Ack) ->
%% Compute indentation and strip trailing spaces
N = Col - M,
- Text = lists:reverse(string:strip(Cs1, left)),
+ Text = lists:reverse(string:trim(Cs1, leading)),
Ack1 = [{L, Col + 1, N, Text} | Ack],
scan_lines(Cs, L + 1, 0, 0, Ack1).
diff --git a/lib/syntax_tools/src/merl_transform.erl b/lib/syntax_tools/src/merl_transform.erl
index b298bc407f..571d7e4d86 100644
--- a/lib/syntax_tools/src/merl_transform.erl
+++ b/lib/syntax_tools/src/merl_transform.erl
@@ -196,7 +196,7 @@ var_name(V) -> V.
var_to_tag(V) when is_integer(V) -> V;
var_to_tag(V) ->
- list_to_atom(string:to_lower(atom_to_list(V))).
+ list_to_atom(string:lowercase(atom_to_list(V))).
pre_expand_case(Expr, Clauses, Line) ->
merl:qquote(Line, "merl:switch(_@expr, _@clauses)",
diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el
index 411e0e13df..429188b028 100644
--- a/lib/tools/emacs/erlang.el
+++ b/lib/tools/emacs/erlang.el
@@ -900,6 +900,11 @@ resulting regexp is surrounded by \\_< and \\_>."
"display"
"display_nl"
"display_string"
+ "dist_get_stat"
+ "dist_ctrl_get_data"
+ "dist_ctrl_get_data_notification"
+ "dist_ctrl_input_handler"
+ "dist_ctrl_put_data"
"dist_exit"
"dlink"
"dmonitor_node"
diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl
index 5517882ffa..801bbc7461 100644
--- a/lib/tools/src/cover.erl
+++ b/lib/tools/src/cover.erl
@@ -2439,11 +2439,11 @@ do_analyse_to_file1(Module, OutFile, ErlFile, HTML) ->
Timestamp =
io_lib:format("~p-~s-~s at ~s:~s:~s",
[Y,
- string:right(integer_to_list(Mo), 2, $0),
- string:right(integer_to_list(D), 2, $0),
- string:right(integer_to_list(H), 2, $0),
- string:right(integer_to_list(Mi), 2, $0),
- string:right(integer_to_list(S), 2, $0)]),
+ string:pad(integer_to_list(Mo), 2, leading, $0),
+ string:pad(integer_to_list(D), 2, leading, $0),
+ string:pad(integer_to_list(H), 2, leading, $0),
+ string:pad(integer_to_list(Mi), 2, leading, $0),
+ string:pad(integer_to_list(S), 2, leading, $0)]),
H2Bin = unicode:characters_to_binary(
["File generated from ",ErlFile," by COVER ",
@@ -2493,12 +2493,12 @@ print_lines(Module, CovLines, InFd, OutFd, L, HTML) ->
if N=:=0, HTML=:=true ->
LineNoNL = Line -- "\n",
Str = " 0",
- %%Str = string:right("0", 6, 32),
+ %%Str = string:pad("0", 6, leading, $\s),
RedLine = ["<font color=red>",Str,fill1(),
LineNoNL,"</font>\n"],
ok = file:write(OutFd, RedLine);
N < 1000000 ->
- Str = string:right(integer_to_list(N), 6, 32),
+ Str = string:pad(integer_to_list(N), 6, leading, $\s),
ok = file:write(OutFd, [Str,fill1(),Line]);
N < 10000000 ->
Str = integer_to_list(N),
diff --git a/lib/tools/src/xref_utils.erl b/lib/tools/src/xref_utils.erl
index 02e207d40c..eca751337b 100644
--- a/lib/tools/src/xref_utils.erl
+++ b/lib/tools/src/xref_utils.erl
@@ -557,12 +557,9 @@ subdir(Dir, SubDir, true) ->
%% Avoid "App-01.01" - the zeroes will be lost.
filename2appl(File) ->
- Pos = string:rstr(File, "-"),
- true = Pos > 1,
- V = string:sub_string(File, Pos+1),
- true = string:len(V) > 0,
- VsnT = string:tokens(V, "."),
- ApplName = string:sub_string(File, 1, Pos-1),
+ [ApplName, V] = string:split(File, "-", trailing),
+ true = string:length(V) > 0,
+ VsnT = string:lexemes(V, "."),
Vsn = [list_to_integer(Vsn) || Vsn <- VsnT],
{list_to_atom(ApplName),Vsn}.
diff --git a/lib/tools/test/xref_SUITE.erl b/lib/tools/test/xref_SUITE.erl
index d651cbcfee..018f632948 100644
--- a/lib/tools/test/xref_SUITE.erl
+++ b/lib/tools/test/xref_SUITE.erl
@@ -1112,27 +1112,16 @@ read_expected(Version) ->
{POS7+2,{FF,{erlang,spawn_opt,4}}},
{POS8+1,{FF,{hej,san,1}}},
{POS8+4,{FF,{a,b,1}}},
- {POS8+4,{FF,{erlang,apply,2}}},
- {POS8+5,{FF,{erlang,apply,2}}},
{POS8+6,{FF,{m,f,1}}},
{POS9+1,{FF,{read,bi,0}}},
{POS9+2,{FF,{a,b,1}}},
- {POS9+2,{FF,{erlang,apply,2}}},
- {POS9+3,{FF,{erlang,apply,2}}},
- {POS9+4,{FF,{erlang,apply,2}}},
{POS9+4,{FF,{erlang,not_a_function,1}}},
{POS9+5,{FF,{mod,func,2}}},
{POS9+6,{FF,{erlang,apply,1}}},
- {POS9+7,{FF,{erlang,apply,2}}},
{POS9+7,{FF,{math,add3,1}}},
{POS9+8,{FF,{q,f,1}}},
- {POS10+4,{FF,{erlang,apply,2}}},
{POS10+5,{FF,{mod1,fun1,1}}},
- {POS11+6,{FF,{erlang,apply,2}}},
- {POS12+1,{FF,{erlang,apply,2}}},
- {POS12+4,{FF,{erlang,apply,2}}},
{POS12+5,{FF,{m3,f3,2}}},
- {POS12+7,{FF,{erlang,apply,2}}},
{POS13+1,{FF,{dm,df,1}}},
{POS13+6,{{read,bi,0},{foo,module_info,0}}},
{POS13+7,{{read,bi,0},{read,module_info,0}}},
@@ -1162,15 +1151,26 @@ read_expected(Version) ->
{POS3+3, {FF,{erlang,spawn_link,3}}},
{POS3+4, {FF,{erlang,spawn_link,3}}},
{POS6+4, {FF,{erlang,spawn,3}}},
+ {POS8+4,{FF,{erlang,apply,2}}},
+ {POS8+5,{FF,{erlang,apply,2}}},
{POS8+6,{FF,{erlang,apply,3}}},
{POS8+7,{FF,{erlang,apply,3}}},
{POS9+1,{FF,{erlang,apply,3}}},
+ {POS9+2,{FF,{erlang,apply,2}}},
+ {POS9+3,{FF,{erlang,apply,2}}},
+ {POS9+4,{FF,{erlang,apply,2}}},
{POS9+5,{FF,{erlang,apply,3}}},
+ {POS9+7,{FF,{erlang,apply,2}}},
+ {POS10+4,{FF,{erlang,apply,2}}},
{POS11+1,{FF,{erlang,apply,3}}},
{POS11+2,{FF,{erlang,apply,3}}},
{POS11+3,{FF,{erlang,apply,3}}},
{POS11+4,{FF,{erlang,apply,3}}},
+ {POS11+6,{FF,{erlang,apply,2}}},
+ {POS12+1,{FF,{erlang,apply,2}}},
+ {POS12+4,{FF,{erlang,apply,2}}},
{POS12+5,{FF,{erlang,apply,3}}},
+ {POS12+7,{FF,{erlang,apply,2}}},
{POS12+8,{FF,{erlang,apply,3}}},
{POS13+5, {{read,bi,0},{erlang,length,1}}},
{POS14+3, {{read,bi,0},{erlang,length,1}}}],
diff --git a/lib/wx/examples/demo/demo.erl b/lib/wx/examples/demo/demo.erl
index 8fb70ad7b0..8f3291305b 100644
--- a/lib/wx/examples/demo/demo.erl
+++ b/lib/wx/examples/demo/demo.erl
@@ -411,7 +411,7 @@ find(Ed) ->
keyWords() ->
L = ["after","begin","case","try","cond","catch","andalso","orelse",
- "end","fun","if","let","of","query","receive","when","bnot","not",
+ "end","fun","if","let","of","receive","when","bnot","not",
"div","rem","band","and","bor","bxor","bsl","bsr","or","xor"],
lists:flatten([K ++ " " || K <- L] ++ [0]).